Compare commits

...

75 Commits

Author SHA1 Message Date
dd645e6da4 Merge #1605
1605: Fix pacic when decoding r=curquiza a=curquiza

Update milli to fix the panic during document deletion

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
2021-08-23 11:06:45 +00:00
149f46c184 Fix pacic when decoding 2021-08-23 12:37:51 +02:00
3e27d5e885 Merge #1596
1596: Update milli and tokenizer version: fix panic during indexation r=curquiza a=curquiza

Fixes #1590 

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
2021-08-18 13:44:30 +00:00
38fc876704 Update tokenizer and new milli version with new tags 2021-08-18 14:55:10 +02:00
39d5a99095 Update milli and tokenizer version 2021-08-18 12:09:34 +02:00
2beb306834 Merge #1577
1577: Update milli dependency: fix facet values bugs r=Kerollmops a=curquiza

Fixes #1576 

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
2021-08-16 16:13:42 +00:00
f3e595e2f0 Update milli dependency 2021-08-16 13:36:42 +02:00
5d80d11b23 Merge #1580
1580: Update telemetry link r=curquiza a=curquiza

Here is the page the user will have: https://dev.docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html
instead of: https://docs.meilisearch.com/reference/features/configuration.html#disable-analytics

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
2021-08-12 17:11:30 +00:00
621529e9dc Update telemetry link 2021-08-12 18:58:07 +02:00
535aff8f7e Merge #1578
1578: Update tokenizer version to v0.2.4 r=ManyTheFish a=curquiza



Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
2021-08-12 15:27:12 +00:00
7531280764 Update tokenizer version to v0.2.4 2021-08-12 13:55:47 +02:00
7e3b2ddff2 Merge #1554
1554: Fix dump v1 (attributesForFaceting, and criteria) r=curquiza a=MarinPostma

close #1553


Co-authored-by: mpostma <postma.marin@protonmail.com>
2021-08-05 19:45:52 +00:00
312d93961a Merge #1556
1556: Update milli to v0.9.0 r=MarinPostma a=curquiza

Fixes #1552 

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
2021-08-05 14:04:55 +00:00
8f05d8d546 fix clippy warnings 2021-08-05 16:00:47 +02:00
f5ddea481a reintroduce exactness 2021-08-05 15:59:39 +02:00
29ca8271b3 test dumpv1 format regression 2021-08-05 15:59:39 +02:00
3084537d1e restore attributes for faceting in dump v1 2021-08-05 15:59:39 +02:00
86ac994543 Merge #1557
1557: Fix docs link anchor r=MarinPostma a=curquiza

thank you `@guimachiavelli` 

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
2021-08-05 13:34:48 +00:00
992b082c6f Fix docs link anchor 2021-08-05 13:28:32 +02:00
31fe263356 Update milli to v0.9.0 2021-08-05 13:08:27 +02:00
7a0b20c740 Merge #1532
1532: Start writing documentation for newcomers r=MarinPostma a=irevoire



Co-authored-by: Tamo <tamo@meilisearch.com>
2021-08-03 09:26:45 +00:00
9810f6b695 Merge #1540
1540: Update milli to version 0.8.1 r=curquiza a=curquiza

Integrates this fix into MeiliSearch https://github.com/meilisearch/milli/pull/296

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
2021-07-29 17:15:52 +00:00
09c74c04a0 Merge #1539
1539: Use serdeval for validating json format. r=curquiza a=MarinPostma

uses [serdeval](https://github.com/MarinPostma/serdeval) to validate that the json payload is valid json, and in the correct format.

fix #1535


Co-authored-by: mpostma <postma.marin@protonmail.com>
2021-07-29 17:05:13 +00:00
b6cc932c09 Merge #1541
1541: Make clippy happy r=curquiza a=curquiza



Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
2021-07-29 16:53:53 +00:00
1b5d918cb9 Fix rustfmt 2021-07-29 18:32:09 +02:00
bf76d4a43c Make clippy happy 2021-07-29 18:14:36 +02:00
53b4b2fcbc Use serdeval for validating json format. 2021-07-29 18:02:54 +02:00
9a8629a6a9 Update milli 2021-07-29 17:45:31 +02:00
78308365ec fix typos 2021-07-29 14:40:41 +02:00
976075578f Merge #1537
1537: Import `.git` to the docker build image to fix vergen r=curquiza a=irevoire

I observed a small difference in the size of the build image, but I think we can allow it:
![image](https://user-images.githubusercontent.com/7032172/127369567-d03f9a41-3ad5-4933-888e-a3777df8c6cf.png)

I was not able to see any difference in build time, though.

Co-authored-by: Tamo <tamo@meilisearch.com>
2021-07-29 12:31:55 +00:00
243233f652 import .git to docker to fix vergen 2021-07-28 19:12:40 +02:00
d66eea42bb Merge #1536
1536: Remove ARMv7 binary publish r=MarinPostma a=curquiza

Fixes #1315 

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
2021-07-28 15:39:34 +00:00
c55f73bbc3 Remove ARMv7 support 2021-07-28 17:29:40 +02:00
3e30d4270b Merge #1533
1533: Update milli version to v0.8.0 r=MarinPostma a=curquiza

- Update milli, heed and obkv
- fix relevancy issue and the `facetsDistribution` display

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
2021-07-28 11:15:31 +00:00
80916baa21 Add FieldId in import 2021-07-28 12:25:13 +02:00
1df8f041bd Update meilisearch-http/src/index/search.rs
Co-authored-by: marin <postma.marin@protonmail.com>
2021-07-28 12:10:25 +02:00
6a6e2a8cd1 Update meilisearch-http/src/index/search.rs
Co-authored-by: marin <postma.marin@protonmail.com>
2021-07-28 12:08:51 +02:00
f9d337b320 Update meilisearch-http/src/index/search.rs
Co-authored-by: marin <postma.marin@protonmail.com>
2021-07-28 12:08:36 +02:00
feb069f604 Update meilisearch-http/src/index/search.rs
Co-authored-by: marin <postma.marin@protonmail.com>
2021-07-28 12:08:28 +02:00
7e0eed5772 Update meilisearch-http/src/index/search.rs
Co-authored-by: marin <postma.marin@protonmail.com>
2021-07-28 12:08:24 +02:00
9bdd040dd0 Update meilisearch-http/src/index/mod.rs
Co-authored-by: marin <postma.marin@protonmail.com>
2021-07-28 12:08:19 +02:00
e5dabf265a Update milli version to v0.8.0 2021-07-28 10:52:47 +02:00
1a1046a0ef start writing some documentation for newcomers 2021-07-27 16:35:42 +02:00
dd18319b44 Merge #1530
1530: Update mini-dashboard version to v0.1.4 r=irevoire a=mdubus



Co-authored-by: Morgane Dubus <30866152+mdubus@users.noreply.github.com>
2021-07-27 10:11:02 +00:00
d3cd7e92d1 Update mini-dashboard version to v0.1.4 2021-07-27 11:44:20 +02:00
553e7d8aaa Merge #1528
1528: Update of the Date Time Format in commitDate  r=MarinPostma a=irevoire

Since we were relying on a [super old version of `vergen`](https://docs.rs/crate/vergen/3.0.1), we could not get the `commit timestamp`, so I updated `vergen` to the latest version.
This also allows us to remove all the features we don't use.

closes #1522

Co-authored-by: Tamo <tamo@meilisearch.com>
2021-07-27 07:49:31 +00:00
f79b8287f5 update vergen 2021-07-26 15:25:30 +02:00
b4c98f6cc3 Merge #1521
1521: Sentry was never sending anything r=Kerollmops a=irevoire

@Kerollmops noticed that we had no log of this release in sentry, and it look like I badly tested my code after ignoring the “No space left on device” errors.

Now it should be fixed.

Co-authored-by: Tamo <tamo@meilisearch.com>
2021-07-21 14:46:56 +00:00
5d4a0ac844 sentry was never sending anything 2021-07-21 11:50:54 +02:00
0136b02e5b Merge #1498
1498: Show the filterable and not the faceted attributes in the settings r=Kerollmops a=Kerollmops

Fixes #1497

Co-authored-by: Clément Renault <clement@meilisearch.com>
2021-07-13 07:27:14 +00:00
f49a01703a Show the filterable and not the faceted attributes in the settings 2021-07-09 16:11:37 +02:00
e4f82aa441 Merge #1494
1494: Add cache to the ci r=irevoire a=irevoire

closes #1446 

Co-authored-by: Tamo <tamo@meilisearch.com>
2021-07-08 14:06:03 +00:00
751d1af2a6 Merge #1492
1492: auth tests r=irevoire a=MarinPostma

add regression tests on route authentication.


Co-authored-by: mpostma <postma.marin@protonmail.com>
2021-07-08 13:13:55 +00:00
076d8fbb84 add cache to the ci 2021-07-08 11:19:12 +02:00
4b2d01a453 Merge #1484
1484: Add MeiliSearch version to issue template r=irevoire a=bidoubiwa

It is relevant to know the version of MeiliSearch before any other additional information that might be important to know.

We could also reduce the number of required information asked to the user. I would like to suggest the following:

Instead of the section of `Desktop` and `Smartphone`  I would just improve the last section

```
**Additional context**
Additional information that may be relevant to the issue.
[e.g. architecture, device, OS, browser]
```

By applying this, the template final look will be the following: 

-----

**Describe the bug**
A clear and concise description of what the bug is.

**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error

**Expected behavior**
A clear and concise description of what you expected to happen.

**Screenshots**
If applicable, add screenshots to help explain your problem.

**MeiliSearch version:** [e.g. v0.20.0]

**Additional context**
Additional information that may be relevant to the issue.
[e.g. architecture, device, OS, browser]

Co-authored-by: Charlotte Vermandel <charlottevermandel@gmail.com>
2021-07-08 08:49:35 +00:00
a71fa25ebe auth tests 2021-07-07 17:47:48 +02:00
b4db54cb1f Merge #1488
1488: fix search permissions r=MarinPostma a=MarinPostma

fix #1485


Co-authored-by: mpostma <postma.marin@protonmail.com>
2021-07-07 13:16:07 +00:00
b2ca600e79 Remove unecessary questions 2021-07-07 11:15:08 +02:00
83725a1330 fix search permissions 2021-07-07 10:39:04 +02:00
587b837a6c Add MeiliSearch version to issue template 2021-07-06 22:04:15 +02:00
2844fe959f Merge #1483
1483: search tests r=MarinPostma a=MarinPostma

adds search tests from #1440.

Co-authored-by: mpostma <postma.marin@protonmail.com>
2021-07-06 15:13:07 +00:00
41e271974a add tests 2021-07-06 16:21:15 +02:00
520d37983c implement index search methods 2021-07-06 11:54:09 +02:00
487d82773a Merge #1481
1481: fix bug in index deletion r=Kerollmops a=MarinPostma

this bug was caused by a heed iterator entry being deleted while still holding a reference to it.


close #1333


Co-authored-by: mpostma <postma.marin@protonmail.com>
2021-07-06 08:07:30 +00:00
066085f6f5 fix index deletion bug 2021-07-05 18:42:13 +02:00
0d1f5b7193 Merge #1469
1469: Return 201 on index creation r=Kerollmops a=MarinPostma

fix #1467


Co-authored-by: mpostma <postma.marin@protonmail.com>
2021-07-05 15:42:13 +00:00
2f3a439566 fix tests 2021-07-05 16:31:52 +02:00
9681ffca52 change index create http code 2021-07-05 16:31:51 +02:00
fddc60f893 Merge #1471
1471: Bump milli to 0.7.2 r=irevoire a=irevoire



Co-authored-by: Tamo <tamo@meilisearch.com>
2021-07-05 13:29:38 +00:00
0f024cc225 Merge #1478
1478: refactor routes r=irevoire a=MarinPostma

refactor the route directory, so the module tree follows the route structure


Co-authored-by: mpostma <postma.marin@protonmail.com>
2021-07-05 12:55:39 +00:00
575ec2a06f refactor routes 2021-07-05 14:33:48 +02:00
83aef0a27d Merge #1473
1473: Update loop r=MarinPostma a=irevoire



Co-authored-by: mpostma <postma.marin@protonmail.com>
2021-07-05 12:32:29 +00:00
bc85d30076 add test 2021-07-05 12:33:28 +02:00
bc417726fc fix update loop bug 2021-07-05 12:33:22 +02:00
9949a2a930 bump milli to 0.7.2 2021-07-05 12:19:27 +02:00
40 changed files with 901 additions and 426 deletions

View File

@ -1,5 +1,4 @@
target
Dockerfile
.dockerignore
.git
.gitignore

View File

@ -23,16 +23,8 @@ A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- OS: [e.g. iOS]
- Browser [e.g. chrome, safari]
- Version [e.g. 22]
**Smartphone (please complete the following information):**
- Device: [e.g. iPhone6]
- OS: [e.g. iOS8.1]
- Browser [e.g. stock browser, safari]
- Version [e.g. 22]
**MeiliSearch version:** [e.g. v0.20.0]
**Additional context**
Add any other context about the problem here.
Additional information that may be relevant to the issue.
[e.g. architecture, device, OS, browser]

View File

@ -37,30 +37,6 @@ jobs:
asset_name: ${{ matrix.asset_name }}
tag: ${{ github.ref }}
publish-armv7:
name: Publish for ARMv7
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v1.0.0
- uses: uraimo/run-on-arch-action@v1.0.7
id: runcmd
with:
architecture: armv7
distribution: ubuntu18.04
run: |
apt update
apt install -y curl gcc make
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
source $HOME/.cargo/env
cargo build --release --locked
- name: Upload the binary to release
uses: svenstaro/upload-release-action@v1-release
with:
repo_token: ${{ secrets.PUBLISH_TOKEN }}
file: target/release/meilisearch
asset_name: meilisearch-linux-armv7
tag: ${{ github.ref }}
publish-armv8:
name: Publish for ARMv8
runs-on: ubuntu-18.04

View File

@ -22,6 +22,13 @@ jobs:
os: [ubuntu-18.04, macos-latest, windows-latest]
steps:
- uses: actions/checkout@v2
- name: Cache dependencies
uses: actions/cache@v2
with:
path: |
~/.cargo
./target
key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
- name: Run cargo check without any default features
uses: actions-rs/cargo@v1
with:
@ -38,6 +45,13 @@ jobs:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
- name: Cache dependencies
uses: actions/cache@v2
with:
path: |
~/.cargo
./target
key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
- uses: actions-rs/toolchain@v1
with:
profile: minimal
@ -55,6 +69,13 @@ jobs:
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
- name: Cache dependencies
uses: actions/cache@v2
with:
path: |
~/.cargo
./target
key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
- uses: actions-rs/toolchain@v1
with:
profile: minimal

180
Cargo.lock generated
View File

@ -620,6 +620,17 @@ dependencies = [
"vec_map",
]
[[package]]
name = "concat-arrays"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1df715824eb382e34b7afb7463b0247bf41538aeba731fba05241ecdb5dc3747"
dependencies = [
"proc-macro2 1.0.27",
"quote 1.0.9",
"syn 1.0.73",
]
[[package]]
name = "const_fn"
version = "0.4.8"
@ -831,6 +842,26 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "enum-iterator"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4eeac5c5edb79e4e39fe8439ef35207780a11f69c52cbe424ce3dfad4cb78de6"
dependencies = [
"enum-iterator-derive",
]
[[package]]
name = "enum-iterator-derive"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c134c37760b27a871ba422106eedbb8247da973a09e82558bf26d619c882b159"
dependencies = [
"proc-macro2 1.0.27",
"quote 1.0.9",
"syn 1.0.73",
]
[[package]]
name = "env_logger"
version = "0.8.4"
@ -1067,12 +1098,37 @@ dependencies = [
"wasi 0.10.0+wasi-snapshot-preview1",
]
[[package]]
name = "getset"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24b328c01a4d71d2d8173daa93562a73ab0fe85616876f02500f53d82948c504"
dependencies = [
"proc-macro-error",
"proc-macro2 1.0.27",
"quote 1.0.9",
"syn 1.0.73",
]
[[package]]
name = "gimli"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189"
[[package]]
name = "git2"
version = "0.13.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9831e983241f8c5591ed53f17d874833e2fa82cac2625f3888c50cbfe136cba"
dependencies = [
"bitflags",
"libc",
"libgit2-sys",
"log",
"url",
]
[[package]]
name = "glob"
version = "0.3.0"
@ -1146,7 +1202,7 @@ dependencies = [
[[package]]
name = "heed"
version = "0.12.0"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
dependencies = [
"byteorder",
"heed-traits",
@ -1164,12 +1220,12 @@ dependencies = [
[[package]]
name = "heed-traits"
version = "0.7.0"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
[[package]]
name = "heed-types"
version = "0.7.2"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.0#6c0b95793a805dc598f05c119494e6c069de0326"
source = "git+https://github.com/Kerollmops/heed?tag=v0.12.1#8e5dc6d71c8166a8d7d0db059e6e51478942b551"
dependencies = [
"bincode",
"heed-traits",
@ -1437,6 +1493,30 @@ version = "0.2.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6"
[[package]]
name = "libgit2-sys"
version = "0.12.21+1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86271bacd72b2b9e854c3dcfb82efd538f15f870e4c11af66900effb462f6825"
dependencies = [
"cc",
"libc",
"libz-sys",
"pkg-config",
]
[[package]]
name = "libz-sys"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "linked-hash-map"
version = "0.5.4"
@ -1578,7 +1658,7 @@ dependencies = [
"log",
"main_error",
"meilisearch-error",
"meilisearch-tokenizer 0.2.3",
"meilisearch-tokenizer",
"memmap",
"milli",
"mime",
@ -1586,7 +1666,6 @@ dependencies = [
"num_cpus",
"obkv",
"once_cell",
"oxidized-json-checker",
"parking_lot",
"paste",
"pin-project",
@ -1599,6 +1678,7 @@ dependencies = [
"serde",
"serde_json",
"serde_url_params",
"serdeval",
"sha-1 0.9.6",
"sha2",
"siphasher",
@ -1619,24 +1699,8 @@ dependencies = [
[[package]]
name = "meilisearch-tokenizer"
version = "0.2.2"
source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.2#eda4ed4968c8ac973cf1707ef89bd7012bb2722f"
dependencies = [
"character_converter",
"cow-utils",
"deunicode",
"fst",
"jieba-rs",
"once_cell",
"slice-group-by",
"unicode-segmentation",
"whatlang",
]
[[package]]
name = "meilisearch-tokenizer"
version = "0.2.3"
source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.3#c2399c3f879144ad92e20ae057e14984dfd22781"
version = "0.2.5"
source = "git+https://github.com/meilisearch/tokenizer.git?tag=v0.2.5#c0b5cf741ed9485147f2cbe523f2214d4fa4c395"
dependencies = [
"character_converter",
"cow-utils",
@ -1676,12 +1740,13 @@ dependencies = [
[[package]]
name = "milli"
version = "0.7.1"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.7.1#b4dcdbf00d232f9bc744a4a6ba4ebaf5cb592b56"
version = "0.10.2"
source = "git+https://github.com/meilisearch/milli.git?tag=v0.10.2#879d5e8799836d93f8995810965b6797be4f69d1"
dependencies = [
"bstr",
"byteorder",
"chrono",
"concat-arrays",
"csv",
"either",
"flate2",
@ -1695,7 +1760,7 @@ dependencies = [
"linked-hash-map",
"log",
"logging_timer",
"meilisearch-tokenizer 0.2.2",
"meilisearch-tokenizer",
"memmap",
"obkv",
"once_cell",
@ -1857,9 +1922,9 @@ dependencies = [
[[package]]
name = "obkv"
version = "0.1.1"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddd8a5a0aa2f3adafe349259a5b3e21a19c388b792414c1161d60a69c1fa48e8"
checksum = "f69e48cd7c8e5bb52a1da1287fdbfd877c32673176583ce664cd63b201aba385"
[[package]]
name = "once_cell"
@ -1888,12 +1953,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "oxidized-json-checker"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "938464aebf563f48ab86d1cfc0e2df952985c0b814d3108f41d1b85e7f5b0dac"
[[package]]
name = "page_size"
version = "0.4.2"
@ -2476,15 +2535,6 @@ dependencies = [
"semver 0.11.0",
]
[[package]]
name = "rustc_version"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
"semver 1.0.3",
]
[[package]]
name = "rustls"
version = "0.19.1"
@ -2498,6 +2548,12 @@ dependencies = [
"webpki",
]
[[package]]
name = "rustversion"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61b3909d758bb75c79f23d4736fac9433868679d3ad2ea7a61e3c25cfda9a088"
[[package]]
name = "ryu"
version = "1.0.5"
@ -2547,12 +2603,6 @@ dependencies = [
"semver-parser 0.10.2",
]
[[package]]
name = "semver"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f3aac57ee7f3272d8395c6e4f502f434f0e289fcd62876f70daa008c20dcabe"
[[package]]
name = "semver-parser"
version = "0.7.0"
@ -2712,6 +2762,15 @@ dependencies = [
"serde",
]
[[package]]
name = "serdeval"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94023adfd3d548a8bd9a1f09c09f44eaab7080c7a9ab20314bb65154bee62bd0"
dependencies = [
"serde",
]
[[package]]
name = "sha-1"
version = "0.8.2"
@ -3315,6 +3374,12 @@ dependencies = [
"serde",
]
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "vec_map"
version = "0.8.2"
@ -3323,13 +3388,18 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "vergen"
version = "3.2.0"
version = "5.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7141e445af09c8919f1d5f8a20dae0b20c3b57a45dee0d5823c6ed5d237f15a"
checksum = "542f37b4798c879409865dde7908e746d836f77839c3a6bea5c8b4e4dcf6620b"
dependencies = [
"bitflags",
"anyhow",
"cfg-if 1.0.0",
"chrono",
"rustc_version 0.4.0",
"enum-iterator",
"getset",
"git2",
"rustversion",
"thiserror",
]
[[package]]
@ -3470,9 +3540,9 @@ dependencies = [
[[package]]
name = "whatlang"
version = "0.9.0"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc0289c1d1548414a5645e6583e118e9c569c579ec2a0c32417cc3dbf7a89075"
checksum = "7a346d2eb29c03618693ed24a29d1acd0c3f2cb08ae58b9669d7461e033cf703"
dependencies = [
"hashbrown 0.7.2",
]

View File

@ -18,7 +18,7 @@ hex = { version = "0.4.3", optional = true }
reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false, optional = true }
sha-1 = { version = "0.9.4", optional = true }
tempfile = { version = "3.1.0", optional = true }
vergen = "3.1.0"
vergen = { version = "5.1.13", default-features = false, features = ["build", "git"] }
zip = { version = "0.5.12", optional = true }
[dependencies]
@ -42,20 +42,19 @@ fst = "0.4.5"
futures = "0.3.7"
futures-util = "0.3.8"
grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
http = "0.2.1"
indexmap = { version = "1.3.2", features = ["serde-1"] }
itertools = "0.10.0"
log = "0.4.8"
main_error = "0.1.0"
meilisearch-error = { path = "../meilisearch-error" }
meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.3" }
meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" }
memmap = "0.7.0"
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.7.1" }
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.10.2" }
mime = "0.3.16"
num_cpus = "1.13.0"
once_cell = "1.5.2"
oxidized-json-checker = "0.3.2"
parking_lot = "0.11.1"
rand = "0.7.3"
rayon = "1.5.0"
@ -73,10 +72,11 @@ thiserror = "1.0.24"
tokio = { version = "1", features = ["full"] }
uuid = { version = "0.8.2", features = ["serde"] }
walkdir = "2.3.2"
obkv = "0.1.1"
obkv = "0.2.0"
pin-project = "1.0.7"
whoami = { version = "1.1.2", optional = true }
reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true }
serdeval = "0.1.0"
[dependencies.sentry]
default-features = false
@ -97,7 +97,7 @@ actix-rt = "2.1.0"
assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" }
mockall = "0.9.1"
paste = "1.0.5"
serde_url_params = "0.2.0"
serde_url_params = "0.2.1"
tempdir = "0.3.7"
urlencoding = "1.1.1"
@ -119,5 +119,5 @@ default = ["analytics", "mini-dashboard"]
jemallocator = "0.3.2"
[package.metadata.mini-dashboard]
assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.3/build.zip"
sha1 = "fea1780e13d8e570e35a1921e7a45cabcd501d5e"
assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.4/build.zip"
sha1 = "750e8a8e56cfa61fbf9ead14b08a5f17ad3f3d37"

View File

@ -1,12 +1,7 @@
use vergen::{generate_cargo_keys, ConstantsFlags};
use vergen::{vergen, Config};
fn main() {
// Setup the flags, toggling off the 'SEMVER_FROM_CARGO_PKG' flag
let mut flags = ConstantsFlags::all();
flags.toggle(ConstantsFlags::SEMVER_FROM_CARGO_PKG);
// Generate the 'cargo:' key output
generate_cargo_keys(ConstantsFlags::all()).expect("Unable to generate the cargo keys!");
vergen(Config::default()).unwrap();
#[cfg(feature = "mini-dashboard")]
mini_dashboard::setup_mini_dashboard().expect("Could not load the mini-dashboard assets");

View File

@ -101,7 +101,7 @@ impl Index {
let index = Self::open(&dst_dir_path, size)?;
let mut txn = index.write_txn()?;
let handler = UpdateHandler::new(&indexing_options)?;
let handler = UpdateHandler::new(indexing_options)?;
index.update_settings_txn(&mut txn, &settings, handler.update_builder(0))?;

View File

@ -6,7 +6,7 @@ use std::path::Path;
use std::sync::Arc;
use heed::{EnvOpenOptions, RoTxn};
use milli::obkv_to_json;
use milli::{obkv_to_json, FieldId};
use serde::{de::Deserializer, Deserialize};
use serde_json::{Map, Value};
@ -62,34 +62,34 @@ impl Index {
pub fn settings_txn(&self, txn: &RoTxn) -> Result<Settings<Checked>> {
let displayed_attributes = self
.displayed_fields(&txn)?
.displayed_fields(txn)?
.map(|fields| fields.into_iter().map(String::from).collect());
let searchable_attributes = self
.searchable_fields(&txn)?
.searchable_fields(txn)?
.map(|fields| fields.into_iter().map(String::from).collect());
let faceted_attributes = self.faceted_fields(&txn)?.into_iter().collect();
let filterable_attributes = self.filterable_fields(txn)?.into_iter().collect();
let criteria = self
.criteria(&txn)?
.criteria(txn)?
.into_iter()
.map(|c| c.to_string())
.collect();
let stop_words = self
.stop_words(&txn)?
.stop_words(txn)?
.map(|stop_words| -> Result<BTreeSet<_>> {
Ok(stop_words.stream().into_strs()?.into_iter().collect())
})
.transpose()?
.unwrap_or_else(BTreeSet::new);
let distinct_field = self.distinct_field(&txn)?.map(String::from);
let distinct_field = self.distinct_field(txn)?.map(String::from);
// in milli each word in the synonyms map were split on their separator. Since we lost
// this information we are going to put space between words.
let synonyms = self
.synonyms(&txn)?
.synonyms(txn)?
.iter()
.map(|(key, values)| {
(
@ -102,7 +102,7 @@ impl Index {
Ok(Settings {
displayed_attributes: Some(displayed_attributes),
searchable_attributes: Some(searchable_attributes),
filterable_attributes: Some(Some(faceted_attributes)),
filterable_attributes: Some(Some(filterable_attributes)),
ranking_rules: Some(Some(criteria)),
stop_words: Some(Some(stop_words)),
distinct_attribute: Some(distinct_field),
@ -174,8 +174,8 @@ impl Index {
txn: &heed::RoTxn,
attributes_to_retrieve: &Option<Vec<S>>,
fields_ids_map: &milli::FieldsIdsMap,
) -> Result<Vec<u8>> {
let mut displayed_fields_ids = match self.displayed_fields_ids(&txn)? {
) -> Result<Vec<FieldId>> {
let mut displayed_fields_ids = match self.displayed_fields_ids(txn)? {
Some(ids) => ids.into_iter().collect::<Vec<_>>(),
None => fields_ids_map.iter().map(|(id, _)| id).collect(),
};

View File

@ -239,7 +239,7 @@ fn compute_matches<A: AsRef<[u8]>>(
for (key, value) in document {
let mut infos = Vec::new();
compute_value_matches(&mut infos, value, matcher, &analyzer);
compute_value_matches(&mut infos, value, matcher, analyzer);
if !infos.is_empty() {
matches.insert(key.clone(), infos);
}
@ -281,9 +281,9 @@ fn compute_formatted_options(
attr_to_highlight: &HashSet<String>,
attr_to_crop: &[String],
query_crop_length: usize,
to_retrieve_ids: &BTreeSet<u8>,
to_retrieve_ids: &BTreeSet<FieldId>,
fields_ids_map: &FieldsIdsMap,
displayed_ids: &BTreeSet<u8>,
displayed_ids: &BTreeSet<FieldId>,
) -> BTreeMap<FieldId, FormatOptions> {
let mut formatted_options = BTreeMap::new();
@ -314,7 +314,7 @@ fn add_highlight_to_formatted_options(
formatted_options: &mut BTreeMap<FieldId, FormatOptions>,
attr_to_highlight: &HashSet<String>,
fields_ids_map: &FieldsIdsMap,
displayed_ids: &BTreeSet<u8>,
displayed_ids: &BTreeSet<FieldId>,
) {
for attr in attr_to_highlight {
let new_format = FormatOptions {
@ -329,7 +329,7 @@ fn add_highlight_to_formatted_options(
break;
}
if let Some(id) = fields_ids_map.id(&attr) {
if let Some(id) = fields_ids_map.id(attr) {
if displayed_ids.contains(&id) {
formatted_options.insert(id, new_format);
}
@ -342,7 +342,7 @@ fn add_crop_to_formatted_options(
attr_to_crop: &[String],
crop_length: usize,
fields_ids_map: &FieldsIdsMap,
displayed_ids: &BTreeSet<u8>,
displayed_ids: &BTreeSet<FieldId>,
) {
for attr in attr_to_crop {
let mut split = attr.rsplitn(2, ':');
@ -366,7 +366,7 @@ fn add_crop_to_formatted_options(
}
}
if let Some(id) = fields_ids_map.id(&attr_name) {
if let Some(id) = fields_ids_map.id(attr_name) {
if displayed_ids.contains(&id) {
formatted_options
.entry(id)
@ -382,7 +382,7 @@ fn add_crop_to_formatted_options(
fn add_non_formatted_ids_to_formatted_options(
formatted_options: &mut BTreeMap<FieldId, FormatOptions>,
to_retrieve_ids: &BTreeSet<u8>,
to_retrieve_ids: &BTreeSet<FieldId>,
) {
for id in to_retrieve_ids {
formatted_options.entry(*id).or_insert(FormatOptions {
@ -395,7 +395,7 @@ fn add_non_formatted_ids_to_formatted_options(
fn make_document(
attributes_to_retrieve: &BTreeSet<FieldId>,
field_ids_map: &FieldsIdsMap,
obkv: obkv::KvReader,
obkv: obkv::KvReaderU16,
) -> Result<Document> {
let mut document = Document::new();
@ -418,7 +418,7 @@ fn make_document(
fn format_fields<A: AsRef<[u8]>>(
field_ids_map: &FieldsIdsMap,
obkv: obkv::KvReader,
obkv: obkv::KvReaderU16,
formatter: &Formatter<A>,
matching_words: &impl Matcher,
formatted_options: &BTreeMap<FieldId, FormatOptions>,
@ -592,7 +592,7 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> {
// we highlight the complete word.
None => {
out.push_str(&self.marks.0);
out.push_str(&word);
out.push_str(word);
out.push_str(&self.marks.1);
}
}

View File

@ -205,7 +205,7 @@ impl Index {
// Set the primary key if not set already, ignore if already set.
if let (None, Some(primary_key)) = (self.primary_key(txn)?, primary_key) {
let mut builder = UpdateBuilder::new(0).settings(txn, &self);
let mut builder = UpdateBuilder::new(0).settings(txn, self);
builder.set_primary_key(primary_key.to_string());
builder.execute(|_, _| ())?;
}

View File

@ -73,7 +73,7 @@ struct Settings {
#[serde(default, deserialize_with = "deserialize_some")]
pub synonyms: Option<Option<BTreeMap<String, Vec<String>>>>,
#[serde(default, deserialize_with = "deserialize_some")]
pub filterable_attributes: Option<Option<Vec<String>>>,
pub attributes_for_faceting: Option<Option<Vec<String>>>,
}
fn load_index(
@ -98,7 +98,7 @@ fn load_index(
let mut txn = index.write_txn()?;
let handler = UpdateHandler::new(&indexer_options)?;
let handler = UpdateHandler::new(indexer_options)?;
index.update_settings_txn(&mut txn, &settings.check(), handler.update_builder(0))?;
@ -142,23 +142,19 @@ impl From<Settings> for index_controller::Settings<Unchecked> {
// representing the name of the faceted field + the type of the field. Since the type
// was not known in the V1 of the dump we are just going to assume everything is a
// String
filterable_attributes: settings.filterable_attributes.map(|o| o.map(|vec| vec.into_iter().collect())),
filterable_attributes: settings.attributes_for_faceting.map(|o| o.map(|vec| vec.into_iter().collect())),
// we need to convert the old `Vec<String>` into a `BTreeSet<String>`
ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter_map(|criterion| {
ranking_rules: settings.ranking_rules.map(|o| o.map(|vec| vec.into_iter().filter(|criterion| {
match criterion.as_str() {
"words" | "typo" | "proximity" | "attribute" => Some(criterion),
s if s.starts_with("asc") || s.starts_with("desc") => Some(criterion),
"words" | "typo" | "proximity" | "attribute" | "exactness" => true,
s if s.starts_with("asc") || s.starts_with("desc") => true,
"wordsPosition" => {
warn!("The criteria `words` and `wordsPosition` have been merged into a single criterion `words` so `wordsPositon` will be ignored");
Some(String::from("words"))
}
"exactness" => {
error!("The criterion `{}` is not implemented currently and thus will be ignored", criterion);
None
warn!("The criteria `attribute` and `wordsPosition` have been merged into a single criterion `attribute` so `wordsPositon` will be ignored");
false
}
s => {
error!("Unknown criterion found in the dump: `{}`, it will be ignored", s);
None
false
}
}
}).collect())),
@ -180,3 +176,17 @@ fn import_settings(dir_path: impl AsRef<Path>) -> anyhow::Result<Settings> {
Ok(metadata)
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn settings_format_regression() {
let settings = Settings::default();
assert_eq!(
r##"{"rankingRules":null,"distinctAttribute":null,"searchableAttributes":null,"displayedAttributes":null,"stopWords":null,"synonyms":null,"attributesForFaceting":null}"##,
serde_json::to_string(&settings).unwrap()
);
}
}

View File

@ -37,7 +37,7 @@ pub trait DumpActorHandle {
async fn create_dump(&self) -> Result<DumpInfo>;
/// Return the status of an already created dump
/// Implementation: [handle_impl::DumpActorHandleImpl::dump_status]
/// Implementation: [handle_impl::DumpActorHandleImpl::dump_info]
async fn dump_info(&self, uid: String) -> Result<DumpInfo>;
}

View File

@ -40,9 +40,9 @@ impl IndexMeta {
}
fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result<Self> {
let created_at = index.created_at(&txn)?;
let updated_at = index.updated_at(&txn)?;
let primary_key = index.primary_key(&txn)?.map(String::from);
let created_at = index.created_at(txn)?;
let updated_at = index.updated_at(txn)?;
let primary_key = index.primary_key(txn)?.map(String::from);
Ok(Self {
created_at,
updated_at,

View File

@ -7,7 +7,7 @@ use std::sync::Arc;
use async_stream::stream;
use futures::StreamExt;
use log::trace;
use oxidized_json_checker::JsonChecker;
use serdeval::*;
use tokio::fs;
use tokio::io::AsyncWriteExt;
use tokio::sync::mpsc;
@ -180,7 +180,7 @@ where
let update_store = self.store.clone();
tokio::task::spawn_blocking(move || {
use std::io::{copy, sink, BufReader, Seek};
use std::io::{BufReader, Seek};
// If the payload is empty, ignore the check.
let update_uuid = if let Some((mut file, uuid)) = file_path {
@ -188,14 +188,10 @@ where
file.seek(SeekFrom::Start(0))?;
// Check that the json payload is valid:
let reader = BufReader::new(&mut file);
let mut checker = JsonChecker::new(reader);
// Validate that the payload is in the correct format.
let _: Seq<Map<Str, Any>> = serde_json::from_reader(reader)
.map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?;
if copy(&mut checker, &mut sink()).is_err() || checker.finish().is_err() {
// The json file is invalid, we use Serde to get a nice error message:
file.seek(SeekFrom::Start(0))?;
let _: serde_json::Value = serde_json::from_reader(file)
.map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?;
}
Some(uuid)
} else {
None

View File

@ -59,8 +59,8 @@ impl UpdateStore {
let update_files_path = path.as_ref().join(super::UPDATE_DIR);
create_dir_all(&update_files_path)?;
self.dump_pending(&txn, uuids, &mut dump_data_file, &path)?;
self.dump_completed(&txn, uuids, &mut dump_data_file)?;
self.dump_pending(txn, uuids, &mut dump_data_file, &path)?;
self.dump_completed(txn, uuids, &mut dump_data_file)?;
Ok(())
}

View File

@ -268,13 +268,12 @@ impl UpdateStore {
self.pending_queue.remap_key_type::<PendingKeyCodec>().put(
wtxn,
&(global_id, index_uuid, enqueued.id()),
&enqueued,
enqueued,
)?;
}
_ => {
let _update_id = self.next_update_id_raw(wtxn, index_uuid)?;
self.updates
.put(wtxn, &(index_uuid, update.id()), &update)?;
self.updates.put(wtxn, &(index_uuid, update.id()), update)?;
}
}
Ok(())
@ -442,13 +441,16 @@ impl UpdateStore {
while let Some(Ok(((_, uuid, _), pending))) = pendings.next() {
if uuid == index_uuid {
unsafe {
pendings.del_current()?;
}
let mut pending = pending.decode()?;
if let Some(update_uuid) = pending.content.take() {
uuids_to_remove.push(update_uuid);
}
// Invariant check: we can only delete the current entry when we don't hold
// references to it anymore. This must be done after we have retrieved its content.
unsafe {
pendings.del_current()?;
}
}
}
@ -470,13 +472,6 @@ impl UpdateStore {
txn.commit()?;
uuids_to_remove
.iter()
.map(|uuid| update_uuid_to_file_path(&self.path, *uuid))
.for_each(|path| {
let _ = remove_file(path);
});
// If the currently processing update is from our index, we wait until it is
// finished before returning. This ensure that no write to the index occurs after we delete it.
if let State::Processing(uuid, _) = *self.state.read() {
@ -486,6 +481,16 @@ impl UpdateStore {
}
}
// Finally, remove any outstanding update files. This must be done after waiting for the
// last update to ensure that the update files are not deleted before the update needs
// them.
uuids_to_remove
.iter()
.map(|uuid| update_uuid_to_file_path(&self.path, *uuid))
.for_each(|path| {
let _ = remove_file(path);
});
Ok(())
}

View File

@ -1,3 +1,43 @@
//! # MeiliSearch
//! Hello there, future contributors. If you are here and see this code, it's probably because you want to add a super new fancy feature in MeiliSearch or fix a bug and first of all, thank you for that!
//!
//! To help you in this task, we'll try to do a little overview of the project.
//! ## Milli
//! [Milli](https://github.com/meilisearch/milli) is the core library of MeiliSearch. It's where we actually index documents and perform searches. Its purpose is to do these two tasks as fast as possible. You can give an update to milli, and it'll uses as many cores as provided to perform it as fast as possible. Nothing more. You can perform searches at the same time (search only uses one core).
//! As you can see, we're missing quite a lot of features here; milli does not handle multiples indexes, it can't queue updates, it doesn't provide any web / API frontend, it doesn't implement dumps or snapshots, etc...
//!
//! ## `Index` module
//! The [index] module is what encapsulates one milli index. It abstracts over its transaction and isolates a task that can be run into a thread. This is the unit of interaction with milli.
//! If you add a feature to milli, you'll probably need to add it in this module too before exposing it to the rest of meilisearch.
//!
//! ## `IndexController` module
//! To handle multiple indexes, we created an [index_controller]. It's in charge of creating new indexes, keeping references to all its indexes, forward asynchronous updates to its indexes, and provide an API to search in its indexes synchronously.
//! To achieves this goal, we use an [actor model](https://en.wikipedia.org/wiki/Actor_model).
//!
//! ### The actor model
//! Every actor is composed of at least three files:
//! - `mod.rs` declare and import all the files used by the actor. We also describe the interface (= all the methods) used to interact with the actor. If you are not modifying anything inside of an actor, this is usually all you need to see.
//! - `handle_impl.rs` implements the interface described in the `mod.rs`; in reality, there is no code logic in this file. Every method is only wrapping its parameters in a structure that is sent to the actor. This is useful for test and futureproofing.
//! - `message.rs` contains an enum that describes all the interactions you can have with the actor.
//! - `actor.rs` is used to create and execute the actor. It's where we'll write the loop looking for new messages and actually perform the tasks.
//!
//! MeiliSearch currently uses four actors:
//! - [`uuid_resolver`](index_controller/uuid_resolver/index.html) hold the association between the user-provided indexes name and the internal [`uuid`](https://en.wikipedia.org/wiki/Universally_unique_identifier) representation we use.
//! - [`index_actor`](index_controller::index_actor) is our representation of multiples indexes. Any request made to MeiliSearch that needs to talk to milli will pass through this actor.
//! - [`update_actor`](index_controller/update_actor/index.html) is in charge of indexes updates. Since updates can take a long time to receive and process, we need to:
//! 1. Store them as fast as possible so we can continue to receive other updates even if nothing has been processed
//! 2. Feed the `index_actor` with a new update every time it finished its current job.
//! - [`dump_actor`](index_controller/dump_actor/index.html) this actor handle the [dumps](https://docs.meilisearch.com/reference/api/dump.html). It needs to contact all the others actors and create a dump of everything that was currently happening.
//!
//! ## Data module
//! The [data] module provide a unified interface to communicate with the index controller and other services (snapshot, dumps, ...), initialize the MeiliSearch instance
//!
//! ## HTTP server
//! To handle the web and API part, we are using [actix-web](https://docs.rs/actix-web/); you can find all routes in the [routes] module.
//! Currently, the configuration of actix-web is made in the [lib.rs](crate).
//! Most of the routes use [extractors] to handle the authentication.
#![allow(rustdoc::private_intra_doc_links)]
pub mod data;
#[macro_use]
pub mod error;
@ -106,20 +146,13 @@ macro_rules! create_app {
use actix_web::middleware::TrailingSlash;
use actix_web::App;
use actix_web::{middleware, web};
use meilisearch_http::routes::*;
use meilisearch_http::routes;
use meilisearch_http::{configure_auth, configure_data, dashboard};
App::new()
.configure(|s| configure_data(s, $data.clone()))
.configure(|s| configure_auth(s, &$data))
.configure(document::services)
.configure(index::services)
.configure(search::services)
.configure(settings::services)
.configure(health::services)
.configure(stats::services)
.configure(key::services)
.configure(dump::services)
.configure(routes::configure)
.configure(|s| dashboard(s, $enable_frontend))
.wrap(
Cors::default()

View File

@ -49,12 +49,12 @@ async fn main() -> Result<(), MainError> {
release: sentry::release_name!(),
dsn: Some(SENTRY_DSN.parse()?),
before_send: Some(Arc::new(|event| {
event
.message
.as_ref()
.map(|msg| msg.to_lowercase().contains("no space left on device"))
.unwrap_or(false)
.then(|| event)
if matches!(event.message, Some(ref msg) if msg.to_lowercase().contains("no space left on device"))
{
None
} else {
Some(event)
}
})),
..Default::default()
});
@ -105,11 +105,11 @@ async fn run_http(data: Data, opt: Opt) -> Result<(), Box<dyn std::error::Error>
pub fn print_launch_resume(opt: &Opt, data: &Data) {
let commit_sha = match option_env!("COMMIT_SHA") {
Some("") | None => env!("VERGEN_SHA"),
Some("") | None => env!("VERGEN_GIT_SHA"),
Some(commit_sha) => commit_sha,
};
let commit_date = match option_env!("COMMIT_DATE") {
Some("") | None => env!("VERGEN_COMMIT_DATE"),
Some("") | None => env!("VERGEN_GIT_COMMIT_TIMESTAMP"),
Some(commit_date) => commit_date,
};
@ -145,7 +145,7 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) {
"
Thank you for using MeiliSearch!
We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/reference/features/configuration.html#analytics
We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html
Anonymous telemetry: \"Enabled\""
);

View File

@ -6,12 +6,12 @@ use crate::error::ResponseError;
use crate::extractors::authentication::{policies::*, GuardedData};
use crate::Data;
pub fn services(cfg: &mut web::ServiceConfig) {
cfg.service(web::resource("/dumps").route(web::post().to(create_dump)))
.service(web::resource("/dumps/{dump_uid}/status").route(web::get().to(get_dump_status)));
pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(web::resource("").route(web::post().to(create_dump)))
.service(web::resource("/{dump_uid}/status").route(web::get().to(get_dump_status)));
}
async fn create_dump(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
pub async fn create_dump(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
let res = data.create_dump().await?;
debug!("returns: {:?}", res);

View File

@ -1,11 +0,0 @@
use actix_web::{web, HttpResponse};
use crate::error::ResponseError;
pub fn services(cfg: &mut web::ServiceConfig) {
cfg.service(web::resource("/health").route(web::get().to(get_health)));
}
async fn get_health() -> Result<HttpResponse, ResponseError> {
Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" })))
}

View File

@ -49,32 +49,29 @@ fn guard_json(head: &actix_web::dev::RequestHead) -> bool {
}
#[derive(Deserialize)]
struct DocumentParam {
pub struct DocumentParam {
index_uid: String,
document_id: String,
}
pub fn services(cfg: &mut web::ServiceConfig) {
pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(
web::scope("/indexes/{index_uid}/documents")
.service(
web::resource("")
.route(web::get().to(get_all_documents))
.route(web::post().guard(guard_json).to(add_documents))
.route(web::put().guard(guard_json).to(update_documents))
.route(web::delete().to(clear_all_documents)),
)
// this route needs to be before the /documents/{document_id} to match properly
.service(web::resource("/delete-batch").route(web::post().to(delete_documents)))
.service(
web::resource("/{document_id}")
.route(web::get().to(get_document))
.route(web::delete().to(delete_document)),
),
web::resource("")
.route(web::get().to(get_all_documents))
.route(web::post().guard(guard_json).to(add_documents))
.route(web::put().guard(guard_json).to(update_documents))
.route(web::delete().to(clear_all_documents)),
)
// this route needs to be before the /documents/{document_id} to match properly
.service(web::resource("/delete-batch").route(web::post().to(delete_documents)))
.service(
web::resource("/{document_id}")
.route(web::get().to(get_document))
.route(web::delete().to(delete_document)),
);
}
async fn get_document(
pub async fn get_document(
data: GuardedData<Public, Data>,
path: web::Path<DocumentParam>,
) -> Result<HttpResponse, ResponseError> {
@ -87,7 +84,7 @@ async fn get_document(
Ok(HttpResponse::Ok().json(document))
}
async fn delete_document(
pub async fn delete_document(
data: GuardedData<Private, Data>,
path: web::Path<DocumentParam>,
) -> Result<HttpResponse, ResponseError> {
@ -100,13 +97,13 @@ async fn delete_document(
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct BrowseQuery {
pub struct BrowseQuery {
offset: Option<usize>,
limit: Option<usize>,
attributes_to_retrieve: Option<String>,
}
async fn get_all_documents(
pub async fn get_all_documents(
data: GuardedData<Public, Data>,
path: web::Path<IndexParam>,
params: web::Query<BrowseQuery>,
@ -137,13 +134,13 @@ async fn get_all_documents(
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct UpdateDocumentsQuery {
pub struct UpdateDocumentsQuery {
primary_key: Option<String>,
}
/// Route used when the payload type is "application/json"
/// Used to add or replace documents
async fn add_documents(
pub async fn add_documents(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
params: web::Query<UpdateDocumentsQuery>,
@ -166,7 +163,7 @@ async fn add_documents(
/// Route used when the payload type is "application/json"
/// Used to add or replace documents
async fn update_documents(
pub async fn update_documents(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
params: web::Query<UpdateDocumentsQuery>,
@ -187,7 +184,7 @@ async fn update_documents(
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() })))
}
async fn delete_documents(
pub async fn delete_documents(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
body: web::Json<Vec<Value>>,
@ -207,7 +204,7 @@ async fn delete_documents(
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
async fn clear_all_documents(
pub async fn clear_all_documents(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
) -> Result<HttpResponse, ResponseError> {

View File

@ -3,42 +3,63 @@ use chrono::{DateTime, Utc};
use log::debug;
use serde::{Deserialize, Serialize};
use super::{IndexParam, UpdateStatusResponse};
use crate::error::ResponseError;
use crate::extractors::authentication::{policies::*, GuardedData};
use crate::routes::IndexParam;
use crate::Data;
pub fn services(cfg: &mut web::ServiceConfig) {
pub mod documents;
pub mod search;
pub mod settings;
pub mod updates;
pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(
web::resource("indexes")
web::resource("")
.route(web::get().to(list_indexes))
.route(web::post().to(create_index)),
)
.service(
web::resource("/indexes/{index_uid}")
.route(web::get().to(get_index))
.route(web::put().to(update_index))
.route(web::delete().to(delete_index)),
)
.service(
web::resource("/indexes/{index_uid}/updates").route(web::get().to(get_all_updates_status)),
)
.service(
web::resource("/indexes/{index_uid}/updates/{update_id}")
.route(web::get().to(get_update_status)),
web::scope("/{index_uid}")
.service(
web::resource("")
.route(web::get().to(get_index))
.route(web::put().to(update_index))
.route(web::delete().to(delete_index)),
)
.service(web::resource("/stats").route(web::get().to(get_index_stats)))
.service(web::scope("/documents").configure(documents::configure))
.service(web::scope("/search").configure(search::configure))
.service(web::scope("/updates").configure(updates::configure))
.service(web::scope("/settings").configure(settings::configure)),
);
}
pub async fn list_indexes(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
let indexes = data.list_indexes().await?;
debug!("returns: {:?}", indexes);
Ok(HttpResponse::Ok().json(indexes))
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct IndexCreateRequest {
pub struct IndexCreateRequest {
uid: String,
primary_key: Option<String>,
}
pub async fn create_index(
data: GuardedData<Private, Data>,
body: web::Json<IndexCreateRequest>,
) -> Result<HttpResponse, ResponseError> {
let body = body.into_inner();
let meta = data.create_index(body.uid, body.primary_key).await?;
Ok(HttpResponse::Created().json(meta))
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct UpdateIndexRequest {
pub struct UpdateIndexRequest {
uid: Option<String>,
primary_key: Option<String>,
}
@ -53,22 +74,7 @@ pub struct UpdateIndexResponse {
primary_key: Option<String>,
}
async fn list_indexes(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
let indexes = data.list_indexes().await?;
debug!("returns: {:?}", indexes);
Ok(HttpResponse::Ok().json(indexes))
}
async fn create_index(
data: GuardedData<Private, Data>,
body: web::Json<IndexCreateRequest>,
) -> Result<HttpResponse, ResponseError> {
let body = body.into_inner();
let meta = data.create_index(body.uid, body.primary_key).await?;
Ok(HttpResponse::Ok().json(meta))
}
async fn get_index(
pub async fn get_index(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
) -> Result<HttpResponse, ResponseError> {
@ -77,7 +83,7 @@ async fn get_index(
Ok(HttpResponse::Ok().json(meta))
}
async fn update_index(
pub async fn update_index(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
body: web::Json<UpdateIndexRequest>,
@ -91,7 +97,7 @@ async fn update_index(
Ok(HttpResponse::Ok().json(meta))
}
async fn delete_index(
pub async fn delete_index(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
) -> Result<HttpResponse, ResponseError> {
@ -99,35 +105,12 @@ async fn delete_index(
Ok(HttpResponse::NoContent().finish())
}
#[derive(Deserialize)]
struct UpdateParam {
index_uid: String,
update_id: u64,
}
async fn get_update_status(
data: GuardedData<Private, Data>,
path: web::Path<UpdateParam>,
) -> Result<HttpResponse, ResponseError> {
let params = path.into_inner();
let meta = data
.get_update_status(params.index_uid, params.update_id)
.await?;
let meta = UpdateStatusResponse::from(meta);
debug!("returns: {:?}", meta);
Ok(HttpResponse::Ok().json(meta))
}
async fn get_all_updates_status(
pub async fn get_index_stats(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
) -> Result<HttpResponse, ResponseError> {
let metas = data.get_updates_status(path.into_inner().index_uid).await?;
let metas = metas
.into_iter()
.map(UpdateStatusResponse::from)
.collect::<Vec<_>>();
let response = data.get_index_stats(path.index_uid.clone()).await?;
debug!("returns: {:?}", metas);
Ok(HttpResponse::Ok().json(metas))
debug!("returns: {:?}", response);
Ok(HttpResponse::Ok().json(response))
}

View File

@ -11,9 +11,9 @@ use crate::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT};
use crate::routes::IndexParam;
use crate::Data;
pub fn services(cfg: &mut web::ServiceConfig) {
pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(
web::resource("/indexes/{index_uid}/search")
web::resource("")
.route(web::get().to(search_with_url_query))
.route(web::post().to(search_with_post)),
);
@ -77,19 +77,24 @@ impl From<SearchQueryGet> for SearchQuery {
}
}
async fn search_with_url_query(
data: GuardedData<Admin, Data>,
pub async fn search_with_url_query(
data: GuardedData<Public, Data>,
path: web::Path<IndexParam>,
params: web::Query<SearchQueryGet>,
) -> Result<HttpResponse, ResponseError> {
debug!("called with params: {:?}", params);
let query = params.into_inner().into();
let search_result = data.search(path.into_inner().index_uid, query).await?;
// Tests that the nb_hits is always set to false
#[cfg(test)]
assert!(!search_result.exhaustive_nb_hits);
debug!("returns: {:?}", search_result);
Ok(HttpResponse::Ok().json(search_result))
}
async fn search_with_post(
pub async fn search_with_post(
data: GuardedData<Public, Data>,
path: web::Path<IndexParam>,
params: web::Json<SearchQuery>,
@ -98,6 +103,11 @@ async fn search_with_post(
let search_result = data
.search(path.into_inner().index_uid, params.into_inner())
.await?;
// Tests that the nb_hits is always set to false
#[cfg(test)]
assert!(!search_result.exhaustive_nb_hits);
debug!("returns: {:?}", search_result);
Ok(HttpResponse::Ok().json(search_result))
}

View File

@ -9,7 +9,7 @@ use crate::{error::ResponseError, index::Unchecked};
#[macro_export]
macro_rules! make_setting_route {
($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => {
mod $attr {
pub mod $attr {
use log::debug;
use actix_web::{web, HttpResponse, Resource};
@ -18,7 +18,7 @@ macro_rules! make_setting_route {
use crate::index::Settings;
use crate::extractors::authentication::{GuardedData, policies::*};
async fn delete(
pub async fn delete(
data: GuardedData<Private, data::Data>,
index_uid: web::Path<String>,
) -> Result<HttpResponse, ResponseError> {
@ -32,7 +32,7 @@ macro_rules! make_setting_route {
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
async fn update(
pub async fn update(
data: GuardedData<Private, data::Data>,
index_uid: actix_web::web::Path<String>,
body: actix_web::web::Json<Option<$type>>,
@ -47,7 +47,7 @@ macro_rules! make_setting_route {
Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
}
async fn get(
pub async fn get(
data: GuardedData<Private, data::Data>,
index_uid: actix_web::web::Path<String>,
) -> std::result::Result<HttpResponse, ResponseError> {
@ -69,68 +69,63 @@ macro_rules! make_setting_route {
}
make_setting_route!(
"/indexes/{index_uid}/settings/filterable-attributes",
"/filterable-attributes",
std::collections::HashSet<String>,
filterable_attributes,
"filterableAttributes"
);
make_setting_route!(
"/indexes/{index_uid}/settings/displayed-attributes",
"/displayed-attributes",
Vec<String>,
displayed_attributes,
"displayedAttributes"
);
make_setting_route!(
"/indexes/{index_uid}/settings/searchable-attributes",
"/searchable-attributes",
Vec<String>,
searchable_attributes,
"searchableAttributes"
);
make_setting_route!(
"/indexes/{index_uid}/settings/stop-words",
"/stop-words",
std::collections::BTreeSet<String>,
stop_words,
"stopWords"
);
make_setting_route!(
"/indexes/{index_uid}/settings/synonyms",
"/synonyms",
std::collections::BTreeMap<String, Vec<String>>,
synonyms,
"synonyms"
);
make_setting_route!(
"/indexes/{index_uid}/settings/distinct-attribute",
"/distinct-attribute",
String,
distinct_attribute,
"distinctAttribute"
);
make_setting_route!(
"/indexes/{index_uid}/settings/ranking-rules",
Vec<String>,
ranking_rules,
"rankingRules"
);
make_setting_route!("/ranking-rules", Vec<String>, ranking_rules, "rankingRules");
macro_rules! create_services {
macro_rules! generate_configure {
($($mod:ident),*) => {
pub fn services(cfg: &mut web::ServiceConfig) {
cfg
.service(web::resource("/indexes/{index_uid}/settings")
.route(web::post().to(update_all))
.route(web::get().to(get_all))
.route(web::delete().to(delete_all)))
pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(
web::resource("")
.route(web::post().to(update_all))
.route(web::get().to(get_all))
.route(web::delete().to(delete_all)))
$(.service($mod::resources()))*;
}
};
}
create_services!(
generate_configure!(
filterable_attributes,
displayed_attributes,
searchable_attributes,
@ -140,7 +135,7 @@ create_services!(
ranking_rules
);
async fn update_all(
pub async fn update_all(
data: GuardedData<Private, Data>,
index_uid: web::Path<String>,
body: web::Json<Settings<Unchecked>>,
@ -154,7 +149,7 @@ async fn update_all(
Ok(HttpResponse::Accepted().json(json))
}
async fn get_all(
pub async fn get_all(
data: GuardedData<Private, Data>,
index_uid: web::Path<String>,
) -> Result<HttpResponse, ResponseError> {
@ -163,7 +158,7 @@ async fn get_all(
Ok(HttpResponse::Ok().json(settings))
}
async fn delete_all(
pub async fn delete_all(
data: GuardedData<Private, Data>,
index_uid: web::Path<String>,
) -> Result<HttpResponse, ResponseError> {

View File

@ -0,0 +1,64 @@
use actix_web::{web, HttpResponse};
use chrono::{DateTime, Utc};
use log::debug;
use serde::{Deserialize, Serialize};
use crate::error::ResponseError;
use crate::extractors::authentication::{policies::*, GuardedData};
use crate::routes::{IndexParam, UpdateStatusResponse};
use crate::Data;
pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(web::resource("").route(web::get().to(get_all_updates_status)))
.service(web::resource("{update_id}").route(web::get().to(get_update_status)));
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct UpdateIndexRequest {
uid: Option<String>,
primary_key: Option<String>,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UpdateIndexResponse {
name: String,
uid: String,
created_at: DateTime<Utc>,
updated_at: DateTime<Utc>,
primary_key: Option<String>,
}
#[derive(Deserialize)]
pub struct UpdateParam {
index_uid: String,
update_id: u64,
}
pub async fn get_update_status(
data: GuardedData<Private, Data>,
path: web::Path<UpdateParam>,
) -> Result<HttpResponse, ResponseError> {
let params = path.into_inner();
let meta = data
.get_update_status(params.index_uid, params.update_id)
.await?;
let meta = UpdateStatusResponse::from(meta);
debug!("returns: {:?}", meta);
Ok(HttpResponse::Ok().json(meta))
}
pub async fn get_all_updates_status(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
) -> Result<HttpResponse, ResponseError> {
let metas = data.get_updates_status(path.into_inner().index_uid).await?;
let metas = metas
.into_iter()
.map(UpdateStatusResponse::from)
.collect::<Vec<_>>();
debug!("returns: {:?}", metas);
Ok(HttpResponse::Ok().json(metas))
}

View File

@ -1,23 +0,0 @@
use actix_web::{web, HttpResponse};
use serde::Serialize;
use crate::extractors::authentication::{policies::*, GuardedData};
use crate::Data;
pub fn services(cfg: &mut web::ServiceConfig) {
cfg.service(web::resource("/keys").route(web::get().to(list)));
}
#[derive(Serialize)]
struct KeysResponse {
private: Option<String>,
public: Option<String>,
}
async fn list(data: GuardedData<Admin, Data>) -> HttpResponse {
let api_keys = data.api_keys.clone();
HttpResponse::Ok().json(&KeysResponse {
private: api_keys.private,
public: api_keys.public,
})
}

View File

@ -1,21 +1,27 @@
use std::time::Duration;
use actix_web::HttpResponse;
use actix_web::{web, HttpResponse};
use chrono::{DateTime, Utc};
use log::debug;
use serde::{Deserialize, Serialize};
use crate::error::ResponseError;
use crate::extractors::authentication::{policies::*, GuardedData};
use crate::index::{Settings, Unchecked};
use crate::index_controller::{UpdateMeta, UpdateResult, UpdateStatus};
use crate::Data;
pub mod document;
pub mod dump;
pub mod health;
pub mod index;
pub mod key;
pub mod search;
pub mod settings;
pub mod stats;
mod dump;
mod indexes;
pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(web::resource("/health").route(web::get().to(get_health)))
.service(web::scope("/dumps").configure(dump::configure))
.service(web::resource("/keys").route(web::get().to(list_keys)))
.service(web::resource("/stats").route(web::get().to(get_stats)))
.service(web::resource("/version").route(web::get().to(get_version)))
.service(web::scope("/indexes").configure(indexes::configure));
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[allow(clippy::large_enum_variant)]
@ -43,7 +49,6 @@ pub enum UpdateType {
impl From<&UpdateStatus> for UpdateType {
fn from(other: &UpdateStatus) -> Self {
use milli::update::IndexDocumentsMethod::*;
match other.meta() {
UpdateMeta::DocumentsAddition { method, .. } => {
let number = match other {
@ -223,3 +228,147 @@ impl IndexUpdateResponse {
pub async fn running() -> HttpResponse {
HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" }))
}
async fn get_stats(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
let response = data.get_all_stats().await?;
debug!("returns: {:?}", response);
Ok(HttpResponse::Ok().json(response))
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct VersionResponse {
commit_sha: String,
commit_date: String,
pkg_version: String,
}
async fn get_version(_data: GuardedData<Private, Data>) -> HttpResponse {
let commit_sha = match option_env!("COMMIT_SHA") {
Some("") | None => env!("VERGEN_GIT_SHA"),
Some(commit_sha) => commit_sha,
};
let commit_date = match option_env!("COMMIT_DATE") {
Some("") | None => env!("VERGEN_GIT_COMMIT_TIMESTAMP"),
Some(commit_date) => commit_date,
};
HttpResponse::Ok().json(VersionResponse {
commit_sha: commit_sha.to_string(),
commit_date: commit_date.to_string(),
pkg_version: env!("CARGO_PKG_VERSION").to_string(),
})
}
#[derive(Serialize)]
struct KeysResponse {
private: Option<String>,
public: Option<String>,
}
pub async fn list_keys(data: GuardedData<Admin, Data>) -> HttpResponse {
let api_keys = data.api_keys.clone();
HttpResponse::Ok().json(&KeysResponse {
private: api_keys.private,
public: api_keys.public,
})
}
pub async fn get_health() -> Result<HttpResponse, ResponseError> {
Ok(HttpResponse::Ok().json(serde_json::json!({ "status": "available" })))
}
#[cfg(test)]
mod test {
use super::*;
use crate::data::Data;
use crate::extractors::authentication::GuardedData;
/// A type implemented for a route that uses a authentication policy `Policy`.
///
/// This trait is used for regression testing of route authenticaton policies.
trait Is<Policy, T> {}
macro_rules! impl_is_policy {
($($param:ident)*) => {
impl<Policy, Func, $($param,)* Res> Is<Policy, (($($param,)*), Res)> for Func
where Func: Fn(GuardedData<Policy, Data>, $($param,)*) -> Res {}
};
}
impl_is_policy! {}
impl_is_policy! {A}
impl_is_policy! {A B}
impl_is_policy! {A B C}
impl_is_policy! {A B C D}
/// Emits a compile error if a route doesn't have the correct authentication policy.
///
/// This works by trying to cast the route function into a Is<Policy, _> type, where Policy it
/// the authentication policy defined for the route.
macro_rules! test_auth_routes {
($($policy:ident => { $($route:expr,)*})*) => {
#[test]
fn test_auth() {
$($(let _: &dyn Is<$policy, _> = &$route;)*)*
}
};
}
test_auth_routes! {
Public => {
indexes::search::search_with_url_query,
indexes::search::search_with_post,
indexes::documents::get_document,
indexes::documents::get_all_documents,
}
Private => {
get_stats,
get_version,
indexes::create_index,
indexes::list_indexes,
indexes::get_index_stats,
indexes::delete_index,
indexes::update_index,
indexes::get_index,
dump::create_dump,
indexes::settings::filterable_attributes::get,
indexes::settings::displayed_attributes::get,
indexes::settings::searchable_attributes::get,
indexes::settings::stop_words::get,
indexes::settings::synonyms::get,
indexes::settings::distinct_attribute::get,
indexes::settings::filterable_attributes::update,
indexes::settings::displayed_attributes::update,
indexes::settings::searchable_attributes::update,
indexes::settings::stop_words::update,
indexes::settings::synonyms::update,
indexes::settings::distinct_attribute::update,
indexes::settings::filterable_attributes::delete,
indexes::settings::displayed_attributes::delete,
indexes::settings::searchable_attributes::delete,
indexes::settings::stop_words::delete,
indexes::settings::synonyms::delete,
indexes::settings::distinct_attribute::delete,
indexes::settings::delete_all,
indexes::settings::get_all,
indexes::settings::update_all,
indexes::documents::clear_all_documents,
indexes::documents::delete_documents,
indexes::documents::update_documents,
indexes::documents::add_documents,
indexes::documents::delete_document,
indexes::updates::get_all_updates_status,
indexes::updates::get_update_status,
}
Admin => { list_keys, }
}
}

View File

@ -1,56 +0,0 @@
use actix_web::{web, HttpResponse};
use log::debug;
use serde::Serialize;
use crate::error::ResponseError;
use crate::extractors::authentication::{policies::*, GuardedData};
use crate::routes::IndexParam;
use crate::Data;
pub fn services(cfg: &mut web::ServiceConfig) {
cfg.service(web::resource("/indexes/{index_uid}/stats").route(web::get().to(get_index_stats)))
.service(web::resource("/stats").route(web::get().to(get_stats)))
.service(web::resource("/version").route(web::get().to(get_version)));
}
async fn get_index_stats(
data: GuardedData<Private, Data>,
path: web::Path<IndexParam>,
) -> Result<HttpResponse, ResponseError> {
let response = data.get_index_stats(path.index_uid.clone()).await?;
debug!("returns: {:?}", response);
Ok(HttpResponse::Ok().json(response))
}
async fn get_stats(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
let response = data.get_all_stats().await?;
debug!("returns: {:?}", response);
Ok(HttpResponse::Ok().json(response))
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
struct VersionResponse {
commit_sha: String,
commit_date: String,
pkg_version: String,
}
async fn get_version(_data: GuardedData<Private, Data>) -> HttpResponse {
let commit_sha = match option_env!("COMMIT_SHA") {
Some("") | None => env!("VERGEN_SHA"),
Some(commit_sha) => commit_sha,
};
let commit_date = match option_env!("COMMIT_DATE") {
Some("") | None => env!("VERGEN_COMMIT_DATE"),
Some(commit_date) => commit_date,
};
HttpResponse::Ok().json(VersionResponse {
commit_sha: commit_sha.to_string(),
commit_date: commit_date.to_string(),
pkg_version: env!("CARGO_PKG_VERSION").to_string(),
})
}

View File

@ -1,4 +1,7 @@
use std::time::Duration;
use std::{
panic::{catch_unwind, resume_unwind, UnwindSafe},
time::Duration,
};
use actix_web::http::StatusCode;
use paste::paste;
@ -185,6 +188,37 @@ impl Index<'_> {
self.service.get(url).await
}
/// Performs both GET and POST search queries
pub async fn search(
&self,
query: Value,
test: impl Fn(Value, StatusCode) + UnwindSafe + Clone,
) {
let (response, code) = self.search_post(query.clone()).await;
let t = test.clone();
if let Err(e) = catch_unwind(move || t(response, code)) {
eprintln!("Error with post search");
resume_unwind(e);
}
let (response, code) = self.search_get(query).await;
if let Err(e) = catch_unwind(move || test(response, code)) {
eprintln!("Error with get search");
resume_unwind(e);
}
}
pub async fn search_post(&self, query: Value) -> (Value, StatusCode) {
let url = format!("/indexes/{}/search", self.uid);
self.service.post(url, query).await
}
pub async fn search_get(&self, query: Value) -> (Value, StatusCode) {
let params = serde_url_params::to_string(&query).unwrap();
let url = format!("/indexes/{}/search?{}", self.uid, params);
self.service.get(url).await
}
make_settings_test_routes!(distinct_attribute);
}

View File

@ -61,7 +61,7 @@ async fn get_no_documents() {
let server = Server::new().await;
let index = server.index("test");
let (_, code) = index.create(None).await;
assert_eq!(code, 200);
assert_eq!(code, 201);
let (response, code) = index
.get_all_documents(GetAllDocumentsOptions::default())

View File

@ -7,7 +7,7 @@ async fn create_index_no_primary_key() {
let index = server.index("test");
let (response, code) = index.create(None).await;
assert_eq!(code, 200);
assert_eq!(code, 201);
assert_eq!(response["uid"], "test");
assert_eq!(response["name"], "test");
assert!(response.get("createdAt").is_some());
@ -23,7 +23,7 @@ async fn create_index_with_primary_key() {
let index = server.index("test");
let (response, code) = index.create(Some("primary")).await;
assert_eq!(code, 200);
assert_eq!(code, 201);
assert_eq!(response["uid"], "test");
assert_eq!(response["name"], "test");
assert!(response.get("createdAt").is_some());
@ -41,7 +41,7 @@ async fn create_existing_index() {
let index = server.index("test");
let (_, code) = index.create(Some("primary")).await;
assert_eq!(code, 200);
assert_eq!(code, 201);
let (_response, code) = index.create(Some("primary")).await;
assert_eq!(code, 400);

View File

@ -1,3 +1,5 @@
use serde_json::json;
use crate::common::Server;
#[actix_rt::test]
@ -6,7 +8,7 @@ async fn create_and_delete_index() {
let index = server.index("test");
let (_response, code) = index.create(None).await;
assert_eq!(code, 200);
assert_eq!(code, 201);
let (_response, code) = index.delete().await;
@ -23,3 +25,16 @@ async fn delete_unexisting_index() {
assert_eq!(code, 404);
}
#[actix_rt::test]
async fn loop_delete_add_documents() {
let server = Server::new().await;
let index = server.index("test");
let documents = json!([{"id": 1, "field1": "hello"}]);
for _ in 0..50 {
let (response, code) = index.add_documents(documents.clone(), None).await;
assert_eq!(code, 202, "{}", response);
let (response, code) = index.delete().await;
assert_eq!(code, 204, "{}", response);
}
}

View File

@ -7,7 +7,7 @@ async fn create_and_get_index() {
let index = server.index("test");
let (_, code) = index.create(None).await;
assert_eq!(code, 200);
assert_eq!(code, 201);
let (response, code) = index.get().await;

View File

@ -8,7 +8,7 @@ async fn stats() {
let index = server.index("test");
let (_, code) = index.create(Some("id")).await;
assert_eq!(code, 200);
assert_eq!(code, 201);
let (response, code) = index.stats().await;

View File

@ -7,7 +7,7 @@ async fn update_primary_key() {
let index = server.index("test");
let (_, code) = index.create(None).await;
assert_eq!(code, 200);
assert_eq!(code, 201);
let (response, code) = index.update(Some("primary")).await;
@ -31,7 +31,7 @@ async fn update_nothing() {
let index = server.index("test");
let (response, code) = index.create(None).await;
assert_eq!(code, 200);
assert_eq!(code, 201);
let (update, code) = index.update(None).await;
@ -47,7 +47,7 @@ async fn update_existing_primary_key() {
let index = server.index("test");
let (_response, code) = index.create(Some("primary")).await;
assert_eq!(code, 200);
assert_eq!(code, 201);
let (_update, code) = index.update(Some("primary2")).await;

View File

@ -0,0 +1,28 @@
use crate::common::Server;
use serde_json::json;
#[actix_rt::test]
async fn search_unexisting_index() {
let server = Server::new().await;
let index = server.index("test");
index
.search(json!({"q": "hello"}), |response, code| {
assert_eq!(code, 404, "{}", response);
assert_eq!(response["errorCode"], "index_not_found");
})
.await;
}
#[actix_rt::test]
async fn search_unexisting_parameter() {
let server = Server::new().await;
let index = server.index("test");
index
.search(json!({"marin": "hello"}), |response, code| {
assert_eq!(code, 400, "{}", response);
assert_eq!(response["errorCode"], "bad_request");
})
.await;
}

View File

@ -1,2 +1,195 @@
// This modules contains all the test concerning search. Each particular feture of the search
// should be tested in its own module to isolate tests and keep the tests readable.
mod errors;
use crate::common::Server;
use once_cell::sync::Lazy;
use serde_json::{json, Value};
static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
json!([
{
"title": "Shazam!",
"id": "287947"
},
{
"title": "Captain Marvel",
"id": "299537"
},
{
"title": "Escape Room",
"id": "522681"
},
{ "title": "How to Train Your Dragon: The Hidden World", "id": "166428"
},
{
"title": "Glass",
"id": "450465"
}
])
});
#[actix_rt::test]
async fn simple_placeholder_search() {
let server = Server::new().await;
let index = server.index("test");
let documents = DOCUMENTS.clone();
index.add_documents(documents, None).await;
index.wait_update_id(0).await;
index
.search(json!({}), |response, code| {
assert_eq!(code, 200, "{}", response);
assert_eq!(response["hits"].as_array().unwrap().len(), 5);
})
.await;
}
#[actix_rt::test]
async fn simple_search() {
let server = Server::new().await;
let index = server.index("test");
let documents = DOCUMENTS.clone();
index.add_documents(documents, None).await;
index.wait_update_id(0).await;
index
.search(json!({"q": "glass"}), |response, code| {
assert_eq!(code, 200, "{}", response);
assert_eq!(response["hits"].as_array().unwrap().len(), 1);
})
.await;
}
#[actix_rt::test]
async fn search_multiple_params() {
let server = Server::new().await;
let index = server.index("test");
let documents = DOCUMENTS.clone();
index.add_documents(documents, None).await;
index.wait_update_id(0).await;
index
.search(
json!({
"q": "glass",
"attributesToCrop": ["title:2"],
"attributesToHighlight": ["title"],
"limit": 1,
"offset": 0,
}),
|response, code| {
assert_eq!(code, 200, "{}", response);
assert_eq!(response["hits"].as_array().unwrap().len(), 1);
},
)
.await;
}
#[actix_rt::test]
async fn search_with_filter_string_notation() {
let server = Server::new().await;
let index = server.index("test");
index
.update_settings(json!({"filterableAttributes": ["title"]}))
.await;
let documents = DOCUMENTS.clone();
index.add_documents(documents, None).await;
index.wait_update_id(1).await;
index
.search(
json!({
"filter": "title = Glass"
}),
|response, code| {
assert_eq!(code, 200, "{}", response);
assert_eq!(response["hits"].as_array().unwrap().len(), 1);
},
)
.await;
}
#[actix_rt::test]
async fn search_with_filter_array_notation() {
let server = Server::new().await;
let index = server.index("test");
index
.update_settings(json!({"filterableAttributes": ["title"]}))
.await;
let documents = DOCUMENTS.clone();
index.add_documents(documents, None).await;
index.wait_update_id(1).await;
let (response, code) = index
.search_post(json!({
"filter": ["title = Glass"]
}))
.await;
assert_eq!(code, 200, "{}", response);
assert_eq!(response["hits"].as_array().unwrap().len(), 1);
let (response, code) = index
.search_post(json!({
"filter": [["title = Glass", "title = \"Shazam!\"", "title = \"Escape Room\""]]
}))
.await;
assert_eq!(code, 200, "{}", response);
assert_eq!(response["hits"].as_array().unwrap().len(), 3);
}
#[actix_rt::test]
async fn search_facet_distribution() {
let server = Server::new().await;
let index = server.index("test");
index
.update_settings(json!({"filterableAttributes": ["title"]}))
.await;
let documents = DOCUMENTS.clone();
index.add_documents(documents, None).await;
index.wait_update_id(1).await;
index
.search(
json!({
"facetsDistribution": ["title"]
}),
|response, code| {
assert_eq!(code, 200, "{}", response);
let dist = response["facetsDistribution"].as_object().unwrap();
assert_eq!(dist.len(), 1);
assert!(dist.get("title").is_some());
},
)
.await;
}
#[actix_rt::test]
async fn displayed_attributes() {
let server = Server::new().await;
let index = server.index("test");
index
.update_settings(json!({ "displayedAttributes": ["title"] }))
.await;
let documents = DOCUMENTS.clone();
index.add_documents(documents, None).await;
index.wait_update_id(1).await;
let (response, code) = index
.search_post(json!({ "attributesToRetrieve": ["title", "id"] }))
.await;
assert_eq!(code, 200, "{}", response);
assert!(response["hits"].get("title").is_none());
}

View File

@ -206,7 +206,7 @@ macro_rules! test_setting_routes {
let server = Server::new().await;
let index = server.index("test");
let (response, code) = index.create(None).await;
assert_eq!(code, 200, "{}", response);
assert_eq!(code, 201, "{}", response);
let url = format!("/indexes/test/settings/{}",
stringify!($setting)
.chars()

View File

@ -28,7 +28,7 @@ async fn stats() {
let index = server.index("test");
let (_, code) = index.create(Some("id")).await;
assert_eq!(code, 200);
assert_eq!(code, 201);
let (response, code) = server.stats().await;