with binary quantization

This commit is contained in:
Tamo
2024-07-08 20:12:56 +02:00
parent 05cc2d1fac
commit 42bfc67871
4 changed files with 28 additions and 6 deletions

View File

@@ -169,7 +169,7 @@ pub struct Index {
/// Maps an embedder name to its id in the arroy store.
pub embedder_category_id: Database<Str, U8>,
/// Vector store based on arroy™.
pub vector_arroy: arroy::Database<arroy::distances::Angular>,
pub vector_arroy: arroy::Database<arroy::distances::BinaryQuantizedEuclidean>,
/// Maps the document id to the document as an obkv store.
pub(crate) documents: Database<BEU32, ObkvCodec>,
@@ -1605,7 +1605,8 @@ impl Index {
&'a self,
rtxn: &'a RoTxn<'a>,
embedder_id: u8,
) -> impl Iterator<Item = Result<arroy::Reader<arroy::distances::Angular>>> + 'a {
) -> impl Iterator<Item = Result<arroy::Reader<arroy::distances::BinaryQuantizedEuclidean>>> + 'a
{
crate::vector::arroy_db_range_for_embedder(embedder_id).map_while(move |k| {
arroy::Reader::open(rtxn, k, self.vector_arroy)
.map(Some)

View File

@@ -990,7 +990,10 @@ impl<'a, 'i> Transform<'a, 'i> {
};
let readers: Result<
BTreeMap<&str, (Vec<arroy::Reader<arroy::distances::Angular>>, &RoaringBitmap)>,
BTreeMap<
&str,
(Vec<arroy::Reader<arroy::distances::BinaryQuantizedEuclidean>>, &RoaringBitmap),
>,
> = settings_diff
.embedding_config_updates
.iter()