diff --git a/server/src/bin/sync-qdrant.rs b/server/src/bin/sync-qdrant.rs index bf53657c67..c42de95017 100644 --- a/server/src/bin/sync-qdrant.rs +++ b/server/src/bin/sync-qdrant.rs @@ -40,10 +40,11 @@ async fn main() -> Result<(), ServiceError> { let mut offset = Some(uuid::Uuid::nil().to_string()); while let Some(cur_offset) = offset { + println!("cur_offset: {}", cur_offset); let (qdrant_point_ids, new_offset) = scroll_qdrant_collection_ids( collection.clone(), Some(cur_offset.to_string()), - Some(1000), + Some(10000), ) .await?; @@ -56,35 +57,29 @@ async fn main() -> Result<(), ServiceError> { .map(|(x, _)| *x) .collect::>(); - let qdrant_point_ids_and_datasets_not_in_pg = qdrant_point_ids - .iter() - .filter(|x| !pg_point_ids.contains(x)) - .filter_map(|x| pg_point_ids_and_datasets.iter().find(|(y, _)| y == x)) - .cloned() - .collect::>(); - - let qdrant_point_ids_not_in_pg = qdrant_point_ids_and_datasets_not_in_pg - .iter() - .map(|(x, _)| *x) - .collect::>(); + total += qdrant_point_ids.len(); - let datasets_out_of_sync = qdrant_point_ids_and_datasets_not_in_pg + let qdrant_points_missing = qdrant_point_ids .iter() - .map(|(_, x)| *x) + .filter(|x| !pg_point_ids.contains(x)) + .map(|x| *x) .collect::>(); - total += qdrant_point_ids.len(); - - if !qdrant_point_ids_not_in_pg.is_empty() { + if !qdrant_points_missing.is_empty() { println!( - "len of qdrant_point_ids_not_in_pg: {:?}, {:?}", - qdrant_point_ids_not_in_pg.len(), - datasets_out_of_sync + "len of qdrant_point_ids_not_in_pg: {:?}", + qdrant_points_missing.len(), ); - delete_points_from_qdrant(qdrant_point_ids_not_in_pg, collection.clone()).await?; + delete_points_from_qdrant(qdrant_points_missing, collection.clone()).await?; } else { - println!("Scrolled {}/{}", qdrant_point_ids.len(), total); + println!( + "{:?} Scrolled {}(qd) {}(pg) /{}", + qdrant_point_ids.get(0), + qdrant_point_ids.len(), + pg_point_ids.len(), + total + ); } offset = new_offset;