Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
chunk deletes for delete-account
things blew up (in c++ with a very unhelpful error abort) when an apparently very-large (spam probably) account was deleted. the batch it was writing to grew to over 1GiB.
+44
-32
+44
-32
link_aggregator/src/storage/rocks_store.rs
+44
-32
link_aggregator/src/storage/rocks_store.rs
···
521
521
};
522
522
self.delete_did_id_value(batch, did);
523
523
524
-
for (i, (record_link_key, links)) in self.iter_links_for_did_id(&did_id).enumerate() {
525
-
self.delete_record_link(batch, &record_link_key); // _could_ use delete range here instead of individual deletes, but since we have to scan anyway it's not obvious if it's better
524
+
// use a separate batch for all their links, since it can be a lot and make us crash at around 1GiB batch size.
525
+
// this should still hopefully be crash-safe: as long as we don't actually delete the DidId entry until after all links are cleared.
526
+
// the above .delete_did_id_value is batched, so it shouldn't be written until we've returned from this fn successfully
527
+
// TODO: queue a background delete task or whatever
528
+
// TODO: test delete account with more links than chunk size
529
+
let stuff: Vec<_> = self.iter_links_for_did_id(&did_id).collect();
530
+
for chunk in stuff.chunks(1024) {
531
+
let mut mini_batch = WriteBatch::default();
526
532
527
-
for (j, RecordLinkTarget(path, target_link_id)) in links.0.iter().enumerate() {
528
-
self.update_target_linkers(batch, target_link_id, |mut linkers| {
529
-
if !linkers.remove_last_linker(&did_id) {
530
-
eprintln!(
531
-
"DELETING ACCOUNT: blowing up: missing linker entry in linked target."
532
-
);
533
-
eprintln!("account: {did:?}");
534
-
eprintln!("did_id: {did_id:?}, was active? {active:?}");
535
-
eprintln!("with links: {links:?}");
536
-
eprintln!("and linkers: {linkers:?}");
537
-
eprintln!(
538
-
"working on #{i}.#{j}: {:?} / {path:?} / {target_link_id:?}",
539
-
record_link_key.collection()
540
-
);
541
-
eprintln!("from record link key {record_link_key:?}");
542
-
eprintln!("but could not find this link :/");
543
-
eprintln!("checking for did_id dups...");
544
-
self.check_for_did_dups(&did_id);
545
-
eprintln!("ok so what the heck. did_id again, for did {did:?}:");
546
-
let did_id_again = self
547
-
.did_id_table
548
-
.get_id_val(&self.db, did)
549
-
.unwrap()
550
-
.unwrap();
551
-
eprintln!("did_id_value (again): {did_id_again:?}");
552
-
panic!("ohnoooo");
553
-
}
554
-
Some(linkers)
555
-
})
556
-
.unwrap();
533
+
for (i, (record_link_key, links)) in chunk.iter().enumerate() {
534
+
self.delete_record_link(&mut mini_batch, &record_link_key); // _could_ use delete range here instead of individual deletes, but since we have to scan anyway it's not obvious if it's better
535
+
536
+
for (j, RecordLinkTarget(path, target_link_id)) in links.0.iter().enumerate() {
537
+
self.update_target_linkers(&mut mini_batch, target_link_id, |mut linkers| {
538
+
if !linkers.remove_last_linker(&did_id) {
539
+
eprintln!(
540
+
"DELETING ACCOUNT: blowing up: missing linker entry in linked target."
541
+
);
542
+
eprintln!("account: {did:?}");
543
+
eprintln!("did_id: {did_id:?}, was active? {active:?}");
544
+
eprintln!("with links: {links:?}");
545
+
eprintln!("and linkers: {linkers:?}");
546
+
eprintln!(
547
+
"working on #{i}.#{j}: {:?} / {path:?} / {target_link_id:?}",
548
+
record_link_key.collection()
549
+
);
550
+
eprintln!("from record link key {record_link_key:?}");
551
+
eprintln!("but could not find this link :/");
552
+
eprintln!("checking for did_id dups...");
553
+
self.check_for_did_dups(&did_id);
554
+
eprintln!("ok so what the heck. did_id again, for did {did:?}:");
555
+
let did_id_again = self
556
+
.did_id_table
557
+
.get_id_val(&self.db, did)
558
+
.unwrap()
559
+
.unwrap();
560
+
eprintln!("did_id_value (again): {did_id_again:?}");
561
+
panic!("ohnoooo");
562
+
}
563
+
Some(linkers)
564
+
})
565
+
.unwrap();
566
+
}
557
567
}
568
+
569
+
self.db.write(mini_batch).unwrap(); // todo
558
570
}
559
571
}
560
572
}