1use std::collections::HashMap;
2
3use super::DeviceSyncError;
4use crate::{
5 context::XmtpSharedContext,
6 groups::{MlsGroup, group_permissions::PolicySet},
7 worker::device_sync::MissingField,
8};
9use futures::StreamExt;
10pub use xmtp_archive::*;
11use xmtp_db::{
12 ConnectionExt, StoreOrIgnore, XmtpMlsStorageProvider,
13 consent_record::StoredConsentRecord,
14 group::{ConversationType, DmIdExt, GroupMembershipState},
15 group_message::StoredGroupMessage,
16 prelude::*,
17};
18use xmtp_mls_common::group::{DMMetadataOptions, GroupMetadataOptions};
19use xmtp_mls_common::group_mutable_metadata::MessageDisappearingSettings;
20use xmtp_proto::xmtp::device_sync::{BackupElement, backup_element::Element};
21
22use xmtp_proto::types::GroupId;
23#[derive(Default)]
24struct ImportContext {
25 group_timestamps: HashMap<Vec<u8>, Option<i64>>,
26}
27
28impl ImportContext {
29 fn post_import(&self, context: &impl XmtpSharedContext) -> Result<(), DeviceSyncError> {
30 use xmtp_db::diesel::prelude::*;
31 use xmtp_db::diesel::sql_types::{BigInt, Nullable};
32 use xmtp_db::schema::groups::dsl;
33
34 for (group_id, timestamp) in &self.group_timestamps {
37 crate::state_tx::state_write(context.mls_storage(), |tx| {
38 let storage = tx.storage();
39 storage.db().raw_query(|conn| {
40 let newest = xmtp_db::diesel::dsl::sql::<Nullable<BigInt>>(
41 "CASE WHEN last_message_ns IS NULL OR last_message_ns < ",
42 )
43 .bind::<Nullable<BigInt>, _>(*timestamp)
44 .sql(" THEN ")
45 .bind::<Nullable<BigInt>, _>(*timestamp)
46 .sql(" ELSE last_message_ns END");
47 xmtp_db::diesel::update(dsl::groups.find(group_id))
48 .set(dsl::last_message_ns.eq(newest))
49 .execute(conn)
50 })?;
51 Ok::<_, xmtp_db::StorageError>(xmtp_db::TransactionOutcome::Continue(()))
52 })?;
53 }
54
55 Ok(())
56 }
57}
58
59pub async fn insert_importer(
60 importer: &mut ArchiveImporter,
61 context: &impl XmtpSharedContext,
62) -> Result<(), DeviceSyncError> {
63 let mut import_ctx = ImportContext::default();
64
65 while let Some(element) = importer.next().await {
66 let element = element?;
67 insert(element, context, &mut import_ctx)?;
69 }
70
71 import_ctx.post_import(context)?;
72
73 Ok(())
74}
75
76fn insert(
77 element: BackupElement,
78 context: &impl XmtpSharedContext,
79 import_context: &mut ImportContext,
80) -> Result<(), DeviceSyncError> {
81 let Some(element) = element.element else {
82 return Ok(());
83 };
84
85 match element {
86 Element::Consent(consent) => {
87 let consent: StoredConsentRecord = consent.try_into()?;
88 context.db().insert_newer_consent_record(consent)?;
89 }
90 Element::Group(save) => {
91 if let Some(existing_group) = context
94 .db()
95 .find_group(&GroupId::try_from(save.id.as_slice())?)?
96 {
97 let timestamp = match (existing_group.last_message_ns, save.last_message_ns) {
98 (Some(e), Some(s)) => Some(e.max(s)),
99 (None, Some(s)) => Some(s),
100 (Some(e), None) => Some(e),
101 (None, None) => None,
102 };
103
104 import_context
105 .group_timestamps
106 .insert(existing_group.id.to_vec(), timestamp);
107 return Ok(());
109 }
110
111 let conversation_type = save.conversation_type().try_into()?;
112 let attributes = save
113 .mutable_metadata
114 .map(|m| m.attributes)
115 .unwrap_or_default();
116
117 import_context
120 .group_timestamps
121 .insert(save.id.clone(), save.last_message_ns);
122 let message_disappearing_settings =
123 match (save.message_disappear_from_ns, save.message_disappear_in_ns) {
124 (Some(from_ns), Some(in_ns)) => {
125 Some(MessageDisappearingSettings::new(from_ns, in_ns))
126 }
127 _ => None,
128 };
129
130 match conversation_type {
131 ConversationType::Dm => {
132 let Some(dm_id) = save.dm_id else {
133 return Err(DeviceSyncError::MissingField(
134 MissingField::Conversation(super::ConversationField::DmId),
135 format!("DM with id of {:?} was missing the dm_id field.", save.id),
136 ));
137 };
138
139 let target_inbox_id = dm_id.other_inbox_id(context.inbox_id());
140
141 MlsGroup::create_dm_and_insert(
142 context,
143 GroupMembershipState::Restored,
144 target_inbox_id,
145 DMMetadataOptions {
146 message_disappearing_settings,
147 },
148 Some(&save.id),
149 )?;
150 }
151 _ => {
152 MlsGroup::insert(
153 context,
154 Some(&save.id),
155 GroupMembershipState::Restored,
156 ConversationType::Group,
157 PolicySet::default(),
158 GroupMetadataOptions {
159 name: attributes.get("group_name").cloned(),
160 image_url_square: attributes.get("group_image_url_square").cloned(),
161 description: attributes.get("description").cloned(),
162 app_data: attributes.get("app_data").cloned(),
163 message_disappearing_settings,
164 },
165 None,
166 )?;
167 }
168 }
169 }
170 Element::GroupMessage(message) => {
171 let message: StoredGroupMessage = message.try_into()?;
172 message.store_or_ignore(&context.db())?;
173 }
174 _ => {}
175 }
176
177 Ok(())
178}
179
180#[cfg(test)]
181mod tests {
182 #![allow(unused)]
183 use super::*;
184 use crate::groups::send_message_opts::SendMessageOpts;
185 use crate::tester;
186 use crate::utils::{LocalTester, Tester};
187 use crate::worker::device_sync::{ArchiveOptions, BackupElementSelection};
188 use crate::{builder::ClientBuilder, utils::test::wait_for_min_intents};
189 use diesel::prelude::*;
190 use futures::AsyncReadExt;
191 use futures::io::{BufReader, Cursor};
192 use std::{path::Path, sync::Arc};
193 use xmtp_archive::exporter::ArchiveExporter;
194 use xmtp_cryptography::utils::generate_local_wallet;
195 use xmtp_db::group_message::MsgQueryArgs;
196 use xmtp_db::{
197 consent_record::StoredConsentRecord,
198 group::StoredGroup,
199 group_message::StoredGroupMessage,
200 schema::{consent_records, group_messages, groups},
201 };
202
203 #[xmtp_common::test(unwrap_try = true)]
204 async fn archive_timestamp_keeps_a_message_received_during_import() {
205 tester!(alix, disable_workers);
206 let group = alix.create_group(None, None)?;
207 let pending_import = ImportContext {
208 group_timestamps: [(group.group_id.to_vec(), Some(0))].into(),
209 };
210
211 group.send_message_optimistic(b"message during import", Default::default())?;
212 let current = alix.db().find_group(&group.group_id)??.last_message_ns;
213 assert!(current > Some(0));
214 pending_import.post_import(&alix.context)?;
215 assert_eq!(
216 alix.db().find_group(&group.group_id)??.last_message_ns,
217 current
218 );
219 }
220
221 #[xmtp_common::test(unwrap_try = true)]
222 async fn test_archive_timestamps() {
223 tester!(alix, disable_workers);
224 tester!(alix2, from: alix);
225 tester!(bo, disable_workers);
226
227 let alix_group = alix
228 .create_group_with_members(&[bo.inbox_id()], None, None)
229 .await?;
230 alix_group.send_message(b"hi", Default::default()).await?;
231
232 alix2.sync_welcomes().await?;
233 bo.sync_welcomes().await?;
234
235 let alix2_group = alix2.group(&alix_group.group_id)?;
236 let bo_group = bo.group(&alix_group.group_id)?;
237
238 alix2_group.sync().await?;
239 bo_group.sync().await?;
240
241 alix_group
243 .send_message(b"Hello again", Default::default())
244 .await?;
245
246 let key = vec![7; 32];
247 let opts = ArchiveOptions {
248 start_ns: None,
249 end_ns: None,
250 elements: vec![
251 BackupElementSelection::Messages,
252 BackupElementSelection::Consent,
253 ],
254 exclude_disappearing_messages: false,
255 };
256 let export = {
257 let mut file = vec![];
258 let mut exporter = ArchiveExporter::new(opts, alix.db(), &key);
259 exporter.read_to_end(&mut file).await?;
260 file
261 };
262
263 tester!(alix3, from: alix);
264
265 let reader = Box::pin(BufReader::new(Cursor::new(export.clone())));
268 let mut importer = ArchiveImporter::load(reader, &key).await?;
269 insert_importer(&mut importer, &alix2.context).await?;
270
271 let reader = Box::pin(BufReader::new(Cursor::new(export)));
272 let mut importer = ArchiveImporter::load(reader, &key).await?;
273 insert_importer(&mut importer, &alix3.context).await?;
274
275 let alix_timestamp = alix
276 .db()
277 .find_group(&alix_group.group_id)??
278 .last_message_ns?;
279 let alix2_timestamp = alix2
280 .db()
281 .find_group(&alix_group.group_id)??
282 .last_message_ns?;
283 let alix3_timestamp = alix3
284 .db()
285 .find_group(&alix_group.group_id)??
286 .last_message_ns?;
287
288 assert_eq!(alix2_timestamp, alix_timestamp);
290 assert_eq!(alix3_timestamp, alix_timestamp);
292 }
293
294 #[xmtp_common::test(unwrap_try = true)]
295 async fn test_dm_archive() {
296 tester!(alix, disable_workers);
297 tester!(bo, disable_workers);
298
299 let alix_bo_dm = alix.find_or_create_dm(bo.inbox_id(), None).await?;
300 let archived_message_id = alix_bo_dm
301 .send_message(b"old group", Default::default())
302 .await?;
303
304 let timestamp = alix
305 .db()
306 .find_group(&alix_bo_dm.group_id)??
307 .last_message_ns?;
308
309 let key = vec![7; 32];
310 let opts = ArchiveOptions {
311 start_ns: None,
312 end_ns: None,
313 elements: vec![
314 BackupElementSelection::Messages,
315 BackupElementSelection::Consent,
316 ],
317 exclude_disappearing_messages: false,
318 };
319 let export = {
320 let mut file = vec![];
321
322 let mut exporter = ArchiveExporter::new(opts, alix.db(), &key);
323 exporter.read_to_end(&mut file).await?;
324 file
325 };
326
327 tester!(alix2, from: alix);
328 let reader = Box::pin(BufReader::new(Cursor::new(export)));
329 let mut importer = ArchiveImporter::load(reader, &key).await?;
330 insert_importer(&mut importer, &alix2.context).await?;
331
332 let restored = alix2.group(&alix_bo_dm.group_id)?;
335 let restored_topic = xmtp_db::incoming_envelope::StreamTopic::group(alix_bo_dm.group_id);
336 let before_join = alix2.db().topic_progress(&restored_topic)?;
337 crate::mls_store::MlsStore::new(alix2.context.clone())
338 .receive_topics_once(
339 &[xmtp_proto::types::Topic::new_group_message(
340 alix_bo_dm.group_id,
341 )],
342 alix2
343 .context
344 .incoming_runtime()
345 .policy()
346 .incoming_limits(xmtp_db::incoming_envelope::NetworkEntityKind::Group),
347 )
348 .await?;
349 let received_before_join = alix2.db().topic_progress(&restored_topic)?;
350 assert!(received_before_join.received > before_join.processed);
351 assert!(matches!(
352 restored.process_pending_group_head(None)?,
353 crate::groups::mls_sync::GroupHeadOutcome::Inactive
354 ));
355 assert_eq!(
356 alix2.db().topic_progress(&restored_topic)?.processed,
357 before_join.processed
358 );
359 assert!(
360 !alix2
361 .db()
362 .pending_states_through(&restored_topic, received_before_join.received)?
363 .is_empty()
364 );
365
366 let alix2_bo_dm = alix2.find_or_create_dm(bo.inbox_id(), None).await?;
367 assert_ne!(alix_bo_dm.group_id, alix2_bo_dm.group_id);
368 let mut msgs = alix2_bo_dm.find_messages(&MsgQueryArgs::default())?;
369 assert_eq!(msgs.len(), 2);
370 assert!(
371 msgs.iter()
372 .any(|m| m.decrypted_message_bytes == b"old group")
373 );
374
375 let timestamp2 = alix2
378 .db()
379 .find_group(&alix_bo_dm.group_id)??
380 .last_message_ns?;
381 assert_eq!(timestamp, timestamp2);
382
383 let live_message_id = alix2_bo_dm
384 .send_message(b"hi bo", Default::default())
385 .await?;
386
387 bo.sync_all_welcomes_and_groups(None).await?;
388 let bo_alix2_dm = bo.group(&alix2_bo_dm.group_id)?;
389 assert_eq!(bo_alix2_dm.test_last_message_bytes().await??, b"hi bo");
390
391 alix2.sync_all_welcomes_and_groups(None).await?;
393 let rejoined_original = alix2.group(&alix_bo_dm.group_id)?;
394 assert!(rejoined_original.is_active()?);
395 let stitched = alix2_bo_dm.find_messages(&MsgQueryArgs::default())?;
396 assert_eq!(stitched.len(), 4);
397 let application_ids: Vec<_> = stitched
398 .iter()
399 .filter(|message| message.kind == xmtp_db::group_message::GroupMessageKind::Application)
400 .map(|message| message.id.clone())
401 .collect();
402 assert_eq!(application_ids, vec![archived_message_id, live_message_id]);
403 assert_eq!(alix2.find_groups(Default::default())?.len(), 1);
404 let bo_original = bo.group(&alix_bo_dm.group_id)?;
405 rejoined_original.test_can_talk_with(&bo_original).await?;
406 bo_original.test_can_talk_with(&rejoined_original).await?;
407 }
408
409 #[rstest::rstest]
410 #[xmtp_common::test]
411 async fn test_buffer_export_import() {
412 use futures::io::BufReader;
413 use futures_util::AsyncReadExt;
414
415 tester!(alix);
416 tester!(bo);
417
418 let alix_group = alix.create_group(None, None).unwrap();
419 alix_group.add_members(&[bo.inbox_id()]).await.unwrap();
420 alix_group
421 .send_message(b"hello there", SendMessageOpts::default())
422 .await
423 .unwrap();
424
425 let opts = ArchiveOptions {
426 start_ns: None,
427 end_ns: None,
428 elements: vec![
429 BackupElementSelection::Messages,
430 BackupElementSelection::Consent,
431 ],
432 exclude_disappearing_messages: false,
433 };
434
435 let key = vec![7; 32];
436
437 let file = {
438 let mut file = Vec::new();
439 let mut exporter = ArchiveExporter::new(opts, alix.db(), &key);
440 exporter.read_to_end(&mut file).await.unwrap();
441 file
442 };
443
444 let alix2_wallet = generate_local_wallet();
445 let alix2 = ClientBuilder::new_test_client(&alix2_wallet).await;
446
447 let messages: Vec<StoredGroupMessage> = alix2
449 .context
450 .db()
451 .raw_query(|conn| {
452 group_messages::table
453 .select(StoredGroupMessage::as_select())
454 .load(conn)
455 })
456 .unwrap();
457 assert_eq!(messages.len(), 0);
458
459 let reader = BufReader::new(Cursor::new(file));
460 let reader = Box::pin(reader);
461 let mut importer = ArchiveImporter::load(reader, &key).await.unwrap();
462 insert_importer(&mut importer, &alix2.context)
463 .await
464 .unwrap();
465
466 let messages: Vec<StoredGroupMessage> = alix2
468 .context
469 .db()
470 .raw_query(|conn| {
471 group_messages::table
472 .select(StoredGroupMessage::as_select())
473 .load(conn)
474 })
475 .unwrap();
476 assert_eq!(messages.len(), 1);
477 }
478
479 #[xmtp_common::test(unwrap_try = true)]
480 #[cfg(not(target_arch = "wasm32"))]
481 async fn test_file_backup() {
482 use crate::{groups::send_message_opts::SendMessageOpts, tester};
483 use diesel::QueryDsl;
484 use xmtp_db::group::{ConversationType, GroupQueryArgs};
485
486 tester!(alix, sync_worker, triggers);
487 tester!(bo);
488
489 let alix_group = alix.create_group(None, None)?;
490
491 wait_for_min_intents(&alix.context.db(), 2).await?;
493
494 alix_group.add_members(&[bo.inbox_id()]).await?;
495 alix_group.update_group_name("My group".to_string()).await?;
496
497 bo.sync_welcomes().await?;
498 let bo_group = bo.group(&alix_group.group_id)?;
499
500 wait_for_min_intents(&alix.context.db(), 1).await?;
502
503 alix_group
504 .send_message(b"hello there", SendMessageOpts::default())
505 .await?;
506
507 wait_for_min_intents(&alix.context.db(), 4).await?;
510
511 let mut consent_records: Vec<StoredConsentRecord> = alix
512 .context
513 .db()
514 .raw_query(|conn| consent_records::table.load(conn))?;
515 assert_eq!(consent_records.len(), 1);
516 let old_consent_record = consent_records.pop()?;
517
518 let mut groups: Vec<StoredGroup> = alix
519 .context
520 .db()
521 .raw_query(|conn| groups::table.load(conn))?;
522 assert_eq!(groups.len(), 2);
523 let old_group = groups.pop()?;
524
525 let old_messages: Vec<StoredGroupMessage> = alix.context.db().raw_query(|conn| {
526 group_messages::table
527 .select(StoredGroupMessage::as_select())
528 .load(conn)
529 })?;
530 assert_eq!(old_messages.len(), 4);
531
532 let opts = ArchiveOptions {
533 start_ns: None,
534 end_ns: None,
535 elements: vec![
536 BackupElementSelection::Messages,
537 BackupElementSelection::Consent,
538 ],
539 exclude_disappearing_messages: false,
540 };
541
542 let key = xmtp_common::rand_vec::<32>();
543 let mut exporter = ArchiveExporter::new(opts, alix.db(), &key);
544 let path = Path::new("archive.xmtp");
545 let _ = tokio::fs::remove_file(path).await;
546 exporter.write_to_file(path).await?;
547
548 tester!(alix2, sync_worker);
549 alix2.device_sync_client().wait_for_sync_worker_init().await;
550
551 let consent_records: Vec<StoredConsentRecord> = alix2
553 .context
554 .db()
555 .raw_query(|conn| consent_records::table.load(conn))?;
556 assert_eq!(consent_records.len(), 0);
557
558 let mut importer = ArchiveImporter::from_file(path, &key).await?;
559 insert_importer(&mut importer, &alix2.context)
560 .await
561 .unwrap();
562
563 let consent_records: Vec<StoredConsentRecord> = alix2
565 .context
566 .db()
567 .raw_query(|conn| consent_records::table.load(conn))?;
568 assert_eq!(consent_records.len(), 1);
569 assert_eq!(consent_records[0], old_consent_record);
571
572 let groups: Vec<StoredGroup> = alix2.context.db().raw_query(|conn| {
573 groups::table
574 .filter(groups::conversation_type.ne_all(ConversationType::virtual_types()))
575 .load(conn)
576 })?;
577 assert_eq!(groups.len(), 1);
578 assert_eq!(groups[0].id, old_group.id);
580
581 let messages: Vec<StoredGroupMessage> = alix2.context.db().raw_query(|conn| {
582 group_messages::table
583 .select(StoredGroupMessage::as_select())
584 .filter(group_messages::group_id.eq(&groups[0].id))
585 .load(conn)
586 })?;
587 assert_eq!(messages.len(), 1);
589 for msg in messages {
590 let old_msg = old_messages.iter().find(|m| msg.id == m.id)?;
591 assert_eq!(old_msg.authority_id, msg.authority_id);
592 assert_eq!(old_msg.decrypted_message_bytes, msg.decrypted_message_bytes);
593 assert_eq!(old_msg.sent_at_ns, msg.sent_at_ns);
594 assert_eq!(old_msg.sender_installation_id, msg.sender_installation_id);
595 assert_eq!(old_msg.sender_inbox_id, msg.sender_inbox_id);
596 assert_eq!(old_msg.group_id, msg.group_id);
597 }
598
599 let alix2_group = alix2.group(&old_group.id)?;
600 let _groups = alix2.find_groups(GroupQueryArgs::default())?;
602 alix2_group.group_name()?;
604 assert!(!alix2_group.is_active()?);
605
606 alix.group(&old_group.id)?
608 .add_members(&[alix2.inbox_id()])
609 .await?;
610 alix2.sync_welcomes().await?;
611
612 let alix2_group = alix2.group(&old_group.id)?;
614 assert!(alix2_group.is_active()?);
615
616 let msgs = alix2_group.find_messages(&MsgQueryArgs::default())?;
618 let old_msg_exists = msgs
619 .iter()
620 .any(|msg| msg.decrypted_message_bytes == b"hello there");
621 assert!(old_msg_exists);
622
623 alix2_group
625 .send_message(b"this should send", SendMessageOpts::default())
626 .await?;
627 bo_group.sync().await?;
628 let msgs = bo_group.find_messages(&MsgQueryArgs::default())?;
629 let new_msg_exists = msgs
630 .iter()
631 .any(|msg| msg.decrypted_message_bytes == b"this should send");
632 assert!(new_msg_exists);
633
634 let _ = tokio::fs::remove_file(path).await;
636 }
637
638 #[xmtp_common::test(unwrap_try = true)]
639 #[cfg(not(target_arch = "wasm32"))]
640 async fn test_legacy_archive_import() {
641 use std::path::PathBuf;
642
643 use crate::tester;
644
645 let key = vec![0; 32];
646 let path = PathBuf::from("tests/assets/archive-legacy.xmtp");
647 let mut importer = ArchiveImporter::from_file(path, &key).await?;
648
649 tester!(alix);
650
651 let result = insert_importer(&mut importer, &alix.context).await;
652 assert!(result.is_ok());
653 }
654
655 #[xmtp_common::test(unwrap_try = true)]
664 async fn test_archive_includes_migrated_groups() {
665 use crate::groups::EnableProposalsOptions;
666
667 tester!(alix, disable_workers);
668 tester!(bo, disable_workers);
669
670 let alix_group = alix
671 .create_group_with_members(&[bo.inbox_id()], None, None)
672 .await?;
673 alix_group.send_message(b"hi", Default::default()).await?;
674
675 alix_group
678 .enable_proposals(EnableProposalsOptions::test_default())
679 .await?;
680 alix_group
681 .update_group_name("post-migration name".to_string())
682 .await?;
683 alix_group
684 .update_group_description("post-migration description".to_string())
685 .await?;
686 alix_group
687 .update_group_image_url_square("https://example.com/post-migration.png".to_string())
688 .await?;
689
690 let legacy_group = alix
694 .create_group_with_members(&[bo.inbox_id()], None, None)
695 .await?;
696 legacy_group
697 .update_group_name("legacy name".to_string())
698 .await?;
699
700 let key = vec![7; 32];
701 let opts = ArchiveOptions {
702 start_ns: None,
703 end_ns: None,
704 elements: vec![
705 BackupElementSelection::Messages,
706 BackupElementSelection::Consent,
707 ],
708 exclude_disappearing_messages: false,
709 };
710 let export = {
711 let mut file = vec![];
712 let mut exporter = ArchiveExporter::new(opts, alix.db(), &key);
713 exporter.read_to_end(&mut file).await?;
714 file
715 };
716
717 tester!(alix2, from: alix);
720 let reader = Box::pin(BufReader::new(Cursor::new(export)));
721 let mut importer = ArchiveImporter::load(reader, &key).await?;
722 insert_importer(&mut importer, &alix2.context).await?;
723
724 let restored = alix2.db().find_group(&alix_group.group_id)?;
725 assert!(
726 restored.is_some(),
727 "migrated group missing from restored archive — the exporter \
728 dropped it (pre-fix behavior: legacy-extension read on a \
729 migrated group)"
730 );
731
732 let restored_group = alix2.group(&alix_group.group_id)?;
736 assert_eq!(restored_group.group_name()?, "post-migration name");
737 assert_eq!(
738 restored_group.group_description()?,
739 "post-migration description"
740 );
741 assert_eq!(
742 restored_group.group_image_url_square()?,
743 "https://example.com/post-migration.png"
744 );
745
746 let restored_legacy = alix2.group(&legacy_group.group_id)?;
748 assert_eq!(restored_legacy.group_name()?, "legacy name");
749 }
750}