From 043faea06a632bf461fcbf0e051aae7a24f01c2d Mon Sep 17 00:00:00 2001 From: Janez T Date: Wed, 1 Apr 2026 13:35:02 +0200 Subject: [PATCH] fix: Dedupe recent messages and wasm --- .github/workflows/build-artifacts.yml | 2 +- Makefile | 3 + lib/providers/messages_provider.dart | 116 ++++++++++++---- lib/services/message_storage_service.dart | 48 ++++--- ...messages_provider_retransmission_test.dart | 127 ++++++++++++++++++ .../message_storage_service_test.dart | 36 +++++ 6 files changed, 289 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 9bbdc1c..2169fca 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -301,7 +301,7 @@ jobs: run: flutter pub get - name: Build web release - run: flutter build web --release --base-href /meshcore-sar/ + run: flutter build web --wasm --release --base-href /meshcore-sar/ - name: Upload pages artifact uses: actions/upload-pages-artifact@v3 diff --git a/Makefile b/Makefile index 4120b54..3d4d8b4 100644 --- a/Makefile +++ b/Makefile @@ -135,6 +135,9 @@ clean: ## Clean build artifacts flutter clean rm -rf $(BUILD_DIR) +build-web: ## Build web with WebAssembly + flutter build web --wasm --release + # Build for all platforms build-all: bump ## Build for Android and iOS flutter build apk --release diff --git a/lib/providers/messages_provider.dart b/lib/providers/messages_provider.dart index cc7dbbc..54e74c0 100644 --- a/lib/providers/messages_provider.dart +++ b/lib/providers/messages_provider.dart @@ -23,6 +23,7 @@ typedef DisplayMessageEntry = ({Message message, int occurrenceCount}); /// Messages Provider - manages message history and SAR markers class MessagesProvider with ChangeNotifier { static const Duration _channelEchoWarningDelay = Duration(seconds: 12); + static const Duration _receivedDuplicateWindow = Duration(seconds: 5); final List _messages = []; final Map _sarMarkers = {}; @@ -731,10 +732,25 @@ class MessagesProvider with ChangeNotifier { return -1; } + final exactDuplicateIndex = _findExactDuplicateMessageIndex(message); + if (exactDuplicateIndex != -1) { + return exactDuplicateIndex; + } + + final lastConversationDuplicateIndex = + _findLastConversationDuplicateMessageIndex(message); + if (lastConversationDuplicateIndex != -1) { + return lastConversationDuplicateIndex; + } + + return -1; + } + + int _findExactDuplicateMessageIndex(Message message) { for (int index = 0; index < _messages.length; index++) { final existing = _messages[index]; if (existing.isSentMessage || - !_matchesDuplicateScope(existing, message) || + !_matchesExactDuplicateScope(existing, message) || existing.text != message.text) { continue; } @@ -751,23 +767,38 @@ class MessagesProvider with ChangeNotifier { return -1; } - bool _matchesDuplicateScope(Message existing, Message message) { + int _findLastConversationDuplicateMessageIndex(Message message) { + for (int index = _messages.length - 1; index >= 0; index--) { + final existing = _messages[index]; + if (!_isSameConversation(existing, message)) { + continue; + } + if (existing.isSentMessage || + existing.isSystemMessage || + existing.text != message.text) { + return -1; + } + final receivedDelta = existing.receivedAt.difference(message.receivedAt).abs(); + if (receivedDelta > _receivedDuplicateWindow) { + return -1; + } + return _matchesDuplicateSenderIdentity(existing, message) ? index : -1; + } + + return -1; + } + + bool _matchesExactDuplicateScope(Message existing, Message message) { if (existing.messageType != message.messageType) { return false; } if (message.isContactMessage) { - // Match by sender key + sender timestamp (matches official app's DB - // uniqueness: contactPublicKey + senderTimestamp + text + txtType). - if (existing.senderKeyShort == message.senderKeyShort && - existing.senderTimestamp == message.senderTimestamp) { - return true; + if (!_isSameConversation(existing, message)) { + return false; } - - // Fallback dedup when retransmits surface as separate inbound rows - // without a stable timestamp/message id, but still carry the same - // visible sender identity and payload. - return _matchesDuplicateSenderIdentity(existing, message); + return existing.senderTimestamp == message.senderTimestamp && + _matchesDuplicateSenderIdentity(existing, message); } if (message.isChannelMessage) { @@ -775,27 +806,40 @@ class MessagesProvider with ChangeNotifier { return false; } - // Primary dedup: same senderTimestamp = same message from mesh repeats. - // Matches official app's DB uniqueness: (channelSecret, senderTimestamp, text). if (existing.senderTimestamp == message.senderTimestamp) { return true; } - - // Secondary: catch near-duplicate repeats within a 30s window - // (clock drift between nodes). - final withinChannelRepeatWindow = - (existing.senderTimestamp - message.senderTimestamp).abs() <= 30; - if (!withinChannelRepeatWindow) { - return false; - } - - return _matchesDuplicateSenderIdentity(existing, message); + return false; } // System messages and other types: never deduplicate by scope alone. return false; } + bool _isSameConversation(Message existing, Message message) { + if (existing.messageType != message.messageType) { + return false; + } + + if (message.isChannelMessage) { + return existing.channelIdx == message.channelIdx; + } + + if (!message.isContactMessage) { + return false; + } + + if (existing.recipientPublicKey != null && message.recipientPublicKey != null) { + return _listEquals(existing.recipientPublicKey!, message.recipientPublicKey!); + } + + if (existing.recipientPublicKey == null && message.recipientPublicKey == null) { + return true; + } + + return false; + } + bool _matchesDuplicateSenderIdentity(Message existing, Message message) { final existingSenderKey = existing.senderKeyShort; final incomingSenderKey = message.senderKeyShort; @@ -825,7 +869,7 @@ class MessagesProvider with ChangeNotifier { continue; } - if (_matchesDuplicateScope(existing, message)) { + if (_matchesSentReplayScope(existing, message)) { return index; } } @@ -833,6 +877,24 @@ class MessagesProvider with ChangeNotifier { return -1; } + bool _matchesSentReplayScope(Message existing, Message message) { + if (!_isSameConversation(existing, message)) { + return false; + } + + if (existing.senderTimestamp == message.senderTimestamp) { + return true; + } + + final withinChannelRepeatWindow = + (existing.senderTimestamp - message.senderTimestamp).abs() <= 30; + if (!withinChannelRepeatWindow) { + return false; + } + + return _matchesDuplicateSenderIdentity(existing, message); + } + /// Add multiple messages void addMessages(List messages) { int addedCount = 0; @@ -1254,7 +1316,9 @@ class MessagesProvider with ChangeNotifier { } return existing.text == message.text && - _matchesDuplicateScope(existing, message); + (_matchesExactDuplicateScope(existing, message) || + (_isSameConversation(existing, message) && + _matchesDuplicateSenderIdentity(existing, message))); } int _messageOccurrenceCount(Message message) => diff --git a/lib/services/message_storage_service.dart b/lib/services/message_storage_service.dart index b4c06e8..12b362a 100644 --- a/lib/services/message_storage_service.dart +++ b/lib/services/message_storage_service.dart @@ -58,10 +58,7 @@ class MessageStorageService { : jsonList; final jsonString = jsonEncode(limitedList); - await prefs.setString( - _key(_messagesKey, namespace: namespace), - jsonString, - ); + final messagesKey = _key(_messagesKey, namespace: namespace); final retainedMessageIds = limitedList .map((entry) => entry['id'] as String) .toSet(); @@ -89,22 +86,41 @@ class MessageStorageService { routeMetadataJson[entry.key] = entry.value.toJson(); } } - await prefs.setString( - _key(_messageContactLocationsKey, namespace: namespace), - jsonEncode(locationJson), + final contactLocationsKey = _key( + _messageContactLocationsKey, + namespace: namespace, ); - await prefs.setString( - _key(_messageReceptionDetailsKey, namespace: namespace), - jsonEncode(receptionJson), + final receptionDetailsKey = _key( + _messageReceptionDetailsKey, + namespace: namespace, ); - await prefs.setString( - _key(_messageTransferDetailsKey, namespace: namespace), - jsonEncode(transferJson), + final transferDetailsKey = _key( + _messageTransferDetailsKey, + namespace: namespace, ); - await prefs.setString( - _key(_messageRouteMetadataKey, namespace: namespace), - jsonEncode(routeMetadataJson), + final routeMetadataKey = _key( + _messageRouteMetadataKey, + namespace: namespace, ); + final locationJsonString = jsonEncode(locationJson); + final receptionJsonString = jsonEncode(receptionJson); + final transferJsonString = jsonEncode(transferJson); + final routeMetadataJsonString = jsonEncode(routeMetadataJson); + final hasChanges = + prefs.getString(messagesKey) != jsonString || + prefs.getString(contactLocationsKey) != locationJsonString || + prefs.getString(receptionDetailsKey) != receptionJsonString || + prefs.getString(transferDetailsKey) != transferJsonString || + prefs.getString(routeMetadataKey) != routeMetadataJsonString; + if (!hasChanges) { + return; + } + + await prefs.setString(messagesKey, jsonString); + await prefs.setString(contactLocationsKey, locationJsonString); + await prefs.setString(receptionDetailsKey, receptionJsonString); + await prefs.setString(transferDetailsKey, transferJsonString); + await prefs.setString(routeMetadataKey, routeMetadataJsonString); debugPrint( '✅ [MessageStorage] Saved ${limitedList.length} messages to storage', diff --git a/test/providers/messages_provider_retransmission_test.dart b/test/providers/messages_provider_retransmission_test.dart index f4540d4..3729ec3 100644 --- a/test/providers/messages_provider_retransmission_test.dart +++ b/test/providers/messages_provider_retransmission_test.dart @@ -423,6 +423,133 @@ void main() { expect(provider.messages.single.id, equals('handle-1')); }); + test('channel duplicates only dedupe against the latest channel message', () { + final provider = MessagesProvider(); + final sender = Uint8List.fromList([9, 8, 7, 6, 5, 4]); + + provider.addMessage( + Message( + id: 'channel-first', + messageType: MessageType.channel, + channelIdx: 2, + pathLen: 1, + textType: MessageTextType.plain, + senderTimestamp: 1700000820, + text: 'same payload', + senderName: 'Radio Alpha', + receivedAt: DateTime.now(), + senderPublicKeyPrefix: sender, + ), + ); + provider.addMessage( + Message( + id: 'channel-middle', + messageType: MessageType.channel, + channelIdx: 2, + pathLen: 1, + textType: MessageTextType.plain, + senderTimestamp: 1700000825, + text: 'different payload', + senderName: 'Radio Bravo', + receivedAt: DateTime.now(), + senderPublicKeyPrefix: Uint8List.fromList([1, 2, 3, 4, 5, 6]), + ), + ); + provider.addMessage( + Message( + id: 'channel-repeat', + messageType: MessageType.channel, + channelIdx: 2, + pathLen: 2, + textType: MessageTextType.plain, + senderTimestamp: 1700000830, + text: 'same payload', + senderName: 'Radio Alpha', + receivedAt: DateTime.now(), + senderPublicKeyPrefix: sender, + ), + ); + + expect(provider.messages, hasLength(3)); + expect(provider.messages.last.id, equals('channel-repeat')); + }); + + test('adjacent channel duplicates still dedupe within 5 seconds', () { + final provider = MessagesProvider(); + final sender = Uint8List.fromList([4, 5, 6, 7, 8, 9]); + final baseTime = DateTime.fromMillisecondsSinceEpoch(1700000840000); + + provider.addMessage( + Message( + id: 'adjacent-1', + messageType: MessageType.channel, + channelIdx: 3, + pathLen: 1, + textType: MessageTextType.plain, + senderTimestamp: 1700000840, + text: 'same payload', + senderName: 'Radio Charlie', + receivedAt: baseTime, + senderPublicKeyPrefix: sender, + ), + ); + provider.addMessage( + Message( + id: 'adjacent-2', + messageType: MessageType.channel, + channelIdx: 3, + pathLen: 2, + textType: MessageTextType.plain, + senderTimestamp: 1700000845, + text: 'same payload', + senderName: 'Radio Charlie', + receivedAt: baseTime.add(const Duration(seconds: 4)), + senderPublicKeyPrefix: sender, + ), + ); + + expect(provider.messages, hasLength(1)); + expect(provider.messages.single.id, equals('adjacent-1')); + }); + + test('channel duplicates outside 5 second window are kept', () { + final provider = MessagesProvider(); + final sender = Uint8List.fromList([6, 7, 8, 9, 0, 1]); + final baseTime = DateTime.fromMillisecondsSinceEpoch(1700000850000); + + provider.addMessage( + Message( + id: 'window-1', + messageType: MessageType.channel, + channelIdx: 4, + pathLen: 1, + textType: MessageTextType.plain, + senderTimestamp: 1700000850, + text: 'same payload', + senderName: 'Radio Delta', + receivedAt: baseTime, + senderPublicKeyPrefix: sender, + ), + ); + provider.addMessage( + Message( + id: 'window-2', + messageType: MessageType.channel, + channelIdx: 4, + pathLen: 2, + textType: MessageTextType.plain, + senderTimestamp: 1700000855, + text: 'same payload', + senderName: 'Radio Delta', + receivedAt: baseTime.add(const Duration(seconds: 6)), + senderPublicKeyPrefix: sender, + ), + ); + + expect(provider.messages, hasLength(2)); + expect(provider.messages.last.id, equals('window-2')); + }); + test('display list collapses stored duplicates and sums copy counts', () { final provider = MessagesProvider(); final sender = Uint8List.fromList([5, 4, 3, 2, 1, 0]); diff --git a/test/services/message_storage_service_test.dart b/test/services/message_storage_service_test.dart index 0844e52..9102839 100644 --- a/test/services/message_storage_service_test.dart +++ b/test/services/message_storage_service_test.dart @@ -1,5 +1,6 @@ import 'dart:typed_data'; +import 'package:flutter/foundation.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:meshcore_client/meshcore_client.dart'; import 'package:meshcore_sar_app/models/message_reception_details.dart'; @@ -8,11 +9,16 @@ import 'package:shared_preferences/shared_preferences.dart'; void main() { TestWidgetsFlutterBinding.ensureInitialized(); + final originalDebugPrint = debugPrint; setUp(() { SharedPreferences.setMockInitialValues({}); }); + tearDown(() { + debugPrint = originalDebugPrint; + }); + test( 'retains path bytes for stored unread message when reception sidecar is missing', () async { @@ -155,4 +161,34 @@ void main() { expect(customMessages.single.id, customMessage.id); expect(customMessages.single.text, customMessage.text); }); + + test('skips unchanged message snapshots', () async { + final storage = MessageStorageService(); + final logs = []; + debugPrint = (String? message, {int? wrapWidth}) { + if (message != null) { + logs.add(message); + } + }; + final message = Message( + id: 'msg-stable', + messageType: MessageType.channel, + senderPublicKeyPrefix: Uint8List.fromList([3, 3, 3, 3, 3, 3]), + channelIdx: 3, + pathLen: 1, + textType: MessageTextType.plain, + senderTimestamp: 1700003000, + text: 'Stable snapshot', + receivedAt: DateTime.fromMillisecondsSinceEpoch(1700003000500), + isRead: true, + ); + + await storage.saveMessages([message]); + await storage.saveMessages([message]); + + expect( + logs.where((log) => log.contains('Saved 1 messages to storage')), + hasLength(1), + ); + }); }