Code
·
1697 lines
·
89174 bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697using MailKit;
using MailKit.Net.Imap;
using MailKit.Search;
using MailSharp.MailClient.Models;
using Microsoft.Extensions.Options;
using MimeKit;
using System.Collections.Concurrent;
using MailFolder = MailSharp.MailClient.Models.MailFolder;
using ModelPriority = MailSharp.MailClient.Models.MessagePriority;
namespace MailSharp.MailClient.Services;
public interface IImapService
{
// includeSizes is false by default on purpose: computing folder sizes can mean a full
// per-message header scan (see ResolveFolderSizeAsync) when the server doesn't support IMAP's
// STATUS SIZE extension, so only the Settings > Mappen beheren page (the only place sizes are
// ever shown) should ask for it - the Mail app's folder sidebar and the login connectivity
// check never display sizes and shouldn't pay for computing them.
//
// includeUnsubscribed is false by default (LSUB - subscribed only) for the same reason: the Mail
// app's sidebar should only ever show what the user has actually subscribed to. Mappen beheren
// is the exception - its whole "Geabonneerd" checkbox column only makes sense if it can show a
// folder that ISN'T subscribed yet (e.g. one just created, which doesn't auto-subscribe - see
// CreateFolderAsync) so the user can toggle it on.
Task<List<MailFolder>> GetFoldersAsync(Account account, string password, bool includeSizes = false, bool forceRefreshSizes = false, bool includeUnsubscribed = false, CancellationToken ct = default);
Task CreateFolderAsync(Account account, string password, string folderName, CancellationToken ct = default);
// Real IMAP SUBSCRIBE/UNSUBSCRIBE, not folderSettingsStore's separate (and, until now, entirely
// decorative) Subscribed field - see GetFoldersAsync's includeUnsubscribed. One connection for
// the whole batch, since Settings > Mappen beheren saves every folder's checkbox state at once.
Task SetFolderSubscriptionsAsync(Account account, string password, Dictionary<string, bool> subscriptions, CancellationToken ct = default);
// Physically re-parents a folder on the server (IMAP RENAME to a new parent, same leaf name) -
// unlike GetOrder/SaveOrder (Settings > Mappen beheren's existing display-only reordering), this
// changes the folder's actual FullName/hierarchy. newParentFullName null/empty moves it to the
// top level.
Task MoveFolderAsync(Account account, string password, string folderFullName, string? newParentFullName, CancellationToken ct = default);
// IMAP RENAME with the same parent, new leaf name - MoveFolderAsync's counterpart for changing
// what a folder is called rather than where it sits in the tree.
Task RenameFolderAsync(Account account, string password, string folderFullName, string newName, CancellationToken ct = default);
// Refuses to delete non-empty folders (checked via IMAP message count, not the cached
// UnreadCount) rather than silently expunging their contents - returns the subset of the
// requested names that were skipped for that reason, so the caller can report it.
Task<List<string>> DeleteFoldersAsync(Account account, string password, IEnumerable<string> folderFullNames, CancellationToken ct = default);
Task<List<MessageListItem>> GetMessagesAsync(Account account, string password, string folderFullName, bool forceRefresh = false, CancellationToken ct = default);
Task<MessageDetail> GetMessageAsync(Account account, string password, string folderFullName, uint uid, CancellationToken ct = default);
Task<Stream> GetAttachmentAsync(Account account, string password, string folderFullName, uint uid, int partIndex, CancellationToken ct = default);
Task SaveDraftAsync(Account account, string password, ComposeModel compose, IEnumerable<(string FileName, Stream Content)> attachments, IEnumerable<(string Cid, string FileName, Stream Content)> inlineImages, CancellationToken ct = default);
Task AppendSentAsync(Account account, string password, MimeMessage message, CancellationToken ct = default);
Task SetSeenAsync(Account account, string password, string folderFullName, IEnumerable<uint> uids, bool seen, CancellationToken ct = default);
Task SetFlaggedAsync(Account account, string password, string folderFullName, IEnumerable<uint> uids, bool flagged, CancellationToken ct = default);
Task MarkAllAsync(Account account, string password, string folderFullName, bool seen, CancellationToken ct = default);
Task MoveAsync(Account account, string password, string folderFullName, IEnumerable<uint> uids, string targetFolderFullName, CancellationToken ct = default);
Task DeleteAsync(Account account, string password, string folderFullName, IEnumerable<uint> uids, CancellationToken ct = default);
Task EmptyFolderAsync(Account account, string password, string folderFullName, CancellationToken ct = default);
Task<List<ContactAddress>> GetSentContactsAsync(Account account, string password, CancellationToken ct = default);
Task<List<MessageSearchResult>> SearchByAddressAsync(Account account, string password, string address, bool forceRefresh = false, CancellationToken ct = default);
}
public sealed record RecentFlagsRefreshResult(int AccountId, string Folder, int CheckedCount, int ChangedCount, long ElapsedMs, DateTime TimestampUtc);
public class ImapService(
IMailCacheStore cacheStore,
IFolderSettingsStore folderSettingsStore,
IMessageIndexStore indexStore,
IOptions<MailSettings> mailSettings,
ImageSanitizer imageSanitizer,
ILogger<ImapService> logger) : IImapService
{
private readonly MailSettings _settings = mailSettings.Value;
// De-dup guard so a folder never gets indexed by two overlapping background builds at once
// (e.g. two browser tabs, or a page reload while the first build is still running). Process-
// wide (static) rather than per-instance since ImapService is scoped per request. Internal (not
// private) so BackgroundIndexingStatus can read it for the Maintenance page without needing to
// route indexing calls through a new service.
internal static readonly ConcurrentDictionary<string, byte> IndexingInProgress = new();
// Message-level progress for the same key as IndexingInProgress ("{accountId}:{folder}") -
// only populated during a full reindex (see EnsureFolderIndexedAsync), since that's the only
// path slow enough for a raw count/percentage to matter to a user watching the toast; the
// incremental sync paths finish fast enough that per-chunk progress wouldn't be visible anyway.
internal static readonly ConcurrentDictionary<string, (int Processed, int Total)> IndexingProgress = new();
// Caps how many background index builds run their IMAP connection at once, process-wide. Most
// IMAP servers cap concurrent connections per account - without this, clicking through several
// never-indexed folders in a row would fire off that many simultaneous background connections,
// competing with (and slowing down) whatever the user is actually waiting on in the foreground.
internal const int MaxConcurrentBackgroundIndexing = 2;
internal static readonly SemaphoreSlim BackgroundIndexingThrottle = new(MaxConcurrentBackgroundIndexing, MaxConcurrentBackgroundIndexing);
// Most read/flag changes made from another client (a phone, another tab) land on recently
// received mail - a user who checks mail daily has their unread messages concentrated in
// roughly the last day or so, not scattered across a 74,000-message mailbox. So instead of the
// old approach (compare the whole folder's unread count and, on any mismatch, FETCH FLAGS for
// every UID - see EnsureFolderIndexedAsync's history), this only ever re-checks the flags of the
// most recent RecentFlagsRefreshCount messages, and only running here in the background so it
// never blocks GetMessagesAsync. Results (and how long the FETCH actually took, so it's easy to
// see from the logs/SSE stream whether this approach is actually cheap in practice) are kept
// per "{accountId}:{folder}" key and pushed to clients over the same SSE channel as the
// background-indexing toast (see MailApiController.IndexingStatusStream).
internal const int RecentFlagsRefreshCount = 200;
internal static readonly ConcurrentDictionary<string, RecentFlagsRefreshResult> RecentFlagsRefreshResults = new();
private static readonly ConcurrentDictionary<string, byte> RecentFlagsRefreshInProgress = new();
// Cheap cooldown so rapid repeated folder opens (a user bouncing between tabs, or a slow
// connection retrying) don't each kick off their own IMAP connection - only useful information
// once per interval anyway, since nothing changes flags that fast.
private static readonly TimeSpan RecentFlagsRefreshCooldown = TimeSpan.FromSeconds(20);
// Serializes StartRecentFlagsRefresh against SetSeenAsync/SetFlaggedAsync for the same
// "{accountId}:{folder}" key - without this, a background refresh that read a message's flags
// from the server just before the user explicitly marked it read/unread/flagged could still be
// mid-flight when that explicit action's own indexStore write lands, and then overwrite it with
// its now-stale snapshot a moment later (indistinguishable from the user's own change silently
// reverting). Whichever of the two starts first now runs to completion - including its local
// index write - before the other's fetch begins, so a background refresh either sees the
// explicit change already applied (server-side too, so it agrees) or the explicit action
// naturally comes after and simply wins.
private static readonly ConcurrentDictionary<string, SemaphoreSlim> FolderMutationLocks = new();
private static SemaphoreSlim GetFolderMutationLock(string key) => FolderMutationLocks.GetOrAdd(key, _ => new SemaphoreSlim(1, 1));
// Building a folder's index for the first time means a full IMAP scan (see
// EnsureFolderIndexedAsync) - too slow to do inline within the request that triggered it (the
// user would stare at a blank screen). Instead this runs it on a background Task using its own
// fresh IMAP connection, while the caller falls back to the old capped-live-fetch path for this
// one request; once the background build finishes, the next request finds the index ready and
// takes the fast path. Deliberately uses no request-scoped state (only the static ConnectAsync
// and the singleton-backed index/settings stores captured on `this`), so it's safe to keep
// running after the HTTP request that started it has already completed.
private void StartBackgroundIndexBuild(Account account, string password, string folderFullName)
{
var key = $"{account.Id}:{folderFullName}";
if (!IndexingInProgress.TryAdd(key, 0))
{
logger.LogInformation("Background index build for account {AccountId} folder {Folder} already in progress, skipping duplicate trigger", account.Id, folderFullName);
return;
}
logger.LogInformation("Background index build queued for account {AccountId} folder {Folder}", account.Id, folderFullName);
var sw = System.Diagnostics.Stopwatch.StartNew();
_ = Task.Run(async () =>
{
try
{
// Let the page load that triggered this finish its own IMAP connections first
// (folder list, live message fallback) instead of immediately competing for the same
// account's connection slots.
await Task.Delay(TimeSpan.FromSeconds(3));
await BackgroundIndexingThrottle.WaitAsync();
try
{
await EnsureFolderIndexedAsync(account, password, folderFullName, CancellationToken.None);
logger.LogInformation("Background index build completed for account {AccountId} folder {Folder} in {DurationMs}ms, {Count} messages",
account.Id, folderFullName, sw.ElapsedMilliseconds, indexStore.GetMessageCount(account.Id, folderFullName));
}
finally
{
BackgroundIndexingThrottle.Release();
}
}
catch (Exception ex)
{
logger.LogWarning(ex, "Background index build failed for account {AccountId} folder {Folder} after {DurationMs}ms", account.Id, folderFullName, sw.ElapsedMilliseconds);
}
finally
{
IndexingInProgress.TryRemove(key, out _);
}
});
}
// See RecentFlagsRefreshResults - re-checks only the most recent RecentFlagsRefreshCount
// messages' flags (a cheap FETCH FLAGS, not the full Envelope/Size fetch a real index rebuild
// needs) so read/flag changes made from another client show up without ever touching the rest
// of a large mailbox. Always backgrounded (called from EnsureFolderIndexedAsync, which must not
// block on it) and skipped entirely if a full reindex is already in flight for this folder -
// that reindex will already pick up current flags for free.
private void StartRecentFlagsRefresh(Account account, string password, string folderFullName)
{
var key = $"{account.Id}:{folderFullName}";
if (IndexingInProgress.ContainsKey(key)) return;
if (RecentFlagsRefreshResults.TryGetValue(key, out var last) && DateTime.UtcNow - last.TimestampUtc < RecentFlagsRefreshCooldown) return;
if (!RecentFlagsRefreshInProgress.TryAdd(key, 0)) return;
var sw = System.Diagnostics.Stopwatch.StartNew();
_ = Task.Run(async () =>
{
try
{
await BackgroundIndexingThrottle.WaitAsync();
try
{
// IMAP fetch happens before the lock is taken (no need to serialize the network
// round trip against explicit actions, only the read-compare-write against the
// local index) - but the local snapshot has to be re-taken *inside* the lock,
// otherwise an explicit SetSeenAsync/SetFlaggedAsync that lands between the first
// snapshot and the fetch completing would get diffed against stale local data here.
var candidateUids = indexStore.GetMessages(account.Id, folderFullName)
.OrderByDescending(m => m.Uid)
.Take(RecentFlagsRefreshCount)
.Select(m => m.Uid)
.ToList();
if (candidateUids.Count == 0) return;
using var client = await ConnectAsync(account, password, CancellationToken.None);
var folder = await client.GetFolderAsync(folderFullName, CancellationToken.None);
await folder.OpenAsync(FolderAccess.ReadOnly, CancellationToken.None);
var summaries = await folder.FetchAsync(
[.. candidateUids.Select(uid => new UniqueId(uid))],
MessageSummaryItems.Flags,
CancellationToken.None);
await folder.CloseAsync(false, CancellationToken.None);
await client.DisconnectAsync(true, CancellationToken.None);
var folderLock = GetFolderMutationLock(key);
await folderLock.WaitAsync();
int changedCount;
List<uint> stillUnreadUids;
try
{
var currentLocal = indexStore.GetMessages(account.Id, folderFullName).ToDictionary(m => m.Uid);
var toMarkRead = new List<uint>();
var toMarkUnread = new List<uint>();
var toMarkFlagged = new List<uint>();
var toMarkUnflagged = new List<uint>();
stillUnreadUids = [];
foreach (var summary in summaries)
{
if (!currentLocal.TryGetValue(summary.UniqueId.Id, out var local)) continue;
var isRead = summary.Flags?.HasFlag(MessageFlags.Seen) ?? local.IsRead;
var isFlagged = summary.Flags?.HasFlag(MessageFlags.Flagged) ?? local.IsFlagged;
if (isRead != local.IsRead) (isRead ? toMarkRead : toMarkUnread).Add(local.Uid);
if (isFlagged != local.IsFlagged) (isFlagged ? toMarkFlagged : toMarkUnflagged).Add(local.Uid);
if (!isRead) stillUnreadUids.Add(local.Uid);
}
if (toMarkRead.Count > 0) indexStore.MarkRead(account.Id, folderFullName, toMarkRead, true);
if (toMarkUnread.Count > 0) indexStore.MarkRead(account.Id, folderFullName, toMarkUnread, false);
if (toMarkFlagged.Count > 0) indexStore.MarkFlagged(account.Id, folderFullName, toMarkFlagged, true);
if (toMarkUnflagged.Count > 0) indexStore.MarkFlagged(account.Id, folderFullName, toMarkUnflagged, false);
changedCount = new HashSet<uint>([.. toMarkRead, .. toMarkUnread, .. toMarkFlagged, .. toMarkUnflagged]).Count;
}
finally
{
folderLock.Release();
}
var result = new RecentFlagsRefreshResult(account.Id, folderFullName, candidateUids.Count, changedCount, sw.ElapsedMilliseconds, DateTime.UtcNow);
RecentFlagsRefreshResults[key] = result;
logger.LogInformation("Recent flags refresh for folder {Folder} account {AccountId}: checked {CheckedCount} of the most recent messages, {ChangedCount} changed, took {ElapsedMs}ms",
folderFullName, account.Id, result.CheckedCount, result.ChangedCount, result.ElapsedMs);
// Directly answers "which message is stuck unread" - per the server's own live
// Flags response just fetched above, not a guess derived from the local index.
if (stillUnreadUids.Count > 0)
{
logger.LogInformation("Recent flags refresh for folder {Folder} account {AccountId}: {Count} still unread among the checked messages (server-confirmed): {Uids}",
folderFullName, account.Id, stillUnreadUids.Count, string.Join(",", stillUnreadUids));
}
}
finally
{
BackgroundIndexingThrottle.Release();
}
}
catch (Exception ex)
{
logger.LogWarning(ex, "Recent flags refresh failed for account {AccountId} folder {Folder} after {ElapsedMs}ms", account.Id, folderFullName, sw.ElapsedMilliseconds);
}
finally
{
RecentFlagsRefreshInProgress.TryRemove(key, out _);
}
});
}
// IMailFolder.GetSubfoldersAsync only returns the immediate children of the folder it's called
// on - it is NOT recursive. Called on just the personal namespace root (as every call site here
// used to do), that means any folder nested two or more levels deep (e.g. "Archief/AliExpress",
// "Willy/okkerse") never showed up anywhere: not in the folder list, not in special-folder
// lookups, not in cross-folder address search. Walking every folder's own children in turn is
// the only way MailKit exposes the full tree.
private static async Task<List<IMailFolder>> GetAllSubfoldersRecursiveAsync(IMailFolder root, bool subscribedOnly, CancellationToken ct)
{
var result = new List<IMailFolder>();
foreach (var folder in await root.GetSubfoldersAsync(subscribedOnly, ct))
{
result.Add(folder);
result.AddRange(await GetAllSubfoldersRecursiveAsync(folder, subscribedOnly, ct));
}
return result;
}
// SpecialFolder.* lookups rely on the server advertising IMAP SPECIAL-USE/XLIST - servers that
// don't (like this one) make MailKit throw NotSupportedException rather than returning null.
// Falls back to matching common folder names (English/Dutch) among the account's actual
// folders, then to a couple of hardcoded full-name guesses; returns null (rather than throwing)
// if nothing matches, since some callers (e.g. "delete" falling back to expunge when there's no
// Trash) treat "no such folder" as a valid, handleable outcome.
private static async Task<IMailFolder?> ResolveFolderOrNullAsync(ImapClient client, SpecialFolder special, string[] nameHints, string[] nameGuesses, CancellationToken ct)
{
IMailFolder? folder = null;
try { folder = client.GetFolder(special); }
catch (NotSupportedException) { /* server doesn't advertise SPECIAL-USE/XLIST - fall through */ }
if (folder != null) return folder;
var personal = client.GetFolder(client.PersonalNamespaces[0]);
var subfolders = await GetAllSubfoldersRecursiveAsync(personal, true, ct);
var byName = subfolders.FirstOrDefault(f => nameHints.Any(h => f.Name.Contains(h, StringComparison.OrdinalIgnoreCase)));
if (byName != null) return byName;
foreach (var guess in nameGuesses)
{
try { return await client.GetFolderAsync(guess, ct); }
catch { /* try next guess */ }
}
return null;
}
private static readonly string[] SentNameHints = ["sent", "verzonden"];
private static readonly string[] SentNameGuesses = ["Sent", "Sent Items", "Sent Messages", "INBOX.Sent", "INBOX/Sent"];
private static readonly string[] DraftsNameHints = ["draft", "concept"];
private static readonly string[] DraftsNameGuesses = ["Drafts", "INBOX.Drafts", "INBOX/Drafts"];
private static readonly string[] TrashNameHints = ["trash", "deleted", "prullenbak"];
private static readonly string[] TrashNameGuesses = ["Trash", "Deleted Items", "INBOX.Trash", "INBOX/Trash"];
private static readonly string[] JunkNameHints = ["spam", "junk", "unwanted", "ongewenst"];
// INBOX (by definition) and whatever folder is filling Sent/Trash/Drafts/Junk's role can't be
// deleted (see DeleteFoldersAsync) since core features assume they exist - detected the same
// way as the Resolve*FolderAsync helpers above: SPECIAL-USE attributes first, common name
// hints as a fallback for servers that don't advertise SPECIAL-USE.
private static bool IsProtectedFolder(string fullName, string name, FolderAttributes attributes)
{
if (string.Equals(fullName, "INBOX", StringComparison.OrdinalIgnoreCase)) return true;
if ((attributes & (FolderAttributes.Inbox | FolderAttributes.Sent | FolderAttributes.Trash | FolderAttributes.Drafts | FolderAttributes.Junk)) != 0) return true;
return SentNameHints.Concat(TrashNameHints).Concat(DraftsNameHints).Concat(JunkNameHints)
.Any(h => name.Contains(h, StringComparison.OrdinalIgnoreCase));
}
// External images are how spam/phishing senders confirm a mailbox is live and being read (a
// tracking pixel that loads means "human opened this") - the per-sender "always show" preference
// (see IMailCacheStore.GetAllowExternalImages) exists for legitimate senders a user trusts, but
// that trust decision shouldn't apply inside the folder that exists specifically to hold mail the
// user (or the server's own spam filter) has flagged as untrustworthy. Only the leaf folder name
// is checked (not the account's other folders sharing a hint substring, e.g. a folder named
// "Sponsors" containing "spo" wouldn't match "spam" as a whole segment).
// Internal (not private) so MailApiController can reject the "always show images from this
// sender" action outright when it's being requested from within the Spam folder - see AllowImages.
internal static bool IsSpamFolder(string folderFullName)
{
var leaf = folderFullName.Split('/', '.') is { Length: > 0 } segments ? segments[^1] : folderFullName;
return JunkNameHints.Any(h => leaf.Contains(h, StringComparison.OrdinalIgnoreCase));
}
private static async Task<IMailFolder> ResolveSentFolderAsync(ImapClient client, CancellationToken ct) =>
await ResolveFolderOrNullAsync(client, SpecialFolder.Sent, SentNameHints, SentNameGuesses, ct)
?? throw new InvalidOperationException("Could not locate a Sent folder on this account.");
private static async Task<IMailFolder> ResolveDraftsFolderAsync(ImapClient client, CancellationToken ct) =>
await ResolveFolderOrNullAsync(client, SpecialFolder.Drafts, DraftsNameHints, DraftsNameGuesses, ct)
?? throw new InvalidOperationException("Could not locate a Drafts folder on this account.");
private static Task<IMailFolder?> ResolveTrashFolderAsync(ImapClient client, CancellationToken ct) =>
ResolveFolderOrNullAsync(client, SpecialFolder.Trash, TrashNameHints, TrashNameGuesses, ct);
// Bounds every socket read/write MailKit does on a client - connect, auth, LIST, STATUS,
// FETCH, disconnect, everywhere this client gets used - not just the initial connect. Without
// this a server that accepts the connection/auth but then stalls mid-protocol (e.g. a firewall
// that lets the handshake through but silently drops later packets) hangs the calling request
// forever with nothing to log; MailKit's own default (120s) is too long to notice quickly and,
// being per-client rather than something we set, easy to forget applies to every code path that
// creates a client, not just this one. A single retry with a fresh client/socket rides out the
// transient case (e.g. one bad connection attempt against a loaded mail server) without
// surfacing an error to the user.
private const int ImapTimeoutSeconds = 20;
private const int ConnectRetryCount = 1;
// Process-wide count of currently-open IMAP connections (across every account/request) - the
// one piece of visibility that was missing while diagnosing the "mappen beheren never comes
// back" issue: nothing before this logged how many connections were actually in flight at once,
// so a pile-up (e.g. one new connection per folder) was invisible until it made the page hang.
// Decremented via MailKit's Disconnected event rather than at the call site, so it stays
// accurate however the client goes away - explicit DisconnectAsync, Dispose() on a `using`, or
// the socket dying under it.
private static int _openConnectionCount;
private async Task<ImapClient> ConnectAsync(Account account, string password, CancellationToken ct)
{
var secureSocket = account.ImapSecurity switch
{
SecurityMode.SslTls => MailKit.Security.SecureSocketOptions.SslOnConnect,
SecurityMode.StartTls => MailKit.Security.SecureSocketOptions.StartTls,
_ => MailKit.Security.SecureSocketOptions.None
};
for (var attempt = 0; ; attempt++)
{
var client = new ImapClient { Timeout = ImapTimeoutSeconds * 1000 };
var sw = System.Diagnostics.Stopwatch.StartNew();
try
{
logger.LogDebug("IMAP connecting: account {AccountId} to {Host}:{Port}, attempt {Attempt}, {OpenConnections} connection(s) currently open",
account.Id, account.ImapHost, account.ImapPort, attempt + 1, _openConnectionCount);
await client.ConnectAsync(account.ImapHost, account.ImapPort, secureSocket, ct);
await client.AuthenticateAsync(account.Username, password, ct);
var opened = Interlocked.Increment(ref _openConnectionCount);
client.Disconnected += (_, _) =>
{
var remaining = Interlocked.Decrement(ref _openConnectionCount);
logger.LogDebug("IMAP disconnected: account {AccountId} from {Host}:{Port}, {OpenConnections} connection(s) still open",
account.Id, account.ImapHost, account.ImapPort, remaining);
};
logger.LogDebug("IMAP connected: account {AccountId} to {Host}:{Port} in {ElapsedMs}ms, {OpenConnections} connection(s) now open",
account.Id, account.ImapHost, account.ImapPort, sw.ElapsedMilliseconds, opened);
return client;
}
catch (Exception ex) when (!ct.IsCancellationRequested)
{
client.Dispose();
if (attempt < ConnectRetryCount)
{
logger.LogWarning(ex, "IMAP connect/authenticate attempt {Attempt} failed for account {AccountId} against {Host}:{Port} after {ElapsedMs}ms, retrying",
attempt + 1, account.Id, account.ImapHost, account.ImapPort, sw.ElapsedMilliseconds);
continue;
}
logger.LogError(ex, "IMAP connect/authenticate failed for account {AccountId} against {Host}:{Port} after {Attempts} attempt(s), {ElapsedMs}ms",
account.Id, account.ImapHost, account.ImapPort, attempt + 1, sw.ElapsedMilliseconds);
throw;
}
}
}
public async Task<List<MailFolder>> GetFoldersAsync(Account account, string password, bool includeSizes = false, bool forceRefreshSizes = false, bool includeUnsubscribed = false, CancellationToken ct = default)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id });
var sw = System.Diagnostics.Stopwatch.StartNew();
using var client = await ConnectAsync(account, password, ct);
var result = new List<MailFolder>();
var personal = client.GetFolder(client.PersonalNamespaces[0]);
var folders = await GetAllSubfoldersRecursiveAsync(personal, !includeUnsubscribed, ct);
var settings = folderSettingsStore.GetAll(account.Id);
// "Subscribed" reported to the client is the real IMAP subscription state, not a locally
// stored preference (folderSettingsStore's own Subscribed field is unrelated - see
// SetFolderSubscriptionAsync) - a folder is subscribed exactly when it appears in the LSUB
// listing. When this call already used LSUB-only (the Mail sidebar's normal path), every
// folder here already IS subscribed by definition; only Mappen beheren's includeUnsubscribed
// pass needs the extra LSUB walk to know which of the (now larger) full LIST is subscribed.
HashSet<string>? subscribedFullNames = includeUnsubscribed
? [.. (await GetAllSubfoldersRecursiveAsync(personal, true, ct)).Select(sf => sf.FullName)]
: null;
logger.LogInformation("GetFoldersAsync: account {AccountId}, includeSizes={IncludeSizes}, forceRefreshSizes={ForceRefreshSizes}, {FolderCount} folder(s) listed after {ElapsedMs}ms",
account.Id, includeSizes, forceRefreshSizes, folders.Count + 1, sw.ElapsedMilliseconds);
foreach (var f in new[] { client.Inbox }.Concat(folders))
{
if (result.Any(x => x.FullName == f.FullName)) continue;
var setting = settings.GetValueOrDefault(f.FullName);
var depth = FolderDepth(f, personal);
var isProtected = IsProtectedFolder(f.FullName, f.Name, f.Attributes);
var folderSw = System.Diagnostics.Stopwatch.StartNew();
try
{
// STATUS rather than SELECT/Open - cheaper (no mailbox actually gets opened). Only
// asks for SIZE (RFC 8438) when the caller actually wants sizes - not every server
// supports that extension, so retry without it rather than losing Count/Unread too.
if (includeSizes)
{
try { await f.StatusAsync(StatusItems.Count | StatusItems.Unread | StatusItems.Size, ct); }
catch { await f.StatusAsync(StatusItems.Count | StatusItems.Unread, ct); }
}
else
{
await f.StatusAsync(StatusItems.Count | StatusItems.Unread, ct);
}
var (sizeBytes, isSizeEstimated) = includeSizes
? await ResolveFolderSizeAsync(account, password, client, f, setting?.SyncMode ?? FolderSyncMode.Direct, forceRefreshSizes, ct)
: (0, false);
if (folderSw.ElapsedMilliseconds > 500)
{
logger.LogInformation("GetFoldersAsync: folder {Folder} for account {AccountId} took {ElapsedMs}ms",
f.FullName, account.Id, folderSw.ElapsedMilliseconds);
}
// Recent-scoped unread, not the folder's raw live total (see GetRecentUnreadCount) -
// a message from years ago, or the tens of thousands sitting untouched in Trash,
// shouldn't move this badge. Falls back to the live IMAP count for NotSynchronized
// folders (which are deliberately never indexed) or a folder not indexed yet.
var syncMode = setting?.SyncMode ?? FolderSyncMode.Direct;
var isIndexed = syncMode != FolderSyncMode.NotSynchronized && indexStore.GetSyncState(account.Id, f.FullName) != null;
var unreadCount = isIndexed
? indexStore.GetRecentUnreadCount(account.Id, f.FullName, RecentFlagsRefreshCount)
: f.Unread;
if (isIndexed && unreadCount > 0)
{
var uids = indexStore.GetRecentUnreadUids(account.Id, f.FullName, RecentFlagsRefreshCount);
logger.LogInformation("GetFoldersAsync: folder {Folder} for account {AccountId} shows {UnreadCount} unread among the most recent {RecentCount} - uids: {Uids}",
f.FullName, account.Id, unreadCount, RecentFlagsRefreshCount, string.Join(",", uids));
}
result.Add(new MailFolder
{
FullName = f.FullName,
DisplayName = f.Name,
UnreadCount = unreadCount,
MessageCount = f.Count,
IsSelectable = true,
Depth = depth,
SyncMode = setting?.SyncMode ?? FolderSyncMode.Direct,
Subscribed = subscribedFullNames?.Contains(f.FullName) ?? true,
SizeBytes = sizeBytes,
IsSizeEstimated = isSizeEstimated,
IsProtected = isProtected
});
}
catch (Exception ex)
{
logger.LogWarning(ex, "GetFoldersAsync: folder {Folder} for account {AccountId} failed STATUS/size resolution after {ElapsedMs}ms, marking unselectable",
f.FullName, account.Id, folderSw.ElapsedMilliseconds);
result.Add(new MailFolder
{
FullName = f.FullName,
DisplayName = f.Name,
IsSelectable = false,
Depth = depth,
SyncMode = setting?.SyncMode ?? FolderSyncMode.Direct,
Subscribed = subscribedFullNames?.Contains(f.FullName) ?? true,
IsProtected = isProtected
});
}
}
await client.DisconnectAsync(true, ct);
logger.LogInformation("GetFoldersAsync: account {AccountId} completed in {ElapsedMs}ms, {FolderCount} folder(s)",
account.Id, sw.ElapsedMilliseconds, result.Count);
return ApplyDisplayOrder(result, folderSettingsStore.GetOrder(account.Id));
}
// Real total size: IMAP STATUS SIZE (RFC 8438) where the server supports it - already checked
// by the caller via f.Size. Otherwise this is the local message index's own total (see
// EnsureFolderIndexedAsync) - exact, not an estimate, and free once the folder is indexed.
// NotSynchronized folders are deliberately never indexed, so their size is simply unknown (0)
// rather than triggering the very full-folder scan that setting exists to avoid. A folder
// that's never been indexed yet also shows 0 for now - its first build runs in the background
// (see StartBackgroundIndexBuild) rather than blocking this page load; a later reload picks up
// the real total once that finishes.
// Takes the already-connected client from GetFoldersAsync's own loop rather than reconnecting -
// resolving sizes for every already-indexed folder on the Settings > Mappen beheren page used to
// open a brand new IMAP connection per folder (via EnsureFolderIndexedAsync's own ConnectAsync),
// which for an account with many folders meant that many sequential connect+auth round trips
// (plus any per-folder flag-drift reconciliation) serialized within one request - individually
// fast enough that nothing ever hit a socket timeout, but the page never came back in practice.
private async Task<(long SizeBytes, bool IsEstimated)> ResolveFolderSizeAsync(Account account, string password, ImapClient client, IMailFolder folder, FolderSyncMode syncMode, bool forceRebuild, CancellationToken ct)
{
if (folder.Size.HasValue) return ((long)folder.Size.Value, false);
if (syncMode == FolderSyncMode.NotSynchronized) return (0, false);
var state = indexStore.GetSyncState(account.Id, folder.FullName);
if (forceRebuild && state != null)
{
indexStore.RemoveFolder(account.Id, folder.FullName);
state = null;
}
if (state == null)
{
StartBackgroundIndexBuild(account, password, folder.FullName);
return (0, false);
}
try
{
await EnsureFolderIndexedAsync(account, password, client, folder.FullName, ct);
return (indexStore.GetTotalSize(account.Id, folder.FullName), false);
}
catch
{
return (0, false);
}
}
// Keeps a local header index (Envelope/Flags/Size per message) up to date without re-scanning
// everything on every access - the same headers GetMessagesAsync/GetFoldersAsync/
// SearchByAddressAsync/GetSentContactsAsync already needed, just fetched once and kept in sync
// instead of re-fetched (fully or partially) every time. No CONDSTORE required: UIDVALIDITY
// changing means the server considers the mailbox rebuilt (full reindex); UIDNEXT advancing
// means new mail arrived (cheap incremental fetch of just the new UID range); and the message
// count not matching the index's own row count is the cheap signal - already free from the
// STATUS call below - that something was removed (by this app or another IMAP client), which is
// the only case expensive enough (a UID-only SEARCH ALL) to want to avoid doing blindly.
//
// Deliberately doesn't fetch BODYSTRUCTURE: some servers send a slightly non-conformant one for
// certain (often malformed/spam) messages, and MailKit doesn't just skip that one message - a
// parse error mid-response desyncs the whole stream, disconnecting the client outright. The only
// thing BODYSTRUCTURE would have bought here is the message list's HasAttachments paperclip icon,
// not worth risking the entire index build over.
private async Task EnsureFolderIndexedAsync(Account account, string password, string folderFullName, CancellationToken ct)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id, ["folder"] = folderFullName });
using var client = await ConnectAsync(account, password, ct);
await EnsureFolderIndexedAsync(account, password, client, folderFullName, ct);
await client.DisconnectAsync(true, ct);
}
// See ResolveFolderSizeAsync - lets a caller that already has an open, authenticated client
// (GetFoldersAsync's per-folder size resolution) reuse it instead of paying for a fresh
// connect+auth round trip per folder. Never disconnects the client itself; that's the caller's
// connection to manage. Takes password (unlike the client itself) purely to hand off to
// StartRecentFlagsRefresh below, which needs its own separate background connection.
private async Task EnsureFolderIndexedAsync(Account account, string password, ImapClient client, string folderFullName, CancellationToken ct)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id, ["folder"] = folderFullName });
var sw = System.Diagnostics.Stopwatch.StartNew();
var folder = await client.GetFolderAsync(folderFullName, ct);
await folder.StatusAsync(StatusItems.Count | StatusItems.UidValidity | StatusItems.UidNext | StatusItems.Unread, ct);
var uidNext = folder.UidNext?.Id ?? 0;
var state = indexStore.GetSyncState(account.Id, folderFullName);
logger.LogDebug("EnsureFolderIndexedAsync: folder {Folder} for account {AccountId}, indexState={IndexState}, serverCount={ServerCount}, serverUidValidity={UidValidity}, serverUidNext={UidNext}",
folderFullName, account.Id, state == null ? "none" : "present", folder.Count, folder.UidValidity, uidNext);
if (state == null || state.UidValidity != folder.UidValidity)
{
logger.LogInformation("Full reindex of folder {Folder} for account {AccountId} ({Reason})",
folderFullName, account.Id, state == null ? "first index" : "UIDVALIDITY changed");
await folder.OpenAsync(FolderAccess.ReadOnly, ct);
var allUids = await folder.SearchAsync(SearchQuery.All, ct);
var progressKey = $"{account.Id}:{folderFullName}";
var messages = new List<IndexedMessage>(allUids.Count);
try
{
if (allUids.Count > 0) IndexingProgress[progressKey] = (0, allUids.Count);
// Fetched in chunks (rather than one FETCH for every UID) purely so IndexingProgress
// can be updated as we go - MailKit has no per-message progress callback for FETCH,
// only per-chunk granularity is achievable this way.
const int ChunkSize = 200;
for (var i = 0; i < allUids.Count; i += ChunkSize)
{
var chunk = allUids.Skip(i).Take(ChunkSize).ToList();
messages.AddRange(await FetchIndexedMessagesAsync(folder, chunk, ct));
IndexingProgress[progressKey] = (messages.Count, allUids.Count);
}
}
finally
{
IndexingProgress.TryRemove(progressKey, out _);
}
await folder.CloseAsync(false, ct);
indexStore.ReplaceFolder(account.Id, folderFullName, messages);
indexStore.SaveSyncState(account.Id, folderFullName, folder.UidValidity, uidNext, folder.Count);
logger.LogInformation("EnsureFolderIndexedAsync: folder {Folder} for account {AccountId} full reindex of {MessageCount} message(s) took {ElapsedMs}ms",
folderFullName, account.Id, messages.Count, sw.ElapsedMilliseconds);
return;
}
if (uidNext > state.UidNext)
{
await folder.OpenAsync(FolderAccess.ReadOnly, ct);
var range = new UniqueIdRange(new UniqueId(state.UidNext), UniqueId.MaxValue);
var newMessages = await FetchIndexedMessagesAsync(folder, range, ct);
await folder.CloseAsync(false, ct);
if (newMessages.Count > 0) indexStore.UpsertMessages(account.Id, folderFullName, newMessages);
}
if (indexStore.GetMessageCount(account.Id, folderFullName) != folder.Count)
{
await folder.OpenAsync(FolderAccess.ReadOnly, ct);
var currentUids = (await folder.SearchAsync(SearchQuery.All, ct)).Select(u => u.Id).ToHashSet();
await folder.CloseAsync(false, ct);
var missing = indexStore.GetMessages(account.Id, folderFullName)
.Select(m => m.Uid)
.Where(uid => !currentUids.Contains(uid))
.ToList();
if (missing.Count > 0) indexStore.RemoveMessages(account.Id, folderFullName, missing);
}
indexStore.SaveSyncState(account.Id, folderFullName, folder.UidValidity, uidNext, folder.Count);
logger.LogDebug("EnsureFolderIndexedAsync: folder {Folder} for account {AccountId} incremental sync took {ElapsedMs}ms",
folderFullName, account.Id, sw.ElapsedMilliseconds);
StartRecentFlagsRefresh(account, password, folderFullName);
}
// Priority isn't covered by MessageSummaryItems' normal flags (Envelope/Flags/Size) - MailKit
// only exposes it by explicitly requesting the two headers senders actually use for it
// (Importance and the older X-Priority), fetched here rather than the whole header block to
// keep this as cheap as the rest of the index build.
private static readonly HeaderId[] PriorityHeaderFields = [HeaderId.Importance, HeaderId.XPriority];
// A message with no Date header (seen in the wild from at least one spam sender) makes both
// MimeKit and, apparently, some IMAP servers' own ENVELOPE response fall back to an implausible
// default (e.g. year 1) rather than something usable - which then sorts that message to the very
// bottom of a folder with tens of thousands of messages, making it effectively unfindable by
// scrolling/paging (see the INBOX uid 95868 investigation). INTERNALDATE - when the server itself
// received the message - is always present and a much more useful fallback than an arbitrary
// sentinel, so anything implausibly old uses that instead.
private static readonly DateTimeOffset PlausibleDateFloor = new(1990, 1, 1, 0, 0, 0, TimeSpan.Zero);
private static DateTimeOffset ResolveMessageDate(IMessageSummary s) =>
s.Envelope?.Date is { } envelopeDate && envelopeDate >= PlausibleDateFloor
? envelopeDate
: s.InternalDate ?? DateTimeOffset.MinValue;
// See GetMessageAsync's date self-heal - a single lightweight FETCH just for INTERNALDATE, its
// own short-lived connection since this only runs for the rare already-broken index entry.
private async Task<DateTimeOffset?> FetchInternalDateAsync(Account account, string password, string folderFullName, uint uid, CancellationToken ct)
{
using var client = await ConnectAsync(account, password, ct);
var folder = await client.GetFolderAsync(folderFullName, ct);
var summaries = await folder.FetchAsync([new UniqueId(uid)], MessageSummaryItems.InternalDate, ct);
await client.DisconnectAsync(true, ct);
return summaries.FirstOrDefault()?.InternalDate;
}
private static async Task<List<IndexedMessage>> FetchIndexedMessagesAsync(IMailFolder folder, IList<UniqueId> uids, CancellationToken ct)
{
var summaries = await folder.FetchAsync(uids, MessageSummaryItems.Envelope | MessageSummaryItems.Flags | MessageSummaryItems.Size | MessageSummaryItems.InternalDate, PriorityHeaderFields, ct);
return [.. summaries.Select(s => new IndexedMessage
{
Uid = s.UniqueId.Id,
Subject = s.Envelope?.Subject ?? "(no subject)",
From = s.Envelope?.From?.ToString() ?? "",
To = s.Envelope?.To?.ToString() ?? "",
Cc = s.Envelope?.Cc?.ToString() ?? "",
Date = ResolveMessageDate(s),
SizeBytes = s.Size ?? 0,
IsRead = s.Flags?.HasFlag(MessageFlags.Seen) ?? false,
IsFlagged = s.Flags?.HasFlag(MessageFlags.Flagged) ?? false,
HasAttachments = false,
Priority = ParsePriorityHeaders(s.Headers)
})];
}
private static ModelPriority ParsePriorityHeaders(HeaderList? headers)
{
var importance = headers?[HeaderId.Importance]?.Trim();
if (string.Equals(importance, "high", StringComparison.OrdinalIgnoreCase)) return ModelPriority.High;
if (string.Equals(importance, "low", StringComparison.OrdinalIgnoreCase)) return ModelPriority.Low;
// X-Priority: 1-2 = High, 3 = Normal, 4-5 = Low (the de-facto convention since Outlook Express).
var xPriority = headers?[HeaderId.XPriority]?.Trim();
if (xPriority != null && xPriority.Length > 0 && char.IsDigit(xPriority[0]))
{
return xPriority[0] switch { '1' or '2' => ModelPriority.High, '4' or '5' => ModelPriority.Low, _ => ModelPriority.Normal };
}
return ModelPriority.Normal;
}
// Presentation-only reordering (Settings > Mappen beheren) - never touches the real IMAP
// folder structure. Folders mentioned in savedOrder come first, in that sequence; anything
// else (new folders discovered since the order was last saved) is appended afterwards in its
// original IMAP discovery order.
private static List<MailFolder> ApplyDisplayOrder(List<MailFolder> folders, List<string> savedOrder)
{
if (savedOrder.Count == 0) return folders;
var position = savedOrder.Select((name, i) => (name, i)).ToDictionary(x => x.name, x => x.i, StringComparer.Ordinal);
// A folder not in savedOrder (e.g. a subfolder GetFoldersAsync's now-recursive walk finds
// that didn't exist - or wasn't visible - when the order was last saved) used to sort via
// int.MaxValue, dumping it at the very end regardless of where it actually belongs in the
// tree - every newly-discovered nested folder clumped together at the bottom, detached from
// its parent. Instead it inherits the position of the nearest PRECEDING ordered folder: since
// `folders` arrives here in parent-before-children discovery order, that's always its own
// ancestor (or an ancestor's already-ordered sibling), so it stays visually attached to where
// it belongs instead of migrating to the end of the whole list.
var inheritedPosition = new int[folders.Count];
var lastKnownPosition = 0;
for (var i = 0; i < folders.Count; i++)
{
if (position.TryGetValue(folders[i].FullName, out var p)) lastKnownPosition = p;
inheritedPosition[i] = lastKnownPosition;
}
return [.. folders
.Select((f, naturalIndex) => (f, naturalIndex))
.OrderBy(x => inheritedPosition[x.naturalIndex])
.ThenBy(x => x.naturalIndex)
.Select(x => x.f)];
}
private static int FolderDepth(IMailFolder folder, IMailFolder root)
{
var depth = 0;
var parent = folder.ParentFolder;
while (parent != null && parent != root)
{
depth++;
parent = parent.ParentFolder;
}
return depth;
}
// Looks up folderName under the personal namespace without creating anything. GetFolderAsync
// throws FolderNotFoundException when it isn't there, which is the normal "go ahead and create
// it" case, so it's swallowed; the subfolder scan is the fallback for servers whose GetFolder
// path building doesn't match their own LIST output (differing separator, namespace prefix).
private static async Task<IMailFolder?> ResolveExistingFolderOrNullAsync(ImapClient client, IMailFolder personal, string folderName, CancellationToken ct)
{
var prefix = string.IsNullOrEmpty(personal.FullName) ? string.Empty : personal.FullName + personal.DirectorySeparator;
try
{
return await client.GetFolderAsync(prefix + folderName, ct);
}
catch (FolderNotFoundException)
{
}
try
{
var existing = await personal.GetSubfoldersAsync(false, ct);
return existing.FirstOrDefault(f => string.Equals(f.Name, folderName, StringComparison.OrdinalIgnoreCase));
}
catch (Exception)
{
return null;
}
}
public async Task CreateFolderAsync(Account account, string password, string folderName, CancellationToken ct = default)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id, ["folder"] = folderName });
using var client = await ConnectAsync(account, password, ct);
var personal = client.GetFolder(client.PersonalNamespaces[0]);
// Creating a folder that already exists makes the server answer NO [ALREADYEXISTS], which
// MailKit throws as ImapCommandException - and that used to surface as a 500. It's a very
// easy state to reach: a folder created before the SubscribeAsync below existed (or created
// by another client) exists on the server but is invisible in the sidebar's LSUB-only
// listing, so the user simply creates it again. Treat "already there" as success and fall
// through to the subscribe, which is exactly what such a folder is missing.
var folder = await ResolveExistingFolderOrNullAsync(client, personal, folderName, ct);
if (folder == null)
{
try
{
folder = await personal.CreateAsync(folderName, true, ct);
}
catch (ImapCommandException ex) when (ex.Response == ImapCommandResponse.No)
{
folder = await ResolveExistingFolderOrNullAsync(client, personal, folderName, ct);
if (folder == null) throw;
logger.LogInformation("Folder {Folder} already existed for account {AccountId}; subscribing to it instead", folderName, account.Id);
}
}
// CreateAsync only creates the folder - it doesn't subscribe to it, and every folder listing
// in this app (GetFoldersAsync, ResolveFolderOrNullAsync, SearchByAddressAsync) asks IMAP for
// subscribed folders only (LSUB, via GetSubfoldersAsync(true, ...)). Without this, a newly
// created folder exists on the server (you can move messages into it, as confirmed by its own
// log line) but is invisible everywhere in the UI - nothing was wrong with the move, the
// folder just never showed up in any list that led back to it.
if (folder != null) await folder.SubscribeAsync(ct);
await client.DisconnectAsync(true, ct);
logger.LogInformation("Created and subscribed to folder {Folder} for account {AccountId}", folderName, account.Id);
}
public async Task SetFolderSubscriptionsAsync(Account account, string password, Dictionary<string, bool> subscriptions, CancellationToken ct = default)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id });
using var client = await ConnectAsync(account, password, ct);
foreach (var (folderFullName, subscribed) in subscriptions)
{
try
{
var folder = await client.GetFolderAsync(folderFullName, ct);
if (subscribed) await folder.SubscribeAsync(ct);
else await folder.UnsubscribeAsync(ct);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to {Action} folder {Folder} for account {AccountId}",
subscribed ? "subscribe to" : "unsubscribe from", folderFullName, account.Id);
}
}
await client.DisconnectAsync(true, ct);
}
public async Task MoveFolderAsync(Account account, string password, string folderFullName, string? newParentFullName, CancellationToken ct = default)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id, ["folder"] = folderFullName });
using var client = await ConnectAsync(account, password, ct);
var folder = await client.GetFolderAsync(folderFullName, ct);
if (IsProtectedFolder(folder.FullName, folder.Name, folder.Attributes))
throw new InvalidOperationException($"Refusing to move protected folder '{folderFullName}'.");
// A folder can't become its own (grand)child - IMAP has no cycle detection of its own, the
// server would just accept a nonsensical RENAME and leave the folder unreachable.
var separator = folder.DirectorySeparator;
if (!string.IsNullOrEmpty(newParentFullName) &&
(newParentFullName == folderFullName || newParentFullName.StartsWith(folderFullName + separator, StringComparison.Ordinal)))
{
throw new InvalidOperationException("Cannot move a folder into itself or one of its own subfolders.");
}
var destination = string.IsNullOrEmpty(newParentFullName)
? client.GetFolder(client.PersonalNamespaces[0])
: await client.GetFolderAsync(newParentFullName, ct);
await folder.RenameAsync(destination, folder.Name, ct);
await client.DisconnectAsync(true, ct);
// The folder's FullName - and therefore every store keyed by it (local index, sync-mode/
// subscription settings) - changes the moment it moves. Rather than migrating those rows to
// the new name, just drop them; the folder re-indexes itself automatically the next time it's
// opened (see EnsureFolderIndexedAsync), same as any folder seen for the first time.
indexStore.RemoveFolder(account.Id, folderFullName);
folderSettingsStore.Remove(account.Id, folderFullName);
logger.LogInformation("Moved folder {Folder} to parent {NewParent} for account {AccountId}",
folderFullName, string.IsNullOrEmpty(newParentFullName) ? "(root)" : newParentFullName, account.Id);
}
public async Task RenameFolderAsync(Account account, string password, string folderFullName, string newName, CancellationToken ct = default)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id, ["folder"] = folderFullName });
if (string.IsNullOrWhiteSpace(newName))
throw new InvalidOperationException("New folder name cannot be empty.");
using var client = await ConnectAsync(account, password, ct);
var folder = await client.GetFolderAsync(folderFullName, ct);
if (IsProtectedFolder(folder.FullName, folder.Name, folder.Attributes))
throw new InvalidOperationException($"Refusing to rename protected folder '{folderFullName}'.");
// Same parent as before, so only the leaf name changes - the destination for RenameAsync is
// deliberately the folder's own current parent, not the personal namespace root.
var destination = folder.ParentFolder ?? client.GetFolder(client.PersonalNamespaces[0]);
await folder.RenameAsync(destination, newName, ct);
await client.DisconnectAsync(true, ct);
// See MoveFolderAsync - the FullName changes, so stores keyed by the old one are dropped
// rather than migrated; the folder re-indexes itself the next time it's opened.
indexStore.RemoveFolder(account.Id, folderFullName);
folderSettingsStore.Remove(account.Id, folderFullName);
logger.LogInformation("Renamed folder {Folder} to {NewName} for account {AccountId}", folderFullName, newName, account.Id);
}
public async Task<List<string>> DeleteFoldersAsync(Account account, string password, IEnumerable<string> folderFullNames, CancellationToken ct = default)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id });
using var client = await ConnectAsync(account, password, ct);
var failed = new List<string>();
foreach (var fullName in folderFullNames)
{
try
{
var folder = await client.GetFolderAsync(fullName, ct);
if (IsProtectedFolder(folder.FullName, folder.Name, folder.Attributes))
{
logger.LogWarning("Refused to delete protected folder {Folder} for account {AccountId}", fullName, account.Id);
failed.Add(fullName);
continue;
}
await folder.OpenAsync(FolderAccess.ReadOnly, ct);
var count = folder.Count;
await folder.CloseAsync(false, ct);
if (count > 0)
{
logger.LogWarning("Refused to delete non-empty folder {Folder} for account {AccountId} ({Count} messages)", fullName, account.Id, count);
failed.Add(fullName);
continue;
}
await folder.DeleteAsync(ct);
indexStore.RemoveFolder(account.Id, fullName);
logger.LogInformation("Deleted folder {Folder} for account {AccountId}", fullName, account.Id);
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to delete folder {Folder} for account {AccountId}", fullName, account.Id);
failed.Add(fullName);
}
}
await client.DisconnectAsync(true, ct);
return failed;
}
public async Task<List<MessageListItem>> GetMessagesAsync(Account account, string password, string folderFullName, bool forceRefresh = false, CancellationToken ct = default)
{
var syncMode = folderSettingsStore.GetAll(account.Id).GetValueOrDefault(folderFullName)?.SyncMode ?? FolderSyncMode.Direct;
// Indexed path: covers every folder except NotSynchronized - the local header index (see
// EnsureFolderIndexedAsync) holds every message, not just the last MessageListFetchLimit, and
// is kept in sync cheaply instead of being re-fetched (fully or partially) on every request.
if (syncMode != FolderSyncMode.NotSynchronized)
{
var state = indexStore.GetSyncState(account.Id, folderFullName);
// forceRefresh is deliberately a no-op here (unlike the NotSynchronized fallback below,
// which uses it to bypass a short TTL cache): EnsureFolderIndexedAsync already re-verifies
// against the live server on every call - cheaply, via STATUS - and mark-read/flag/move/
// delete already update the index immediately for their own uids. There's nothing a
// "refresh" needs to force here; treating it as "wipe and rebuild the whole folder" would
// only throw away an accurate index (and re-trigger the full-folder header scan) on every
// single mark-read/flag/move/delete, since the UI always requests a refresh after those.
if (state == null)
{
// First time this folder is seen (or just reset above): a full rebuild means scanning
// every message, far too slow to do inline - the user would stare at a blank screen.
// Kick it off in the background and fall through to the fast capped-live-fetch path
// below for this one request; once the background build lands, later requests take
// the fast indexed path above instead.
StartBackgroundIndexBuild(account, password, folderFullName);
}
else
{
await EnsureFolderIndexedAsync(account, password, folderFullName, ct);
return [.. indexStore.GetMessages(account.Id, folderFullName)
.OrderByDescending(m => m.Date)
.Select(m => new MessageListItem
{
Uid = m.Uid,
Subject = m.Subject,
From = m.From,
Date = m.Date,
SizeBytes = m.SizeBytes,
IsRead = m.IsRead,
IsFlagged = m.IsFlagged,
HasAttachments = m.HasAttachments,
Priority = m.Priority
})];
}
}
// Fallback path: used for NotSynchronized folders (unchanged, permanent behaviour for that
// mode) and for indexed folders whose very first index build is still running in the
// background above - capped live fetch (last MessageListFetchLimit messages) behind a short
// TTL cache, same as before the index existed.
if (!forceRefresh)
{
var cached = cacheStore.GetMessageList(account.Id, folderFullName, TimeSpan.FromMinutes(_settings.MessageListCacheTtlMinutes));
if (cached != null) return cached;
}
var indexed = await FetchLiveMessagesAsync(account, password, folderFullName, ct);
var items = indexed.Select(m => new MessageListItem
{
Uid = m.Uid,
Subject = m.Subject,
From = m.From,
Date = m.Date,
SizeBytes = m.SizeBytes,
IsRead = m.IsRead,
IsFlagged = m.IsFlagged,
HasAttachments = m.HasAttachments,
Priority = m.Priority
});
var result = new List<MessageListItem>([.. items.OrderByDescending(x => x.Date)]);
cacheStore.SaveMessageList(account.Id, folderFullName, result);
return result;
}
// Used for the NotSynchronized capped-live-fetch path.
private async Task<List<IndexedMessage>> FetchLiveMessagesAsync(Account account, string password, string folderFullName, CancellationToken ct)
{
using var client = await ConnectAsync(account, password, ct);
var folder = await client.GetFolderAsync(folderFullName, ct);
await folder.OpenAsync(FolderAccess.ReadOnly, ct);
var uids = await folder.SearchAsync(SearchQuery.All, ct);
var take = uids.Skip(Math.Max(0, uids.Count - _settings.MessageListFetchLimit)).ToList();
var messages = take.Count > 0 ? await FetchIndexedMessagesAsync(folder, take, ct) : [];
await client.DisconnectAsync(true, ct);
return messages;
}
public async Task<MessageDetail> GetMessageAsync(Account account, string password, string folderFullName, uint uid, CancellationToken ct = default)
{
var message = await LoadMimeMessageAsync(account, password, folderFullName, uid, ct);
// LoadMimeMessageAsync only sets the Seen flag on IMAP (and only on the very first open,
// before the .eml is cached - a cache hit skips even that) and marks the short-TTL live
// message-list cache - it never touches the persistent index (see IMessageIndexStore),
// which is what indexed folders actually read their read/unread state from (see
// GetMessagesAsync). Without this, an opened message's "read" state only ever showed up as
// a client-side-only flag that reverted to unread on the next list reload. No-ops harmlessly
// for folders that aren't indexed (message not present in the index) or already marked read.
// Serialized against StartRecentFlagsRefresh (see FolderMutationLocks) for the same reason as
// SetSeenAsync below - without it, a background refresh reading stale flags could revert this
// right after it lands.
var openMutationLock = GetFolderMutationLock($"{account.Id}:{folderFullName}");
await openMutationLock.WaitAsync(ct);
try { indexStore.MarkRead(account.Id, folderFullName, [uid], true); }
finally { openMutationLock.Release(); }
// Self-heal for a message indexed before ResolveMessageDate's INTERNALDATE fallback existed -
// a normal reindex only happens on UIDVALIDITY change, so an already-broken date otherwise
// never gets corrected on its own (see the uid 95868/46823/46821 investigation). Opening the
// message is the one moment it's cheap to fix: one extra lightweight FETCH, only when the
// stored date actually looks wrong, and it never needs fixing again after this.
var indexedEntry = indexStore.GetMessage(account.Id, folderFullName, uid);
if (indexedEntry != null && indexedEntry.Date < PlausibleDateFloor)
{
var internalDate = await FetchInternalDateAsync(account, password, folderFullName, uid, ct);
if (internalDate is { } resolvedDate)
{
indexStore.UpdateDate(account.Id, folderFullName, uid, resolvedDate);
}
}
// MimeMessage.HtmlBody/TextBody only match the exact "text/html"/"text/plain" subtypes - a
// message whose body part uses a non-standard Content-Type (e.g. "text/text", seen in the
// wild from at least one spam sender - see the uid 95868 investigation) is a TextPart by
// MimeKit's own classification (MediaType "text", any subtype) but neither convenience
// property picks it up, so the message rendered as completely empty even though it has
// content. Falling back to the first TextPart at all covers that case without needing to
// special-case every possible malformed subtype.
var fallbackText = message.HtmlBody == null && message.TextBody == null
? message.BodyParts.OfType<TextPart>().FirstOrDefault()?.Text
: null;
var html = message.HtmlBody ?? (fallbackText != null ? System.Net.WebUtility.HtmlEncode(fallbackText).Replace("\n", "<br>") : "");
if (!string.IsNullOrEmpty(html))
{
foreach (var part in message.BodyParts.OfType<MimePart>())
{
if (string.IsNullOrEmpty(part.ContentId) || part.Content == null) continue;
if (!html.Contains($"cid:{part.ContentId}", StringComparison.OrdinalIgnoreCase)) continue;
using var ms = new MemoryStream();
part.Content.DecodeTo(ms, ct);
var dataUri = $"data:{part.ContentType.MimeType};base64,{Convert.ToBase64String(ms.ToArray())}";
html = html.Replace($"cid:{part.ContentId}", dataUri, StringComparison.OrdinalIgnoreCase);
}
}
var senderEmail = message.From.Mailboxes.FirstOrDefault()?.Address ?? "";
// Never in the Spam folder, regardless of any saved per-sender preference - see IsSpamFolder.
var allowImages = !IsSpamFolder(folderFullName) && cacheStore.GetAllowExternalImages(account.Id, senderEmail);
var messageId = ResolveMessageId(message, folderFullName, uid);
var (transformedHtml, hasExternal) = imageSanitizer.ApplyPolicy(html, allowImages, account.EmailAddress, messageId);
transformedHtml = imageSanitizer.ApplyLinkPolicy(transformedHtml);
var detail = new MessageDetail
{
Uid = uid,
Subject = message.Subject ?? "(no subject)",
From = message.From.ToString(),
SenderEmail = senderEmail,
To = message.To.ToString(),
Cc = message.Cc.ToString(),
// Prefer the index's already-resolved date (see ResolveMessageDate/FetchIndexedMessagesAsync)
// over the raw MimeMessage.Date - a message with no Date header parses to an implausible
// default here too, and showing a different (wrong) date in the detail pane than what the
// message list/sort already fell back to would just be a fresh source of confusion.
Date = indexStore.GetMessage(account.Id, folderFullName, uid)?.Date ?? message.Date,
HtmlBody = transformedHtml,
TextBody = message.TextBody ?? fallbackText ?? "",
HasExternalImages = hasExternal,
ImagesAllowed = allowImages,
Priority = message.Priority switch
{
MimeKit.MessagePriority.NonUrgent => ModelPriority.Low,
MimeKit.MessagePriority.Urgent => ModelPriority.High,
_ => ModelPriority.Normal
},
Sensitivity = ParseSensitivity(message.Headers["Sensitivity"]),
// Recent-scoped, not the folder's raw unread total (see GetRecentUnreadCount) - matches
// what StartRecentFlagsRefresh actually keeps accurate, and is the number that's actually
// useful (a message from years ago sitting unread, or Trash's tens of thousands of
// untouched messages, shouldn't move this badge).
FolderUnreadCount = indexStore.GetSyncState(account.Id, folderFullName) != null
? indexStore.GetRecentUnreadCount(account.Id, folderFullName, RecentFlagsRefreshCount)
: null
};
int idx = 0;
foreach (var att in message.Attachments)
{
var fileName = att.ContentDisposition?.FileName ?? att.ContentType.Name ?? $"attachment{idx}";
long size = 0;
if (att is MimePart part && part.Content != null)
{
using var ms = new MemoryStream();
part.Content.DecodeTo(ms, ct);
size = ms.Length;
}
detail.Attachments.Add(new AttachmentInfo
{
FileName = fileName,
ContentType = att.ContentType.MimeType,
SizeBytes = size,
PartIndex = idx
});
idx++;
}
return detail;
}
// Reads the cached .eml if we already have one; otherwise fetches the message's exact raw
// bytes from IMAP (not a MimeMessage re-serialized via WriteToAsync, which would only be a
// logical reproduction - original header order/whitespace could differ, which matters e.g.
// for DKIM verification), marks it Seen, and writes those raw bytes to disk as-is - keyed by
// the message's own Message-ID rather than its (folder, uid), since a uid is only unique
// within one folder and changes when a message is moved. (folder, uid) is all we know before
// contacting IMAP though, so a small mapping to the Message-ID is kept for that lookup; a
// fresh IMAP round trip is still needed the first time a given (folder, uid) pair is seen
// (e.g. right after a move), but every later read - here and in GetAttachmentAsync - is then
// a local file read instead.
private async Task<MimeMessage> LoadMimeMessageAsync(Account account, string password, string folderFullName, uint uid, CancellationToken ct)
{
var messageId = cacheStore.GetMessageIdForUid(account.Id, folderFullName, uid);
if (messageId != null)
{
var cachedEml = cacheStore.GetMessageEml(account.EmailAddress, messageId);
if (cachedEml != null)
{
// A cached .eml skips the IMAP round trip entirely for speed, but that means the
// Seen-flag AddFlagsAsync below (the cache-miss path) never runs either. That's fine
// the first time a message is read (Seen already got set then, which is what put it
// in the cache), but if it's since been marked unread again - by this app or another
// client - and reopened, the local index gets marked read again (see GetMessageAsync)
// while the server's flag silently stays unset, so the live unread count this app's
// own folder sidebar shows (straight from IMAP STATUS UNSEEN) never reflects it. Only
// worth the extra connection for that specific case; the common case (reopening an
// already-read message) stays free of any IMAP round trip.
if (indexStore.GetMessage(account.Id, folderFullName, uid) is { IsRead: false })
{
await SetSeenAsync(account, password, folderFullName, [uid], true, ct);
}
using var cachedStream = new MemoryStream(cachedEml);
return await MimeMessage.LoadAsync(cachedStream, ct);
}
}
using var client = await ConnectAsync(account, password, ct);
var folder = await client.GetFolderAsync(folderFullName, ct);
await folder.OpenAsync(FolderAccess.ReadWrite, ct);
byte[] rawBytes;
using (var rawStream = await folder.GetStreamAsync(new UniqueId(uid), ct))
using (var buffer = new MemoryStream())
{
await rawStream.CopyToAsync(buffer, ct);
rawBytes = buffer.ToArray();
}
await folder.AddFlagsAsync(new UniqueId(uid), MessageFlags.Seen, true, ct);
await client.DisconnectAsync(true, ct);
MimeMessage message;
using (var parseStream = new MemoryStream(rawBytes))
{
message = await MimeMessage.LoadAsync(parseStream, ct);
}
var resolvedMessageId = ResolveMessageId(message, folderFullName, uid);
cacheStore.SaveMessageEml(account.EmailAddress, resolvedMessageId, rawBytes);
cacheStore.SaveMessageIdForUid(account.Id, folderFullName, uid, resolvedMessageId);
cacheStore.MarkMessageRead(account.Id, folderFullName, uid);
return message;
}
// Shared by the .eml cache key and the image cache directory name, so both always agree on
// the same identity for a given message even when its Message-ID header is missing.
private static string ResolveMessageId(MimeMessage message, string folderFullName, uint uid) =>
string.IsNullOrWhiteSpace(message.MessageId) ? $"no-message-id-{folderFullName}-{uid}" : message.MessageId;
// "Sensitivity" isn't a MimeKit first-class property (unlike Priority/X-Priority) - it's a
// plain header this app itself writes on send (see MimeMessageFactory.Build) using the same
// values Outlook/OWA use, so parsing it back here just has to match that same vocabulary.
private static MessageSensitivity ParseSensitivity(string? headerValue) => headerValue?.Trim() switch
{
"Personal" => MessageSensitivity.Personal,
"Private" => MessageSensitivity.Private,
"Company-Confidential" => MessageSensitivity.Confidential,
_ => MessageSensitivity.Nothing
};
public async Task<Stream> GetAttachmentAsync(Account account, string password, string folderFullName, uint uid, int partIndex, CancellationToken ct = default)
{
// The uid -> Message-ID mapping alone (cheap LiteDB lookup) is enough to check the disk
// cache, so a cached attachment never requires loading/parsing the .eml at all.
var knownMessageId = cacheStore.GetMessageIdForUid(account.Id, folderFullName, uid);
if (knownMessageId != null)
{
var cached = cacheStore.GetAttachmentContent(account.EmailAddress, knownMessageId, partIndex);
if (cached != null) return new MemoryStream(cached);
}
var message = await LoadMimeMessageAsync(account, password, folderFullName, uid, ct);
var messageId = ResolveMessageId(message, folderFullName, uid);
var att = message.Attachments.ElementAt(partIndex);
var result = new MemoryStream();
if (att is MimePart part && part.Content != null)
{
part.Content.DecodeTo(result, ct);
result.Position = 0;
var fileName = att.ContentDisposition?.FileName ?? att.ContentType.Name ?? $"attachment{partIndex}";
cacheStore.SaveAttachmentContent(account.EmailAddress, messageId, partIndex, fileName, result.ToArray());
result.Position = 0;
}
return result;
}
public async Task SaveDraftAsync(Account account, string password, ComposeModel compose, IEnumerable<(string FileName, Stream Content)> attachments, IEnumerable<(string Cid, string FileName, Stream Content)> inlineImages, CancellationToken ct = default)
{
var message = MimeMessageFactory.Build(account, compose, attachments, inlineImages);
using var client = await ConnectAsync(account, password, ct);
var drafts = await ResolveDraftsFolderAsync(client, ct);
await drafts.AppendAsync(message, MessageFlags.Draft | MessageFlags.Seen, ct);
await client.DisconnectAsync(true, ct);
}
// Most SMTP servers never store a copy of what they relay - the client is expected to save its
// own copy to the Sent folder over IMAP, the same way any other webmail/desktop mail client does.
public async Task AppendSentAsync(Account account, string password, MimeMessage message, CancellationToken ct = default)
{
using var client = await ConnectAsync(account, password, ct);
try
{
var sent = await ResolveSentFolderAsync(client, ct);
await sent.AppendAsync(message, MessageFlags.Seen, ct);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to save sent copy to the Sent folder for account {AccountId}", account.Id);
throw;
}
finally
{
await client.DisconnectAsync(true, ct);
}
}
public async Task SetSeenAsync(Account account, string password, string folderFullName, IEnumerable<uint> uids, bool seen, CancellationToken ct = default)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id, ["folder"] = folderFullName });
using var client = await ConnectAsync(account, password, ct);
var folder = await client.GetFolderAsync(folderFullName, ct);
await folder.OpenAsync(FolderAccess.ReadWrite, ct);
var uidList = uids.Select(u => new UniqueId(u)).ToList();
var applied = await ApplySeenFlagWithVerificationAsync(folder, uidList, seen, ct);
await client.DisconnectAsync(true, ct);
// See FolderMutationLocks / StartRecentFlagsRefresh - the server-side flag is already set by
// this point, but the local index write still has to be serialized against a concurrent
// background flag refresh for this folder, otherwise that refresh's own (now-stale) snapshot
// could land right after this and silently revert what the user just explicitly did.
var mutationLock = GetFolderMutationLock($"{account.Id}:{folderFullName}");
await mutationLock.WaitAsync(ct);
try { indexStore.MarkRead(account.Id, folderFullName, applied, seen); }
finally { mutationLock.Release(); }
}
public async Task SetFlaggedAsync(Account account, string password, string folderFullName, IEnumerable<uint> uids, bool flagged, CancellationToken ct = default)
{
using var client = await ConnectAsync(account, password, ct);
var folder = await client.GetFolderAsync(folderFullName, ct);
await folder.OpenAsync(FolderAccess.ReadWrite, ct);
var uidList = uids.Select(u => new UniqueId(u)).ToList();
await folder.AddFlagsAsync(uidList, MessageFlags.Flagged, flagged, ct);
if (!flagged) await folder.RemoveFlagsAsync(uidList, MessageFlags.Flagged, true, ct);
await client.DisconnectAsync(true, ct);
// See SetSeenAsync above for why this is serialized against StartRecentFlagsRefresh.
var mutationLock = GetFolderMutationLock($"{account.Id}:{folderFullName}");
await mutationLock.WaitAsync(ct);
try { indexStore.MarkFlagged(account.Id, folderFullName, uidList.Select(u => u.Id), flagged); }
finally { mutationLock.Release(); }
}
// One giant STORE command covering every UID in the folder has been observed (see the Vmtux
// investigation) to silently not apply to a handful of UIDs on at least one real server, even
// though the same command against a single UID at a time works fine for those exact messages -
// no exception, no server error, just a subset quietly not taking effect. Root cause is outside
// this app (server-side UID-set/command-length handling), so instead of trusting one big batch
// this chunks the request and then verifies + individually retries whatever didn't stick,
// mirroring the code path that's confirmed to work. Only UIDs verified to actually have the
// target flag afterward get written into the local index - never assumed from the request.
private const int FlagUpdateBatchSize = 50;
public async Task MarkAllAsync(Account account, string password, string folderFullName, bool seen, CancellationToken ct = default)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id, ["folder"] = folderFullName });
using var client = await ConnectAsync(account, password, ct);
var folder = await client.GetFolderAsync(folderFullName, ct);
await folder.OpenAsync(FolderAccess.ReadWrite, ct);
var uids = await folder.SearchAsync(SearchQuery.All, ct);
var applied = await ApplySeenFlagWithVerificationAsync(folder, uids, seen, ct);
await client.DisconnectAsync(true, ct);
logger.LogInformation("Marked {Applied}/{Requested} messages as {SeenState} in folder {Folder} for account {AccountId}",
applied.Count, uids.Count, seen ? "read" : "unread", folderFullName, account.Id);
// See SetSeenAsync for why this is serialized against StartRecentFlagsRefresh.
var mutationLock = GetFolderMutationLock($"{account.Id}:{folderFullName}");
await mutationLock.WaitAsync(ct);
try { indexStore.MarkRead(account.Id, folderFullName, applied, seen); }
finally { mutationLock.Release(); }
}
// Applies the Seen flag in batches, then verifies against a fresh SEARCH and individually
// retries any UID that didn't stick - see MarkAllAsync's remarks. Returns the UIDs confirmed to
// actually be in the target state afterward (a strict subset of the input on a flaky server).
private async Task<List<uint>> ApplySeenFlagWithVerificationAsync(IMailFolder folder, IList<UniqueId> uids, bool seen, CancellationToken ct)
{
if (uids.Count == 0) return [];
for (var offset = 0; offset < uids.Count; offset += FlagUpdateBatchSize)
{
var batch = uids.Skip(offset).Take(FlagUpdateBatchSize).ToList();
try
{
if (seen) await folder.AddFlagsAsync(batch, MessageFlags.Seen, true, ct);
else await folder.RemoveFlagsAsync(batch, MessageFlags.Seen, true, ct);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Batch flag update failed for {Count} messages in folder {Folder}, will retry individually", batch.Count, folder.FullName);
}
}
var mismatchQuery = seen ? SearchQuery.NotSeen : SearchQuery.Seen;
var stillMismatched = (await folder.SearchAsync(mismatchQuery, ct)).Where(uids.Contains).ToList();
var confirmed = uids.Except(stillMismatched).Select(u => u.Id).ToList();
if (stillMismatched.Count == 0) return confirmed;
logger.LogWarning("{Count} messages did not pick up the bulk flag update in folder {Folder}, retrying individually: {Uids}",
stillMismatched.Count, folder.FullName, string.Join(",", stillMismatched.Select(u => u.Id)));
foreach (var uid in stillMismatched)
{
try
{
if (seen) await folder.AddFlagsAsync(uid, MessageFlags.Seen, true, ct);
else await folder.RemoveFlagsAsync(uid, MessageFlags.Seen, true, ct);
}
catch (Exception ex)
{
logger.LogError(ex, "Individual flag update also failed for uid {Uid} in folder {Folder} - leaving as-is", uid.Id, folder.FullName);
}
}
// The command not throwing doesn't mean the server actually applied it - that's the exact
// silent-no-op behaviour this whole verify/retry path exists to work around in the first
// place (see the Vmtux investigation), so trusting "no exception" here would just reintroduce
// the same bug one level down: the caller would mark these read/flagged locally, only for the
// next background flags refresh to discover the server disagrees and quietly revert it -
// which looked like "marking as read does nothing" no matter how many times it was retried.
// A final SEARCH is the only way to know it actually stuck.
var stillMismatchedAfterRetry = (await folder.SearchAsync(mismatchQuery, ct)).Where(stillMismatched.Contains).ToHashSet();
confirmed.AddRange(stillMismatched.Where(u => !stillMismatchedAfterRetry.Contains(u)).Select(u => u.Id));
if (stillMismatchedAfterRetry.Count > 0)
{
logger.LogError("{Count} message(s) in folder {Folder} still did not pick up the flag update after individual retry - leaving as-is: {Uids}",
stillMismatchedAfterRetry.Count, folder.FullName, string.Join(",", stillMismatchedAfterRetry.Select(u => u.Id)));
}
return confirmed;
}
public async Task MoveAsync(Account account, string password, string folderFullName, IEnumerable<uint> uids, string targetFolderFullName, CancellationToken ct = default)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id, ["folder"] = folderFullName });
using var client = await ConnectAsync(account, password, ct);
var folder = await client.GetFolderAsync(folderFullName, ct);
await folder.OpenAsync(FolderAccess.ReadWrite, ct);
var target = await client.GetFolderAsync(targetFolderFullName, ct);
var uidList = uids.Select(u => new UniqueId(u)).ToList();
await folder.MoveToAsync(uidList, target, ct);
await client.DisconnectAsync(true, ct);
logger.LogInformation("Moved {Count} messages from {Folder} to {TargetFolder} for account {AccountId}", uidList.Count, folderFullName, targetFolderFullName, account.Id);
// The moved messages get new UIDs in targetFolderFullName - rather than guess at those, just
// drop them from the source folder's index now; the target folder picks them up as "new"
// UIDs (with flags intact, since IMAP MOVE preserves them) next time it's indexed.
indexStore.RemoveMessages(account.Id, folderFullName, uidList.Select(u => u.Id));
}
public async Task DeleteAsync(Account account, string password, string folderFullName, IEnumerable<uint> uids, CancellationToken ct = default)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id, ["folder"] = folderFullName });
using var client = await ConnectAsync(account, password, ct);
var folder = await client.GetFolderAsync(folderFullName, ct);
await folder.OpenAsync(FolderAccess.ReadWrite, ct);
var uidList = uids.Select(u => new UniqueId(u)).ToList();
var trash = await ResolveTrashFolderAsync(client, ct);
if (trash != null && folder.FullName != trash.FullName)
{
await folder.MoveToAsync(uidList, trash, ct);
logger.LogInformation("Moved {Count} messages from {Folder} to Trash for account {AccountId}", uidList.Count, folderFullName, account.Id);
}
else
{
await folder.AddFlagsAsync(uidList, MessageFlags.Deleted, true, ct);
await folder.ExpungeAsync(ct);
logger.LogInformation("Expunged {Count} messages from {Folder} for account {AccountId}", uidList.Count, folderFullName, account.Id);
}
await client.DisconnectAsync(true, ct);
// Same reasoning as MoveAsync above: moved to Trash (new UIDs there, picked up on next
// index) or actually expunged (gone for good either way) - either way, gone from here now.
indexStore.RemoveMessages(account.Id, folderFullName, uidList.Select(u => u.Id));
}
public async Task EmptyFolderAsync(Account account, string password, string folderFullName, CancellationToken ct = default)
{
using var scope = logger.BeginScope(new Dictionary<string, object?> { ["accountId"] = account.Id, ["folder"] = folderFullName });
using var client = await ConnectAsync(account, password, ct);
var folder = await client.GetFolderAsync(folderFullName, ct);
await folder.OpenAsync(FolderAccess.ReadWrite, ct);
var uids = await folder.SearchAsync(SearchQuery.All, ct);
if (uids.Count > 0)
{
await folder.AddFlagsAsync(uids, MessageFlags.Deleted, true, ct);
await folder.ExpungeAsync(ct);
}
await client.DisconnectAsync(true, ct);
indexStore.RemoveFolder(account.Id, folderFullName);
logger.LogInformation("Emptied folder {Folder} for account {AccountId} ({Count} messages)", folderFullName, account.Id, uids.Count);
}
public async Task<List<ContactAddress>> GetSentContactsAsync(Account account, string password, CancellationToken ct = default)
{
string sentFullName;
using (var client = await ConnectAsync(account, password, ct))
{
var sent = await ResolveSentFolderAsync(client, ct);
sentFullName = sent.FullName;
await client.DisconnectAsync(true, ct);
}
var syncMode = folderSettingsStore.GetAll(account.Id).GetValueOrDefault(sentFullName)?.SyncMode ?? FolderSyncMode.Direct;
var state = syncMode != FolderSyncMode.NotSynchronized ? indexStore.GetSyncState(account.Id, sentFullName) : null;
var result = new Dictionary<string, ContactAddress>(StringComparer.OrdinalIgnoreCase);
if (state != null)
{
// Already indexed: no IMAP fetch needed at all beyond the cheap sync check - To/Cc are
// stored as the same InternetAddressList.ToString() form MimeKit itself produces, so they
// parse straight back into MailboxAddress for extraction, same as the live path below.
await EnsureFolderIndexedAsync(account, password, sentFullName, ct);
foreach (var m in indexStore.GetMessages(account.Id, sentFullName))
AddContactRecipients(result, ParseMailboxes(m.To).Concat(ParseMailboxes(m.Cc)));
return [.. result.Values.OrderBy(c => c.Email)];
}
// NotSynchronized, or simply never indexed yet: do the (correct, if slower) live scan this
// once, same as before the index existed - and if it's the latter case, also kick off a
// background build so the next call takes the fast indexed path above instead.
if (syncMode != FolderSyncMode.NotSynchronized) StartBackgroundIndexBuild(account, password, sentFullName);
using var liveClient = await ConnectAsync(account, password, ct);
var liveSent = await liveClient.GetFolderAsync(sentFullName, ct);
await liveSent.OpenAsync(FolderAccess.ReadOnly, ct);
var uids = await liveSent.SearchAsync(SearchQuery.All, ct);
var summaries = await liveSent.FetchAsync(uids, MessageSummaryItems.Envelope, ct);
await liveClient.DisconnectAsync(true, ct);
foreach (var s in summaries)
AddContactRecipients(result, (s.Envelope?.To ?? []).Concat(s.Envelope?.Cc ?? []).OfType<MailboxAddress>());
return [.. result.Values.OrderBy(c => c.Email)];
}
private static void AddContactRecipients(Dictionary<string, ContactAddress> result, IEnumerable<MailboxAddress> recipients)
{
foreach (var mb in recipients)
{
if (string.IsNullOrWhiteSpace(mb.Address)) continue;
if (!result.ContainsKey(mb.Address))
{
result[mb.Address] = new ContactAddress { Email = mb.Address, Name = mb.Name ?? "" };
}
}
}
private static IEnumerable<MailboxAddress> ParseMailboxes(string addressListString)
{
if (string.IsNullOrWhiteSpace(addressListString)) return [];
try { return InternetAddressList.Parse(addressListString).OfType<MailboxAddress>(); }
catch { return []; }
}
public async Task<List<MessageSearchResult>> SearchByAddressAsync(Account account, string password, string address, bool forceRefresh = false, CancellationToken ct = default)
{
if (!forceRefresh)
{
var cached = cacheStore.GetContactMailSearch(account.Id, address, TimeSpan.FromMinutes(_settings.ContactMailSearchCacheTtlMinutes));
if (cached != null) return cached;
}
using var client = await ConnectAsync(account, password, ct);
var personal = client.GetFolder(client.PersonalNamespaces[0]);
var folders = await GetAllSubfoldersRecursiveAsync(personal, true, ct);
var settings = folderSettingsStore.GetAll(account.Id);
var result = new List<MessageSearchResult>();
var query = SearchQuery.FromContains(address).Or(SearchQuery.ToContains(address));
foreach (var f in new[] { client.Inbox }.Concat(folders).DistinctBy(f => f.FullName))
{
var syncMode = settings.GetValueOrDefault(f.FullName)?.SyncMode ?? FolderSyncMode.Direct;
var alreadyIndexed = syncMode != FolderSyncMode.NotSynchronized && indexStore.GetSyncState(account.Id, f.FullName) != null;
// Already indexed: just keep it current and let the bulk SearchByAddress query below
// pick up this folder's matches - no per-folder IMAP search/fetch needed at all.
if (alreadyIndexed)
{
try { await EnsureFolderIndexedAsync(account, password, f.FullName, ct); }
catch { /* couldn't refresh right now - fall through using whatever's already indexed */ }
continue;
}
// Never indexed yet (or NotSynchronized): do the live per-folder scan this once, same as
// before the index existed, so this search isn't missing results while the background
// build (kicked off below, for the non-NotSynchronized case) is still running.
if (syncMode != FolderSyncMode.NotSynchronized) StartBackgroundIndexBuild(account, password, f.FullName);
try
{
await f.OpenAsync(FolderAccess.ReadOnly, ct);
var uids = await f.SearchAsync(query, ct);
if (uids.Count == 0) { await f.CloseAsync(false, ct); continue; }
var summaries = await f.FetchAsync(uids, MessageSummaryItems.Envelope, ct);
foreach (var s in summaries)
{
result.Add(new MessageSearchResult
{
Folder = f.FullName,
Uid = s.UniqueId.Id,
Subject = s.Envelope?.Subject ?? "(no subject)",
From = s.Envelope?.From?.ToString() ?? "",
Date = s.Envelope?.Date ?? DateTimeOffset.MinValue
});
}
await f.CloseAsync(false, ct);
}
catch { /* non-selectable folder */ }
}
await client.DisconnectAsync(true, ct);
result.AddRange(indexStore.SearchByAddress(account.Id, address).Select(m => new MessageSearchResult
{
Folder = m.Folder,
Uid = m.Uid,
Subject = m.Subject,
From = m.From,
Date = m.Date
}));
var ordered = new List<MessageSearchResult>([.. result.OrderByDescending(r => r.Date)]);
cacheStore.SaveContactMailSearch(account.Id, address, ordered);
return ordered;
}
}