@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Begin improving the soundness of received mail
Summary:
We/I broke a couple of things here recently (see D5911) and are doing some work here in general (see D5912, etc.).
Generally, this code is pretty oldschool and not especially well architected for modern application-oriented Phabricator. It hardcodes a lot of stuff which should be applications' responsibilites.
Take the first steps toward making it more solid to reduce the risk here. In particular:
- Factor out the "self mail" and "duplicate mail" checks and add unit tests.
- Make Message-ID hash handling automatic.
Test Plan: Ran unit tests.
Reviewers: btrahan, chad
Reviewed By: btrahan
CC: aran
Differential Revision: https://secure.phabricator.com/D5915
+180
-44
+2
resources/sql/patches/20130513.receviedmailstatus.sql
+2
resources/sql/patches/20130513.receviedmailstatus.sql
-3
scripts/mail/mail_handler.php
-3
scripts/mail/mail_handler.php
+6
src/__phutil_library_map__.php
+6
src/__phutil_library_map__.php
···
653
653
'ManiphestView' => 'applications/maniphest/view/ManiphestView.php',
654
654
'MetaMTAConstants' => 'applications/metamta/constants/MetaMTAConstants.php',
655
655
'MetaMTANotificationType' => 'applications/metamta/constants/MetaMTANotificationType.php',
656
+
'MetaMTAReceivedMailStatus' => 'applications/metamta/constants/MetaMTAReceivedMailStatus.php',
656
657
'ObjectHandleLoader' => 'applications/phid/handle/ObjectHandleLoader.php',
657
658
'OwnersPackageReplyHandler' => 'applications/owners/OwnersPackageReplyHandler.php',
658
659
'PHUI' => 'view/phui/PHUI.php',
···
1107
1108
'PhabricatorMetaMTAReceiveController' => 'applications/metamta/controller/PhabricatorMetaMTAReceiveController.php',
1108
1109
'PhabricatorMetaMTAReceivedListController' => 'applications/metamta/controller/PhabricatorMetaMTAReceivedListController.php',
1109
1110
'PhabricatorMetaMTAReceivedMail' => 'applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php',
1111
+
'PhabricatorMetaMTAReceivedMailProcessingException' => 'applications/metamta/exception/PhabricatorMetaMTAReceivedMailProcessingException.php',
1112
+
'PhabricatorMetaMTAReceivedMailTestCase' => 'applications/metamta/storage/__tests__/PhabricatorMetaMTAReceivedMailTestCase.php',
1110
1113
'PhabricatorMetaMTASendController' => 'applications/metamta/controller/PhabricatorMetaMTASendController.php',
1111
1114
'PhabricatorMetaMTASendGridReceiveController' => 'applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php',
1112
1115
'PhabricatorMetaMTAViewController' => 'applications/metamta/controller/PhabricatorMetaMTAViewController.php',
···
2391
2394
'ManiphestTransactionType' => 'ManiphestConstants',
2392
2395
'ManiphestView' => 'AphrontView',
2393
2396
'MetaMTANotificationType' => 'MetaMTAConstants',
2397
+
'MetaMTAReceivedMailStatus' => 'MetaMTAConstants',
2394
2398
'OwnersPackageReplyHandler' => 'PhabricatorMailReplyHandler',
2395
2399
'PHUIBoxExample' => 'PhabricatorUIExample',
2396
2400
'PHUIBoxView' => 'AphrontTagView',
···
2832
2836
'PhabricatorMetaMTAReceiveController' => 'PhabricatorMetaMTAController',
2833
2837
'PhabricatorMetaMTAReceivedListController' => 'PhabricatorMetaMTAController',
2834
2838
'PhabricatorMetaMTAReceivedMail' => 'PhabricatorMetaMTADAO',
2839
+
'PhabricatorMetaMTAReceivedMailProcessingException' => 'Exception',
2840
+
'PhabricatorMetaMTAReceivedMailTestCase' => 'PhabricatorTestCase',
2835
2841
'PhabricatorMetaMTASendController' => 'PhabricatorMetaMTAController',
2836
2842
'PhabricatorMetaMTASendGridReceiveController' => 'PhabricatorMetaMTAController',
2837
2843
'PhabricatorMetaMTAViewController' => 'PhabricatorMetaMTAController',
+9
src/applications/metamta/constants/MetaMTAReceivedMailStatus.php
+9
src/applications/metamta/constants/MetaMTAReceivedMailStatus.php
+3
-6
src/applications/metamta/controller/PhabricatorMetaMTAReceiveController.php
+3
-6
src/applications/metamta/controller/PhabricatorMetaMTAReceiveController.php
···
10
10
11
11
if ($request->isFormPost()) {
12
12
$received = new PhabricatorMetaMTAReceivedMail();
13
-
$header_content = array();
13
+
$header_content = array(
14
+
'Message-ID' => Filesystem::readRandomBytes(12),
15
+
);
14
16
$from = $request->getStr('sender');
15
17
$to = $request->getStr('receiver');
16
18
$uri = '/mail/received/';
···
41
43
array(
42
44
'text' => $request->getStr('body'),
43
45
));
44
-
45
-
// Make up some unique value, since this column isn't nullable.
46
-
$received->setMessageIDHash(
47
-
PhabricatorHash::digestForIndex(
48
-
Filesystem::readRandomBytes(12)));
49
46
50
47
$received->save();
51
48
-2
src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php
-2
src/applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php
+20
src/applications/metamta/exception/PhabricatorMetaMTAReceivedMailProcessingException.php
+20
src/applications/metamta/exception/PhabricatorMetaMTAReceivedMailProcessingException.php
···
1
+
<?php
2
+
3
+
final class PhabricatorMetaMTAReceivedMailProcessingException
4
+
extends Exception {
5
+
6
+
private $statusCode;
7
+
8
+
public function getStatusCode() {
9
+
return $this->statusCode;
10
+
}
11
+
12
+
public function __construct($status_code /* ... */) {
13
+
$args = func_get_args();
14
+
$this->statusCode = $args[0];
15
+
16
+
$args = array_slice($args, 1);
17
+
call_user_func_array(array('parent', '__construct'), $args);
18
+
}
19
+
20
+
}
+84
-33
src/applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php
+84
-33
src/applications/metamta/storage/PhabricatorMetaMTAReceivedMail.php
···
5
5
protected $headers = array();
6
6
protected $bodies = array();
7
7
protected $attachments = array();
8
+
protected $status = '';
8
9
9
10
protected $relatedPHID;
10
11
protected $authorPHID;
11
12
protected $message;
12
-
protected $messageIDHash;
13
+
protected $messageIDHash = '';
13
14
14
15
public function getConfiguration() {
15
16
return array(
···
25
26
// Normalize headers to lowercase.
26
27
$normalized = array();
27
28
foreach ($headers as $name => $value) {
28
-
$normalized[strtolower($name)] = $value;
29
+
$name = $this->normalizeMailHeaderName($name);
30
+
if ($name == 'message-id') {
31
+
$this->setMessageIDHash(PhabricatorHash::digestForIndex($value));
32
+
}
33
+
$normalized[$name] = $value;
29
34
}
30
35
$this->headers = $normalized;
31
36
return $this;
32
37
}
33
38
39
+
public function getHeader($key, $default = null) {
40
+
$key = $this->normalizeMailHeaderName($key);
41
+
return idx($this->headers, $key, $default);
42
+
}
43
+
44
+
private function normalizeMailHeaderName($name) {
45
+
return strtolower($name);
46
+
}
47
+
34
48
public function getMessageID() {
35
-
return idx($this->headers, 'message-id');
49
+
return $this->getHeader('Message-ID');
36
50
}
37
51
38
52
public function getSubject() {
39
-
return idx($this->headers, 'subject');
53
+
return $this->getHeader('Subject');
40
54
}
41
55
42
56
public function getCCAddresses() {
···
156
170
157
171
public function processReceivedMail() {
158
172
159
-
// If Phabricator sent the mail, always drop it immediately. This prevents
160
-
// loops where, e.g., the public bug address is also a user email address
161
-
// and creating a bug sends them an email, which loops.
162
-
$is_phabricator_mail = idx(
163
-
$this->headers,
164
-
'x-phabricator-sent-this-message');
165
-
if ($is_phabricator_mail) {
166
-
$message = "Ignoring email with 'X-Phabricator-Sent-This-Message' ".
167
-
"header to avoid loops.";
168
-
return $this->setMessage($message)->save();
169
-
}
170
-
171
-
$message_id_hash = $this->getMessageIDHash();
172
-
if ($message_id_hash) {
173
-
$messages = $this->loadAllWhere(
174
-
'messageIDHash = %s',
175
-
$message_id_hash);
176
-
$messages_count = count($messages);
177
-
if ($messages_count > 1) {
178
-
$first_message = reset($messages);
179
-
if ($first_message->getID() != $this->getID()) {
180
-
$message = sprintf(
181
-
'Ignoring email with message id hash "%s" that has been seen %d '.
182
-
'times, including this message.',
183
-
$message_id_hash,
184
-
$messages_count);
185
-
return $this->setMessage($message)->save();
186
-
}
187
-
}
173
+
try {
174
+
$this->dropMailFromPhabricator();
175
+
$this->dropMailAlreadyReceived();
176
+
} catch (PhabricatorMetaMTAReceivedMailProcessingException $ex) {
177
+
$this
178
+
->setStatus($ex->getStatusCode())
179
+
->setMessage($ex->getMessage())
180
+
->save();
181
+
return $this;
188
182
}
189
183
190
184
list($to,
···
458
452
}
459
453
460
454
return $user;
455
+
}
456
+
457
+
/**
458
+
* If Phabricator sent the mail, always drop it immediately. This prevents
459
+
* loops where, e.g., the public bug address is also a user email address
460
+
* and creating a bug sends them an email, which loops.
461
+
*/
462
+
private function dropMailFromPhabricator() {
463
+
if (!$this->getHeader('x-phabricator-sent-this-message')) {
464
+
return;
465
+
}
466
+
467
+
throw new PhabricatorMetaMTAReceivedMailProcessingException(
468
+
MetaMTAReceivedMailStatus::STATUS_FROM_PHABRICATOR,
469
+
"Ignoring email with 'X-Phabricator-Sent-This-Message' header to avoid ".
470
+
"loops.");
471
+
}
472
+
473
+
/**
474
+
* If this mail has the same message ID as some other mail, and isn't the
475
+
* first mail we we received with that message ID, we drop it as a duplicate.
476
+
*/
477
+
private function dropMailAlreadyReceived() {
478
+
$message_id_hash = $this->getMessageIDHash();
479
+
if (!$message_id_hash) {
480
+
// No message ID hash, so we can't detect duplicates. This should only
481
+
// happen with very old messages.
482
+
return;
483
+
}
484
+
485
+
$messages = $this->loadAllWhere(
486
+
'messageIDHash = %s ORDER BY id ASC LIMIT 2',
487
+
$message_id_hash);
488
+
$messages_count = count($messages);
489
+
if ($messages_count <= 1) {
490
+
// If we only have one copy of this message, we're good to process it.
491
+
return;
492
+
}
493
+
494
+
$first_message = reset($messages);
495
+
if ($first_message->getID() == $this->getID()) {
496
+
// If this is the first copy of the message, it is okay to process it.
497
+
// We may not have been able to to process it immediately when we received
498
+
// it, and could may have received several copies without processing any
499
+
// yet.
500
+
return;
501
+
}
502
+
503
+
$message = sprintf(
504
+
'Ignoring email with message id hash "%s" that has been seen %d '.
505
+
'times, including this message.',
506
+
$message_id_hash,
507
+
$messages_count);
508
+
509
+
throw new PhabricatorMetaMTAReceivedMailProcessingException(
510
+
MetaMTAReceivedMailStatus::STATUS_DUPLICATE,
511
+
$message);
461
512
}
462
513
463
514
}
+52
src/applications/metamta/storage/__tests__/PhabricatorMetaMTAReceivedMailTestCase.php
+52
src/applications/metamta/storage/__tests__/PhabricatorMetaMTAReceivedMailTestCase.php
···
1
+
<?php
2
+
3
+
final class PhabricatorMetaMTAReceivedMailTestCase extends PhabricatorTestCase {
4
+
5
+
protected function getPhabricatorTestCaseConfiguration() {
6
+
return array(
7
+
self::PHABRICATOR_TESTCONFIG_BUILD_STORAGE_FIXTURES => true,
8
+
);
9
+
}
10
+
11
+
public function testDropSelfMail() {
12
+
$mail = new PhabricatorMetaMTAReceivedMail();
13
+
$mail->setHeaders(
14
+
array(
15
+
'X-Phabricator-Sent-This-Message' => 'yes',
16
+
));
17
+
$mail->save();
18
+
19
+
$mail->processReceivedMail();
20
+
21
+
$this->assertEqual(
22
+
MetaMTAReceivedMailStatus::STATUS_FROM_PHABRICATOR,
23
+
$mail->getStatus());
24
+
}
25
+
26
+
27
+
public function testDropDuplicateMail() {
28
+
$mail_a = new PhabricatorMetaMTAReceivedMail();
29
+
$mail_a->setHeaders(
30
+
array(
31
+
'Message-ID' => 'test@example.com',
32
+
));
33
+
$mail_a->save();
34
+
35
+
$mail_b = new PhabricatorMetaMTAReceivedMail();
36
+
$mail_b->setHeaders(
37
+
array(
38
+
'Message-ID' => 'test@example.com',
39
+
));
40
+
$mail_b->save();
41
+
42
+
$mail_a->processReceivedMail();
43
+
$mail_b->processReceivedMail();
44
+
45
+
$this->assertEqual(
46
+
MetaMTAReceivedMailStatus::STATUS_DUPLICATE,
47
+
$mail_b->getStatus());
48
+
}
49
+
50
+
51
+
52
+
}
+4
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php
+4
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php
···
1294
1294
'type' => 'php',
1295
1295
'name' => $this->getPatchPath('20130508.releephtransactionsmig.php'),
1296
1296
),
1297
+
'20130513.receviedmailstatus.sql' => array(
1298
+
'type' => 'sql',
1299
+
'name' => $this->getPatchPath('20130513.receviedmailstatus.sql'),
1300
+
),
1297
1301
);
1298
1302
}
1299
1303
}