@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Allow "SMTP" and "Sendmail" mailers to have "Message-ID" behavior configured in "cluster.mailers"
Summary:
Fixes T13265. See that task for discussion. Briefly:
- For mailers that use other mailers (SMTP, Sendmail), optionally let administrators set `"message-id": false` to improve threading behavior if their local Postfix is ultimately sending through SES or some other mailer which will replace the "Message-ID" header.
Also:
- Postmark is currently marked as supporting "Message-ID", but it does not actually support "Message-ID" on `secure.phabricator.com` (mail arrives with a non-Phabricator message ID). I suspect this was just an oversight in building or refactoring the adapter; correct it.
- Remove the "encoding" parameter from "sendmail". It think this was just missed in the cleanup a couple months ago; it is no longer used or documented.
Test Plan: Added and ran unit tests. (These feel like overkill, but this is super hard to test on real code.) See T13265 for evidence that this overall approach improves behavior.
Reviewers: amckinley
Reviewed By: amckinley
Maniphest Tasks: T13265
Differential Revision: https://secure.phabricator.com/D20285
+196
-15
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
3464
3464
'PhabricatorMacroTransactionType' => 'applications/macro/xaction/PhabricatorMacroTransactionType.php',
3465
3465
'PhabricatorMacroViewController' => 'applications/macro/controller/PhabricatorMacroViewController.php',
3466
3466
'PhabricatorMailAdapter' => 'applications/metamta/adapter/PhabricatorMailAdapter.php',
3467
+
'PhabricatorMailAdapterTestCase' => 'applications/metamta/adapter/__tests__/PhabricatorMailAdapterTestCase.php',
3467
3468
'PhabricatorMailAmazonSESAdapter' => 'applications/metamta/adapter/PhabricatorMailAmazonSESAdapter.php',
3468
3469
'PhabricatorMailAmazonSNSAdapter' => 'applications/metamta/adapter/PhabricatorMailAmazonSNSAdapter.php',
3469
3470
'PhabricatorMailAttachment' => 'applications/metamta/message/PhabricatorMailAttachment.php',
···
9425
9426
'PhabricatorMacroTransactionType' => 'PhabricatorModularTransactionType',
9426
9427
'PhabricatorMacroViewController' => 'PhabricatorMacroController',
9427
9428
'PhabricatorMailAdapter' => 'Phobject',
9429
+
'PhabricatorMailAdapterTestCase' => 'PhabricatorTestCase',
9428
9430
'PhabricatorMailAmazonSESAdapter' => 'PhabricatorMailAdapter',
9429
9431
'PhabricatorMailAmazonSNSAdapter' => 'PhabricatorMailAdapter',
9430
9432
'PhabricatorMailAttachment' => 'Phobject',
+33
src/applications/metamta/adapter/PhabricatorMailAdapter.php
+33
src/applications/metamta/adapter/PhabricatorMailAdapter.php
···
137
137
138
138
abstract public function newDefaultOptions();
139
139
140
+
final protected function guessIfHostSupportsMessageID($config, $host) {
141
+
// See T13265. Mailers like "SMTP" and "sendmail" usually allow us to
142
+
// set the "Message-ID" header to a value we choose, but we may not be
143
+
// able to if the mailer is being used as API glue and the outbound
144
+
// pathway ends up routing to a service with an SMTP API that selects
145
+
// its own "Message-ID" header, like Amazon SES.
146
+
147
+
// If users configured a behavior explicitly, use that behavior.
148
+
if ($config !== null) {
149
+
return $config;
150
+
}
151
+
152
+
// If the server we're connecting to is part of a service that we know
153
+
// does not support "Message-ID", guess that we don't support "Message-ID".
154
+
if ($host !== null) {
155
+
$host_blocklist = array(
156
+
'/\.amazonaws\.com\z/',
157
+
'/\.postmarkapp\.com\z/',
158
+
'/\.sendgrid\.net\z/',
159
+
);
160
+
161
+
$host = phutil_utf8_strtolower($host);
162
+
foreach ($host_blocklist as $regexp) {
163
+
if (preg_match($regexp, $host)) {
164
+
return false;
165
+
}
166
+
}
167
+
}
168
+
169
+
return true;
170
+
}
171
+
172
+
140
173
}
-4
src/applications/metamta/adapter/PhabricatorMailAmazonSESAdapter.php
-4
src/applications/metamta/adapter/PhabricatorMailAmazonSESAdapter.php
-4
src/applications/metamta/adapter/PhabricatorMailPostmarkAdapter.php
-4
src/applications/metamta/adapter/PhabricatorMailPostmarkAdapter.php
+5
-1
src/applications/metamta/adapter/PhabricatorMailSMTPAdapter.php
+5
-1
src/applications/metamta/adapter/PhabricatorMailSMTPAdapter.php
···
12
12
}
13
13
14
14
public function supportsMessageIDHeader() {
15
-
return true;
15
+
return $this->guessIfHostSupportsMessageID(
16
+
$this->getOption('message-id'),
17
+
$this->getOption('host'));
16
18
}
17
19
18
20
protected function validateOptions(array $options) {
···
24
26
'user' => 'string|null',
25
27
'password' => 'string|null',
26
28
'protocol' => 'string|null',
29
+
'message-id' => 'bool|null',
27
30
));
28
31
}
29
32
···
34
37
'user' => null,
35
38
'password' => null,
36
39
'protocol' => null,
40
+
'message-id' => null,
37
41
);
38
42
}
39
43
+5
-4
src/applications/metamta/adapter/PhabricatorMailSendmailAdapter.php
+5
-4
src/applications/metamta/adapter/PhabricatorMailSendmailAdapter.php
···
5
5
6
6
const ADAPTERTYPE = 'sendmail';
7
7
8
-
9
8
public function getSupportedMessageTypes() {
10
9
return array(
11
10
PhabricatorMailEmailMessage::MESSAGETYPE,
···
13
12
}
14
13
15
14
public function supportsMessageIDHeader() {
16
-
return true;
15
+
return $this->guessIfHostSupportsMessageID(
16
+
$this->getOption('message-id'),
17
+
null);
17
18
}
18
19
19
20
protected function validateOptions(array $options) {
20
21
PhutilTypeSpec::checkMap(
21
22
$options,
22
23
array(
23
-
'encoding' => 'string',
24
+
'message-id' => 'bool|null',
24
25
));
25
26
}
26
27
27
28
public function newDefaultOptions() {
28
29
return array(
29
-
'encoding' => 'base64',
30
+
'message-id' => null,
30
31
);
31
32
}
32
33
+96
src/applications/metamta/adapter/__tests__/PhabricatorMailAdapterTestCase.php
+96
src/applications/metamta/adapter/__tests__/PhabricatorMailAdapterTestCase.php
···
1
+
<?php
2
+
3
+
final class PhabricatorMailAdapterTestCase
4
+
extends PhabricatorTestCase {
5
+
6
+
public function testSupportsMessageID() {
7
+
$cases = array(
8
+
array(
9
+
pht('Amazon SES'),
10
+
false,
11
+
new PhabricatorMailAmazonSESAdapter(),
12
+
array(
13
+
'access-key' => 'test',
14
+
'secret-key' => 'test',
15
+
'endpoint' => 'test',
16
+
),
17
+
),
18
+
19
+
array(
20
+
pht('Mailgun'),
21
+
true,
22
+
new PhabricatorMailMailgunAdapter(),
23
+
array(
24
+
'api-key' => 'test',
25
+
'domain' => 'test',
26
+
'api-hostname' => 'test',
27
+
),
28
+
),
29
+
30
+
array(
31
+
pht('Sendmail'),
32
+
true,
33
+
new PhabricatorMailSendmailAdapter(),
34
+
array(),
35
+
),
36
+
37
+
array(
38
+
pht('Sendmail (Explicit Config)'),
39
+
false,
40
+
new PhabricatorMailSendmailAdapter(),
41
+
array(
42
+
'message-id' => false,
43
+
),
44
+
),
45
+
46
+
array(
47
+
pht('SMTP (Local)'),
48
+
true,
49
+
new PhabricatorMailSMTPAdapter(),
50
+
array(),
51
+
),
52
+
53
+
array(
54
+
pht('SMTP (Local + Explicit)'),
55
+
false,
56
+
new PhabricatorMailSMTPAdapter(),
57
+
array(
58
+
'message-id' => false,
59
+
),
60
+
),
61
+
62
+
array(
63
+
pht('SMTP (AWS)'),
64
+
false,
65
+
new PhabricatorMailSMTPAdapter(),
66
+
array(
67
+
'host' => 'test.amazonaws.com',
68
+
),
69
+
),
70
+
71
+
array(
72
+
pht('SMTP (AWS + Explicit)'),
73
+
true,
74
+
new PhabricatorMailSMTPAdapter(),
75
+
array(
76
+
'host' => 'test.amazonaws.com',
77
+
'message-id' => true,
78
+
),
79
+
),
80
+
81
+
);
82
+
83
+
foreach ($cases as $case) {
84
+
list($label, $expect, $mailer, $options) = $case;
85
+
86
+
$defaults = $mailer->newDefaultOptions();
87
+
$mailer->setOptions($options + $defaults);
88
+
89
+
$actual = $mailer->supportsMessageIDHeader();
90
+
91
+
$this->assertEqual($expect, $actual, pht('Message-ID: %s', $label));
92
+
}
93
+
}
94
+
95
+
96
+
}
+55
-2
src/docs/user/configuration/configuring_outbound_email.diviner
+55
-2
src/docs/user/configuration/configuring_outbound_email.diviner
···
339
339
how to configure it, this option is straightforward. If you have no idea how to
340
340
do any of this, strongly consider using Postmark or Mailgun instead.
341
341
342
-
To use this mailer, set `type` to `sendmail`. There are no `options` to
343
-
configure.
342
+
To use this mailer, set `type` to `sendmail`, then configure these `options`:
344
343
344
+
- `message-id`: Optional bool. Set to `false` if Phabricator will not be
345
+
able to select a custom "Message-ID" header when sending mail via this
346
+
mailer. See "Message-ID Headers" below.
345
347
346
348
Mailer: SMTP
347
349
============
···
361
363
- `password`: Optional string. Password for authentication.
362
364
- `protocol`: Optional string. Set to `tls` or `ssl` if necessary. Use
363
365
`ssl` for Gmail.
366
+
- `message-id`: Optional bool. Set to `false` if Phabricator will not be
367
+
able to select a custom "Message-ID" header when sending mail via this
368
+
mailer. See "Message-ID Headers" below.
364
369
365
370
366
371
Disable Mail
···
444
449
If it still hasn't sent the mail, Phabricator will try servers which are not
445
450
in any priority group, in the configured order. In this example there is
446
451
only one such server, so it will try to send via Mailgun.
452
+
453
+
454
+
Message-ID Headers
455
+
==================
456
+
457
+
Email has a "Message-ID" header which is important for threading messages
458
+
correctly in mail clients. Normally, Phabricator is free to select its own
459
+
"Message-ID" header values for mail it sends.
460
+
461
+
However, some mailers (including Amazon SES) do not allow selection of custom
462
+
"Message-ID" values and will ignore or replace the "Message-ID" in mail that
463
+
is submitted through them.
464
+
465
+
When Phabricator adds other mail headers which affect threading, like
466
+
"In-Reply-To", it needs to know if its "Message-ID" headers will be respected
467
+
or not to select header values which will produce good threading behavior. If
468
+
we guess wrong and think we can set a "Message-ID" header when we can't, you
469
+
may get poor threading behavior in mail clients.
470
+
471
+
For most mailers (like Postmark, Mailgun, and Amazon SES), the correct setting
472
+
will be selected for you automatically, because the behavior of the mailer
473
+
is knowable ahead of time. For example, we know Amazon SES will never respect
474
+
our "Message-ID" headers.
475
+
476
+
However, if you're sending mail indirectly through a mailer like SMTP or
477
+
Sendmail, the mail might or might not be routing through some mail service
478
+
which will ignore or replace the "Message-ID" header.
479
+
480
+
For example, your local mailer might submit mail to Mailgun (so "Message-ID"
481
+
will work), or to Amazon SES (so "Message-ID" will not work), or to some other
482
+
mail service (which we may not know anything about). We can't make a reliable
483
+
guess about whether "Message-ID" will be respected or not based only on
484
+
the local mailer configuration.
485
+
486
+
By default, we check if the mailer has a hostname we recognize as belonging
487
+
to a service which does not allow us to set a "Message-ID" header. If we don't
488
+
recognize the hostname (which is very common, since these services are most
489
+
often configured against the localhost or some other local machine), we assume
490
+
we can set a "Message-ID" header.
491
+
492
+
If the outbound pathway does not actually allow selection of a "Message-ID"
493
+
header, you can set the `message-id` option on the mailer to `false` to tell
494
+
Phabricator that it should not assume it can select a value for this header.
495
+
496
+
For example, if you are sending mail via a local Postfix server which then
497
+
forwards the mail to Amazon SES (a service which does not allow selection of
498
+
a "Message-ID" header), your `smtp` configuration in Phabricator should
499
+
specify `"message-id": false`.
447
500
448
501
449
502
Next Steps