@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Improve routing of "/robots.txt", "/favicon.ico", and "/status/" on Short and Blog sites
Summary: Ref T13636. Add routing for "/robots.txt", "favicon.ico", and "/status/" on the ShortSite and BlogSite.
Test Plan: Visted all resources (and 404 pages) on Short and Blog sites, and Platform site.
Maniphest Tasks: T13636
Differential Revision: https://secure.phabricator.com/D21607
+81
-33
+7
-3
src/__phutil_library_map__.php
+7
-3
src/__phutil_library_map__.php
···
4689
4689
'PhabricatorRequestExceptionHandler' => 'aphront/handler/PhabricatorRequestExceptionHandler.php',
4690
4690
'PhabricatorResetPasswordUserLogType' => 'applications/people/userlog/PhabricatorResetPasswordUserLogType.php',
4691
4691
'PhabricatorResourceSite' => 'aphront/site/PhabricatorResourceSite.php',
4692
-
'PhabricatorRobotsController' => 'applications/system/controller/PhabricatorRobotsController.php',
4693
-
'PhabricatorRobotsPlatformController' => 'applications/system/controller/PhabricatorRobotsPlatformController.php',
4694
-
'PhabricatorRobotsResourceController' => 'applications/system/controller/PhabricatorRobotsResourceController.php',
4692
+
'PhabricatorRobotsBlogController' => 'applications/system/controller/robots/PhabricatorRobotsBlogController.php',
4693
+
'PhabricatorRobotsController' => 'applications/system/controller/robots/PhabricatorRobotsController.php',
4694
+
'PhabricatorRobotsPlatformController' => 'applications/system/controller/robots/PhabricatorRobotsPlatformController.php',
4695
+
'PhabricatorRobotsResourceController' => 'applications/system/controller/robots/PhabricatorRobotsResourceController.php',
4696
+
'PhabricatorRobotsShortController' => 'applications/system/controller/robots/PhabricatorRobotsShortController.php',
4695
4697
'PhabricatorS3FileStorageEngine' => 'applications/files/engine/PhabricatorS3FileStorageEngine.php',
4696
4698
'PhabricatorSMSAuthFactor' => 'applications/auth/factor/PhabricatorSMSAuthFactor.php',
4697
4699
'PhabricatorSQLPatchList' => 'infrastructure/storage/patch/PhabricatorSQLPatchList.php',
···
11474
11476
'PhabricatorRequestExceptionHandler' => 'AphrontRequestExceptionHandler',
11475
11477
'PhabricatorResetPasswordUserLogType' => 'PhabricatorUserLogType',
11476
11478
'PhabricatorResourceSite' => 'PhabricatorSite',
11479
+
'PhabricatorRobotsBlogController' => 'PhabricatorRobotsController',
11477
11480
'PhabricatorRobotsController' => 'PhabricatorController',
11478
11481
'PhabricatorRobotsPlatformController' => 'PhabricatorRobotsController',
11479
11482
'PhabricatorRobotsResourceController' => 'PhabricatorRobotsController',
11483
+
'PhabricatorRobotsShortController' => 'PhabricatorRobotsController',
11480
11484
'PhabricatorS3FileStorageEngine' => 'PhabricatorFileStorageEngine',
11481
11485
'PhabricatorSMSAuthFactor' => 'PhabricatorAuthFactor',
11482
11486
'PhabricatorSQLPatchList' => 'Phobject',
+5
-1
src/applications/phame/application/PhabricatorPhameApplication.php
+5
-1
src/applications/phame/application/PhabricatorPhameApplication.php
···
70
70
}
71
71
72
72
public function getBlogRoutes() {
73
-
return $this->getLiveRoutes();
73
+
return $this->getLiveRoutes() + array(
74
+
'/status/' => 'PhabricatorStatusController',
75
+
'/favicon.ico' => 'PhabricatorFaviconController',
76
+
'/robots.txt' => 'PhabricatorRobotsBlogController',
77
+
);
74
78
}
75
79
76
80
private function getLiveRoutes() {
+4
src/applications/phurl/application/PhabricatorPhurlApplication.php
+4
src/applications/phurl/application/PhabricatorPhurlApplication.php
···
55
55
56
56
public function getShortRoutes() {
57
57
return array(
58
+
'/status/' => 'PhabricatorStatusController',
59
+
'/favicon.ico' => 'PhabricatorFaviconController',
60
+
'/robots.txt' => 'PhabricatorRobotsShortController',
61
+
58
62
'/u/(?P<append>[^/]+)' => 'PhabricatorPhurlShortURLController',
59
63
'.*' => 'PhabricatorPhurlShortURLDefaultController',
60
64
);
-29
src/applications/system/controller/PhabricatorRobotsController.php
-29
src/applications/system/controller/PhabricatorRobotsController.php
···
1
-
<?php
2
-
3
-
abstract class PhabricatorRobotsController extends PhabricatorController {
4
-
5
-
public function shouldRequireLogin() {
6
-
return false;
7
-
}
8
-
9
-
final public function processRequest() {
10
-
$out = $this->newRobotsRules();
11
-
12
-
// Add a small crawl delay (number of seconds between requests) for spiders
13
-
// which respect it. The intent here is to prevent spiders from affecting
14
-
// performance for users. The possible cost is slower indexing, but that
15
-
// seems like a reasonable tradeoff, since most Phabricator installs are
16
-
// probably not hugely concerned about cutting-edge SEO.
17
-
$out[] = 'Crawl-delay: 1';
18
-
19
-
$content = implode("\n", $out)."\n";
20
-
21
-
return id(new AphrontPlainTextResponse())
22
-
->setContent($content)
23
-
->setCacheDurationInSeconds(phutil_units('2 hours in seconds'))
24
-
->setCanCDN(true);
25
-
}
26
-
27
-
abstract protected function newRobotsRules();
28
-
29
-
}
+7
src/applications/system/controller/PhabricatorRobotsPlatformController.php
src/applications/system/controller/robots/PhabricatorRobotsPlatformController.php
+7
src/applications/system/controller/PhabricatorRobotsPlatformController.php
src/applications/system/controller/robots/PhabricatorRobotsPlatformController.php
···
19
19
$out[] = 'Disallow: /diffusion/';
20
20
$out[] = 'Disallow: /source/';
21
21
22
+
// Add a small crawl delay (number of seconds between requests) for spiders
23
+
// which respect it. The intent here is to prevent spiders from affecting
24
+
// performance for users. The possible cost is slower indexing, but that
25
+
// seems like a reasonable tradeoff, since most Phabricator installs are
26
+
// probably not hugely concerned about cutting-edge SEO.
27
+
$out[] = 'Crawl-delay: 1';
28
+
22
29
return $out;
23
30
}
24
31
+1
src/applications/system/controller/PhabricatorRobotsResourceController.php
src/applications/system/controller/robots/PhabricatorRobotsResourceController.php
+1
src/applications/system/controller/PhabricatorRobotsResourceController.php
src/applications/system/controller/robots/PhabricatorRobotsResourceController.php
+17
src/applications/system/controller/robots/PhabricatorRobotsBlogController.php
+17
src/applications/system/controller/robots/PhabricatorRobotsBlogController.php
···
1
+
<?php
2
+
3
+
final class PhabricatorRobotsBlogController
4
+
extends PhabricatorRobotsController {
5
+
6
+
protected function newRobotsRules() {
7
+
$out = array();
8
+
9
+
// Allow everything on blog domains to be indexed.
10
+
11
+
$out[] = 'User-Agent: *';
12
+
$out[] = 'Crawl-delay: 1';
13
+
14
+
return $out;
15
+
}
16
+
17
+
}
+22
src/applications/system/controller/robots/PhabricatorRobotsController.php
+22
src/applications/system/controller/robots/PhabricatorRobotsController.php
···
1
+
<?php
2
+
3
+
abstract class PhabricatorRobotsController extends PhabricatorController {
4
+
5
+
public function shouldRequireLogin() {
6
+
return false;
7
+
}
8
+
9
+
final public function processRequest() {
10
+
$out = $this->newRobotsRules();
11
+
12
+
$content = implode("\n", $out)."\n";
13
+
14
+
return id(new AphrontPlainTextResponse())
15
+
->setContent($content)
16
+
->setCacheDurationInSeconds(phutil_units('2 hours in seconds'))
17
+
->setCanCDN(true);
18
+
}
19
+
20
+
abstract protected function newRobotsRules();
21
+
22
+
}
+18
src/applications/system/controller/robots/PhabricatorRobotsShortController.php
+18
src/applications/system/controller/robots/PhabricatorRobotsShortController.php
···
1
+
<?php
2
+
3
+
final class PhabricatorRobotsShortController
4
+
extends PhabricatorRobotsController {
5
+
6
+
protected function newRobotsRules() {
7
+
$out = array();
8
+
9
+
// See T13636. Prevent indexing of any content on short domains.
10
+
11
+
$out[] = 'User-Agent: *';
12
+
$out[] = 'Disallow: /';
13
+
$out[] = 'Crawl-delay: 1';
14
+
15
+
return $out;
16
+
}
17
+
18
+
}