@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add a robots.txt file to disallow /diffusion/
Summary:
Fixes T4610. Open to suggestions, etc., if there's anything I'm missing.
Also:
- Moves these "system" endpoints into a real application.
- Makes `isUnlisted()` work a little more consistently.
Test Plan: Accessed `/robots.txt`, `/status/` and `/debug/`.
Reviewers: chad, btrahan
Reviewed By: btrahan
Subscribers: aran, epriestley
Maniphest Tasks: T4610
Differential Revision: https://secure.phabricator.com/D8532
+69
-19
+6
-2
src/__phutil_library_map__.php
+6
-2
src/__phutil_library_map__.php
···
1129
1129
'PhabricatorApplicationSlowvote' => 'applications/slowvote/application/PhabricatorApplicationSlowvote.php',
1130
1130
'PhabricatorApplicationStatusView' => 'applications/meta/view/PhabricatorApplicationStatusView.php',
1131
1131
'PhabricatorApplicationSubscriptions' => 'applications/subscriptions/application/PhabricatorApplicationSubscriptions.php',
1132
+
'PhabricatorApplicationSystem' => 'applications/system/application/PhabricatorApplicationSystem.php',
1132
1133
'PhabricatorApplicationTest' => 'applications/base/controller/__tests__/PhabricatorApplicationTest.php',
1133
1134
'PhabricatorApplicationTokens' => 'applications/tokens/application/PhabricatorApplicationTokens.php',
1134
1135
'PhabricatorApplicationTransaction' => 'applications/transactions/storage/PhabricatorApplicationTransaction.php',
···
1420
1421
'PhabricatorDashboardTransactionQuery' => 'applications/dashboard/query/PhabricatorDashboardTransactionQuery.php',
1421
1422
'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php',
1422
1423
'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php',
1423
-
'PhabricatorDebugController' => 'applications/system/PhabricatorDebugController.php',
1424
+
'PhabricatorDebugController' => 'applications/system/controller/PhabricatorDebugController.php',
1424
1425
'PhabricatorDefaultFileStorageEngineSelector' => 'applications/files/engineselector/PhabricatorDefaultFileStorageEngineSelector.php',
1425
1426
'PhabricatorDefaultSearchEngineSelector' => 'applications/search/selector/PhabricatorDefaultSearchEngineSelector.php',
1426
1427
'PhabricatorDeveloperConfigOptions' => 'applications/config/option/PhabricatorDeveloperConfigOptions.php',
···
1960
1961
'PhabricatorRepositoryURINormalizerTestCase' => 'applications/repository/data/__tests__/PhabricatorRepositoryURINormalizerTestCase.php',
1961
1962
'PhabricatorRepositoryURITestCase' => 'applications/repository/storage/__tests__/PhabricatorRepositoryURITestCase.php',
1962
1963
'PhabricatorRepositoryVCSPassword' => 'applications/repository/storage/PhabricatorRepositoryVCSPassword.php',
1964
+
'PhabricatorRobotsController' => 'applications/system/controller/PhabricatorRobotsController.php',
1963
1965
'PhabricatorS3FileStorageEngine' => 'applications/files/engine/PhabricatorS3FileStorageEngine.php',
1964
1966
'PhabricatorSQLPatchList' => 'infrastructure/storage/patch/PhabricatorSQLPatchList.php',
1965
1967
'PhabricatorSSHKeyGenerator' => 'infrastructure/util/PhabricatorSSHKeyGenerator.php',
···
2077
2079
'PhabricatorStandardCustomFieldText' => 'infrastructure/customfield/standard/PhabricatorStandardCustomFieldText.php',
2078
2080
'PhabricatorStandardCustomFieldUsers' => 'infrastructure/customfield/standard/PhabricatorStandardCustomFieldUsers.php',
2079
2081
'PhabricatorStandardPageView' => 'view/page/PhabricatorStandardPageView.php',
2080
-
'PhabricatorStatusController' => 'applications/system/PhabricatorStatusController.php',
2082
+
'PhabricatorStatusController' => 'applications/system/controller/PhabricatorStatusController.php',
2081
2083
'PhabricatorStorageFixtureScopeGuard' => 'infrastructure/testing/fixture/PhabricatorStorageFixtureScopeGuard.php',
2082
2084
'PhabricatorStorageManagementAPI' => 'infrastructure/storage/management/PhabricatorStorageManagementAPI.php',
2083
2085
'PhabricatorStorageManagementDatabasesWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDatabasesWorkflow.php',
···
3785
3787
'PhabricatorApplicationSlowvote' => 'PhabricatorApplication',
3786
3788
'PhabricatorApplicationStatusView' => 'AphrontView',
3787
3789
'PhabricatorApplicationSubscriptions' => 'PhabricatorApplication',
3790
+
'PhabricatorApplicationSystem' => 'PhabricatorApplication',
3788
3791
'PhabricatorApplicationTest' => 'PhabricatorApplication',
3789
3792
'PhabricatorApplicationTokens' => 'PhabricatorApplication',
3790
3793
'PhabricatorApplicationTransaction' =>
···
4759
4762
'PhabricatorRepositoryURINormalizerTestCase' => 'PhabricatorTestCase',
4760
4763
'PhabricatorRepositoryURITestCase' => 'PhabricatorTestCase',
4761
4764
'PhabricatorRepositoryVCSPassword' => 'PhabricatorRepositoryDAO',
4765
+
'PhabricatorRobotsController' => 'PhabricatorController',
4762
4766
'PhabricatorS3FileStorageEngine' => 'PhabricatorFileStorageEngine',
4763
4767
'PhabricatorSSHKeyGenerator' => 'Phobject',
4764
4768
'PhabricatorSSHLog' => 'Phobject',
-6
src/aphront/configuration/AphrontDefaultApplicationConfiguration.php
-6
src/aphront/configuration/AphrontDefaultApplicationConfiguration.php
···
23
23
'' => 'DarkConsoleController',
24
24
'data/(?P<key>[^/]+)/' => 'DarkConsoleDataController',
25
25
),
26
-
27
-
'/status/' => 'PhabricatorStatusController',
28
-
29
-
30
26
'/help/' => array(
31
27
'keyboardshortcut/' => 'PhabricatorHelpKeyboardShortcutController',
32
28
),
33
-
34
-
'/debug/' => 'PhabricatorDebugController',
35
29
);
36
30
}
37
31
+1
src/applications/home/controller/PhabricatorHomeController.php
+1
src/applications/home/controller/PhabricatorHomeController.php
+4
-11
src/applications/settings/panel/PhabricatorSettingsPanelHomePreferences.php
+4
-11
src/applications/settings/panel/PhabricatorSettingsPanelHomePreferences.php
···
59
59
60
60
$output = array();
61
61
62
-
$applications = PhabricatorApplication::getAllInstalledApplications();
63
-
64
-
$applications = mgroup($applications, 'getApplicationGroup');
62
+
$app_groups = mgroup($apps, 'getApplicationGroup');
63
+
$app_groups = array_select_keys($app_groups, array_keys($group_map));
65
64
66
-
$applications = array_select_keys(
67
-
$applications,
68
-
array_keys($group_map));
69
-
70
-
foreach ($applications as $group => $apps) {
65
+
foreach ($app_groups as $group => $apps) {
71
66
$group_name = $group_map[$group];
72
67
$rows = array();
73
68
74
69
foreach ($apps as $app) {
75
70
if (!$app->shouldAppearInLaunchView()) {
76
-
continue;
71
+
continue;
77
72
}
78
73
79
74
$default = $app->getDefaultTileDisplay($user);
80
75
if ($default == PhabricatorApplication::TILE_INVISIBLE) {
81
76
continue;
82
77
}
83
-
84
-
85
78
86
79
$default_name = PhabricatorApplication::getTileDisplayName($default);
87
80
src/applications/system/PhabricatorDebugController.php
src/applications/system/controller/PhabricatorDebugController.php
src/applications/system/PhabricatorDebugController.php
src/applications/system/controller/PhabricatorDebugController.php
src/applications/system/PhabricatorStatusController.php
src/applications/system/controller/PhabricatorStatusController.php
src/applications/system/PhabricatorStatusController.php
src/applications/system/controller/PhabricatorStatusController.php
+21
src/applications/system/application/PhabricatorApplicationSystem.php
+21
src/applications/system/application/PhabricatorApplicationSystem.php
···
1
+
<?php
2
+
3
+
final class PhabricatorApplicationSystem extends PhabricatorApplication {
4
+
5
+
public function canUninstall() {
6
+
return false;
7
+
}
8
+
9
+
public function isUnlisted() {
10
+
return true;
11
+
}
12
+
13
+
public function getRoutes() {
14
+
return array(
15
+
'/status/' => 'PhabricatorStatusController',
16
+
'/debug/' => 'PhabricatorDebugController',
17
+
'/robots.txt' => 'PhabricatorRobotsController',
18
+
);
19
+
}
20
+
21
+
}
+37
src/applications/system/controller/PhabricatorRobotsController.php
+37
src/applications/system/controller/PhabricatorRobotsController.php
···
1
+
<?php
2
+
3
+
final class PhabricatorRobotsController extends PhabricatorController {
4
+
5
+
public function shouldRequireLogin() {
6
+
return false;
7
+
}
8
+
9
+
public function processRequest() {
10
+
$out = array();
11
+
12
+
// Prevent indexing of '/diffusion/', since the content is not generally
13
+
// useful to index, web spiders get stuck scraping the history of every
14
+
// file, and much of the content is Ajaxed in anyway so spiders won't even
15
+
// see it. These pages are also relatively expensive to generate.
16
+
17
+
// Note that this still allows commits (at '/rPxxxxx') to be indexed.
18
+
// They're probably not hugely useful, but suffer fewer of the problems
19
+
// Diffusion suffers and are hard to omit with 'robots.txt'.
20
+
21
+
$out[] = 'User-Agent: *';
22
+
$out[] = 'Disallow: /diffusion/';
23
+
24
+
// Add a small crawl delay (number of seconds between requests) for spiders
25
+
// which respect it. The intent here is to prevent spiders from affecting
26
+
// performance for users. The possible cost is slower indexing, but that
27
+
// seems like a reasonable tradeoff, since most Phabricator installs are
28
+
// probably not hugely concerned about cutting-edge SEO.
29
+
$out[] = 'Crawl-delay: 1';
30
+
31
+
$content = implode("\n", $out)."\n";
32
+
33
+
return id(new AphrontPlainTextResponse())
34
+
->setContent($content)
35
+
->setCacheDurationInSeconds(phutil_units('2 hours in seconds'));
36
+
}
37
+
}