@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Mostly modernize Conduit logs
Summary:
- Add GC support to conduit logs.
- Add Query support to conduit logs.
- Record the actual user PHID.
- Show client name.
- Support querying by specific method, so I can link to this from a setup issue.
@wez, this migration may not be fast. It took about 8 seconds for me to migrate 800,000 rows in the `conduit_methodcalllog` table. This adds a GC which should keep the table at a more manageable size in the future.
You can safely delete all data older than 30 days from this table, although you should do it by `id` instead of `dateCreated` since there's no key on `dateCreated` until this patch.
Test Plan:
- Ran GC.
- Looked at log UI.
- Ran Conduit methods.
Reviewers: btrahan
Reviewed By: btrahan
CC: wez, aran
Differential Revision: https://secure.phabricator.com/D6332
+209
-43
+14
resources/sql/patches/20130701.conduitlog.sql
+14
resources/sql/patches/20130701.conduitlog.sql
···
1
+
ALTER TABLE {$NAMESPACE}_conduit.conduit_methodcalllog
2
+
ADD callerPHID VARCHAR(64);
3
+
4
+
ALTER TABLE {$NAMESPACE}_conduit.conduit_methodcalllog
5
+
ADD KEY `key_created` (dateCreated);
6
+
7
+
ALTER TABLE {$NAMESPACE}_conduit.conduit_methodcalllog
8
+
ADD KEY `key_method` (method);
9
+
10
+
ALTER TABLE {$NAMESPACE}_conduit.conduit_methodcalllog
11
+
ADD KEY `key_callermethod` (callerPHID, method);
12
+
13
+
ALTER TABLE {$NAMESPACE}_conduit.conduit_connectionlog
14
+
ADD KEY `key_created` (dateCreated);
+7
-1
src/__phutil_library_map__.php
+7
-1
src/__phutil_library_map__.php
···
920
920
'PhabricatorConduitDAO' => 'applications/conduit/storage/PhabricatorConduitDAO.php',
921
921
'PhabricatorConduitListController' => 'applications/conduit/controller/PhabricatorConduitListController.php',
922
922
'PhabricatorConduitLogController' => 'applications/conduit/controller/PhabricatorConduitLogController.php',
923
+
'PhabricatorConduitLogQuery' => 'applications/conduit/query/PhabricatorConduitLogQuery.php',
923
924
'PhabricatorConduitMethodCallLog' => 'applications/conduit/storage/PhabricatorConduitMethodCallLog.php',
924
925
'PhabricatorConduitMethodQuery' => 'applications/conduit/query/PhabricatorConduitMethodQuery.php',
925
926
'PhabricatorConduitSearchEngine' => 'applications/conduit/query/PhabricatorConduitSearchEngine.php',
···
2820
2821
1 => 'PhabricatorApplicationSearchResultsControllerInterface',
2821
2822
),
2822
2823
'PhabricatorConduitLogController' => 'PhabricatorConduitController',
2823
-
'PhabricatorConduitMethodCallLog' => 'PhabricatorConduitDAO',
2824
+
'PhabricatorConduitLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
2825
+
'PhabricatorConduitMethodCallLog' =>
2826
+
array(
2827
+
0 => 'PhabricatorConduitDAO',
2828
+
1 => 'PhabricatorPolicyInterface',
2829
+
),
2824
2830
'PhabricatorConduitMethodQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
2825
2831
'PhabricatorConduitSearchEngine' => 'PhabricatorApplicationSearchEngine',
2826
2832
'PhabricatorConduitTokenController' => 'PhabricatorConduitController',
+11
-11
src/applications/conduit/controller/PhabricatorConduitAPIController.php
+11
-11
src/applications/conduit/controller/PhabricatorConduitAPIController.php
···
124
124
$connection_id = idx($result, 'connectionID');
125
125
}
126
126
127
-
$log->setConnectionID($connection_id);
128
-
$log->setError((string)$error_code);
129
-
$log->setDuration(1000000 * ($time_end - $time_start));
127
+
$log
128
+
->setCallerPHID(
129
+
isset($conduit_user)
130
+
? $conduit_user->getPHID()
131
+
: null)
132
+
->setConnectionID($connection_id)
133
+
->setError((string)$error_code)
134
+
->setDuration(1000000 * ($time_end - $time_start));
130
135
131
-
// TODO: This is a hack, but the insert is comparatively expensive and
132
-
// we only really care about having these logs for real CLI clients, if
133
-
// even that.
134
-
if (empty($metadata['authToken'])) {
135
-
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
136
-
$log->save();
137
-
unset($unguarded);
138
-
}
136
+
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
137
+
$log->save();
138
+
unset($unguarded);
139
139
140
140
$response = id(new ConduitAPIResponse())
141
141
->setResult($result)
+57
-24
src/applications/conduit/controller/PhabricatorConduitLogController.php
+57
-24
src/applications/conduit/controller/PhabricatorConduitLogController.php
···
8
8
9
9
public function processRequest() {
10
10
$request = $this->getRequest();
11
+
$viewer = $request->getUser();
11
12
12
13
$conn_table = new PhabricatorConduitConnectionLog();
13
14
$call_table = new PhabricatorConduitMethodCallLog();
14
15
15
16
$conn_r = $call_table->establishConnection('r');
16
17
17
-
$pager = new AphrontPagerView();
18
-
$pager->setOffset($request->getInt('page'));
19
-
$calls = $call_table->loadAllWhere(
20
-
'1 = 1 ORDER BY id DESC LIMIT %d, %d',
21
-
$pager->getOffset(),
22
-
$pager->getPageSize() + 1);
23
-
$calls = $pager->sliceResults($calls);
24
-
$pager->setURI(new PhutilURI('/conduit/log/'), 'page');
25
-
$pager->setEnableKeyboardShortcuts(true);
18
+
$pager = new AphrontCursorPagerView();
19
+
$pager->readFromRequest($request);
20
+
$pager->setPageSize(500);
21
+
22
+
$query = id(new PhabricatorConduitLogQuery())
23
+
->setViewer($viewer);
24
+
25
+
$methods = $request->getStrList('methods');
26
+
if ($methods) {
27
+
$query->withMethods($methods);
28
+
}
26
29
27
-
$min = $pager->getOffset() + 1;
28
-
$max = ($min + count($calls) - 1);
30
+
$calls = $query->executeWithCursorPager($pager);
29
31
30
32
$conn_ids = array_filter(mpull($calls, 'getConnectionID'));
31
33
$conns = array();
···
36
38
}
37
39
38
40
$table = $this->renderCallTable($calls, $conns);
39
-
$panel = new AphrontPanelView();
40
-
$panel->setHeader('Conduit Method Calls ('.$min.'-'.$max.')');
41
-
$panel->appendChild($table);
42
-
$panel->appendChild($pager);
43
41
44
42
$crumbs = $this->buildApplicationCrumbs();
45
43
$crumbs->addCrumb(
···
49
47
return $this->buildApplicationPage(
50
48
array(
51
49
$crumbs,
52
-
$panel,
50
+
$table,
51
+
$pager,
53
52
),
54
53
array(
55
54
'title' => 'Conduit Logs',
55
+
'device' => true,
56
+
'dust' => true,
56
57
));
57
58
}
58
59
···
60
61
assert_instances_of($calls, 'PhabricatorConduitMethodCallLog');
61
62
assert_instances_of($conns, 'PhabricatorConduitConnectionLog');
62
63
63
-
$user = $this->getRequest()->getUser();
64
+
$viewer = $this->getRequest()->getUser();
65
+
66
+
$methods = id(new PhabricatorConduitMethodQuery())
67
+
->setViewer($viewer)
68
+
->execute();
69
+
$methods = mpull($methods, null, 'getAPIMethodName');
64
70
65
71
$rows = array();
66
72
foreach ($calls as $call) {
67
73
$conn = idx($conns, $call->getConnectionID());
68
-
if (!$conn) {
69
-
// If there's no connection, use an empty object.
70
-
$conn = new PhabricatorConduitConnectionLog();
74
+
if ($conn) {
75
+
$name = $conn->getUserName();
76
+
$client = ' (via '.$conn->getClient().')';
77
+
} else {
78
+
$name = null;
79
+
$client = null;
71
80
}
81
+
82
+
$method = idx($methods, $call->getMethod());
83
+
if ($method) {
84
+
switch ($method->getMethodStatus()) {
85
+
case ConduitAPIMethod::METHOD_STATUS_STABLE:
86
+
$status = null;
87
+
break;
88
+
case ConduitAPIMethod::METHOD_STATUS_UNSTABLE:
89
+
$status = pht('Unstable');
90
+
break;
91
+
case ConduitAPIMethod::METHOD_STATUS_DEPRECATED:
92
+
$status = pht('Deprecated');
93
+
break;
94
+
}
95
+
} else {
96
+
$status = pht('Unknown');
97
+
}
98
+
72
99
$rows[] = array(
73
100
$call->getConnectionID(),
74
-
$conn->getUserName(),
75
-
$call->getMethod(),
101
+
$name,
102
+
array($call->getMethod(), $client),
103
+
$status,
76
104
$call->getError(),
77
105
number_format($call->getDuration()).' us',
78
-
phabricator_datetime($call->getDateCreated(), $user),
106
+
phabricator_datetime($call->getDateCreated(), $viewer),
79
107
);
80
108
}
81
-
$table = new AphrontTableView($rows);
109
+
110
+
$table = id(new AphrontTableView($rows))
111
+
->setDeviceReadyTable(true);
112
+
82
113
$table->setHeaders(
83
114
array(
84
115
'Connection',
85
116
'User',
86
117
'Method',
118
+
'Status',
87
119
'Error',
88
120
'Duration',
89
121
'Date',
···
93
125
'',
94
126
'',
95
127
'wide',
128
+
'',
96
129
'',
97
130
'n',
98
131
'right',
+43
src/applications/conduit/query/PhabricatorConduitLogQuery.php
+43
src/applications/conduit/query/PhabricatorConduitLogQuery.php
···
1
+
<?php
2
+
3
+
final class PhabricatorConduitLogQuery
4
+
extends PhabricatorCursorPagedPolicyAwareQuery {
5
+
6
+
private $methods;
7
+
8
+
public function withMethods(array $methods) {
9
+
$this->methods = $methods;
10
+
return $this;
11
+
}
12
+
13
+
14
+
public function loadPage() {
15
+
$table = new PhabricatorConduitMethodCallLog();
16
+
$conn_r = $table->establishConnection('r');
17
+
18
+
$data = queryfx_all(
19
+
$conn_r,
20
+
'SELECT * FROM %T %Q %Q %Q',
21
+
$table->getTableName(),
22
+
$this->buildWhereClause($conn_r),
23
+
$this->buildOrderClause($conn_r),
24
+
$this->buildLimitClause($conn_r));
25
+
26
+
return $table->loadAllFromArray($data);;
27
+
}
28
+
29
+
private function buildWhereClause(AphrontDatabaseConnection $conn_r) {
30
+
$where = array();
31
+
32
+
if ($this->methods) {
33
+
$where[] = qsprintf(
34
+
$conn_r,
35
+
'method IN (%Ls)',
36
+
$this->methods);
37
+
}
38
+
39
+
$where[] = $this->buildPagingClause($conn_r);
40
+
return $this->formatWhereClause($where);
41
+
}
42
+
43
+
}
+7
-6
src/applications/conduit/ssh/ConduitSSHWorkflow.php
+7
-6
src/applications/conduit/ssh/ConduitSSHWorkflow.php
···
72
72
$time_end = microtime(true);
73
73
74
74
$connection_id = idx($metadata, 'connectionID');
75
-
$log = new PhabricatorConduitMethodCallLog();
76
-
$log->setConnectionID($connection_id);
77
-
$log->setMethod($method);
78
-
$log->setError((string)$error_code);
79
-
$log->setDuration(1000000 * ($time_end - $time_start));
80
-
$log->save();
75
+
$log = id(new PhabricatorConduitMethodCallLog())
76
+
->setCallerPHID($this->getUser()->getPHID())
77
+
->setConnectionID($connection_id)
78
+
->setMethod($method)
79
+
->setError((string)$error_code)
80
+
->setDuration(1000000 * ($time_end - $time_start))
81
+
->save();
81
82
}
82
83
}
+21
-1
src/applications/conduit/storage/PhabricatorConduitMethodCallLog.php
+21
-1
src/applications/conduit/storage/PhabricatorConduitMethodCallLog.php
···
3
3
/**
4
4
* @group conduit
5
5
*/
6
-
final class PhabricatorConduitMethodCallLog extends PhabricatorConduitDAO {
6
+
final class PhabricatorConduitMethodCallLog extends PhabricatorConduitDAO
7
+
implements PhabricatorPolicyInterface {
7
8
9
+
protected $callerPHID;
8
10
protected $connectionID;
9
11
protected $method;
10
12
protected $error;
11
13
protected $duration;
14
+
15
+
16
+
/* -( PhabricatorPolicyInterface )----------------------------------------- */
17
+
18
+
19
+
public function getCapabilities() {
20
+
return array(
21
+
PhabricatorPolicyCapability::CAN_VIEW,
22
+
);
23
+
}
24
+
25
+
public function getPolicy($capability) {
26
+
return PhabricatorPolicies::POLICY_USER;
27
+
}
28
+
29
+
public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
30
+
return false;
31
+
}
12
32
13
33
}
+3
src/applications/config/option/PhabricatorGarbageCollectorConfigOptions.php
+3
src/applications/config/option/PhabricatorGarbageCollectorConfigOptions.php
+42
src/infrastructure/daemon/PhabricatorGarbageCollectorDaemon.php
+42
src/infrastructure/daemon/PhabricatorGarbageCollectorDaemon.php
···
18
18
$n_cache_ttl = $this->collectGeneralCacheTTL();
19
19
$n_cache = $this->collectGeneralCaches();
20
20
$n_files = $this->collectExpiredFiles();
21
+
$n_clogs = $this->collectExpiredConduitLogs();
22
+
$n_ccons = $this->collectExpiredConduitConnections();
21
23
22
24
$collected = array(
23
25
'Herald Transcript' => $n_herald,
···
28
30
'General Cache TTL' => $n_cache_ttl,
29
31
'General Cache Entries' => $n_cache,
30
32
'Temporary Files' => $n_files,
33
+
'Conduit Logs' => $n_clogs,
34
+
'Conduit Connections' => $n_ccons,
31
35
);
32
36
$collected = array_filter($collected);
33
37
···
217
221
}
218
222
219
223
return count($files);
224
+
}
225
+
226
+
private function collectExpiredConduitLogs() {
227
+
$key = 'gcdaemon.ttl.conduit-logs';
228
+
$ttl = PhabricatorEnv::getEnvConfig($key);
229
+
if ($ttl <= 0) {
230
+
return 0;
231
+
}
232
+
233
+
$table = new PhabricatorConduitMethodCallLog();
234
+
$conn_w = $table->establishConnection('w');
235
+
queryfx(
236
+
$conn_w,
237
+
'DELETE FROM %T WHERE dateCreated < %d
238
+
ORDER BY dateCreated ASC LIMIT 100',
239
+
$table->getTableName(),
240
+
time() - $ttl);
241
+
242
+
return $conn_w->getAffectedRows();
243
+
}
244
+
245
+
private function collectExpiredConduitConnections() {
246
+
$key = 'gcdaemon.ttl.conduit-logs';
247
+
$ttl = PhabricatorEnv::getEnvConfig($key);
248
+
if ($ttl <= 0) {
249
+
return 0;
250
+
}
251
+
252
+
$table = new PhabricatorConduitConnectionLog();
253
+
$conn_w = $table->establishConnection('w');
254
+
queryfx(
255
+
$conn_w,
256
+
'DELETE FROM %T WHERE dateCreated < %d
257
+
ORDER BY dateCreated ASC LIMIT 100',
258
+
$table->getTableName(),
259
+
time() - $ttl);
260
+
261
+
return $conn_w->getAffectedRows();
220
262
}
221
263
222
264
}
+4
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php
+4
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php