@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add a "lock log" for debugging where locks are being held
Summary: Depends on D19173. Ref T13096. Adds an optional, disabled-by-default lock log to make it easier to figure out what is acquiring and holding locks.
Test Plan: Ran `bin/lock log --enable`, `--disable`, `--name`, etc. Saw sensible-looking output with log enabled and daemons restarted. Saw no additional output with log disabled and daemons restarted.
Maniphest Tasks: T13096
Differential Revision: https://secure.phabricator.com/D19174
+389
-6
+1
bin/lock
+1
bin/lock
···
1
+
../scripts/setup/manage_lock.php
+9
resources/sql/autopatches/20180305.lock.01.locklog.sql
+9
resources/sql/autopatches/20180305.lock.01.locklog.sql
···
1
+
CREATE TABLE {$NAMESPACE}_daemon.daemon_locklog (
2
+
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
3
+
lockName VARCHAR(64) NOT NULL COLLATE {$COLLATE_TEXT},
4
+
lockReleased INT UNSIGNED,
5
+
lockParameters LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT},
6
+
lockContext LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT},
7
+
dateCreated INT UNSIGNED NOT NULL,
8
+
dateModified INT UNSIGNED NOT NULL
9
+
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};
+21
scripts/setup/manage_lock.php
+21
scripts/setup/manage_lock.php
···
1
+
#!/usr/bin/env php
2
+
<?php
3
+
4
+
$root = dirname(dirname(dirname(__FILE__)));
5
+
require_once $root.'/scripts/init/init-script.php';
6
+
7
+
$args = new PhutilArgumentParser($argv);
8
+
$args->setTagline(pht('manage locks'));
9
+
$args->setSynopsis(<<<EOSYNOPSIS
10
+
**lock** __command__ [__options__]
11
+
Manage locks.
12
+
13
+
EOSYNOPSIS
14
+
);
15
+
$args->parseStandardArguments();
16
+
17
+
$workflows = id(new PhutilClassMapQuery())
18
+
->setAncestorClass('PhabricatorLockManagementWorkflow')
19
+
->execute();
20
+
$workflows[] = new PhutilHelpArgumentWorkflow();
21
+
$args->parseWorkflows($workflows);
+8
src/__phutil_library_map__.php
+8
src/__phutil_library_map__.php
···
2670
2670
'PhabricatorDaemonController' => 'applications/daemon/controller/PhabricatorDaemonController.php',
2671
2671
'PhabricatorDaemonDAO' => 'applications/daemon/storage/PhabricatorDaemonDAO.php',
2672
2672
'PhabricatorDaemonEventListener' => 'applications/daemon/event/PhabricatorDaemonEventListener.php',
2673
+
'PhabricatorDaemonLockLog' => 'applications/daemon/storage/PhabricatorDaemonLockLog.php',
2674
+
'PhabricatorDaemonLockLogGarbageCollector' => 'applications/daemon/garbagecollector/PhabricatorDaemonLockLogGarbageCollector.php',
2673
2675
'PhabricatorDaemonLog' => 'applications/daemon/storage/PhabricatorDaemonLog.php',
2674
2676
'PhabricatorDaemonLogEvent' => 'applications/daemon/storage/PhabricatorDaemonLogEvent.php',
2675
2677
'PhabricatorDaemonLogEventGarbageCollector' => 'applications/daemon/garbagecollector/PhabricatorDaemonLogEventGarbageCollector.php',
···
3204
3206
'PhabricatorLocalTimeTestCase' => 'view/__tests__/PhabricatorLocalTimeTestCase.php',
3205
3207
'PhabricatorLocaleScopeGuard' => 'infrastructure/internationalization/scope/PhabricatorLocaleScopeGuard.php',
3206
3208
'PhabricatorLocaleScopeGuardTestCase' => 'infrastructure/internationalization/scope/__tests__/PhabricatorLocaleScopeGuardTestCase.php',
3209
+
'PhabricatorLockLogManagementWorkflow' => 'applications/daemon/management/PhabricatorLockLogManagementWorkflow.php',
3210
+
'PhabricatorLockManagementWorkflow' => 'applications/daemon/management/PhabricatorLockManagementWorkflow.php',
3207
3211
'PhabricatorLogTriggerAction' => 'infrastructure/daemon/workers/action/PhabricatorLogTriggerAction.php',
3208
3212
'PhabricatorLogoutController' => 'applications/auth/controller/PhabricatorLogoutController.php',
3209
3213
'PhabricatorLunarPhasePolicyRule' => 'applications/policy/rule/PhabricatorLunarPhasePolicyRule.php',
···
8194
8198
'PhabricatorDaemonController' => 'PhabricatorController',
8195
8199
'PhabricatorDaemonDAO' => 'PhabricatorLiskDAO',
8196
8200
'PhabricatorDaemonEventListener' => 'PhabricatorEventListener',
8201
+
'PhabricatorDaemonLockLog' => 'PhabricatorDaemonDAO',
8202
+
'PhabricatorDaemonLockLogGarbageCollector' => 'PhabricatorGarbageCollector',
8197
8203
'PhabricatorDaemonLog' => array(
8198
8204
'PhabricatorDaemonDAO',
8199
8205
'PhabricatorPolicyInterface',
···
8794
8800
'PhabricatorLocalTimeTestCase' => 'PhabricatorTestCase',
8795
8801
'PhabricatorLocaleScopeGuard' => 'Phobject',
8796
8802
'PhabricatorLocaleScopeGuardTestCase' => 'PhabricatorTestCase',
8803
+
'PhabricatorLockLogManagementWorkflow' => 'PhabricatorLockManagementWorkflow',
8804
+
'PhabricatorLockManagementWorkflow' => 'PhabricatorManagementWorkflow',
8797
8805
'PhabricatorLogTriggerAction' => 'PhabricatorTriggerAction',
8798
8806
'PhabricatorLogoutController' => 'PhabricatorAuthController',
8799
8807
'PhabricatorLunarPhasePolicyRule' => 'PhabricatorPolicyRule',
+29
src/applications/daemon/garbagecollector/PhabricatorDaemonLockLogGarbageCollector.php
+29
src/applications/daemon/garbagecollector/PhabricatorDaemonLockLogGarbageCollector.php
···
1
+
<?php
2
+
3
+
final class PhabricatorDaemonLockLogGarbageCollector
4
+
extends PhabricatorGarbageCollector {
5
+
6
+
const COLLECTORCONST = 'daemon.lock-log';
7
+
8
+
public function getCollectorName() {
9
+
return pht('Lock Logs');
10
+
}
11
+
12
+
public function getDefaultRetentionPolicy() {
13
+
return 0;
14
+
}
15
+
16
+
protected function collectGarbage() {
17
+
$table = new PhabricatorDaemonLockLog();
18
+
$conn = $table->establishConnection('w');
19
+
20
+
queryfx(
21
+
$conn,
22
+
'DELETE FROM %T WHERE dateCreated < %d LIMIT 100',
23
+
$table->getTableName(),
24
+
$this->getGarbageEpoch());
25
+
26
+
return ($conn->getAffectedRows() == 100);
27
+
}
28
+
29
+
}
+222
src/applications/daemon/management/PhabricatorLockLogManagementWorkflow.php
+222
src/applications/daemon/management/PhabricatorLockLogManagementWorkflow.php
···
1
+
<?php
2
+
3
+
final class PhabricatorLockLogManagementWorkflow
4
+
extends PhabricatorLockManagementWorkflow {
5
+
6
+
protected function didConstruct() {
7
+
$this
8
+
->setName('log')
9
+
->setSynopsis(pht('Enable, disable, or show the lock log.'))
10
+
->setArguments(
11
+
array(
12
+
array(
13
+
'name' => 'enable',
14
+
'help' => pht('Enable the lock log.'),
15
+
),
16
+
array(
17
+
'name' => 'disable',
18
+
'help' => pht('Disable the lock log.'),
19
+
),
20
+
array(
21
+
'name' => 'name',
22
+
'param' => 'name',
23
+
'help' => pht('Review logs for a specific lock.'),
24
+
),
25
+
));
26
+
}
27
+
28
+
public function execute(PhutilArgumentParser $args) {
29
+
$is_enable = $args->getArg('enable');
30
+
$is_disable = $args->getArg('disable');
31
+
32
+
if ($is_enable && $is_disable) {
33
+
throw new PhutilArgumentUsageException(
34
+
pht(
35
+
'You can not both "--enable" and "--disable" the lock log.'));
36
+
}
37
+
38
+
$with_name = $args->getArg('name');
39
+
40
+
if ($is_enable || $is_disable) {
41
+
if (strlen($with_name)) {
42
+
throw new PhutilArgumentUsageException(
43
+
pht(
44
+
'You can not both "--enable" or "--disable" with search '.
45
+
'parameters like "--name".'));
46
+
}
47
+
48
+
$gc = new PhabricatorDaemonLockLogGarbageCollector();
49
+
$is_enabled = (bool)$gc->getRetentionPolicy();
50
+
51
+
$config_key = 'phd.garbage-collection';
52
+
$const = $gc->getCollectorConstant();
53
+
$value = PhabricatorEnv::getEnvConfig($config_key);
54
+
55
+
if ($is_disable) {
56
+
if (!$is_enabled) {
57
+
echo tsprintf(
58
+
"%s\n",
59
+
pht('Lock log is already disabled.'));
60
+
return 0;
61
+
}
62
+
echo tsprintf(
63
+
"%s\n",
64
+
pht('Disabling the lock log.'));
65
+
66
+
unset($value[$const]);
67
+
} else {
68
+
if ($is_enabled) {
69
+
echo tsprintf(
70
+
"%s\n",
71
+
pht('Lock log is already enabled.'));
72
+
return 0;
73
+
}
74
+
echo tsprintf(
75
+
"%s\n",
76
+
pht('Enabling the lock log.'));
77
+
78
+
$value[$const] = phutil_units('24 hours in seconds');
79
+
}
80
+
81
+
id(new PhabricatorConfigLocalSource())
82
+
->setKeys(
83
+
array(
84
+
$config_key => $value,
85
+
));
86
+
87
+
echo tsprintf(
88
+
"%s\n",
89
+
pht('Done.'));
90
+
91
+
echo tsprintf(
92
+
"%s\n",
93
+
pht('Restart daemons to apply changes.'));
94
+
95
+
return 0;
96
+
}
97
+
98
+
$table = new PhabricatorDaemonLockLog();
99
+
$conn = $table->establishConnection('r');
100
+
101
+
$parts = array();
102
+
if (strlen($with_name)) {
103
+
$parts[] = qsprintf(
104
+
$conn,
105
+
'lockName = %s',
106
+
$with_name);
107
+
}
108
+
109
+
if (!$parts) {
110
+
$constraint = '1 = 1';
111
+
} else {
112
+
$constraint = '('.implode(') AND (', $parts).')';
113
+
}
114
+
115
+
$logs = $table->loadAllWhere(
116
+
'%Q ORDER BY id DESC LIMIT 100',
117
+
$constraint);
118
+
$logs = array_reverse($logs);
119
+
120
+
if (!$logs) {
121
+
echo tsprintf(
122
+
"%s\n",
123
+
pht('No matching lock logs.'));
124
+
return 0;
125
+
}
126
+
127
+
$table = id(new PhutilConsoleTable())
128
+
->setBorders(true)
129
+
->addColumn(
130
+
'id',
131
+
array(
132
+
'title' => pht('Lock'),
133
+
))
134
+
->addColumn(
135
+
'name',
136
+
array(
137
+
'title' => pht('Name'),
138
+
))
139
+
->addColumn(
140
+
'acquired',
141
+
array(
142
+
'title' => pht('Acquired'),
143
+
))
144
+
->addColumn(
145
+
'released',
146
+
array(
147
+
'title' => pht('Released'),
148
+
))
149
+
->addColumn(
150
+
'held',
151
+
array(
152
+
'title' => pht('Held'),
153
+
))
154
+
->addColumn(
155
+
'parameters',
156
+
array(
157
+
'title' => pht('Parameters'),
158
+
))
159
+
->addColumn(
160
+
'context',
161
+
array(
162
+
'title' => pht('Context'),
163
+
));
164
+
165
+
$viewer = $this->getViewer();
166
+
167
+
foreach ($logs as $log) {
168
+
$created = $log->getDateCreated();
169
+
$released = $log->getLockReleased();
170
+
171
+
if ($released) {
172
+
$held = '+'.($released - $created);
173
+
} else {
174
+
$held = null;
175
+
}
176
+
177
+
$created = phabricator_datetime($created, $viewer);
178
+
$released = phabricator_datetime($released, $viewer);
179
+
180
+
$parameters = $log->getLockParameters();
181
+
$context = $log->getLockContext();
182
+
183
+
$table->addRow(
184
+
array(
185
+
'id' => $log->getID(),
186
+
'name' => $log->getLockName(),
187
+
'acquired' => $created,
188
+
'released' => $released,
189
+
'held' => $held,
190
+
'parameters' => $this->flattenParameters($parameters),
191
+
'context' => $this->flattenParameters($context),
192
+
));
193
+
}
194
+
195
+
$table->draw();
196
+
197
+
return 0;
198
+
}
199
+
200
+
private function flattenParameters(array $params, $keys = true) {
201
+
$flat = array();
202
+
foreach ($params as $key => $value) {
203
+
if (is_array($value)) {
204
+
$value = $this->flattenParameters($value, false);
205
+
}
206
+
if ($keys) {
207
+
$flat[] = "{$key}={$value}";
208
+
} else {
209
+
$flat[] = "{$value}";
210
+
}
211
+
}
212
+
213
+
if ($keys) {
214
+
$flat = implode(', ', $flat);
215
+
} else {
216
+
$flat = implode(' ', $flat);
217
+
}
218
+
219
+
return $flat;
220
+
}
221
+
222
+
}
+4
src/applications/daemon/management/PhabricatorLockManagementWorkflow.php
+4
src/applications/daemon/management/PhabricatorLockManagementWorkflow.php
+32
src/applications/daemon/storage/PhabricatorDaemonLockLog.php
+32
src/applications/daemon/storage/PhabricatorDaemonLockLog.php
···
1
+
<?php
2
+
3
+
final class PhabricatorDaemonLockLog
4
+
extends PhabricatorDaemonDAO {
5
+
6
+
protected $lockName;
7
+
protected $lockReleased;
8
+
protected $lockParameters = array();
9
+
protected $lockContext = array();
10
+
11
+
protected function getConfiguration() {
12
+
return array(
13
+
self::CONFIG_SERIALIZATION => array(
14
+
'lockParameters' => self::SERIALIZATION_JSON,
15
+
'lockContext' => self::SERIALIZATION_JSON,
16
+
),
17
+
self::CONFIG_COLUMN_SCHEMA => array(
18
+
'lockName' => 'text64',
19
+
'lockReleased' => 'epoch?',
20
+
),
21
+
self::CONFIG_KEY_SCHEMA => array(
22
+
'key_lock' => array(
23
+
'columns' => array('lockName'),
24
+
),
25
+
'key_created' => array(
26
+
'columns' => array('dateCreated'),
27
+
),
28
+
),
29
+
) + parent::getConfiguration();
30
+
}
31
+
32
+
}
+4
-2
src/infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php
+4
-2
src/infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php
···
100
100
101
101
// Hold a lock while performing collection to avoid racing other daemons
102
102
// running the same collectors.
103
-
$lock_name = 'gc:'.$this->getCollectorConstant();
104
-
$lock = PhabricatorGlobalLock::newLock($lock_name);
103
+
$params = array(
104
+
'collector' => $this->getCollectorConstant(),
105
+
);
106
+
$lock = PhabricatorGlobalLock::newLock('gc', $params);
105
107
106
108
try {
107
109
$lock->lock(5);
+8
-4
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
+8
-4
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
···
1163
1163
// Although we're holding this lock on different databases so it could
1164
1164
// have the same name on each as far as the database is concerned, the
1165
1165
// locks would be the same within this process.
1166
-
$ref_key = $api->getRef()->getRefKey();
1167
-
$ref_hash = PhabricatorHash::digestForIndex($ref_key);
1168
-
$lock_name = 'adjust('.$ref_hash.')';
1166
+
$parameters = array(
1167
+
'refKey' => $api->getRef()->getRefKey(),
1168
+
);
1169
1169
1170
-
return PhabricatorGlobalLock::newLock($lock_name)
1170
+
// We disable logging for this lock because we may not have created the
1171
+
// log table yet, or may need to adjust it.
1172
+
1173
+
return PhabricatorGlobalLock::newLock('adjust', $parameters)
1171
1174
->useSpecificConnection($api->getConn(null))
1175
+
->setDisableLogging(true)
1172
1176
->lock();
1173
1177
}
1174
1178
+51
src/infrastructure/util/PhabricatorGlobalLock.php
+51
src/infrastructure/util/PhabricatorGlobalLock.php
···
31
31
private $parameters;
32
32
private $conn;
33
33
private $isExternalConnection = false;
34
+
private $log;
35
+
private $disableLogging;
34
36
35
37
private static $pool = array();
36
38
···
95
97
return $this;
96
98
}
97
99
100
+
public function setDisableLogging($disable) {
101
+
$this->disableLogging = $disable;
102
+
return $this;
103
+
}
104
+
98
105
99
106
/* -( Implementation )----------------------------------------------------- */
100
107
···
143
150
$conn->rememberLock($lock_name);
144
151
145
152
$this->conn = $conn;
153
+
154
+
if ($this->shouldLogLock()) {
155
+
global $argv;
156
+
157
+
$lock_context = array(
158
+
'pid' => getmypid(),
159
+
'host' => php_uname('n'),
160
+
'argv' => $argv,
161
+
);
162
+
163
+
$log = id(new PhabricatorDaemonLockLog())
164
+
->setLockName($lock_name)
165
+
->setLockParameters($this->parameters)
166
+
->setLockContext($lock_context)
167
+
->save();
168
+
169
+
$this->log = $log;
170
+
}
146
171
}
147
172
148
173
protected function doUnlock() {
···
175
200
$conn->close();
176
201
self::$pool[] = $conn;
177
202
}
203
+
204
+
if ($this->log) {
205
+
$log = $this->log;
206
+
$this->log = null;
207
+
208
+
$conn = $log->establishConnection('w');
209
+
queryfx(
210
+
$conn,
211
+
'UPDATE %T SET lockReleased = UNIX_TIMESTAMP() WHERE id = %d',
212
+
$log->getTableName(),
213
+
$log->getID());
214
+
}
215
+
}
216
+
217
+
private function shouldLogLock() {
218
+
if ($this->disableLogging) {
219
+
return false;
220
+
}
221
+
222
+
$policy = id(new PhabricatorDaemonLockLogGarbageCollector())
223
+
->getRetentionPolicy();
224
+
if (!$policy) {
225
+
return false;
226
+
}
227
+
228
+
return true;
178
229
}
179
230
180
231
}