@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Provide an IDS_COUNTER mechanism for ID assignment
Summary: See D3912 for discussion. InnoDB may reuse autoincrement IDs after restart; provide a way to avoid it.
Test Plan: Unit tests. Scheduled and executed tasks through `drydock lease --type host` and `phd debug taskmaster`.
Reviewers: vrana, btrahan
Reviewed By: vrana
CC: aran
Differential Revision: https://secure.phabricator.com/D3914
+161
-6
+42
resources/sql/patches/liskcounters.php
+42
resources/sql/patches/liskcounters.php
···
1
+
<?php
2
+
3
+
// Switch PhabricatorWorkerActiveTask from autoincrement IDs to counter IDs.
4
+
// Set the initial counter ID to be larger than any known task ID.
5
+
6
+
$active_table = new PhabricatorWorkerActiveTask();
7
+
$archive_table = new PhabricatorWorkerArchiveTask();
8
+
9
+
$conn_w = $active_table->establishConnection('w');
10
+
11
+
$active_auto = head(queryfx_one(
12
+
$conn_w,
13
+
'SELECT auto_increment FROM information_schema.tables
14
+
WHERE table_name = %s
15
+
AND table_schema = DATABASE()',
16
+
$active_table->getTableName()));
17
+
18
+
$active_max = head(queryfx_one(
19
+
$conn_w,
20
+
'SELECT MAX(id) FROM %T',
21
+
$active_table->getTableName()));
22
+
23
+
$archive_max = head(queryfx_one(
24
+
$conn_w,
25
+
'SELECT MAX(id) FROM %T',
26
+
$archive_table->getTableName()));
27
+
28
+
$initial_counter = max((int)$active_auto, (int)$active_max, (int)$archive_max);
29
+
30
+
queryfx(
31
+
$conn_w,
32
+
'INSERT IGNORE INTO %T (counterName, counterValue)
33
+
VALUES (%s, %d)',
34
+
LiskDAO::COUNTER_TABLE_NAME,
35
+
$active_table->getTableName(),
36
+
$initial_counter + 1);
37
+
38
+
// Drop AUTO_INCREMENT from the ID column.
39
+
queryfx(
40
+
$conn_w,
41
+
'ALTER TABLE %T CHANGE id id INT UNSIGNED NOT NULL',
42
+
$active_table->getTableName());
+9
resources/sql/patches/liskcounters.sql
+9
resources/sql/patches/liskcounters.sql
···
1
+
CREATE TABLE `{$NAMESPACE}_harbormaster`.`lisk_counter` (
2
+
counterName VARCHAR(64) COLLATE utf8_bin PRIMARY KEY,
3
+
counterValue BIGINT UNSIGNED NOT NULL
4
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
5
+
6
+
CREATE TABLE `{$NAMESPACE}_worker`.`lisk_counter` (
7
+
counterName VARCHAR(64) COLLATE utf8_bin PRIMARY KEY,
8
+
counterValue BIGINT UNSIGNED NOT NULL
9
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+1
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
+1
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
+66
-6
src/infrastructure/storage/lisk/LiskDAO.php
+66
-6
src/infrastructure/storage/lisk/LiskDAO.php
···
177
177
const SERIALIZATION_PHP = 'php';
178
178
179
179
const IDS_AUTOINCREMENT = 'ids-auto';
180
+
const IDS_COUNTER = 'ids-counter';
180
181
const IDS_PHID = 'ids-phid';
181
182
const IDS_MANUAL = 'ids-manual';
183
+
184
+
const COUNTER_TABLE_NAME = 'lisk_counter';
182
185
183
186
private $__dirtyFields = array();
184
187
private $__missingFields = array();
···
327
330
* Lisk objects need to have a unique identifying ID. The three mechanisms
328
331
* available for generating this ID are IDS_AUTOINCREMENT (default, assumes
329
332
* the ID column is an autoincrement primary key), IDS_PHID (to generate a
330
-
* unique PHID for each object) or IDS_MANUAL (you are taking full
331
-
* responsibility for ID management).
333
+
* unique PHID for each object), IDS_MANUAL (you are taking full
334
+
* responsibility for ID management), or IDS_COUNTER (see below).
335
+
*
336
+
* InnoDB does not persist the value of `auto_increment` across restarts,
337
+
* and instead initializes it to `MAX(id) + 1` during startup. This means it
338
+
* may reissue the same autoincrement ID more than once, if the row is deleted
339
+
* and then the database is restarted. To avoid this, you can set an object to
340
+
* use a counter table with IDS_COUNTER. This will generally behave like
341
+
* IDS_AUTOINCREMENT, except that the counter value will persist across
342
+
* restarts and inserts will be slightly slower. If a database stores any
343
+
* DAOs which use this mechanism, you must create a table there with this
344
+
* schema:
345
+
*
346
+
* CREATE TABLE lisk_counter (
347
+
* counterName VARCHAR(64) COLLATE utf8_bin PRIMARY KEY,
348
+
* counterValue BIGINT UNSIGNED NOT NULL
349
+
* ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
332
350
*
333
351
* CONFIG_TIMESTAMPS
334
352
* Lisk can automatically handle keeping track of a `dateCreated' and
···
364
382
* by Lisk (use readField and writeField instead), and you should not
365
383
* directly access or assign protected members of your class (use the getters
366
384
* and setters).
367
-
*
368
385
*
369
386
* @return dictionary Map of configuration options to values.
370
387
*
···
1181
1198
return $this;
1182
1199
}
1183
1200
1184
-
1185
1201
/**
1186
1202
* Internal implementation of INSERT and REPLACE.
1187
1203
*
···
1193
1209
$this->willSaveObject();
1194
1210
$data = $this->getPropertyValues();
1195
1211
1212
+
$conn = $this->establishConnection('w');
1213
+
1196
1214
$id_mechanism = $this->getConfigOption(self::CONFIG_IDS);
1197
1215
switch ($id_mechanism) {
1198
1216
case self::IDS_AUTOINCREMENT:
···
1204
1222
unset($data[$id_key]);
1205
1223
}
1206
1224
break;
1225
+
case self::IDS_COUNTER:
1226
+
// If we are using counter IDs, assign a new ID if we don't already have
1227
+
// one.
1228
+
$id_key = $this->getIDKeyForUse();
1229
+
if (empty($data[$id_key])) {
1230
+
$counter_name = $this->getTableName();
1231
+
$id = self::loadNextCounterID($conn, $counter_name);
1232
+
$this->setID($id);
1233
+
$data[$id_key] = $id;
1234
+
}
1235
+
break;
1207
1236
case self::IDS_PHID:
1208
1237
if (empty($data[$this->getIDKeyForUse()])) {
1209
1238
$phid = $this->generatePHID();
···
1218
1247
}
1219
1248
1220
1249
$this->willWriteData($data);
1221
-
1222
-
$conn = $this->establishConnection('w');
1223
1250
1224
1251
$columns = array_keys($data);
1225
1252
···
1759
1786
public function __set($name, $value) {
1760
1787
phlog('Wrote to undeclared property '.get_class($this).'::$'.$name.'.');
1761
1788
$this->$name = $value;
1789
+
}
1790
+
1791
+
/**
1792
+
* Increments a named counter and returns the next value.
1793
+
*
1794
+
* @param AphrontDatabaseConnection Database where the counter resides.
1795
+
* @param string Counter name to create or increment.
1796
+
* @return int Next counter value.
1797
+
*
1798
+
* @task util
1799
+
*/
1800
+
public static function loadNextCounterID(
1801
+
AphrontDatabaseConnection $conn_w,
1802
+
$counter_name) {
1803
+
1804
+
// NOTE: If an insert does not touch an autoincrement row or call
1805
+
// LAST_INSERT_ID(), MySQL normally does not change the value of
1806
+
// LAST_INSERT_ID(). This can cause a counter's value to leak to a
1807
+
// new counter if the second counter is created after the first one is
1808
+
// updated. To avoid this, we insert LAST_INSERT_ID(1), to ensure the
1809
+
// LAST_INSERT_ID() is always updated and always set correctly after the
1810
+
// query completes.
1811
+
1812
+
queryfx(
1813
+
$conn_w,
1814
+
'INSERT INTO %T (counterName, counterValue) VALUES
1815
+
(%s, LAST_INSERT_ID(1))
1816
+
ON DUPLICATE KEY UPDATE
1817
+
counterValue = LAST_INSERT_ID(counterValue + 1)',
1818
+
self::COUNTER_TABLE_NAME,
1819
+
$counter_name);
1820
+
1821
+
return $conn_w->getInsertID();
1762
1822
}
1763
1823
1764
1824
}
+35
src/infrastructure/storage/lisk/__tests__/LiskFixtureTestCase.php
+35
src/infrastructure/storage/lisk/__tests__/LiskFixtureTestCase.php
···
95
95
$this->assertEqual(true, (bool)$load->load((string)$id));
96
96
}
97
97
98
+
public function testCounters() {
99
+
$obj = new HarbormasterObject();
100
+
$conn_w = $obj->establishConnection('w');
101
+
102
+
// Test that the counter bascially behaves as expected.
103
+
$this->assertEqual(1, LiskDAO::loadNextCounterID($conn_w, 'a'));
104
+
$this->assertEqual(2, LiskDAO::loadNextCounterID($conn_w, 'a'));
105
+
$this->assertEqual(3, LiskDAO::loadNextCounterID($conn_w, 'a'));
106
+
107
+
// This first insert is primarily a test that the previous LAST_INSERT_ID()
108
+
// value does not bleed into the creation of a new counter.
109
+
$this->assertEqual(1, LiskDAO::loadNextCounterID($conn_w, 'b'));
110
+
$this->assertEqual(2, LiskDAO::loadNextCounterID($conn_w, 'b'));
111
+
112
+
// These inserts alternate database connections. Since unit tests are
113
+
// transactional by default, we need to break out of them or we'll deadlock
114
+
// since the transactions don't normally close until we exit the test.
115
+
LiskDAO::endIsolateAllLiskEffectsToTransactions();
116
+
try {
117
+
118
+
$conn_1 = $obj->establishConnection('w', $force_new = true);
119
+
$conn_2 = $obj->establishConnection('w', $force_new = true);
120
+
121
+
$this->assertEqual(1, LiskDAO::loadNextCounterID($conn_1, 'z'));
122
+
$this->assertEqual(2, LiskDAO::loadNextCounterID($conn_2, 'z'));
123
+
$this->assertEqual(3, LiskDAO::loadNextCounterID($conn_1, 'z'));
124
+
$this->assertEqual(4, LiskDAO::loadNextCounterID($conn_2, 'z'));
125
+
$this->assertEqual(5, LiskDAO::loadNextCounterID($conn_1, 'z'));
126
+
127
+
LiskDAO::beginIsolateAllLiskEffectsToTransactions();
128
+
} catch (Exception $ex) {
129
+
LiskDAO::beginIsolateAllLiskEffectsToTransactions();
130
+
throw $ex;
131
+
}
132
+
}
98
133
99
134
}
+8
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php
+8
src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php
···
1012
1012
'type' => 'sql',
1013
1013
'name' => $this->getPatchPath('drydockresourcetype.sql'),
1014
1014
),
1015
+
'liskcounters.sql' => array(
1016
+
'type' => 'sql',
1017
+
'name' => $this->getPatchPath('liskcounters.sql'),
1018
+
),
1019
+
'liskcounters.php' => array(
1020
+
'type' => 'php',
1021
+
'name' => $this->getPatchPath('liskcounters.php'),
1022
+
),
1015
1023
);
1016
1024
}
1017
1025