@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Improve performance of "phabricator:20210215.changeset.02.phid-populate.php"
Summary: Ref T13613. Improve the performance of this migration by using a temporary table and an "UPDATE x JOIN y ..." pattern.
Test Plan:
- Ran on `secure`, got exit after a few seconds since the migration is idempotent and changesets already had PHIDs.
- Ran on `secure` with the `continue;` commented out, got valid new PHIDs in 53s (from 153s).
- Tried a larger page size (16K), didn't see any improvement.
- From "--trace", client PHID generation seems to be the limiting factor.
Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam
Maniphest Tasks: T13613
Differential Revision: https://secure.phabricator.com/D21570
+53
-8
+53
-8
resources/sql/autopatches/20210215.changeset.02.phid-populate.php
+53
-8
resources/sql/autopatches/20210215.changeset.02.phid-populate.php
···
7
7
$conn = $changeset_table->establishConnection('w');
8
8
$table_name = $changeset_table->getTableName();
9
9
10
-
$iterator = new LiskRawMigrationIterator($conn, $table_name);
11
-
foreach ($iterator as $changeset_row) {
12
-
$phid = $changeset_row['phid'];
10
+
$chunk_size = 4096;
11
+
12
+
$temporary_table = 'tmp_20210215_changeset_id_map';
13
+
14
+
queryfx(
15
+
$conn,
16
+
'CREATE TEMPORARY TABLE %T (
17
+
changeset_id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
18
+
changeset_phid VARBINARY(64) NOT NULL)',
19
+
$temporary_table);
20
+
21
+
$table_iterator = id(new LiskRawMigrationIterator($conn, $table_name))
22
+
->setPageSize($chunk_size);
23
+
24
+
$chunk_iterator = new PhutilChunkedIterator($table_iterator, $chunk_size);
25
+
foreach ($chunk_iterator as $chunk) {
26
+
27
+
$map = array();
28
+
foreach ($chunk as $changeset_row) {
29
+
$phid = $changeset_row['phid'];
30
+
31
+
if (strlen($phid)) {
32
+
continue;
33
+
}
34
+
35
+
$phid = PhabricatorPHID::generateNewPHID($phid_type);
36
+
$id = $changeset_row['id'];
37
+
38
+
$map[(int)$id] = $phid;
39
+
}
13
40
14
-
if (strlen($phid)) {
41
+
if (!$map) {
15
42
continue;
16
43
}
17
44
18
-
$phid = PhabricatorPHID::generateNewPHID($phid_type);
45
+
$sql = array();
46
+
foreach ($map as $changeset_id => $changeset_phid) {
47
+
$sql[] = qsprintf(
48
+
$conn,
49
+
'(%d, %s)',
50
+
$changeset_id,
51
+
$changeset_phid);
52
+
}
19
53
20
54
queryfx(
21
55
$conn,
22
-
'UPDATE %T SET phid = %s WHERE id = %d',
56
+
'TRUNCATE TABLE %T',
57
+
$temporary_table);
58
+
59
+
queryfx(
60
+
$conn,
61
+
'INSERT INTO %T (changeset_id, changeset_phid) VALUES %LQ',
62
+
$temporary_table,
63
+
$sql);
64
+
65
+
queryfx(
66
+
$conn,
67
+
'UPDATE %T c JOIN %T x ON c.id = x.changeset_id
68
+
SET c.phid = x.changeset_phid',
23
69
$table_name,
24
-
$phid,
25
-
$changeset_row['id']);
70
+
$temporary_table);
26
71
}