@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Modified script to commit smaller batches of symbols to the database.
Summary:
Modified the import script so it will only try to load a configurable
number of symbols at a time to avoid exhausting memory for large project
imports.
I haven't written a line of PHP in more than a decade, so please forgive
any stylistic or technical errors.
Test Plan: Ran the script on symbol table generated from linux kernel.
Reviewers: epriestley, #blessed_reviewers
Reviewed By: epriestley
CC: Korvin, epriestley, aran
Maniphest Tasks: T4117
Differential Revision: https://secure.phabricator.com/D7596
authored by
William R. Otte
and committed by
epriestley
7d43e591
965c2e67
+71
-39
+71
-39
scripts/symbols/import_project_symbols.php
+71
-39
scripts/symbols/import_project_symbols.php
···
1
1
#!/usr/bin/env php
2
2
<?php
3
3
4
+
4
5
$root = dirname(dirname(dirname(__FILE__)));
5
6
require_once $root.'/scripts/__init_script__.php';
6
7
···
25
26
'continue instead of exiting.',
26
27
),
27
28
array(
29
+
'name' => 'max-transaction',
30
+
'param' => 'num-syms',
31
+
'default' => '100000',
32
+
'help' => 'Maximum number of symbols that should '.
33
+
'be part of a single transaction',
34
+
),
35
+
array(
28
36
'name' => 'more',
29
37
'wildcard' => true,
30
38
),
···
53
61
$input = trim($input);
54
62
$input = explode("\n", $input);
55
63
64
+
65
+
function commit_symbols ($syms, $project, $no_purge) {
66
+
echo "Looking up path IDs...\n";
67
+
$path_map =
68
+
PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths(
69
+
ipull($syms, 'path'));
70
+
71
+
$symbol = new PhabricatorRepositorySymbol();
72
+
$conn_w = $symbol->establishConnection('w');
73
+
74
+
echo "Preparing queries...\n";
75
+
$sql = array();
76
+
foreach ($syms as $dict) {
77
+
$sql[] = qsprintf(
78
+
$conn_w,
79
+
'(%d, %s, %s, %s, %s, %d, %d)',
80
+
$project->getID(),
81
+
$dict['ctxt'],
82
+
$dict['name'],
83
+
$dict['type'],
84
+
$dict['lang'],
85
+
$dict['line'],
86
+
$path_map[$dict['path']]);
87
+
}
88
+
89
+
if (!$no_purge) {
90
+
echo "Purging old syms...\n";
91
+
queryfx($conn_w,
92
+
'DELETE FROM %T WHERE arcanistProjectID = %d',
93
+
$symbol->getTableName(),
94
+
$project->getID());
95
+
}
96
+
97
+
echo "Loading ".number_format(count($sql))." syms...\n";
98
+
foreach (array_chunk($sql, 128) as $chunk) {
99
+
queryfx($conn_w,
100
+
'INSERT INTO %T
101
+
(arcanistProjectID, symbolContext, symbolName, symbolType,
102
+
symbolLanguage, lineNumber, pathID) VALUES %Q',
103
+
$symbol->getTableName(),
104
+
implode(', ', $chunk));
105
+
}
106
+
107
+
}
108
+
109
+
$no_purge = $args->getArg('no-purge');
56
110
$symbols = array();
57
111
foreach ($input as $key => $line) {
58
112
try {
···
129
183
throw $e;
130
184
}
131
185
}
132
-
}
133
186
134
-
echo "Looking up path IDs...\n";
135
-
$path_map = PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths(
136
-
ipull($symbols, 'path'));
137
-
138
-
$symbol = new PhabricatorRepositorySymbol();
139
-
$conn_w = $symbol->establishConnection('w');
140
-
141
-
echo "Preparing queries...\n";
142
-
$sql = array();
143
-
foreach ($symbols as $dict) {
144
-
$sql[] = qsprintf(
145
-
$conn_w,
146
-
'(%d, %s, %s, %s, %s, %d, %d)',
147
-
$project->getID(),
148
-
$dict['ctxt'],
149
-
$dict['name'],
150
-
$dict['type'],
151
-
$dict['lang'],
152
-
$dict['line'],
153
-
$path_map[$dict['path']]);
187
+
if (count ($symbols) >= $args->getArg('max-transaction')) {
188
+
try {
189
+
echo "Committing {$args->getArg('max-transaction')} symbols....\n";
190
+
commit_symbols($symbols, $project, $no_purge);
191
+
$no_purge = true;
192
+
unset($symbols);
193
+
$symbols = array();
194
+
} catch (Exception $e) {
195
+
if ($args->getArg('ignore-errors')) {
196
+
continue;
197
+
} else {
198
+
throw $e;
199
+
}
200
+
}
201
+
}
154
202
}
155
203
156
-
if (!$args->getArg('no-purge')) {
157
-
echo "Purging old symbols...\n";
158
-
queryfx(
159
-
$conn_w,
160
-
'DELETE FROM %T WHERE arcanistProjectID = %d',
161
-
$symbol->getTableName(),
162
-
$project->getID());
163
-
}
164
-
165
-
echo "Loading ".number_format(count($sql))." symbols...\n";
166
-
foreach (array_chunk($sql, 128) as $chunk) {
167
-
queryfx(
168
-
$conn_w,
169
-
'INSERT INTO %T
170
-
(arcanistProjectID, symbolContext, symbolName, symbolType,
171
-
symbolLanguage, lineNumber, pathID) VALUES %Q',
172
-
$symbol->getTableName(),
173
-
implode(', ', $chunk));
204
+
if (count($symbols)) {
205
+
commit_symbols($symbols, $project, $args->getArg('no-purge'));
174
206
}
175
207
176
208
echo "Done.\n";