@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Use binary collations for most text
Summary:
Ref T1191. For most text columns, we either don't care if "a" and "A" are the same, or we expect them to be different (for example: keys, domains, secrets, etc). Default text columns to the `_bin` collation so they are compared by strict character value. This is safer in cases where we aren't sure.
For some text columns, we allow the user to sort by the column in the UI (like Maniphest task titles) or we do care that "A" and "a" are the same (for example: project names). Introduce a new class of virtual data types, the "sort..." types, to cover these columns. These are like the "text..." types but use sorting collations which treat "A" and "a" the same.
Test Plan:
- Made an effort to identify all columns where the UI relies on database collation.
- Ran `bin/storage adjust` and cleared all warnings.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: beng, epriestley
Maniphest Tasks: T1191
Differential Revision: https://secure.phabricator.com/D10602
+73
-32
+6
-3
src/applications/config/schema/PhabricatorConfigSchemaQuery.php
+6
-3
src/applications/config/schema/PhabricatorConfigSchemaQuery.php
···
154
154
// collation. This is most correct, and will sort properly.
155
155
156
156
$utf8_charset = 'utf8mb4';
157
-
$utf8_collation = 'utf8mb4_unicode_ci';
157
+
$utf8_binary_collation = 'utf8mb4_bin';
158
+
$utf8_sorting_collation = 'utf8mb4_unicode_ci';
158
159
} else {
159
160
// If utf8mb4 is not available, we use binary. This allows us to store
160
161
// 4-byte unicode characters. This has some tradeoffs:
···
167
168
// to prevent this.
168
169
169
170
$utf8_charset = 'binary';
170
-
$utf8_collation = 'binary';
171
+
$utf8_binary_collation = 'binary';
172
+
$utf8_sorting_collation = 'binary';
171
173
}
172
174
173
175
$specs = id(new PhutilSymbolLoader())
···
177
179
$server_schema = new PhabricatorConfigServerSchema();
178
180
foreach ($specs as $spec) {
179
181
$spec
180
-
->setUTF8Collation($utf8_collation)
181
182
->setUTF8Charset($utf8_charset)
183
+
->setUTF8BinaryCollation($utf8_binary_collation)
184
+
->setUTF8SortingCollation($utf8_sorting_collation)
182
185
->setServer($server_schema)
183
186
->buildSchemata($server_schema);
184
187
}
+55
-20
src/applications/config/schema/PhabricatorConfigSchemaSpec.php
+55
-20
src/applications/config/schema/PhabricatorConfigSchemaSpec.php
···
4
4
5
5
private $server;
6
6
private $utf8Charset;
7
-
private $utf8Collation;
7
+
private $utf8BinaryCollation;
8
+
private $utf8SortingCollation;
8
9
9
-
public function setUTF8Collation($utf8_collation) {
10
-
$this->utf8Collation = $utf8_collation;
10
+
public function setUTF8SortingCollation($utf8_sorting_collation) {
11
+
$this->utf8SortingCollation = $utf8_sorting_collation;
11
12
return $this;
12
13
}
13
14
14
-
public function getUTF8Collation() {
15
-
return $this->utf8Collation;
15
+
public function getUTF8SortingCollation() {
16
+
return $this->utf8SortingCollation;
17
+
}
18
+
19
+
public function setUTF8BinaryCollation($utf8_binary_collation) {
20
+
$this->utf8BinaryCollation = $utf8_binary_collation;
21
+
return $this;
22
+
}
23
+
24
+
public function getUTF8BinaryCollation() {
25
+
return $this->utf8BinaryCollation;
16
26
}
17
27
18
28
public function setUTF8Charset($utf8_charset) {
···
195
205
return id(new PhabricatorConfigDatabaseSchema())
196
206
->setName($this->getNamespacedDatabase($name))
197
207
->setCharacterSet($this->getUTF8Charset())
198
-
->setCollation($this->getUTF8Collation());
208
+
->setCollation($this->getUTF8BinaryCollation());
199
209
}
200
210
201
211
protected function getNamespacedDatabase($name) {
···
206
216
protected function newTable($name) {
207
217
return id(new PhabricatorConfigTableSchema())
208
218
->setName($name)
209
-
->setCollation($this->getUTF8Collation());
219
+
->setCollation($this->getUTF8BinaryCollation());
210
220
}
211
221
212
222
protected function newColumn($name) {
···
276
286
case 'bytes':
277
287
$column_type = 'longblob';
278
288
break;
289
+
case 'sort255':
290
+
$column_type = 'varchar(255)';
291
+
$charset = $this->getUTF8Charset();
292
+
$collation = $this->getUTF8SortingCollation();
293
+
break;
294
+
case 'sort128':
295
+
$column_type = 'varchar(128)';
296
+
$charset = $this->getUTF8Charset();
297
+
$collation = $this->getUTF8SortingCollation();
298
+
break;
299
+
case 'sort64':
300
+
$column_type = 'varchar(64)';
301
+
$charset = $this->getUTF8Charset();
302
+
$collation = $this->getUTF8SortingCollation();
303
+
break;
304
+
case 'sort32':
305
+
$column_type = 'varchar(32)';
306
+
$charset = $this->getUTF8Charset();
307
+
$collation = $this->getUTF8SortingCollation();
308
+
break;
309
+
case 'sort':
310
+
$column_type = 'longtext';
311
+
$charset = $this->getUTF8Charset();
312
+
$collation = $this->getUTF8SortingCollation();
313
+
break;
279
314
case 'text255':
280
315
$column_type = 'varchar(255)';
281
316
$charset = $this->getUTF8Charset();
282
-
$collation = $this->getUTF8Collation();
317
+
$collation = $this->getUTF8BinaryCollation();
283
318
break;
284
319
case 'text160':
285
320
$column_type = 'varchar(160)';
286
321
$charset = $this->getUTF8Charset();
287
-
$collation = $this->getUTF8Collation();
322
+
$collation = $this->getUTF8BinaryCollation();
288
323
break;
289
324
case 'text128':
290
325
$column_type = 'varchar(128)';
291
326
$charset = $this->getUTF8Charset();
292
-
$collation = $this->getUTF8Collation();
327
+
$collation = $this->getUTF8BinaryCollation();
293
328
break;
294
329
case 'text80':
295
330
$column_type = 'varchar(80)';
296
331
$charset = $this->getUTF8Charset();
297
-
$collation = $this->getUTF8Collation();
332
+
$collation = $this->getUTF8BinaryCollation();
298
333
break;
299
334
case 'text64':
300
335
$column_type = 'varchar(64)';
301
336
$charset = $this->getUTF8Charset();
302
-
$collation = $this->getUTF8Collation();
337
+
$collation = $this->getUTF8BinaryCollation();
303
338
break;
304
339
case 'text40':
305
340
$column_type = 'varchar(40)';
306
341
$charset = $this->getUTF8Charset();
307
-
$collation = $this->getUTF8Collation();
342
+
$collation = $this->getUTF8BinaryCollation();
308
343
break;
309
344
case 'text32':
310
345
$column_type = 'varchar(32)';
311
346
$charset = $this->getUTF8Charset();
312
-
$collation = $this->getUTF8Collation();
347
+
$collation = $this->getUTF8BinaryCollation();
313
348
break;
314
349
case 'text20':
315
350
$column_type = 'varchar(20)';
316
351
$charset = $this->getUTF8Charset();
317
-
$collation = $this->getUTF8Collation();
352
+
$collation = $this->getUTF8BinaryCollation();
318
353
break;
319
354
case 'text16':
320
355
$column_type = 'varchar(16)';
321
356
$charset = $this->getUTF8Charset();
322
-
$collation = $this->getUTF8Collation();
357
+
$collation = $this->getUTF8BinaryCollation();
323
358
break;
324
359
case 'text12':
325
360
$column_type = 'varchar(12)';
326
361
$charset = $this->getUTF8Charset();
327
-
$collation = $this->getUTF8Collation();
362
+
$collation = $this->getUTF8BinaryCollation();
328
363
break;
329
364
case 'text8':
330
365
$column_type = 'varchar(8)';
331
366
$charset = $this->getUTF8Charset();
332
-
$collation = $this->getUTF8Collation();
367
+
$collation = $this->getUTF8BinaryCollation();
333
368
break;
334
369
case 'text4':
335
370
$column_type = 'varchar(4)';
336
371
$charset = $this->getUTF8Charset();
337
-
$collation = $this->getUTF8Collation();
372
+
$collation = $this->getUTF8BinaryCollation();
338
373
break;
339
374
case 'text':
340
375
$column_type = 'longtext';
341
376
$charset = $this->getUTF8Charset();
342
-
$collation = $this->getUTF8Collation();
377
+
$collation = $this->getUTF8BinaryCollation();
343
378
break;
344
379
case 'bool':
345
380
$column_type = 'tinyint(1)';
+1
-1
src/applications/maniphest/storage/ManiphestNameIndex.php
+1
-1
src/applications/maniphest/storage/ManiphestNameIndex.php
+4
-1
src/applications/maniphest/storage/ManiphestTask.php
+4
-1
src/applications/maniphest/storage/ManiphestTask.php
···
71
71
'ownerPHID' => 'phid?',
72
72
'status' => 'text12',
73
73
'priority' => 'uint32',
74
-
'title' => 'text',
74
+
'title' => 'sort',
75
75
'originalTitle' => 'text',
76
76
'description' => 'text',
77
77
'mailKey' => 'bytes20',
···
113
113
),
114
114
'key_dateModified' => array(
115
115
'columns' => array('dateModified'),
116
+
),
117
+
'key_title' => array(
118
+
'columns' => array('title(64)'),
116
119
),
117
120
),
118
121
) + parent::getConfiguration();
+1
-1
src/applications/phame/storage/PhamePost.php
+1
-1
src/applications/phame/storage/PhamePost.php
+1
-1
src/applications/phriction/storage/PhrictionContent.php
+1
-1
src/applications/phriction/storage/PhrictionContent.php
+1
-1
src/applications/phriction/storage/PhrictionDocument.php
+1
-1
src/applications/phriction/storage/PhrictionDocument.php
+1
-1
src/applications/project/storage/PhabricatorProject.php
+1
-1
src/applications/project/storage/PhabricatorProject.php
+2
-2
src/applications/repository/storage/PhabricatorRepository.php
+2
-2
src/applications/repository/storage/PhabricatorRepository.php
···
79
79
'details' => self::SERIALIZATION_JSON,
80
80
),
81
81
self::CONFIG_COLUMN_SCHEMA => array(
82
-
'name' => 'text255',
83
-
'callsign' => 'text32',
82
+
'name' => 'sort255',
83
+
'callsign' => 'sort32',
84
84
'versionControlSystem' => 'text32',
85
85
'uuid' => 'text64?',
86
86
'pushPolicy' => 'policy',
+1
-1
src/infrastructure/customfield/storage/PhabricatorCustomFieldStringIndexStorage.php
+1
-1
src/infrastructure/customfield/storage/PhabricatorCustomFieldStringIndexStorage.php