@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Begin generating meaningful expected schemata
Summary:
Ref T1191. This lays some groundwork for generating the expected schemata, so we can compare them to the actual schemata and produce a meaningful diff.
- In general, each application will subclass `PhabricatorConfigSchemaSpec` and provide a definition of the tables it expects.
- This class has helper methods to mostly-automatically build table definitions for Lisk and (in the future) edges.
- When building expected schema, we specify a "data type", like "epoch". This is the type of data the application stores in the column, from the application's point of view. The SchemaSpec converts this into the best avilable storage type: for example, "text" will translate to `utf8mb4` if it's availalbe, or `binary` if not. This gives us a layer of indirection to insulate us from craziness.
Test Plan: See screenshots.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: epriestley
Maniphest Tasks: T1191
Differential Revision: https://secure.phabricator.com/D10497
+378
-14
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
336
336
'DifferentialRevisionStatus' => 'applications/differential/constants/DifferentialRevisionStatus.php',
337
337
'DifferentialRevisionUpdateHistoryView' => 'applications/differential/view/DifferentialRevisionUpdateHistoryView.php',
338
338
'DifferentialRevisionViewController' => 'applications/differential/controller/DifferentialRevisionViewController.php',
339
+
'DifferentialSchemaSpec' => 'applications/differential/storage/DifferentialSchemaSpec.php',
339
340
'DifferentialSearchIndexer' => 'applications/differential/search/DifferentialSearchIndexer.php',
340
341
'DifferentialSetDiffPropertyConduitAPIMethod' => 'applications/differential/conduit/DifferentialSetDiffPropertyConduitAPIMethod.php',
341
342
'DifferentialStoredCustomField' => 'applications/differential/customfield/DifferentialStoredCustomField.php',
···
3138
3139
'DifferentialRevisionSearchEngine' => 'PhabricatorApplicationSearchEngine',
3139
3140
'DifferentialRevisionUpdateHistoryView' => 'AphrontView',
3140
3141
'DifferentialRevisionViewController' => 'DifferentialController',
3142
+
'DifferentialSchemaSpec' => 'PhabricatorConfigSchemaSpec',
3141
3143
'DifferentialSearchIndexer' => 'PhabricatorSearchDocumentIndexer',
3142
3144
'DifferentialSetDiffPropertyConduitAPIMethod' => 'DifferentialConduitAPIMethod',
3143
3145
'DifferentialStoredCustomField' => 'DifferentialCustomField',
+143
-7
src/applications/config/controller/PhabricatorConfigDatabaseController.php
+143
-7
src/applications/config/controller/PhabricatorConfigDatabaseController.php
···
77
77
$crumbs->addTextCrumb(
78
78
$this->database,
79
79
$this->getApplicationURI('database/'.$this->database.'/'));
80
-
$crumbs->addTextCrumb($this->table);
80
+
if ($this->column) {
81
+
$crumbs->addTextCrumb(
82
+
$this->table,
83
+
$this->getApplicationURI(
84
+
'database/'.$this->database.'/'.$this->table.'/'));
85
+
$crumbs->addTextCrumb($this->column);
86
+
} else {
87
+
$crumbs->addTextCrumb($this->table);
88
+
}
81
89
} else {
82
90
$crumbs->addTextCrumb($this->database);
83
91
}
···
169
177
PhabricatorConfigServerSchema $actual,
170
178
$database_name) {
171
179
180
+
$collation_issue = PhabricatorConfigStorageSchema::ISSUE_COLLATION;
181
+
172
182
$database = $comp->getDatabase($database_name);
173
183
if (!$database) {
174
184
return new Aphront404Response();
···
176
186
177
187
$rows = array();
178
188
foreach ($database->getTables() as $table_name => $table) {
179
-
180
189
$status = $table->getStatus();
181
-
$issues = $table->getIssues();
182
190
183
191
$rows[] = array(
184
192
$this->renderIcon($status),
···
189
197
'/database/'.$database_name.'/'.$table_name.'/'),
190
198
),
191
199
$table_name),
192
-
$table->getCollation(),
200
+
$this->renderAttr(
201
+
$table->getCollation(),
202
+
$table->hasIssue($collation_issue)),
193
203
);
194
204
}
195
205
···
263
273
$database_name,
264
274
$table_name) {
265
275
276
+
$type_issue = PhabricatorConfigStorageSchema::ISSUE_COLUMNTYPE;
277
+
$charset_issue = PhabricatorConfigStorageSchema::ISSUE_CHARSET;
278
+
$collation_issue = PhabricatorConfigStorageSchema::ISSUE_COLLATION;
279
+
266
280
$database = $comp->getDatabase($database_name);
267
281
if (!$database) {
268
282
return new Aphront404Response();
···
273
287
return new Aphront404Response();
274
288
}
275
289
290
+
$actual_database = $actual->getDatabase($database_name);
291
+
$actual_table = null;
292
+
if ($actual_database) {
293
+
$actual_table = $actual_database->getTable($table_name);
294
+
}
295
+
296
+
$expect_database = $expect->getDatabase($database_name);
297
+
$expect_table = null;
298
+
if ($expect_database) {
299
+
$expect_table = $expect_database->getTable($table_name);
300
+
}
301
+
276
302
$rows = array();
277
303
foreach ($table->getColumns() as $column_name => $column) {
304
+
$expect_column = null;
305
+
if ($expect_table) {
306
+
$expect_column = $expect_table->getColumn($column_name);
307
+
}
308
+
278
309
$status = $column->getStatus();
310
+
311
+
$data_type = null;
312
+
if ($expect_column) {
313
+
$data_type = $expect_column->getDataType();
314
+
}
279
315
280
316
$rows[] = array(
281
317
$this->renderIcon($status),
···
289
325
$column_name.'/'),
290
326
),
291
327
$column_name),
292
-
$column->getColumnType(),
293
-
$column->getCharacterSet(),
294
-
$column->getCollation(),
328
+
$data_type,
329
+
$this->renderAttr(
330
+
$column->getColumnType(),
331
+
$column->hasIssue($type_issue)),
332
+
$this->renderAttr(
333
+
$column->getCharacterSet(),
334
+
$column->hasIssue($charset_issue)),
335
+
$this->renderAttr(
336
+
$column->getCollation(),
337
+
$column->hasIssue($collation_issue)),
295
338
);
296
339
}
297
340
···
300
343
array(
301
344
null,
302
345
pht('Table'),
346
+
pht('Data Type'),
303
347
pht('Column Type'),
304
348
pht('Character Set'),
305
349
pht('Collation'),
···
310
354
'wide pri',
311
355
null,
312
356
null,
357
+
null,
313
358
null
314
359
));
315
360
316
361
$title = pht('Database Status: %s.%s', $database_name, $table_name);
317
362
363
+
if ($actual_table) {
364
+
$actual_collation = $actual_table->getCollation();
365
+
} else {
366
+
$actual_collation = null;
367
+
}
368
+
369
+
if ($expect_table) {
370
+
$expect_collation = $expect_table->getCollation();
371
+
} else {
372
+
$expect_collation = null;
373
+
}
374
+
318
375
$properties = $this->buildProperties(
319
376
array(
377
+
array(
378
+
pht('Collation'),
379
+
$actual_collation,
380
+
),
381
+
array(
382
+
pht('Expected Collation'),
383
+
$expect_collation,
384
+
),
320
385
),
321
386
$table->getIssues());
322
387
···
351
416
return new Aphront404Response();
352
417
}
353
418
419
+
$actual_database = $actual->getDatabase($database_name);
420
+
$actual_table = null;
421
+
$actual_column = null;
422
+
if ($actual_database) {
423
+
$actual_table = $actual_database->getTable($table_name);
424
+
if ($actual_table) {
425
+
$actual_column = $actual_table->getColumn($column_name);
426
+
}
427
+
}
428
+
429
+
$expect_database = $expect->getDatabase($database_name);
430
+
$expect_table = null;
431
+
$expect_column = null;
432
+
if ($expect_database) {
433
+
$expect_table = $expect_database->getTable($table_name);
434
+
if ($expect_table) {
435
+
$expect_column = $expect_table->getColumn($column_name);
436
+
}
437
+
}
438
+
439
+
if ($actual_column) {
440
+
$actual_coltype = $actual_column->getColumnType();
441
+
$actual_charset = $actual_column->getCharacterSet();
442
+
$actual_collation = $actual_column->getCollation();
443
+
} else {
444
+
$actual_coltype = null;
445
+
$actual_charset = null;
446
+
$actual_collation = null;
447
+
}
448
+
449
+
if ($expect_column) {
450
+
$data_type = $expect_column->getDataType();
451
+
$expect_coltype = $expect_column->getColumnType();
452
+
$expect_charset = $expect_column->getCharacterSet();
453
+
$expect_collation = $expect_column->getCollation();
454
+
} else {
455
+
$data_type = null;
456
+
$expect_coltype = null;
457
+
$expect_charset = null;
458
+
$expect_collation = null;
459
+
}
460
+
461
+
354
462
$title = pht(
355
463
'Database Status: %s.%s.%s',
356
464
$database_name,
···
359
467
360
468
$properties = $this->buildProperties(
361
469
array(
470
+
array(
471
+
pht('Data Type'),
472
+
$data_type,
473
+
),
474
+
array(
475
+
pht('Column Type'),
476
+
$actual_coltype,
477
+
),
478
+
array(
479
+
pht('Expected Column Type'),
480
+
$expect_coltype,
481
+
),
482
+
array(
483
+
pht('Character Set'),
484
+
$actual_charset,
485
+
),
486
+
array(
487
+
pht('Expected Character Set'),
488
+
$expect_charset,
489
+
),
490
+
array(
491
+
pht('Collation'),
492
+
$actual_collation,
493
+
),
494
+
array(
495
+
pht('Expected Collation'),
496
+
$expect_collation,
497
+
),
362
498
),
363
499
$column->getIssues());
364
500
+10
src/applications/config/schema/PhabricatorConfigColumnSchema.php
+10
src/applications/config/schema/PhabricatorConfigColumnSchema.php
···
6
6
private $characterSet;
7
7
private $collation;
8
8
private $columnType;
9
+
private $dataType;
9
10
10
11
public function setColumnType($column_type) {
11
12
$this->columnType = $column_type;
···
18
19
19
20
protected function getSubschemata() {
20
21
return array();
22
+
}
23
+
24
+
public function setDataType($data_type) {
25
+
$this->dataType = $data_type;
26
+
return $this;
27
+
}
28
+
29
+
public function getDataType() {
30
+
return $this->dataType;
21
31
}
22
32
23
33
public function setCollation($collation) {
+7
-6
src/applications/config/schema/PhabricatorConfigSchemaQuery.php
+7
-6
src/applications/config/schema/PhabricatorConfigSchemaQuery.php
···
102
102
// collation. This is most correct, and will sort properly.
103
103
104
104
$utf8_charset = 'utf8mb4';
105
-
$utf8_collate = 'utf8mb4_unicode_ci';
105
+
$utf8_collation = 'utf8mb4_unicode_ci';
106
106
} else {
107
107
// If utf8mb4 is not available, we use binary. This allows us to store
108
108
// 4-byte unicode characters. This has some tradeoffs:
···
115
115
// to prevent this.
116
116
117
117
$utf8_charset = 'binary';
118
-
$utf8_collate = 'binary';
118
+
$utf8_collation = 'binary';
119
119
}
120
120
121
121
$specs = id(new PhutilSymbolLoader())
···
124
124
125
125
$server_schema = new PhabricatorConfigServerSchema();
126
126
foreach ($specs as $spec) {
127
-
$spec->setUTF8Charset($utf8_charset);
128
-
$spec->setUTF8Collate($utf8_collate);
129
-
130
-
$spec->buildSchemata($server_schema);
127
+
$spec
128
+
->setUTF8Collation($utf8_collation)
129
+
->setUTF8Charset($utf8_charset)
130
+
->setServer($server_schema)
131
+
->buildSchemata($server_schema);
131
132
}
132
133
133
134
return $server_schema;
+133
-1
src/applications/config/schema/PhabricatorConfigSchemaSpec.php
+133
-1
src/applications/config/schema/PhabricatorConfigSchemaSpec.php
···
2
2
3
3
abstract class PhabricatorConfigSchemaSpec extends Phobject {
4
4
5
-
abstract public function buildSchemata(PhabricatorConfigServerSchema $server);
5
+
private $server;
6
+
private $utf8Charset;
7
+
private $utf8Collation;
8
+
9
+
public function setUTF8Collation($utf8_collation) {
10
+
$this->utf8Collation = $utf8_collation;
11
+
return $this;
12
+
}
13
+
14
+
public function getUTF8Collation() {
15
+
return $this->utf8Collation;
16
+
}
17
+
18
+
public function setUTF8Charset($utf8_charset) {
19
+
$this->utf8Charset = $utf8_charset;
20
+
return $this;
21
+
}
22
+
23
+
public function getUTF8Charset() {
24
+
return $this->utf8Charset;
25
+
}
26
+
27
+
public function setServer(PhabricatorConfigServerSchema $server) {
28
+
$this->server = $server;
29
+
return $this;
30
+
}
31
+
32
+
public function getServer() {
33
+
return $this->server;
34
+
}
35
+
36
+
abstract public function buildSchemata();
37
+
38
+
protected function buildLiskSchemata($base) {
39
+
40
+
$objects = id(new PhutilSymbolLoader())
41
+
->setAncestorClass($base)
42
+
->loadObjects();
43
+
44
+
foreach ($objects as $object) {
45
+
$database = $this->getDatabase($object->getApplicationName());
46
+
47
+
$table = $this->newTable($object->getTableName());
48
+
49
+
$cols = $object->getSchemaColumns();
50
+
foreach ($cols as $name => $type) {
51
+
$details = $this->getDetailsForDataType($type);
52
+
list($column_type, $charset, $collation) = $details;
53
+
54
+
$column = $this->newColumn($name)
55
+
->setDataType($type)
56
+
->setColumnType($column_type)
57
+
->setCharacterSet($charset)
58
+
->setCollation($collation);
59
+
60
+
$table->addColumn($column);
61
+
}
62
+
63
+
$database->addTable($table);
64
+
}
65
+
}
66
+
67
+
protected function buildEdgeSchemata(PhabricatorLiskDAO $object) {}
68
+
69
+
protected function getDatabase($name) {
70
+
$server = $this->getServer();
71
+
72
+
$database = $server->getDatabase($this->getNamespacedDatabase($name));
73
+
if (!$database) {
74
+
$database = $this->newDatabase($name);
75
+
$server->addDatabase($database);
76
+
}
77
+
78
+
return $database;
79
+
}
80
+
81
+
protected function newDatabase($name) {
82
+
return id(new PhabricatorConfigDatabaseSchema())
83
+
->setName($this->getNamespacedDatabase($name))
84
+
->setCharacterSet($this->getUTF8Charset())
85
+
->setCollation($this->getUTF8Collation());
86
+
}
87
+
88
+
protected function getNamespacedDatabase($name) {
89
+
$namespace = PhabricatorLiskDAO::getStorageNamespace();
90
+
return $namespace.'_'.$name;
91
+
}
92
+
93
+
protected function newTable($name) {
94
+
return id(new PhabricatorConfigTableSchema())
95
+
->setName($name)
96
+
->setCollation($this->getUTF8Collation());
97
+
}
98
+
99
+
protected function newColumn($name) {
100
+
return id(new PhabricatorConfigColumnSchema())
101
+
->setName($name);
102
+
}
103
+
104
+
private function getDetailsForDataType($data_type) {
105
+
$column_type = null;
106
+
$charset = null;
107
+
$collation = null;
108
+
109
+
switch ($data_type) {
110
+
case 'id':
111
+
case 'epoch':
112
+
$column_type = 'int(10) unsigned';
113
+
break;
114
+
case 'phid':
115
+
$column_type = 'varchar(64)';
116
+
$charset = 'binary';
117
+
$collation = 'binary';
118
+
break;
119
+
case 'blob':
120
+
$column_type = 'longblob';
121
+
$charset = 'binary';
122
+
$collation = 'binary';
123
+
break;
124
+
case 'text':
125
+
$column_type = 'longtext';
126
+
$charset = $this->getUTF8Charset();
127
+
$collation = $this->getUTF8Collation();
128
+
break;
129
+
default:
130
+
$column_type = pht('<unknown>');
131
+
$charset = pht('<unknown>');
132
+
$collation = pht('<unknown>');
133
+
break;
134
+
}
135
+
136
+
return array($column_type, $charset, $collation);
137
+
}
6
138
7
139
}
+10
src/applications/differential/storage/DifferentialSchemaSpec.php
+10
src/applications/differential/storage/DifferentialSchemaSpec.php
+73
src/infrastructure/storage/lisk/LiskDAO.php
+73
src/infrastructure/storage/lisk/LiskDAO.php
···
169
169
const CONFIG_AUX_PHID = 'auxiliary-phid';
170
170
const CONFIG_SERIALIZATION = 'col-serialization';
171
171
const CONFIG_BINARY = 'binary';
172
+
const CONFIG_COLUMN_SCHEMA = 'col-schema';
172
173
173
174
const SERIALIZATION_NONE = 'id';
174
175
const SERIALIZATION_JSON = 'json';
···
342
343
* You can optionally provide a map of columns to a flag indicating that
343
344
* they store binary data. These columns will not raise an error when
344
345
* handling binary writes.
346
+
*
347
+
* CONFIG_COLUMN_SCHEMA
348
+
* Provide a map of columns to schema column types.
345
349
*
346
350
* @return dictionary Map of configuration options to values.
347
351
*
···
1710
1714
1711
1715
private function getBinaryColumns() {
1712
1716
return $this->getConfigOption(self::CONFIG_BINARY);
1717
+
}
1718
+
1719
+
1720
+
public function getSchemaColumns() {
1721
+
$custom_map = $this->getConfigOption(self::CONFIG_COLUMN_SCHEMA);
1722
+
if (!$custom_map) {
1723
+
$custom_map = array();
1724
+
}
1725
+
1726
+
$serialization = $this->getConfigOption(self::CONFIG_SERIALIZATION);
1727
+
if (!$serialization) {
1728
+
$serialization = array();
1729
+
}
1730
+
1731
+
$serialization_map = array(
1732
+
self::SERIALIZATION_JSON => 'text',
1733
+
self::SERIALIZATION_PHP => 'blob',
1734
+
);
1735
+
1736
+
$builtin = array(
1737
+
'id' => 'id',
1738
+
'phid' => 'phid',
1739
+
'dateCreated' => 'epoch',
1740
+
'dateModified' => 'epoch',
1741
+
);
1742
+
1743
+
$map = array();
1744
+
foreach ($this->getAllLiskProperties() as $property) {
1745
+
// First, use types specified explicitly in the table configuration.
1746
+
$type = idx($custom_map, $property);
1747
+
if ($type) {
1748
+
$map[$property] = $type;
1749
+
continue;
1750
+
}
1751
+
1752
+
// If we don't have an explicit type, try a builtin type for the
1753
+
// column.
1754
+
$type = idx($builtin, $property);
1755
+
if ($type) {
1756
+
$map[$property] = $type;
1757
+
continue;
1758
+
}
1759
+
1760
+
// If the column has serialization, we can infer the column type.
1761
+
if (isset($serialization[$property])) {
1762
+
$type = idx($serialization_map, $serialization[$property]);
1763
+
if ($type) {
1764
+
$map[$property] = $type;
1765
+
continue;
1766
+
}
1767
+
}
1768
+
1769
+
// If the column is named `somethingPHID`, infer it is a PHID.
1770
+
if (preg_match('/[a-z]PHID$/', $property)) {
1771
+
$map[$property] = 'phid';
1772
+
continue;
1773
+
}
1774
+
1775
+
// If the column is named `somethingID`, infer it is an ID.
1776
+
if (preg_match('/[a-z]ID$/', $property)) {
1777
+
$map[$property] = 'id';
1778
+
continue;
1779
+
}
1780
+
1781
+
// We don't know the type of this column.
1782
+
$map[$property] = null;
1783
+
}
1784
+
1785
+
return $map;
1713
1786
}
1714
1787
1715
1788
}