@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add "persistence" types (data, cache, or index) to tables, and tweak what "storage dump" dumps
Summary:
Ref T13000. This marks each table as either "data" (normal data), "cache" (automatically rebuilt, no need to ever dump) or "index" (can be manually rebuilt).
By default, `bin/storage dump` dumps data and index tables, but not cache tables.
With `--no-indexes`, it dumps only data tables. Indexes can be rebuilt after a restore with `bin/search index --all ...`.
Test Plan:
- Ran `--no-indexes` and normal dumps with `--trace`, verified that cache and index (former case) or cache only (latter case) tables were dumped with `--no-data`.
- Verified dump has the same number of `CREATE TABLE` statements as before the changes.
- Reviewed persistence tags in the web UI (note Ferret engine tables are "Index"):
{F5210886}
Reviewers: amckinley
Reviewed By: amckinley
Maniphest Tasks: T13000
Differential Revision: https://secure.phabricator.com/D18682
+138
-9
+3
src/applications/cache/storage/PhabricatorCacheSchemaSpec.php
+3
src/applications/cache/storage/PhabricatorCacheSchemaSpec.php
+3
src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php
+3
src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php
···
261
261
$this->renderAttr(
262
262
$table->getCollation(),
263
263
$table->hasIssue($collation_issue)),
264
+
$table->getPersistenceTypeDisplayName(),
264
265
);
265
266
}
266
267
···
270
271
null,
271
272
pht('Table'),
272
273
pht('Collation'),
274
+
pht('Persistence'),
273
275
))
274
276
->setColumnClasses(
275
277
array(
276
278
null,
277
279
'wide pri',
280
+
null,
278
281
null,
279
282
));
280
283
+2
src/applications/config/schema/PhabricatorConfigSchemaQuery.php
+2
src/applications/config/schema/PhabricatorConfigSchemaQuery.php
+26
-5
src/applications/config/schema/PhabricatorConfigSchemaSpec.php
+26
-5
src/applications/config/schema/PhabricatorConfigSchemaSpec.php
···
56
56
}
57
57
58
58
protected function buildFerretIndexSchema(PhabricatorFerretEngine $engine) {
59
+
$index_options = array(
60
+
'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_INDEX,
61
+
);
62
+
59
63
$this->buildRawSchema(
60
64
$engine->getApplicationName(),
61
65
$engine->getDocumentTableName(),
62
66
$engine->getDocumentSchemaColumns(),
63
-
$engine->getDocumentSchemaKeys());
67
+
$engine->getDocumentSchemaKeys(),
68
+
$index_options);
64
69
65
70
$this->buildRawSchema(
66
71
$engine->getApplicationName(),
67
72
$engine->getFieldTableName(),
68
73
$engine->getFieldSchemaColumns(),
69
-
$engine->getFieldSchemaKeys());
74
+
$engine->getFieldSchemaKeys(),
75
+
$index_options);
70
76
71
77
$this->buildRawSchema(
72
78
$engine->getApplicationName(),
73
79
$engine->getNgramsTableName(),
74
80
$engine->getNgramsSchemaColumns(),
75
-
$engine->getNgramsSchemaKeys());
81
+
$engine->getNgramsSchemaKeys(),
82
+
$index_options);
76
83
77
84
$this->buildRawSchema(
78
85
$engine->getApplicationName(),
79
86
$engine->getCommonNgramsTableName(),
80
87
$engine->getCommonNgramsSchemaColumns(),
81
-
$engine->getCommonNgramsSchemaKeys());
88
+
$engine->getCommonNgramsSchemaKeys(),
89
+
$index_options);
82
90
}
83
91
84
92
protected function buildRawSchema(
85
93
$database_name,
86
94
$table_name,
87
95
array $columns,
88
-
array $keys) {
96
+
array $keys,
97
+
array $options = array()) {
98
+
99
+
PhutilTypeSpec::checkMap(
100
+
$options,
101
+
array(
102
+
'persistence' => 'optional string',
103
+
));
104
+
89
105
$database = $this->getDatabase($database_name);
90
106
91
107
$table = $this->newTable($table_name);
···
142
158
$key->setIndexType(idx($key_spec, 'type', 'BTREE'));
143
159
144
160
$table->addKey($key);
161
+
}
162
+
163
+
$persistence_type = idx($options, 'persistence');
164
+
if ($persistence_type !== null) {
165
+
$table->setPersistenceType($persistence_type);
145
166
}
146
167
147
168
$database->addTable($table);
+26
src/applications/config/schema/PhabricatorConfigTableSchema.php
+26
src/applications/config/schema/PhabricatorConfigTableSchema.php
···
7
7
private $engine;
8
8
private $columns = array();
9
9
private $keys = array();
10
+
private $persistenceType = self::PERSISTENCE_DATA;
11
+
12
+
const PERSISTENCE_DATA = 'data';
13
+
const PERSISTENCE_CACHE = 'cache';
14
+
const PERSISTENCE_INDEX = 'index';
10
15
11
16
public function addColumn(PhabricatorConfigColumnSchema $column) {
12
17
$key = $column->getName();
···
43
48
44
49
public function getKey($key) {
45
50
return idx($this->getKeys(), $key);
51
+
}
52
+
53
+
public function setPersistenceType($persistence_type) {
54
+
$this->persistenceType = $persistence_type;
55
+
return $this;
56
+
}
57
+
58
+
public function getPersistenceType() {
59
+
return $this->persistenceType;
60
+
}
61
+
62
+
public function getPersistenceTypeDisplayName() {
63
+
$map = array(
64
+
self::PERSISTENCE_DATA => pht('Data'),
65
+
self::PERSISTENCE_CACHE => pht('Cache'),
66
+
self::PERSISTENCE_INDEX => pht('Index'),
67
+
);
68
+
69
+
$type = $this->getPersistenceType();
70
+
71
+
return idx($map, $type, $type);
46
72
}
47
73
48
74
protected function getSubschemata() {
+3
src/applications/differential/storage/DifferentialSchemaSpec.php
+3
src/applications/differential/storage/DifferentialSchemaSpec.php
+18
src/docs/user/configuration/configuring_backups.diviner
+18
src/docs/user/configuration/configuring_backups.diviner
···
145
145
should also restrict access to the backups.
146
146
147
147
148
+
Skipping Indexes
149
+
================
150
+
151
+
By default, `bin/storage dump` does not dump all of the data in the database:
152
+
it skips some caches which can be rebuilt automatically and do not need to be
153
+
backed up. Some of these caches are very large, so the size of the dump may
154
+
be significantly smaller than the size of the databases.
155
+
156
+
If you have a large amount of data, you can specify `--no-indexes` when taking
157
+
a database dump to skip additional tables which contain search indexes. This
158
+
will reduce the size (and increase the speed) of the backup. This is an
159
+
advanced option which most installs will not benefit from.
160
+
161
+
This index data can be rebuilt after a restore, but will not be rebuilt
162
+
automatically. If you choose to use this flag, you must manually rebuild
163
+
indexes after a restore (for details, see ((reindex))).
164
+
165
+
148
166
Next Steps
149
167
==========
150
168
+57
-4
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php
+57
-4
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php
···
31
31
'of a plaintext file.'),
32
32
),
33
33
array(
34
+
'name' => 'no-indexes',
35
+
'help' => pht(
36
+
'Do not dump data in rebuildable index tables. This means '.
37
+
'backups are smaller and faster, but you will need to manually '.
38
+
'rebuild indexes after performing a restore.'),
39
+
),
40
+
array(
34
41
'name' => 'overwrite',
35
42
'help' => pht(
36
43
'With __--output__, overwrite the output file if it already '.
···
49
56
50
57
$console = PhutilConsole::getConsole();
51
58
59
+
$with_indexes = !$args->getArg('no-indexes');
60
+
52
61
$applied = $api->getAppliedPatches();
53
62
if ($applied === null) {
54
63
$namespace = $api->getNamespace();
···
65
74
$ref = $api->getRef();
66
75
$ref_key = $ref->getRefKey();
67
76
68
-
$schemata_map = id(new PhabricatorConfigSchemaQuery())
77
+
$schemata_query = id(new PhabricatorConfigSchemaQuery())
69
78
->setAPIs(array($api))
70
-
->setRefs(array($ref))
71
-
->loadActualSchemata();
72
-
$schemata = $schemata_map[$ref_key];
79
+
->setRefs(array($ref));
80
+
81
+
$actual_map = $schemata_query->loadActualSchemata();
82
+
$expect_map = $schemata_query->loadExpectedSchemata();
83
+
84
+
$schemata = $actual_map[$ref_key];
85
+
$expect = $expect_map[$ref_key];
73
86
74
87
$targets = array();
75
88
foreach ($schemata->getDatabases() as $database_name => $database) {
89
+
$expect_database = $expect->getDatabase($database_name);
76
90
foreach ($database->getTables() as $table_name => $table) {
91
+
92
+
// NOTE: It's possible for us to find tables in these database which
93
+
// we don't expect to be there. For example, an older version of
94
+
// Phabricator may have had a table that was later dropped. We assume
95
+
// these are data tables and always dump them, erring on the side of
96
+
// caution.
97
+
98
+
$persistence = PhabricatorConfigTableSchema::PERSISTENCE_DATA;
99
+
if ($expect_database) {
100
+
$expect_table = $expect_database->getTable($table_name);
101
+
if ($expect_table) {
102
+
$persistence = $expect_table->getPersistenceType();
103
+
}
104
+
}
105
+
106
+
switch ($persistence) {
107
+
case PhabricatorConfigTableSchema::PERSISTENCE_CACHE:
108
+
// When dumping tables, leave the data in cache tables in the
109
+
// database. This will be automatically rebuild after the data
110
+
// is restored and does not need to be persisted in backups.
111
+
$with_data = false;
112
+
break;
113
+
case PhabricatorConfigTableSchema::PERSISTENCE_INDEX:
114
+
// When dumping tables, leave index data behind of the caller
115
+
// specified "--no-indexes". These tables can be rebuilt manually
116
+
// from other tables, but do not rebuild automatically.
117
+
$with_data = $with_indexes;
118
+
break;
119
+
case PhabricatorConfigTableSchema::PERSISTENCE_DATA:
120
+
default:
121
+
$with_data = true;
122
+
break;
123
+
}
124
+
77
125
$targets[] = array(
78
126
'database' => $database_name,
79
127
'table' => $table_name,
128
+
'data' => $with_data,
80
129
);
81
130
}
82
131
}
···
146
195
$commands = array();
147
196
foreach ($targets as $target) {
148
197
$target_argv = $argv;
198
+
199
+
if (!$target['data']) {
200
+
$target_argv[] = '--no-data';
201
+
}
149
202
150
203
if ($has_password) {
151
204
$commands[] = csprintf(