@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add an alternate "modern" hunk datastore
Summary:
Ref T4045. Ref T5179. Hunk storage has two major issues:
- It's utf8, but actual diffs are binary.
- It's huge and can't be compressed or archived.
This introduces a second datastore which solves these problems: by recording hunk encoding, supporting compression, and supporting alternate storage. There's no actual compression or storage support yet, but there's space in the table for them.
Since nothing actually uses hunk IDs, it's fine to have these tables exist at the same time and use the same IDs. We can migrate data between the tables gradually without requiring downtime or disrupting installs.
Test Plan:
- There are no writes to the new table yet.
- The only effect this has is making us issue one extra query when looking for hunks.
- Observed the query issue, but everything else continue working fine.
- Created a new diff.
- Ran unit tests.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: epriestley
Maniphest Tasks: T4045, T5179
Differential Revision: https://secure.phabricator.com/D9290
+139
-14
+18
resources/sql/autopatches/20140525.hunkmodern.sql
+18
resources/sql/autopatches/20140525.hunkmodern.sql
···
1
+
CREATE TABLE {$NAMESPACE}_differential.differential_hunk_modern (
2
+
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
3
+
changesetID INT UNSIGNED NOT NULL,
4
+
oldOffset INT UNSIGNED NOT NULL,
5
+
oldLen INT UNSIGNED NOT NULL,
6
+
newOffset INT UNSIGNED NOT NULL,
7
+
newLen INT UNSIGNED NOT NULL,
8
+
dataType CHAR(4) NOT NULL COLLATE latin1_bin,
9
+
dataEncoding VARCHAR(16) COLLATE latin1_bin,
10
+
dataFormat CHAR(4) NOT NULL COLLATE latin1_bin,
11
+
data LONGBLOB NOT NULL,
12
+
dateCreated INT UNSIGNED NOT NULL,
13
+
dateModified INT UNSIGNED NOT NULL,
14
+
15
+
KEY `key_changeset` (changesetID),
16
+
KEY `key_created` (dateCreated)
17
+
18
+
) ENGINE=InnoDB, COLLATE utf8_general_ci;
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
385
385
'DifferentialHostField' => 'applications/differential/customfield/DifferentialHostField.php',
386
386
'DifferentialHovercardEventListener' => 'applications/differential/event/DifferentialHovercardEventListener.php',
387
387
'DifferentialHunk' => 'applications/differential/storage/DifferentialHunk.php',
388
+
'DifferentialHunkLegacy' => 'applications/differential/storage/DifferentialHunkLegacy.php',
389
+
'DifferentialHunkModern' => 'applications/differential/storage/DifferentialHunkModern.php',
388
390
'DifferentialHunkParser' => 'applications/differential/parser/DifferentialHunkParser.php',
389
391
'DifferentialHunkParserTestCase' => 'applications/differential/parser/__tests__/DifferentialHunkParserTestCase.php',
390
392
'DifferentialHunkQuery' => 'applications/differential/query/DifferentialHunkQuery.php',
···
3069
3071
0 => 'DifferentialDAO',
3070
3072
1 => 'PhabricatorPolicyInterface',
3071
3073
),
3074
+
'DifferentialHunkLegacy' => 'DifferentialHunk',
3075
+
'DifferentialHunkModern' => 'DifferentialHunk',
3072
3076
'DifferentialHunkParserTestCase' => 'PhabricatorTestCase',
3073
3077
'DifferentialHunkQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
3074
3078
'DifferentialHunkTestCase' => 'ArcanistPhutilTestCase',
+2
-2
src/applications/differential/parser/__tests__/DifferentialChangesetParserTestCase.php
+2
-2
src/applications/differential/parser/__tests__/DifferentialChangesetParserTestCase.php
···
3
3
final class DifferentialChangesetParserTestCase extends PhabricatorTestCase {
4
4
5
5
public function testDiffChangesets() {
6
-
$hunk = new DifferentialHunk();
6
+
$hunk = new DifferentialHunkLegacy();
7
7
$hunk->setChanges("+a\n b\n-c");
8
8
$hunk->setNewOffset(1);
9
9
$hunk->setNewLen(2);
···
20
20
);
21
21
22
22
foreach ($tests as $changes => $expected) {
23
-
$hunk = new DifferentialHunk();
23
+
$hunk = new DifferentialHunkLegacy();
24
24
$hunk->setChanges($changes);
25
25
$hunk->setNewOffset(11);
26
26
$hunk->setNewLen(3);
+1
-1
src/applications/differential/parser/__tests__/DifferentialHunkParserTestCase.php
+1
-1
src/applications/differential/parser/__tests__/DifferentialHunkParserTestCase.php
+23
-3
src/applications/differential/query/DifferentialHunkQuery.php
+23
-3
src/applications/differential/query/DifferentialHunkQuery.php
···
31
31
}
32
32
33
33
public function loadPage() {
34
-
$table = new DifferentialHunk();
34
+
$all_results = array();
35
+
36
+
// Load modern hunks.
37
+
$table = new DifferentialHunkModern();
35
38
$conn_r = $table->establishConnection('r');
36
39
37
-
$data = queryfx_all(
40
+
$modern_data = queryfx_all(
38
41
$conn_r,
39
42
'SELECT * FROM %T %Q %Q %Q',
40
43
$table->getTableName(),
41
44
$this->buildWhereClause($conn_r),
42
45
$this->buildOrderClause($conn_r),
43
46
$this->buildLimitClause($conn_r));
47
+
$modern_results = $table->loadAllFromArray($modern_data);
48
+
44
49
45
-
return $table->loadAllFromArray($data);
50
+
// Now, load legacy hunks.
51
+
$table = new DifferentialHunkLegacy();
52
+
$conn_r = $table->establishConnection('r');
53
+
54
+
$legacy_data = queryfx_all(
55
+
$conn_r,
56
+
'SELECT * FROM %T %Q %Q %Q',
57
+
$table->getTableName(),
58
+
$this->buildWhereClause($conn_r),
59
+
$this->buildOrderClause($conn_r),
60
+
$this->buildLimitClause($conn_r));
61
+
$legacy_results = $table->loadAllFromArray($legacy_data);
62
+
63
+
// Strip all the IDs off since they're not unique and nothing should be
64
+
// using them.
65
+
return array_values(array_merge($legacy_results, $modern_results));
46
66
}
47
67
48
68
public function willFilterPage(array $hunks) {
+1
-1
src/applications/differential/storage/DifferentialChangeset.php
+1
-1
src/applications/differential/storage/DifferentialChangeset.php
+1
-1
src/applications/differential/storage/DifferentialDiff.php
+1
-1
src/applications/differential/storage/DifferentialDiff.php
···
132
132
$hunks = $change->getHunks();
133
133
if ($hunks) {
134
134
foreach ($hunks as $hunk) {
135
-
$dhunk = new DifferentialHunk();
135
+
$dhunk = new DifferentialHunkLegacy();
136
136
$dhunk->setOldOffset($hunk->getOldOffset());
137
137
$dhunk->setOldLen($hunk->getOldLength());
138
138
$dhunk->setNewOffset($hunk->getNewOffset());
+1
-2
src/applications/differential/storage/DifferentialHunk.php
+1
-2
src/applications/differential/storage/DifferentialHunk.php
···
1
1
<?php
2
2
3
-
final class DifferentialHunk extends DifferentialDAO
3
+
abstract class DifferentialHunk extends DifferentialDAO
4
4
implements PhabricatorPolicyInterface {
5
5
6
6
protected $changesetID;
7
-
protected $changes;
8
7
protected $oldOffset;
9
8
protected $oldLen;
10
9
protected $newOffset;
+11
src/applications/differential/storage/DifferentialHunkLegacy.php
+11
src/applications/differential/storage/DifferentialHunkLegacy.php
+73
src/applications/differential/storage/DifferentialHunkModern.php
+73
src/applications/differential/storage/DifferentialHunkModern.php
···
1
+
<?php
2
+
3
+
final class DifferentialHunkModern extends DifferentialHunk {
4
+
5
+
const DATATYPE_TEXT = 'text';
6
+
const DATATYPE_FILE = 'file';
7
+
8
+
const DATAFORMAT_RAW = 'byte';
9
+
const DATAFORMAT_DEFLATE = 'gzde';
10
+
11
+
protected $dataType;
12
+
protected $dataEncoding;
13
+
protected $dataFormat;
14
+
protected $data;
15
+
16
+
public function getTableName() {
17
+
return 'differential_hunk_modern';
18
+
}
19
+
20
+
public function getConfiguration() {
21
+
return array(
22
+
self::CONFIG_BINARY => array(
23
+
'data' => true,
24
+
),
25
+
) + parent::getConfiguration();
26
+
}
27
+
28
+
public function setChanges($text) {
29
+
$this->dataEncoding = $this->detectEncodingForStorage($text);
30
+
$this->dataType = self::DATATYPE_TEXT;
31
+
$this->dataFormat = self::DATAFORMAT_RAW;
32
+
$this->data = $text;
33
+
34
+
return $this;
35
+
}
36
+
37
+
public function getChanges() {
38
+
return $this->getUTF8StringFromStorage(
39
+
$this->getRawData(),
40
+
$this->getDataEncoding());
41
+
}
42
+
43
+
private function getRawData() {
44
+
$type = $this->getDataType();
45
+
$data = $this->getData();
46
+
47
+
switch ($type) {
48
+
case self::DATATYPE_TEXT:
49
+
// In this storage type, the changes are stored on the object.
50
+
$data = $data;
51
+
break;
52
+
case self::DATATYPE_FILE:
53
+
default:
54
+
throw new Exception(
55
+
pht('Hunk has unsupported data type "%s"!', $type));
56
+
}
57
+
58
+
$format = $this->getDataFormat();
59
+
switch ($format) {
60
+
case self::DATAFORMAT_RAW:
61
+
// In this format, the changes are stored as-is.
62
+
$data = $data;
63
+
break;
64
+
case self::DATAFORMAT_DEFLATE:
65
+
default:
66
+
throw new Exception(
67
+
pht('Hunk has unsupported data encoding "%s"!', $type));
68
+
}
69
+
70
+
return $data;
71
+
}
72
+
73
+
}
+2
-2
src/applications/differential/storage/__tests__/DifferentialHunkTestCase.php
+2
-2
src/applications/differential/storage/__tests__/DifferentialHunkTestCase.php
···
5
5
public function testMakeChanges() {
6
6
$root = dirname(__FILE__).'/hunk/';
7
7
8
-
$hunk = new DifferentialHunk();
8
+
$hunk = new DifferentialHunkLegacy();
9
9
$hunk->setChanges(Filesystem::readFile($root.'basic.diff'));
10
10
$hunk->setOldOffset(1);
11
11
$hunk->setNewOffset(11);
···
23
23
);
24
24
$this->assertEqual($added, $hunk->getAddedLines());
25
25
26
-
$hunk = new DifferentialHunk();
26
+
$hunk = new DifferentialHunkLegacy();
27
27
$hunk->setChanges(Filesystem::readFile($root.'newline.diff'));
28
28
$hunk->setOldOffset(1);
29
29
$hunk->setNewOffset(11);
+2
-2
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php
+2
-2
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php
···
382
382
// -echo "test";
383
383
// -(empty line)
384
384
385
-
$hunk = id(new DifferentialHunk())->setChanges($context);
386
-
$vs_hunk = id(new DifferentialHunk())->setChanges($vs_context);
385
+
$hunk = id(new DifferentialHunkLegacy())->setChanges($context);
386
+
$vs_hunk = id(new DifferentialHunkLegacy())->setChanges($vs_context);
387
387
if ($hunk->makeOldFile() != $vs_hunk->makeOldFile() ||
388
388
$hunk->makeNewFile() != $vs_hunk->makeNewFile()) {
389
389
return $vs_diff;