recaptime.dev / phorge
@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.) hq.recaptime.dev/wiki/Phorge
phorge phabricator
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Add `bin/garbage compact-edges` to compact edges into the new format

Summary:
Depends on D18947. Ref T13051. This goes through transaction tables and compacts the edge storage into the slim format.

I put this on `bin/garbage` instead of `bin/storage` because `bin/storage` has a lot of weird stuff about how it manages databases so that it can run before configuration (e.g., all the `--user`, `--password` type flags for configuring DB connections).

Test Plan:
Loaded an object with a bunch of transactions. Ran migration. Spot checked table for sanity. Loaded another copy of the object in the web UI, compared the two pages, saw no user-visible changes.

Here's a concrete example of the migration effect -- old row:

```
*************************** 44. row ***************************
id: 757
phid: PHID-XACT-PSTE-5gnaaway2vnyen5
authorPHID: PHID-USER-cvfydnwadpdj7vdon36z
objectPHID: PHID-PSTE-5uj6oqv4kmhtr6ctwcq7
viewPolicy: public
editPolicy: PHID-USER-cvfydnwadpdj7vdon36z
commentPHID: NULL
commentVersion: 0
transactionType: core:edge
oldValue: {"PHID-PROJ-wh32nih7q5scvc5lvipv":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-wh32nih7q5scvc5lvipv","dateCreated":"1449170691","seq":"0","dataID":null,"data":[]},"PHID-PROJ-5r2ed5v27xrgltvou5or":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-5r2ed5v27xrgltvou5or","dateCreated":"1449170683","seq":"0","dataID":null,"data":[]},"PHID-PROJ-zfp44q7loir643b5i4v4":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-zfp44q7loir643b5i4v4","dateCreated":"1449170668","seq":"0","dataID":null,"data":[]},"PHID-PROJ-okljqs7prifhajtvia3t":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-okljqs7prifhajtvia3t","dateCreated":"1448902756","seq":"0","dataID":null,"data":[]},"PHID-PROJ-3cuwfuuh4pwqyuof2hhr":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-3cuwfuuh4pwqyuof2hhr","dateCreated":"1448899367","seq":"0","dataID":null,"data":[]},"PHID-PROJ-amvkc5zw2gsy7tyvocug":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-amvkc5zw2gsy7tyvocug","dateCreated":"1448833330","seq":"0","dataID":null,"data":[]}}
newValue: {"PHID-PROJ-wh32nih7q5scvc5lvipv":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-wh32nih7q5scvc5lvipv","dateCreated":"1449170691","seq":"0","dataID":null,"data":[]},"PHID-PROJ-5r2ed5v27xrgltvou5or":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-5r2ed5v27xrgltvou5or","dateCreated":"1449170683","seq":"0","dataID":null,"data":[]},"PHID-PROJ-zfp44q7loir643b5i4v4":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-zfp44q7loir643b5i4v4","dateCreated":"1449170668","seq":"0","dataID":null,"data":[]},"PHID-PROJ-okljqs7prifhajtvia3t":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-okljqs7prifhajtvia3t","dateCreated":"1448902756","seq":"0","dataID":null,"data":[]},"PHID-PROJ-3cuwfuuh4pwqyuof2hhr":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-3cuwfuuh4pwqyuof2hhr","dateCreated":"1448899367","seq":"0","dataID":null,"data":[]},"PHID-PROJ-amvkc5zw2gsy7tyvocug":{"src":"PHID-PSTE-5uj6oqv4kmhtr6ctwcq7","type":"41","dst":"PHID-PROJ-amvkc5zw2gsy7tyvocug","dateCreated":"1448833330","seq":"0","dataID":null,"data":[]},"PHID-PROJ-tbowhnwinujwhb346q36":{"dst":"PHID-PROJ-tbowhnwinujwhb346q36","type":41,"data":[]},"PHID-PROJ-izrto7uflimduo6uw2tp":{"dst":"PHID-PROJ-izrto7uflimduo6uw2tp","type":41,"data":[]}}
contentSource: {"source":"web","params":[]}
metadata: {"edge:type":41}
dateCreated: 1450197571
dateModified: 1450197571
```

New row:

```
*************************** 44. row ***************************
id: 757
phid: PHID-XACT-PSTE-5gnaaway2vnyen5
authorPHID: PHID-USER-cvfydnwadpdj7vdon36z
objectPHID: PHID-PSTE-5uj6oqv4kmhtr6ctwcq7
viewPolicy: public
editPolicy: PHID-USER-cvfydnwadpdj7vdon36z
commentPHID: NULL
commentVersion: 0
transactionType: core:edge
oldValue: []
newValue: ["PHID-PROJ-tbowhnwinujwhb346q36","PHID-PROJ-izrto7uflimduo6uw2tp"]
contentSource: {"source":"web","params":[]}
metadata: {"edge:type":41}
dateCreated: 1450197571
dateModified: 1450197571
```

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T13051

Differential Revision: https://secure.phabricator.com/D18948

epriestley 6d2d1d3a e5639a8e

+105
+2
src/__phutil_library_map__.php
··· 3017 3017 'PhabricatorGDSetupCheck' => 'applications/config/check/PhabricatorGDSetupCheck.php', 3018 3018 'PhabricatorGarbageCollector' => 'infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php', 3019 3019 'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCollectWorkflow.php', 3020 + 'PhabricatorGarbageCollectorManagementCompactEdgesWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php', 3020 3021 'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementSetPolicyWorkflow.php', 3021 3022 'PhabricatorGarbageCollectorManagementWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementWorkflow.php', 3022 3023 'PhabricatorGeneralCachePurger' => 'applications/cache/purger/PhabricatorGeneralCachePurger.php', ··· 8484 8485 'PhabricatorGDSetupCheck' => 'PhabricatorSetupCheck', 8485 8486 'PhabricatorGarbageCollector' => 'Phobject', 8486 8487 'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow', 8488 + 'PhabricatorGarbageCollectorManagementCompactEdgesWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow', 8487 8489 'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow', 8488 8490 'PhabricatorGarbageCollectorManagementWorkflow' => 'PhabricatorManagementWorkflow', 8489 8491 'PhabricatorGeneralCachePurger' => 'PhabricatorCachePurger',
+103
src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCompactEdgesWorkflow.php
··· 1 + <?php 2 + 3 + final class PhabricatorGarbageCollectorManagementCompactEdgesWorkflow 4 + extends PhabricatorGarbageCollectorManagementWorkflow { 5 + 6 + protected function didConstruct() { 7 + $this 8 + ->setName('compact-edges') 9 + ->setExamples('**compact-edges**') 10 + ->setSynopsis( 11 + pht( 12 + 'Rebuild old edge transactions storage to use a more compact '. 13 + 'format.')) 14 + ->setArguments(array()); 15 + } 16 + 17 + public function execute(PhutilArgumentParser $args) { 18 + $tables = id(new PhutilClassMapQuery()) 19 + ->setAncestorClass('PhabricatorApplicationTransaction') 20 + ->execute(); 21 + 22 + foreach ($tables as $table) { 23 + $this->compactEdges($table); 24 + } 25 + 26 + return 0; 27 + } 28 + 29 + private function compactEdges(PhabricatorApplicationTransaction $table) { 30 + $conn = $table->establishConnection('w'); 31 + $class = get_class($table); 32 + 33 + echo tsprintf( 34 + "%s\n", 35 + pht( 36 + 'Rebuilding transactions for "%s"...', 37 + $class)); 38 + 39 + $cursor = 0; 40 + $updated = 0; 41 + while (true) { 42 + $rows = $table->loadAllWhere( 43 + 'transactionType = %s 44 + AND id > %d 45 + AND (oldValue LIKE %> OR newValue LIKE %>) 46 + ORDER BY id ASC LIMIT 100', 47 + PhabricatorTransactions::TYPE_EDGE, 48 + $cursor, 49 + // We're looking for transactions with JSON objects in their value 50 + // fields: the new style transactions have JSON lists instead and 51 + // start with "[" rather than "{". 52 + '{', 53 + '{'); 54 + 55 + if (!$rows) { 56 + break; 57 + } 58 + 59 + foreach ($rows as $row) { 60 + $id = $row->getID(); 61 + 62 + $old = $row->getOldValue(); 63 + $new = $row->getNewValue(); 64 + 65 + if (!is_array($old) || !is_array($new)) { 66 + echo tsprintf( 67 + "%s\n", 68 + pht( 69 + 'Transaction %s (of type %s) has unexpected data, skipping.', 70 + $id, 71 + $class)); 72 + } 73 + 74 + $record = PhabricatorEdgeChangeRecord::newFromTransaction($row); 75 + 76 + $old_data = $record->getModernOldEdgeTransactionData(); 77 + $old_json = phutil_json_encode($old_data); 78 + 79 + $new_data = $record->getModernNewEdgeTransactionData(); 80 + $new_json = phutil_json_encode($new_data); 81 + 82 + queryfx( 83 + $conn, 84 + 'UPDATE %T SET oldValue = %s, newValue = %s WHERE id = %d', 85 + $table->getTableName(), 86 + $old_json, 87 + $new_json, 88 + $id); 89 + 90 + $updated++; 91 + 92 + $cursor = $row->getID(); 93 + } 94 + } 95 + 96 + echo tsprintf( 97 + "%s\n", 98 + pht( 99 + 'Done, compacted %s edge transactions.', 100 + new PhutilNumber($updated))); 101 + } 102 + 103 + }