@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add an intracluster synchronization log for cluster repositories
Summary:
Depends on D19778. Ref T13216. See PHI943, PHI889, et al.
We currently have a push log and a pull log, but do not separately log intracluster synchronization events. We've encountered several specific cases where having this kind of log would be helpful:
- In PHI943, an install was accidentally aborting locks early. Having timing information in the sync log would let us identify this more quickly.
- In PHI889, an install hit an issue with `MaxStartups` configuration in `sshd`. A log would let us identify when this is an issue.
- In PHI889, I floated a "push the linux kernel + fetch timeout" theory. A sync log would let us see sync/fetch timeouts to confirm this being a problem in practice.
- A sync log will help us assess, develop, test, and monitor intracluster routing sync changes (likely those in T13211) in the future.
Some of these events are present in the pull log already, but only if they make it as far as running a `git upload-pack` subprocess (not the case with `MaxStartups` problems) -- and they can't record end-to-end timing.
No UI yet, I'll add that in a future change.
Test Plan:
- Forced all operations to synchronize by adding `|| true` to the version check.
- Pulled, got a sync log in the database.
Reviewers: amckinley
Reviewed By: amckinley
Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam
Maniphest Tasks: T13216
Differential Revision: https://secure.phabricator.com/D19779
+338
-6
+14
resources/sql/autopatches/20181106.repo.01.sync.sql
+14
resources/sql/autopatches/20181106.repo.01.sync.sql
···
1
+
CREATE TABLE {$NAMESPACE}_repository.repository_syncevent (
2
+
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
3
+
phid VARBINARY(64) NOT NULL,
4
+
repositoryPHID VARBINARY(64) NOT NULL,
5
+
epoch INT UNSIGNED NOT NULL,
6
+
devicePHID VARBINARY(64) NOT NULL,
7
+
fromDevicePHID VARBINARY(64) NOT NULL,
8
+
deviceVersion INT UNSIGNED,
9
+
fromDeviceVersion INT UNSIGNED,
10
+
resultType VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT},
11
+
resultCode INT UNSIGNED NOT NULL,
12
+
syncWait BIGINT UNSIGNED NOT NULL,
13
+
properties LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT}
14
+
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};
+9
src/__phutil_library_map__.php
+9
src/__phutil_library_map__.php
···
4163
4163
'PhabricatorRepositorySvnCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/PhabricatorRepositorySvnCommitChangeParserWorker.php',
4164
4164
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/PhabricatorRepositorySvnCommitMessageParserWorker.php',
4165
4165
'PhabricatorRepositorySymbol' => 'applications/repository/storage/PhabricatorRepositorySymbol.php',
4166
+
'PhabricatorRepositorySyncEvent' => 'applications/repository/storage/PhabricatorRepositorySyncEvent.php',
4167
+
'PhabricatorRepositorySyncEventPHIDType' => 'applications/repository/phid/PhabricatorRepositorySyncEventPHIDType.php',
4168
+
'PhabricatorRepositorySyncEventQuery' => 'applications/repository/query/PhabricatorRepositorySyncEventQuery.php',
4166
4169
'PhabricatorRepositoryTestCase' => 'applications/repository/storage/__tests__/PhabricatorRepositoryTestCase.php',
4167
4170
'PhabricatorRepositoryTransaction' => 'applications/repository/storage/PhabricatorRepositoryTransaction.php',
4168
4171
'PhabricatorRepositoryTransactionQuery' => 'applications/repository/query/PhabricatorRepositoryTransactionQuery.php',
···
10111
10114
'PhabricatorRepositorySvnCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
10112
10115
'PhabricatorRepositorySvnCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
10113
10116
'PhabricatorRepositorySymbol' => 'PhabricatorRepositoryDAO',
10117
+
'PhabricatorRepositorySyncEvent' => array(
10118
+
'PhabricatorRepositoryDAO',
10119
+
'PhabricatorPolicyInterface',
10120
+
),
10121
+
'PhabricatorRepositorySyncEventPHIDType' => 'PhabricatorPHIDType',
10122
+
'PhabricatorRepositorySyncEventQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
10114
10123
'PhabricatorRepositoryTestCase' => 'PhabricatorTestCase',
10115
10124
'PhabricatorRepositoryTransaction' => 'PhabricatorApplicationTransaction',
10116
10125
'PhabricatorRepositoryTransactionQuery' => 'PhabricatorApplicationTransactionQuery',
+62
-6
src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php
+62
-6
src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php
···
206
206
}
207
207
}
208
208
209
-
$this->synchronizeWorkingCopyFromDevices($fetchable);
209
+
$this->synchronizeWorkingCopyFromDevices(
210
+
$fetchable,
211
+
$this_version,
212
+
$max_version);
210
213
} else {
211
214
$this->synchronizeWorkingCopyFromRemote();
212
215
}
···
653
656
/**
654
657
* @task internal
655
658
*/
656
-
private function synchronizeWorkingCopyFromDevices(array $device_phids) {
659
+
private function synchronizeWorkingCopyFromDevices(
660
+
array $device_phids,
661
+
$local_version,
662
+
$remote_version) {
663
+
657
664
$repository = $this->getRepository();
658
665
659
666
$service = $repository->loadAlmanacService();
···
694
701
$caught = null;
695
702
foreach ($fetchable as $binding) {
696
703
try {
697
-
$this->synchronizeWorkingCopyFromBinding($binding);
704
+
$this->synchronizeWorkingCopyFromBinding(
705
+
$binding,
706
+
$local_version,
707
+
$remote_version);
698
708
$caught = null;
699
709
break;
700
710
} catch (Exception $ex) {
···
711
721
/**
712
722
* @task internal
713
723
*/
714
-
private function synchronizeWorkingCopyFromBinding($binding) {
724
+
private function synchronizeWorkingCopyFromBinding(
725
+
AlmanacBinding $binding,
726
+
$local_version,
727
+
$remote_version) {
728
+
715
729
$repository = $this->getRepository();
716
730
$device = AlmanacKeys::getLiveDevice();
717
731
718
732
$this->logLine(
719
733
pht(
720
-
'Synchronizing this device ("%s") from cluster leader ("%s") before '.
721
-
'read.',
734
+
'Synchronizing this device ("%s") from cluster leader ("%s").',
722
735
$device->getName(),
723
736
$binding->getDevice()->getName()));
724
737
···
746
759
747
760
$future->setCWD($local_path);
748
761
762
+
$log = PhabricatorRepositorySyncEvent::initializeNewEvent()
763
+
->setRepositoryPHID($repository->getPHID())
764
+
->setEpoch(PhabricatorTime::getNow())
765
+
->setDevicePHID($device->getPHID())
766
+
->setFromDevicePHID($binding->getDevice()->getPHID())
767
+
->setDeviceVersion($local_version)
768
+
->setFromDeviceVersion($remote_version);
769
+
770
+
$sync_start = microtime(true);
771
+
749
772
try {
750
773
$future->resolvex();
751
774
} catch (Exception $ex) {
775
+
$sync_end = microtime(true);
776
+
$log->setSyncWait((int)(1000000 * ($sync_end - $sync_start)));
777
+
778
+
if ($ex instanceof CommandException) {
779
+
if ($future->getWasKilledByTimeout()) {
780
+
$result_type = PhabricatorRepositorySyncEvent::RESULT_TIMEOUT;
781
+
} else {
782
+
$result_type = PhabricatorRepositorySyncEvent::RESULT_ERROR;
783
+
}
784
+
785
+
$log
786
+
->setResultCode($ex->getError())
787
+
->setResultType($result_type)
788
+
->setProperty('stdout', $ex->getStdout())
789
+
->setProperty('stderr', $ex->getStderr());
790
+
} else {
791
+
$log
792
+
->setResultCode(1)
793
+
->setResultType(PhabricatorRepositorySyncEvent::RESULT_EXCEPTION)
794
+
->setProperty('message', $ex->getMessage());
795
+
}
796
+
797
+
$log->save();
798
+
752
799
$this->logLine(
753
800
pht(
754
801
'Synchronization of "%s" from leader "%s" failed: %s',
755
802
$device->getName(),
756
803
$binding->getDevice()->getName(),
757
804
$ex->getMessage()));
805
+
758
806
throw $ex;
759
807
}
808
+
809
+
$sync_end = microtime(true);
810
+
811
+
$log
812
+
->setSyncWait((int)(1000000 * ($sync_end - $sync_start)))
813
+
->setResultCode(0)
814
+
->setResultType(PhabricatorRepositorySyncEvent::RESULT_SYNC)
815
+
->save();
760
816
}
761
817
762
818
+39
src/applications/repository/phid/PhabricatorRepositorySyncEventPHIDType.php
+39
src/applications/repository/phid/PhabricatorRepositorySyncEventPHIDType.php
···
1
+
<?php
2
+
3
+
final class PhabricatorRepositorySyncEventPHIDType extends PhabricatorPHIDType {
4
+
5
+
const TYPECONST = 'SYNE';
6
+
7
+
public function getTypeName() {
8
+
return pht('Sync Event');
9
+
}
10
+
11
+
public function newObject() {
12
+
return new PhabricatorRepositorySyncEvent();
13
+
}
14
+
15
+
public function getPHIDTypeApplicationClass() {
16
+
return 'PhabricatorDiffusionApplication';
17
+
}
18
+
19
+
protected function buildQueryForObjects(
20
+
PhabricatorObjectQuery $query,
21
+
array $phids) {
22
+
23
+
return id(new PhabricatorRepositorySyncEventQuery())
24
+
->withPHIDs($phids);
25
+
}
26
+
27
+
public function loadHandles(
28
+
PhabricatorHandleQuery $query,
29
+
array $handles,
30
+
array $objects) {
31
+
32
+
foreach ($handles as $phid => $handle) {
33
+
$event = $objects[$phid];
34
+
35
+
$handle->setName(pht('Sync Event %d', $event->getID()));
36
+
}
37
+
}
38
+
39
+
}
+115
src/applications/repository/query/PhabricatorRepositorySyncEventQuery.php
+115
src/applications/repository/query/PhabricatorRepositorySyncEventQuery.php
···
1
+
<?php
2
+
3
+
final class PhabricatorRepositorySyncEventQuery
4
+
extends PhabricatorCursorPagedPolicyAwareQuery {
5
+
6
+
private $ids;
7
+
private $phids;
8
+
private $repositoryPHIDs;
9
+
private $epochMin;
10
+
private $epochMax;
11
+
12
+
public function withIDs(array $ids) {
13
+
$this->ids = $ids;
14
+
return $this;
15
+
}
16
+
17
+
public function withPHIDs(array $phids) {
18
+
$this->phids = $phids;
19
+
return $this;
20
+
}
21
+
22
+
public function withRepositoryPHIDs(array $repository_phids) {
23
+
$this->repositoryPHIDs = $repository_phids;
24
+
return $this;
25
+
}
26
+
27
+
public function withEpochBetween($min, $max) {
28
+
$this->epochMin = $min;
29
+
$this->epochMax = $max;
30
+
return $this;
31
+
}
32
+
33
+
public function newResultObject() {
34
+
return new PhabricatorRepositoryPullEvent();
35
+
}
36
+
37
+
protected function loadPage() {
38
+
return $this->loadStandardPage($this->newResultObject());
39
+
}
40
+
41
+
protected function willFilterPage(array $events) {
42
+
$repository_phids = mpull($events, 'getRepositoryPHID');
43
+
$repository_phids = array_filter($repository_phids);
44
+
45
+
if ($repository_phids) {
46
+
$repositories = id(new PhabricatorRepositoryQuery())
47
+
->setViewer($this->getViewer())
48
+
->withPHIDs($repository_phids)
49
+
->execute();
50
+
$repositories = mpull($repositories, null, 'getPHID');
51
+
} else {
52
+
$repositories = array();
53
+
}
54
+
55
+
foreach ($events as $key => $event) {
56
+
$phid = $event->getRepositoryPHID();
57
+
58
+
if (empty($repositories[$phid])) {
59
+
unset($events[$key]);
60
+
$this->didRejectResult($event);
61
+
continue;
62
+
}
63
+
64
+
$event->attachRepository($repositories[$phid]);
65
+
}
66
+
67
+
return $events;
68
+
}
69
+
70
+
protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) {
71
+
$where = parent::buildWhereClauseParts($conn);
72
+
73
+
if ($this->ids !== null) {
74
+
$where[] = qsprintf(
75
+
$conn,
76
+
'id IN (%Ld)',
77
+
$this->ids);
78
+
}
79
+
80
+
if ($this->phids !== null) {
81
+
$where[] = qsprintf(
82
+
$conn,
83
+
'phid IN (%Ls)',
84
+
$this->phids);
85
+
}
86
+
87
+
if ($this->repositoryPHIDs !== null) {
88
+
$where[] = qsprintf(
89
+
$conn,
90
+
'repositoryPHID IN (%Ls)',
91
+
$this->repositoryPHIDs);
92
+
}
93
+
94
+
if ($this->epochMin !== null) {
95
+
$where[] = qsprintf(
96
+
$conn,
97
+
'epoch >= %d',
98
+
$this->epochMin);
99
+
}
100
+
101
+
if ($this->epochMax !== null) {
102
+
$where[] = qsprintf(
103
+
$conn,
104
+
'epoch <= %d',
105
+
$this->epochMax);
106
+
}
107
+
108
+
return $where;
109
+
}
110
+
111
+
public function getQueryApplicationClass() {
112
+
return 'PhabricatorDiffusionApplication';
113
+
}
114
+
115
+
}
+99
src/applications/repository/storage/PhabricatorRepositorySyncEvent.php
+99
src/applications/repository/storage/PhabricatorRepositorySyncEvent.php
···
1
+
<?php
2
+
3
+
final class PhabricatorRepositorySyncEvent
4
+
extends PhabricatorRepositoryDAO
5
+
implements PhabricatorPolicyInterface {
6
+
7
+
protected $repositoryPHID;
8
+
protected $epoch;
9
+
protected $devicePHID;
10
+
protected $fromDevicePHID;
11
+
protected $deviceVersion;
12
+
protected $fromDeviceVersion;
13
+
protected $resultType;
14
+
protected $resultCode;
15
+
protected $syncWait;
16
+
protected $properties = array();
17
+
18
+
private $repository = self::ATTACHABLE;
19
+
20
+
const RESULT_SYNC = 'sync';
21
+
const RESULT_ERROR = 'error';
22
+
const RESULT_TIMEOUT = 'timeout';
23
+
const RESULT_EXCEPTION = 'exception';
24
+
25
+
public static function initializeNewEvent() {
26
+
return new self();
27
+
}
28
+
29
+
protected function getConfiguration() {
30
+
return array(
31
+
self::CONFIG_AUX_PHID => true,
32
+
self::CONFIG_TIMESTAMPS => false,
33
+
self::CONFIG_SERIALIZATION => array(
34
+
'properties' => self::SERIALIZATION_JSON,
35
+
),
36
+
self::CONFIG_COLUMN_SCHEMA => array(
37
+
'deviceVersion' => 'uint32?',
38
+
'fromDeviceVersion' => 'uint32?',
39
+
'resultType' => 'text32',
40
+
'resultCode' => 'uint32',
41
+
'syncWait' => 'uint64',
42
+
),
43
+
self::CONFIG_KEY_SCHEMA => array(
44
+
'key_repository' => array(
45
+
'columns' => array('repositoryPHID'),
46
+
),
47
+
'key_epoch' => array(
48
+
'columns' => array('epoch'),
49
+
),
50
+
),
51
+
) + parent::getConfiguration();
52
+
}
53
+
54
+
public function getPHIDType() {
55
+
return PhabricatorRepositorySyncEventPHIDType::TYPECONST;
56
+
}
57
+
58
+
public function attachRepository(PhabricatorRepository $repository) {
59
+
$this->repository = $repository;
60
+
return $this;
61
+
}
62
+
63
+
public function getRepository() {
64
+
return $this->assertAttached($this->repository);
65
+
}
66
+
67
+
public function setProperty($key, $value) {
68
+
$this->properites[$key] = $value;
69
+
return $this;
70
+
}
71
+
72
+
public function getProperty($key, $default = null) {
73
+
return idx($this->properties, $key, $default);
74
+
}
75
+
76
+
/* -( PhabricatorPolicyInterface )----------------------------------------- */
77
+
78
+
79
+
public function getCapabilities() {
80
+
return array(
81
+
PhabricatorPolicyCapability::CAN_VIEW,
82
+
);
83
+
}
84
+
85
+
public function getPolicy($capability) {
86
+
return $this->getRepository()->getPolicy($capability);
87
+
}
88
+
89
+
public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
90
+
return $this->getRepository()->hasAutomaticCapability($capability, $viewer);
91
+
}
92
+
93
+
public function describeAutomaticCapability($capability) {
94
+
return pht(
95
+
"A repository's sync events are visible to users who can see the ".
96
+
"repository.");
97
+
}
98
+
99
+
}