@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Synchronize (hosted, git, clustered, SSH) repositories prior to reads
Summary:
Ref T4292. Before we write or read a hosted, clustered Git repository over SSH, check if another version of the repository exists on another node that is more up-to-date.
If such a version does exist, fetch that version first. This allows reads and writes of any node to always act on the most up-to-date code.
Test Plan: Faked my way through this and got a fetch via `bin/repository update`; this is difficult to test locally and needs more work before we can put it in production.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T4292
Differential Revision: https://secure.phabricator.com/D15757
+178
-34
+1
-1
src/applications/almanac/controller/AlmanacServiceViewController.php
+1
-1
src/applications/almanac/controller/AlmanacServiceViewController.php
+5
-1
src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php
+5
-1
src/applications/diffusion/ssh/DiffusionGitUploadPackSSHWorkflow.php
···
16
16
protected function executeRepositoryOperations() {
17
17
$repository = $this->getRepository();
18
18
19
+
$skip_sync = $this->shouldSkipReadSynchronization();
20
+
19
21
if ($this->shouldProxy()) {
20
22
$command = $this->getProxyCommand();
21
23
} else {
22
24
$command = csprintf('git-upload-pack -- %s', $repository->getLocalPath());
23
-
$repository->synchronizeWorkingCopyBeforeRead();
25
+
if (!$skip_sync) {
26
+
$repository->synchronizeWorkingCopyBeforeRead();
27
+
}
24
28
}
25
29
$command = PhabricatorDaemon::sudoCommandAsDaemonUser($command);
26
30
+22
src/applications/diffusion/ssh/DiffusionSSHWorkflow.php
+22
src/applications/diffusion/ssh/DiffusionSSHWorkflow.php
···
201
201
$repository = $this->getRepository();
202
202
$viewer = $this->getUser();
203
203
204
+
if ($viewer->isOmnipotent()) {
205
+
throw new Exception(
206
+
pht(
207
+
'This request is authenticated as a cluster device, but is '.
208
+
'performing a write. Writes must be performed with a real '.
209
+
'user account.'));
210
+
}
211
+
204
212
switch ($repository->getServeOverSSH()) {
205
213
case PhabricatorRepository::SERVE_READONLY:
206
214
if ($protocol_command !== null) {
···
235
243
$this->hasWriteAccess = true;
236
244
return $this->hasWriteAccess;
237
245
}
246
+
247
+
protected function shouldSkipReadSynchronization() {
248
+
$viewer = $this->getUser();
249
+
250
+
// Currently, the only case where devices interact over SSH without
251
+
// assuming user credentials is when synchronizing before a read. These
252
+
// synchronizing reads do not themselves need to be synchronized.
253
+
if ($viewer->isOmnipotent()) {
254
+
return true;
255
+
}
256
+
257
+
return false;
258
+
}
259
+
238
260
239
261
}
+2
src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
+2
src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
+148
-32
src/applications/repository/storage/PhabricatorRepository.php
+148
-32
src/applications/repository/storage/PhabricatorRepository.php
···
1349
1349
}
1350
1350
1351
1351
$ssh_user = AlmanacKeys::getClusterSSHUser();
1352
-
if ($ssh_user !== null) {
1352
+
if (strlen($ssh_user)) {
1353
1353
$uri->setUser($ssh_user);
1354
1354
}
1355
1355
···
1927
1927
$never_proxy,
1928
1928
array $protocols) {
1929
1929
1930
-
$service_phid = $this->getAlmanacServicePHID();
1931
-
if (!$service_phid) {
1932
-
// No service, so this is a local repository.
1933
-
return null;
1934
-
}
1935
-
1936
-
$service = id(new AlmanacServiceQuery())
1937
-
->setViewer(PhabricatorUser::getOmnipotentUser())
1938
-
->withPHIDs(array($service_phid))
1939
-
->needBindings(true)
1940
-
->needProperties(true)
1941
-
->executeOne();
1930
+
$service = $this->loadAlmanacService();
1942
1931
if (!$service) {
1943
-
throw new Exception(
1944
-
pht(
1945
-
'The Almanac service for this repository is invalid or could not '.
1946
-
'be loaded.'));
1947
-
}
1948
-
1949
-
$service_type = $service->getServiceImplementation();
1950
-
if (!($service_type instanceof AlmanacClusterRepositoryServiceType)) {
1951
-
throw new Exception(
1952
-
pht(
1953
-
'The Almanac service for this repository does not have the correct '.
1954
-
'service type.'));
1932
+
return null;
1955
1933
}
1956
1934
1957
1935
$bindings = $service->getBindings();
···
1990
1968
}
1991
1969
}
1992
1970
1993
-
$protocol = $binding->getAlmanacPropertyValue('protocol');
1994
-
if ($protocol === null) {
1995
-
$protocol = 'https';
1996
-
}
1971
+
$uri = $this->getClusterRepositoryURIFromBinding($binding);
1997
1972
1973
+
$protocol = $uri->getProtocol();
1998
1974
if (empty($protocol_map[$protocol])) {
1999
1975
continue;
2000
1976
}
2001
1977
2002
-
$uris[] = $protocol.'://'.$iface->renderDisplayAddress().'/';
1978
+
$uris[] = $uri;
2003
1979
}
2004
1980
2005
1981
if (!$uris) {
···
2223
2199
private function shouldEnableSynchronization() {
2224
2200
$device = AlmanacKeys::getLiveDevice();
2225
2201
if (!$device) {
2202
+
return false;
2203
+
}
2204
+
2205
+
$service_phid = $this->getAlmanacServicePHID();
2206
+
if (!$service_phid) {
2207
+
return false;
2208
+
}
2209
+
2210
+
// TODO: For now, this is only supported for Git.
2211
+
if (!$this->isGit()) {
2226
2212
return false;
2227
2213
}
2228
2214
···
2275
2261
}
2276
2262
}
2277
2263
2278
-
// TODO: Actualy fetch the newer version from one of the nodes which has
2279
-
// it.
2264
+
$this->synchronizeWorkingCopyFromDevices($fetchable);
2280
2265
2281
2266
PhabricatorRepositoryWorkingCopyVersion::updateVersion(
2282
2267
$repository_phid,
···
2391
2376
$this->clusterWriteLock->unlock();
2392
2377
$this->clusterWriteLock = null;
2393
2378
}
2379
+
2380
+
2381
+
/**
2382
+
* @task sync
2383
+
*/
2384
+
private function synchronizeWorkingCopyFromDevices(array $device_phids) {
2385
+
$service = $this->loadAlmanacService();
2386
+
if (!$service) {
2387
+
throw new Exception(pht('Failed to load repository cluster service.'));
2388
+
}
2389
+
2390
+
$device_map = array_fuse($device_phids);
2391
+
$bindings = $service->getBindings();
2392
+
2393
+
$fetchable = array();
2394
+
foreach ($bindings as $binding) {
2395
+
// We can't fetch from disabled nodes.
2396
+
if ($binding->getIsDisabled()) {
2397
+
continue;
2398
+
}
2399
+
2400
+
// We can't fetch from nodes which don't have the newest version.
2401
+
$device_phid = $binding->getDevicePHID();
2402
+
if (empty($device_map[$device_phid])) {
2403
+
continue;
2404
+
}
2405
+
2406
+
// TODO: For now, only fetch over SSH. We could support fetching over
2407
+
// HTTP eventually.
2408
+
if ($binding->getAlmanacPropertyValue('protocol') != 'ssh') {
2409
+
continue;
2410
+
}
2411
+
2412
+
$fetchable[] = $binding;
2413
+
}
2414
+
2415
+
if (!$fetchable) {
2416
+
throw new Exception(
2417
+
pht(
2418
+
'Leader lost: no up-to-date nodes in repository cluster are '.
2419
+
'fetchable.'));
2420
+
}
2421
+
2422
+
$caught = null;
2423
+
foreach ($fetchable as $binding) {
2424
+
try {
2425
+
$this->synchronizeWorkingCopyFromBinding($binding);
2426
+
$caught = null;
2427
+
break;
2428
+
} catch (Exception $ex) {
2429
+
$caught = $ex;
2430
+
}
2431
+
}
2432
+
2433
+
if ($caught) {
2434
+
throw $caught;
2435
+
}
2436
+
}
2437
+
2438
+
private function synchronizeWorkingCopyFromBinding($binding) {
2439
+
$fetch_uri = $this->getClusterRepositoryURIFromBinding($binding);
2440
+
2441
+
if ($this->isGit()) {
2442
+
$argv = array(
2443
+
'fetch --prune -- %s %s',
2444
+
$fetch_uri,
2445
+
'+refs/*:refs/*',
2446
+
);
2447
+
} else {
2448
+
throw new Exception(pht('Binding sync only supported for git!'));
2449
+
}
2450
+
2451
+
$future = DiffusionCommandEngine::newCommandEngine($this)
2452
+
->setArgv($argv)
2453
+
->setConnectAsDevice(true)
2454
+
->setProtocol($fetch_uri->getProtocol())
2455
+
->newFuture();
2456
+
2457
+
$future->setCWD($this->getLocalPath());
2458
+
2459
+
$future->resolvex();
2460
+
}
2461
+
2462
+
private function getClusterRepositoryURIFromBinding(
2463
+
AlmanacBinding $binding) {
2464
+
$protocol = $binding->getAlmanacPropertyValue('protocol');
2465
+
if ($protocol === null) {
2466
+
$protocol = 'https';
2467
+
}
2468
+
2469
+
$iface = $binding->getInterface();
2470
+
$address = $iface->renderDisplayAddress();
2471
+
2472
+
$path = $this->getURI();
2473
+
2474
+
return id(new PhutilURI("{$protocol}://{$address}"))
2475
+
->setPath($path);
2476
+
}
2477
+
2478
+
private function loadAlmanacService() {
2479
+
$service_phid = $this->getAlmanacServicePHID();
2480
+
if (!$service_phid) {
2481
+
// No service, so this is a local repository.
2482
+
return null;
2483
+
}
2484
+
2485
+
$service = id(new AlmanacServiceQuery())
2486
+
->setViewer(PhabricatorUser::getOmnipotentUser())
2487
+
->withPHIDs(array($service_phid))
2488
+
->needBindings(true)
2489
+
->needProperties(true)
2490
+
->executeOne();
2491
+
if (!$service) {
2492
+
throw new Exception(
2493
+
pht(
2494
+
'The Almanac service for this repository is invalid or could not '.
2495
+
'be loaded.'));
2496
+
}
2497
+
2498
+
$service_type = $service->getServiceImplementation();
2499
+
if (!($service_type instanceof AlmanacClusterRepositoryServiceType)) {
2500
+
throw new Exception(
2501
+
pht(
2502
+
'The Almanac service for this repository does not have the correct '.
2503
+
'service type.'));
2504
+
}
2505
+
2506
+
return $service;
2507
+
}
2508
+
2509
+
2394
2510
2395
2511
2396
2512
/* -( Symbols )-------------------------------------------------------------*/