Improve UX for repository updates · recaptime.dev/phorge@fca8b5a

+2

src/__phutil_library_map__.php

··· 475 475 'DiffusionRepositoryEditLocalController' => 'applications/diffusion/controller/DiffusionRepositoryEditLocalController.php', 476 476 'DiffusionRepositoryEditMainController' => 'applications/diffusion/controller/DiffusionRepositoryEditMainController.php', 477 477 'DiffusionRepositoryEditSubversionController' => 'applications/diffusion/controller/DiffusionRepositoryEditSubversionController.php', 478 + 'DiffusionRepositoryEditUpdateController' => 'applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php', 478 479 'DiffusionRepositoryListController' => 'applications/diffusion/controller/DiffusionRepositoryListController.php', 479 480 'DiffusionRepositoryNewController' => 'applications/diffusion/controller/DiffusionRepositoryNewController.php', 480 481 'DiffusionRepositoryPath' => 'applications/diffusion/data/DiffusionRepositoryPath.php', ··· 3223 3224 'DiffusionRepositoryEditLocalController' => 'DiffusionRepositoryEditController', 3224 3225 'DiffusionRepositoryEditMainController' => 'DiffusionRepositoryEditController', 3225 3226 'DiffusionRepositoryEditSubversionController' => 'DiffusionRepositoryEditController', 3227 + 'DiffusionRepositoryEditUpdateController' => 'DiffusionRepositoryEditController', 3226 3228 'DiffusionRepositoryListController' => 'DiffusionController', 3227 3229 'DiffusionRepositoryNewController' => 'DiffusionController', 3228 3230 'DiffusionRepositoryRef' => 'Phobject',

+1

src/applications/diffusion/application/PhabricatorDiffusionApplication.php

··· 92 92 'delete/' => 'DiffusionRepositoryEditDeleteController', 93 93 'hosting/' => 'DiffusionRepositoryEditHostingController', 94 94 '(?P<serve>serve)/' => 'DiffusionRepositoryEditHostingController', 95 + 'update/' => 'DiffusionRepositoryEditUpdateController', 95 96 ), 96 97 'pathtree/(?P<dblob>.*)' => 'DiffusionPathTreeController', 97 98 'mirror/' => array(

+39 -3

src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php

··· 207 207 ->setHref($this->getRepositoryControllerURI($repository, 'edit/basic/')); 208 208 $view->addAction($edit); 209 209 210 + $edit = id(new PhabricatorActionView()) 211 + ->setIcon('fa-refresh') 212 + ->setName(pht('Update Now')) 213 + ->setWorkflow(true) 214 + ->setHref( 215 + $this->getRepositoryControllerURI($repository, 'edit/update/')); 216 + $view->addAction($edit); 217 + 210 218 $activate = id(new PhabricatorActionView()) 211 219 ->setHref( 212 220 $this->getRepositoryControllerURI($repository, 'edit/activate/')) ··· 279 287 $view->addProperty( 280 288 pht('Status'), 281 289 $this->buildRepositoryStatus($repository)); 290 + 291 + $view->addProperty( 292 + pht('Update Frequency'), 293 + $this->buildRepositoryUpdateInterval($repository)); 282 294 283 295 $description = $repository->getDetail('description'); 284 296 $view->addSectionHeader(pht('Description')); ··· 942 954 ->setNote($message->getParameter('message'))); 943 955 return $view; 944 956 case PhabricatorRepositoryStatusMessage::CODE_OKAY: 957 + $ago = (PhabricatorTime::getNow() - $message->getEpoch()); 945 958 $view->addItem( 946 959 id(new PHUIStatusItemView()) 947 960 ->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green') 948 961 ->setTarget(pht('Updates OK')) 949 962 ->setNote( 950 963 pht( 951 - 'Last updated %s.', 952 - phabricator_datetime($message->getEpoch(), $viewer)))); 964 + 'Last updated %s (%s ago).', 965 + phabricator_datetime($message->getEpoch(), $viewer), 966 + phutil_format_relative_time_detailed($ago)))); 953 967 break; 954 968 } 955 969 } else { ··· 1020 1034 id(new PHUIStatusItemView()) 1021 1035 ->setIcon(PHUIStatusItemView::ICON_UP, 'indigo') 1022 1036 ->setTarget(pht('Prioritized')) 1023 - ->setNote(pht('This repository will be updated soon.'))); 1037 + ->setNote(pht('This repository will be updated soon!'))); 1024 1038 } 1025 1039 1026 1040 return $view; 1027 1041 } 1042 + 1043 + private function buildRepositoryUpdateInterval( 1044 + PhabricatorRepository $repository) { 1045 + 1046 + $smart_wait = $repository->loadUpdateInterval(); 1047 + 1048 + $doc_href = PhabricatorEnv::getDoclink( 1049 + 'Diffusion User Guide: Repository Updates'); 1050 + 1051 + return array( 1052 + phutil_format_relative_time_detailed($smart_wait), 1053 + " \xC2\xB7 ", 1054 + phutil_tag( 1055 + 'a', 1056 + array( 1057 + 'href' => $doc_href, 1058 + 'target' => '_blank', 1059 + ), 1060 + pht('Learn More')), 1061 + ); 1062 + } 1063 + 1028 1064 1029 1065 private function buildMirrorActions( 1030 1066 PhabricatorRepository $repository) {

+75

src/applications/diffusion/controller/DiffusionRepositoryEditUpdateController.php

··· 1 + <?php 2 + 3 + final class DiffusionRepositoryEditUpdateController 4 + extends DiffusionRepositoryEditController { 5 + 6 + public function processRequest() { 7 + $request = $this->getRequest(); 8 + $viewer = $request->getUser(); 9 + $drequest = $this->diffusionRequest; 10 + $repository = $drequest->getRepository(); 11 + 12 + $repository = id(new PhabricatorRepositoryQuery()) 13 + ->setViewer($viewer) 14 + ->requireCapabilities( 15 + array( 16 + PhabricatorPolicyCapability::CAN_VIEW, 17 + PhabricatorPolicyCapability::CAN_EDIT, 18 + )) 19 + ->withIDs(array($repository->getID())) 20 + ->executeOne(); 21 + if (!$repository) { 22 + return new Aphront404Response(); 23 + } 24 + 25 + $edit_uri = $this->getRepositoryControllerURI($repository, 'edit/'); 26 + 27 + if ($request->isFormPost()) { 28 + $params = array( 29 + 'callsigns' => array( 30 + $repository->getCallsign(), 31 + ), 32 + ); 33 + 34 + id(new ConduitCall('diffusion.looksoon', $params)) 35 + ->setUser($viewer) 36 + ->execute(); 37 + 38 + return id(new AphrontRedirectResponse())->setURI($edit_uri); 39 + } 40 + 41 + $doc_name = 'Diffusion User Guide: Repository Updates'; 42 + $doc_href = PhabricatorEnv::getDoclink($doc_name); 43 + $doc_link = phutil_tag( 44 + 'a', 45 + array( 46 + 'href' => $doc_href, 47 + 'target' => '_blank', 48 + ), 49 + $doc_name); 50 + 51 + return $this->newDialog() 52 + ->setTitle(pht('Update Repository Now')) 53 + ->appendParagraph( 54 + pht( 55 + 'Normally, Phabricator automatically updates repositories '. 56 + 'based on how much time has elapsed since the last commit. '. 57 + 'This helps reduce load if you have a large number of mostly '. 58 + 'inactive repositories, which is common.')) 59 + ->appendParagraph( 60 + pht( 61 + 'You can manually schedule an update for this repository. The '. 62 + 'daemons will perform the update as soon as possible. This may '. 63 + 'be helpful if you have just made a commit to a rarely used '. 64 + 'repository.')) 65 + ->appendParagraph( 66 + pht( 67 + 'To learn more about how Phabricator updates repositories, '. 68 + 'read %s in the documentation.', 69 + $doc_link)) 70 + ->addCancelButton($edit_uri) 71 + ->addSubmitButton(pht('Schedule Update')); 72 + } 73 + 74 + 75 + }

+14 -46

src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php

··· 345 345 phlog($stderr_msg); 346 346 } 347 347 348 - $sleep_for = (int)$repository->getDetail('pull-frequency', $min_sleep); 349 - 350 - // Smart wait: pull rarely used repositories less frequently. Find the 351 - // most recent commit which is older than the current time (this keeps us 352 - // from spinning on repositories with a silly commit post-dated to some time 353 - // in 2037), and adjust how frequently we pull based on how frequently this 354 - // repository updates. 355 - 356 - $table = id(new PhabricatorRepositoryCommit()); 357 - $last_commit = queryfx_one( 358 - $table->establishConnection('w'), 359 - 'SELECT epoch FROM %T 360 - WHERE repositoryID = %d AND epoch <= %d 361 - ORDER BY epoch DESC LIMIT 1', 362 - $table->getTableName(), 363 - $repository->getID(), 364 - time() + $min_sleep); 365 - if ($last_commit) { 366 - $time_since_commit = (time() + $min_sleep) - $last_commit['epoch']; 367 - 368 - // Wait 0.5% of the time since the last commit before we pull. This gives 369 - // us these wait times: 370 - // 371 - // 50 minutes or less: 15 seconds 372 - // about 3 hours: 1 minute 373 - // about 16 hours: 5 minutes 374 - // about 2 days: 15 minutes 375 - // 50 days or more: 6 hours 376 - 377 - $smart_wait = ($time_since_commit / 200); 378 - $smart_wait = min($smart_wait, phutil_units('6 hours in seconds')); 379 - 380 - $this->log( 381 - pht( 382 - 'Last commit to repository "%s" was %s seconds ago; considering '. 383 - 'a wait of %s seconds before update.', 384 - $repository->getMonogram(), 385 - new PhutilNumber($time_since_commit), 386 - new PhutilNumber($smart_wait))); 348 + // For now, continue respecting this deprecated setting for raising the 349 + // minimum pull frequency. 350 + // TODO: Remove this some day once this code has been completely stable 351 + // for a while. 352 + $sleep_for = (int)$repository->getDetail('pull-frequency'); 353 + $min_sleep = max($sleep_for, $min_sleep); 387 354 388 - $smart_wait = max(15, $smart_wait); 389 - $sleep_for = max($smart_wait, $sleep_for); 390 - } 355 + $smart_wait = $repository->loadUpdateInterval($min_sleep); 391 356 392 - if ($sleep_for < $min_sleep) { 393 - $sleep_for = $min_sleep; 394 - } 357 + $this->log( 358 + pht( 359 + 'Based on activity in repository "%s", considering a wait of %s '. 360 + 'seconds before update.', 361 + $repository->getMonogram(), 362 + new PhutilNumber($smart_wait))); 395 363 396 - return time() + $sleep_for; 364 + return time() + $smart_wait; 397 365 } 398 366 399 367

+63

src/applications/repository/storage/PhabricatorRepository.php

··· 1319 1319 } 1320 1320 1321 1321 1322 + /** 1323 + * Load the pull frequency for this repository, based on the time since the 1324 + * last activity. 1325 + * 1326 + * We pull rarely used repositories less frequently. This finds the most 1327 + * recent commit which is older than the current time (which prevents us from 1328 + * spinning on repositories with a silly commit post-dated to some time in 1329 + * 2037). We adjust the pull frequency based on when the most recent commit 1330 + * occurred. 1331 + * 1332 + * @param int The minimum update interval to use, in seconds. 1333 + * @return int Repository update interval, in seconds. 1334 + */ 1335 + public function loadUpdateInterval($minimum = 15) { 1336 + // If a repository is still importing, always pull it as frequently as 1337 + // possible. This prevents us from hanging for a long time at 99.9% when 1338 + // importing an inactive repository. 1339 + if ($this->isImporting()) { 1340 + return $minimum; 1341 + } 1342 + 1343 + $window_start = (PhabricatorTime::getNow() + $minimum); 1344 + 1345 + $table = id(new PhabricatorRepositoryCommit()); 1346 + $last_commit = queryfx_one( 1347 + $table->establishConnection('r'), 1348 + 'SELECT epoch FROM %T 1349 + WHERE repositoryID = %d AND epoch <= %d 1350 + ORDER BY epoch DESC LIMIT 1', 1351 + $table->getTableName(), 1352 + $this->getID(), 1353 + $window_start); 1354 + if ($last_commit) { 1355 + $time_since_commit = ($window_start - $last_commit['epoch']); 1356 + 1357 + $last_few_days = phutil_units('3 days in seconds'); 1358 + 1359 + if ($time_since_commit <= $last_few_days) { 1360 + // For repositories with activity in the recent past, we wait one 1361 + // extra second for every 10 minutes since the last commit. This 1362 + // shorter backoff is intended to handle weekends and other short 1363 + // breaks from development. 1364 + $smart_wait = ($time_since_commit / 600); 1365 + } else { 1366 + // For repositories without recent activity, we wait one extra second 1367 + // for every 4 minutes since the last commit. This longer backoff 1368 + // handles rarely used repositories, up to the maximum. 1369 + $smart_wait = ($time_since_commit / 240); 1370 + } 1371 + 1372 + // We'll never wait more than 6 hours to pull a repository. 1373 + $longest_wait = phutil_units('6 hours in seconds'); 1374 + $smart_wait = min($smart_wait, $longest_wait); 1375 + 1376 + $smart_wait = max($minimum, $smart_wait); 1377 + } else { 1378 + $smart_wait = $minimum; 1379 + } 1380 + 1381 + return $smart_wait; 1382 + } 1383 + 1384 + 1322 1385 /* -( PhabricatorPolicyInterface )----------------------------------------- */ 1323 1386 1324 1387

+103

src/docs/user/userguide/diffusion_updates.diviner

··· 1 + @title Diffusion User Guide: Repository Updates 2 + @group userguide 3 + 4 + Explains how Diffusion updates repositories to discover new changes. 5 + 6 + Overview 7 + ======== 8 + 9 + When Phabricator is configured to import repositories which are hosted 10 + elsewhere, it needs to poll those repositories for changes. If it polls too 11 + frequently, it can create too much load locally and on remote services. If it 12 + polls too rarely, it may take a long time for commits to show up in the web 13 + interface. 14 + 15 + This document describes the rules around polling and how to understand and 16 + adjust the behavior. In general: 17 + 18 + - Phabricator chooses a default poll interval based on repository 19 + activity. These intervals range from every 15 seconds (for active 20 + repositories) to every 6 hours (for repositories with no commits in two 21 + months). 22 + - If you use `arc` to push commits, or you host repositories on Phabricator, 23 + repositories automatically update after changes are pushed. 24 + - If you don't use `arc` and your repository is hosted elsewhere, this 25 + document describes ways you can make polling more responsive. 26 + 27 + 28 + Default Behavior 29 + ================ 30 + 31 + By default, Phabricator determines how frequently to poll repositories by 32 + examining how long it has been since the last commit. In most cases this is 33 + fairly accurate and produces good behavior. In particular, it automatically 34 + reduces the polling frequency for rarely-used repositories. This dramatically 35 + reduces load for installs with a large number of inactive repositories, which 36 + is common. 37 + 38 + For repositories with activity in the last 3 days, we wait 1 second for every 39 + 10 minutes without activity. The table below has some examples. 40 + 41 + | Time Since Commit | Poll Interval | 42 + |-------------------|------------------| 43 + | //Minimum// | 15 seconds | 44 + | 6h | about 30 seconds | 45 + | 12h | about 1 minute | 46 + | 1 day | about 2 minutes | 47 + | 2 days | about 5 minutes | 48 + | 3 days | about 7 minutes | 49 + 50 + This means that you may need to wait about 2 minutes for the first commit to 51 + be imported in the morning, and about 5 minutes after a long weekend, but other 52 + commits to active repositories should usually be recognized in 30 seconds or 53 + less. 54 + 55 + For repositories with no activity in the last 3 days, we wait longer between 56 + updates (1 second for every 4 minutes without activity). The table below has 57 + some examples. 58 + 59 + | Time Since Commit | Poll Interval | 60 + |-------------------|------------------| 61 + | 4 days | about 30 minutes | 62 + | 7 days | about 45 minutes | 63 + | 10 days | about 1 hour | 64 + | 20 days | about 2 hours | 65 + | 30 days | about 3 hours | 66 + | //Maximum// | 6 hours | 67 + 68 + You can find the exact default poll frequency of a repository in 69 + Diffusion > (Choose a Repository) > Edit Repository, under "Update Frequency". 70 + You can also see the time when the repository was last updated in this 71 + interface. 72 + 73 + Repositories that are currently importing are always updated at the minimum 74 + update frequency so the import finishes as quickly as possible. 75 + 76 + 77 + Triggering Repository Updates 78 + ============================= 79 + 80 + If you want Phabricator to update a repository more quickly than the default 81 + update frequency (for example, because you just pushed a commit to it), you can 82 + tell Phabricator that it should schedule an update as soon as possible. 83 + 84 + There are several ways to do this: 85 + 86 + - If you push changes with `arc land` or `arc commit`, this will be done 87 + for you automatically. These commits should normally be recognized within 88 + a few seconds. 89 + - If your repository is hosted on Phabricator, this will also be done for you 90 + automatically. 91 + - You can schedule an update from the web interface, in Diffusion > 92 + (Choose a Repository) > Edit Repository > Update Now. 93 + - You can make a call to the Conduit API method `diffusion.looksoon`. This 94 + hints to Phabricator that it should poll a repository as soon as it can. 95 + All of the other mechanisms do this under the hood. 96 + 97 + In particular, you may be able to add a commit hook to your external repository 98 + which calls `diffusion.looksoon`. This should make an external repository about 99 + as responsive as a hosted repository. 100 + 101 + If a repository has an update scheduled, the Diffusion > (Choose a 102 + Repository) > Edit Repository interface will show that the the repository is 103 + prioritized and will be updated soon.

Configure Feed

Configure Feed