@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.) hq.recaptime.dev/wiki/Phorge
phorge phabricator
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Push feed publishing deeper into the task queue

Summary:
Ref T2852. I want to model Asana integration as a response to feed events. Currently, we queue one feed event for each HTTP hook.

Instead, always queue one feed event and then have it queue any necessary followup events (now, http hooks; soon, asana).

Add a script to make it easy to reproducibly fire feed event publishing.

Test Plan:
Republished a feed event and verified it hit configured HTTP hooks correctly.

$ ./bin/feed republish 5765774156541908292 --trace
>>> [2] <connect> phabricator2_feed
<<< [2] <connect> 1,660 us
>>> [3] <query> SELECT story.* FROM `feed_storydata` story JOIN `feed_storyreference` ref ON ref.chronologicalKey = story.chronologicalKey WHERE (ref.chronologicalKey IN (5765774156541908292)) GROUP BY story.chronologicalKey ORDER BY story.chronologicalKey DESC
<<< [3] <query> 595 us
>>> [4] <connect> phabricator2_differential
<<< [4] <connect> 760 us
>>> [5] <query> SELECT * FROM `differential_revision` WHERE phid IN ('PHID-DREV-ywqmrj5zgkdloqh5p3c5')
<<< [5] <query> 478 us
>>> [6] <query> SELECT * FROM `differential_revision` WHERE phid IN ('PHID-DREV-ywqmrj5zgkdloqh5p3c5')
<<< [6] <query> 449 us
>>> [7] <connect> phabricator2_user
<<< [7] <connect> 1,062 us
>>> [8] <query> SELECT * FROM `user` WHERE phid in ('PHID-USER-lqiz3yd7wmk64ejugvov')
<<< [8] <query> 540 us
>>> [9] <connect> phabricator2_file
<<< [9] <connect> 951 us
>>> [10] <query> SELECT * FROM `file` WHERE phid IN ('PHID-FILE-gq6dlsysvxbn3dgwvky7')
<<< [10] <query> 498 us
>>> [11] <query> SELECT * FROM `user_status` WHERE userPHID IN ('PHID-USER-lqiz3yd7wmk64ejugvov') AND UNIX_TIMESTAMP() BETWEEN dateFrom AND dateTo
<<< [11] <query> 507 us
Republishing story...
>>> [12] <query> SELECT story.* FROM `feed_storydata` story JOIN `feed_storyreference` ref ON ref.chronologicalKey = story.chronologicalKey WHERE (ref.chronologicalKey IN (5765774156541908292)) GROUP BY story.chronologicalKey ORDER BY story.chronologicalKey DESC
<<< [12] <query> 685 us
>>> [13] <query> SELECT * FROM `differential_revision` WHERE phid IN ('PHID-DREV-ywqmrj5zgkdloqh5p3c5')
<<< [13] <query> 489 us
>>> [14] <query> SELECT * FROM `differential_revision` WHERE phid IN ('PHID-DREV-ywqmrj5zgkdloqh5p3c5')
<<< [14] <query> 512 us
>>> [15] <query> SELECT * FROM `user` WHERE phid in ('PHID-USER-lqiz3yd7wmk64ejugvov')
<<< [15] <query> 601 us
>>> [16] <query> SELECT * FROM `file` WHERE phid IN ('PHID-FILE-gq6dlsysvxbn3dgwvky7')
<<< [16] <query> 405 us
>>> [17] <query> SELECT * FROM `user_status` WHERE userPHID IN ('PHID-USER-lqiz3yd7wmk64ejugvov') AND UNIX_TIMESTAMP() BETWEEN dateFrom AND dateTo
<<< [17] <query> 551 us
>>> [18] <query> SELECT story.* FROM `feed_storydata` story JOIN `feed_storyreference` ref ON ref.chronologicalKey = story.chronologicalKey WHERE (ref.chronologicalKey IN (5765774156541908292)) GROUP BY story.chronologicalKey ORDER BY story.chronologicalKey DESC
<<< [18] <query> 507 us
>>> [19] <query> SELECT * FROM `differential_revision` WHERE phid IN ('PHID-DREV-ywqmrj5zgkdloqh5p3c5')
<<< [19] <query> 428 us
>>> [20] <query> SELECT * FROM `differential_revision` WHERE phid IN ('PHID-DREV-ywqmrj5zgkdloqh5p3c5')
<<< [20] <query> 419 us
>>> [21] <query> SELECT * FROM `user` WHERE phid in ('PHID-USER-lqiz3yd7wmk64ejugvov')
<<< [21] <query> 591 us
>>> [22] <query> SELECT * FROM `file` WHERE phid IN ('PHID-FILE-gq6dlsysvxbn3dgwvky7')
<<< [22] <query> 406 us
>>> [23] <query> SELECT * FROM `user_status` WHERE userPHID IN ('PHID-USER-lqiz3yd7wmk64ejugvov') AND UNIX_TIMESTAMP() BETWEEN dateFrom AND dateTo
<<< [23] <query> 593 us
>>> [24] <http> http://127.0.0.1/derp/
<<< [24] <http> 746,157 us
[2013-06-24 20:23:26] EXCEPTION: (HTTPFutureResponseStatusHTTP) [HTTP/500] Internal Server Error

Reviewers: btrahan

Reviewed By: btrahan

CC: aran

Maniphest Tasks: T2852

Differential Revision: https://secure.phabricator.com/D6291

+212 -38
+1
bin/feed
··· 1 + ../scripts/setup/manage_feed.php
+22
scripts/setup/manage_feed.php
··· 1 + #!/usr/bin/env php 2 + <?php 3 + 4 + $root = dirname(dirname(dirname(__FILE__))); 5 + require_once $root.'/scripts/__init_script__.php'; 6 + 7 + $args = new PhutilArgumentParser($argv); 8 + $args->setTagline('manage feed'); 9 + $args->setSynopsis(<<<EOSYNOPSIS 10 + **feed** __command__ [__options__] 11 + Test and debug feed events. 12 + 13 + EOSYNOPSIS 14 + ); 15 + $args->parseStandardArguments(); 16 + 17 + $workflows = array( 18 + new PhabricatorFeedManagementRepublishWorkflow(), 19 + new PhutilHelpArgumentWorkflow(), 20 + ); 21 + 22 + $args->parseWorkflows($workflows);
+9 -1
src/__phutil_library_map__.php
··· 575 575 'DrydockSSHCommandInterface' => 'applications/drydock/interface/command/DrydockSSHCommandInterface.php', 576 576 'DrydockWebrootInterface' => 'applications/drydock/interface/webroot/DrydockWebrootInterface.php', 577 577 'DrydockWorkingCopyBlueprint' => 'applications/drydock/blueprint/DrydockWorkingCopyBlueprint.php', 578 + 'FeedPublisherHTTPWorker' => 'applications/feed/worker/FeedPublisherHTTPWorker.php', 578 579 'FeedPublisherWorker' => 'applications/feed/worker/FeedPublisherWorker.php', 580 + 'FeedPushWorker' => 'applications/feed/worker/FeedPushWorker.php', 579 581 'HarbormasterDAO' => 'applications/harbormaster/storage/HarbormasterDAO.php', 580 582 'HarbormasterObject' => 'applications/harbormaster/storage/HarbormasterObject.php', 581 583 'HarbormasterRunnerWorker' => 'applications/harbormaster/worker/HarbormasterRunnerWorker.php', ··· 1051 1053 'PhabricatorFeedController' => 'applications/feed/controller/PhabricatorFeedController.php', 1052 1054 'PhabricatorFeedDAO' => 'applications/feed/storage/PhabricatorFeedDAO.php', 1053 1055 'PhabricatorFeedMainController' => 'applications/feed/controller/PhabricatorFeedMainController.php', 1056 + 'PhabricatorFeedManagementRepublishWorkflow' => 'applications/feed/management/PhabricatorFeedManagementRepublishWorkflow.php', 1057 + 'PhabricatorFeedManagementWorkflow' => 'applications/feed/management/PhabricatorFeedManagementWorkflow.php', 1054 1058 'PhabricatorFeedPublicStreamController' => 'applications/feed/controller/PhabricatorFeedPublicStreamController.php', 1055 1059 'PhabricatorFeedQuery' => 'applications/feed/PhabricatorFeedQuery.php', 1056 1060 'PhabricatorFeedStory' => 'applications/feed/story/PhabricatorFeedStory.php', ··· 2456 2460 'DrydockSSHCommandInterface' => 'DrydockCommandInterface', 2457 2461 'DrydockWebrootInterface' => 'DrydockInterface', 2458 2462 'DrydockWorkingCopyBlueprint' => 'DrydockBlueprint', 2459 - 'FeedPublisherWorker' => 'PhabricatorWorker', 2463 + 'FeedPublisherHTTPWorker' => 'FeedPushWorker', 2464 + 'FeedPublisherWorker' => 'FeedPushWorker', 2465 + 'FeedPushWorker' => 'PhabricatorWorker', 2460 2466 'HarbormasterDAO' => 'PhabricatorLiskDAO', 2461 2467 'HarbormasterObject' => 'HarbormasterDAO', 2462 2468 'HarbormasterRunnerWorker' => 'PhabricatorWorker', ··· 2929 2935 'PhabricatorFeedController' => 'PhabricatorController', 2930 2936 'PhabricatorFeedDAO' => 'PhabricatorLiskDAO', 2931 2937 'PhabricatorFeedMainController' => 'PhabricatorFeedController', 2938 + 'PhabricatorFeedManagementRepublishWorkflow' => 'PhabricatorFeedManagementWorkflow', 2939 + 'PhabricatorFeedManagementWorkflow' => 'PhutilArgumentWorkflow', 2932 2940 'PhabricatorFeedPublicStreamController' => 'PhabricatorFeedController', 2933 2941 'PhabricatorFeedQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', 2934 2942 'PhabricatorFeedStory' => 'PhabricatorPolicyInterface',
+24
src/applications/feed/PhabricatorFeedQuery.php
··· 4 4 extends PhabricatorCursorPagedPolicyAwareQuery { 5 5 6 6 private $filterPHIDs; 7 + private $chronologicalKeys; 7 8 8 9 public function setFilterPHIDs(array $phids) { 9 10 $this->filterPHIDs = $phids; 11 + return $this; 12 + } 13 + 14 + public function withChronologicalKeys(array $keys) { 15 + $this->chronologicalKeys = $keys; 10 16 return $this; 11 17 } 12 18 ··· 52 58 $conn_r, 53 59 'ref.objectPHID IN (%Ls)', 54 60 $this->filterPHIDs); 61 + } 62 + 63 + if ($this->chronologicalKeys) { 64 + // NOTE: We want to use integers in the query so we can take advantage 65 + // of keys, but can't use %d on 32-bit systems. Make sure all the keys 66 + // are integers and then format them raw. 67 + 68 + $keys = $this->chronologicalKeys; 69 + foreach ($keys as $key) { 70 + if (!ctype_digit($key)) { 71 + throw new Exception("Key '{$key}' is not a valid chronological key!"); 72 + } 73 + } 74 + 75 + $where[] = qsprintf( 76 + $conn_r, 77 + 'ref.chronologicalKey IN (%Q)', 78 + implode(', ', $keys)); 55 79 } 56 80 57 81 $where[] = $this->buildPagingClause($conn_r);
+5 -6
src/applications/feed/PhabricatorFeedStoryPublisher.php
··· 105 105 $this->sendNotification($chrono_key); 106 106 } 107 107 108 - $uris = PhabricatorEnv::getEnvConfig('feed.http-hooks'); 109 - foreach ($uris as $uri) { 110 - $task = PhabricatorWorker::scheduleTask( 111 - 'FeedPublisherWorker', 112 - array('chrono_key' => $chrono_key, 'uri' => $uri)); 113 - } 108 + PhabricatorWorker::scheduleTask( 109 + 'FeedPublisherWorker', 110 + array( 111 + 'key' => $chrono_key, 112 + )); 114 113 115 114 return $story; 116 115 }
+61
src/applications/feed/management/PhabricatorFeedManagementRepublishWorkflow.php
··· 1 + <?php 2 + 3 + final class PhabricatorFeedManagementRepublishWorkflow 4 + extends PhabricatorFeedManagementWorkflow { 5 + 6 + protected function didConstruct() { 7 + $this 8 + ->setName('republish') 9 + ->setExamples('**republish** __story_key__') 10 + ->setSynopsis( 11 + pht( 12 + 'Republish a feed event to all consumers.')) 13 + ->setArguments( 14 + array( 15 + array( 16 + 'name' => 'key', 17 + 'wildcard' => true, 18 + ), 19 + )); 20 + } 21 + 22 + public function execute(PhutilArgumentParser $args) { 23 + $console = PhutilConsole::getConsole(); 24 + $viewer = PhabricatorUser::getOmnipotentUser(); 25 + 26 + $key = $args->getArg('key'); 27 + if (count($key) < 1) { 28 + throw new PhutilArgumentUsageException( 29 + pht("Specify a story key to republish.")); 30 + } else if (count($key) > 1) { 31 + throw new PhutilArgumentUsageException( 32 + pht("Specify exactly one story key to republish.")); 33 + } 34 + $key = head($key); 35 + 36 + $story = id(new PhabricatorFeedQuery()) 37 + ->setViewer($viewer) 38 + ->withChronologicalKeys(array($key)) 39 + ->executeOne(); 40 + 41 + if (!$story) { 42 + throw new PhutilArgumentUsageException( 43 + pht('No story exists with key "%s"!', $key)); 44 + } 45 + 46 + $console->writeOut("%s\n", pht("Republishing story...")); 47 + 48 + PhabricatorWorker::setRunAllTasksInProcess(true); 49 + 50 + PhabricatorWorker::scheduleTask( 51 + 'FeedPublisherWorker', 52 + array( 53 + 'key' => $key, 54 + )); 55 + 56 + $console->writeOut("%s\n", pht("Done.")); 57 + 58 + return 0; 59 + } 60 + 61 + }
+10
src/applications/feed/management/PhabricatorFeedManagementWorkflow.php
··· 1 + <?php 2 + 3 + abstract class PhabricatorFeedManagementWorkflow 4 + extends PhutilArgumentWorkflow { 5 + 6 + final public function isExecutable() { 7 + return true; 8 + } 9 + 10 + }
+29
src/applications/feed/worker/FeedPublisherHTTPWorker.php
··· 1 + <?php 2 + 3 + final class FeedPublisherHTTPWorker extends FeedPushWorker { 4 + 5 + protected function doWork() { 6 + $story = $this->loadFeedStory(); 7 + $data = $story->getStoryData(); 8 + 9 + $uri = idx($this->getTaskData(), 'uri'); 10 + 11 + $post_data = array( 12 + 'storyID' => $data->getID(), 13 + 'storyType' => $data->getStoryType(), 14 + 'storyData' => $data->getStoryData(), 15 + 'storyAuthorPHID' => $data->getAuthorPHID(), 16 + 'epoch' => $data->getEpoch(), 17 + ); 18 + 19 + id(new HTTPFuture($uri, $post_data)) 20 + ->setMethod('POST') 21 + ->setTimeout(30) 22 + ->resolvex(); 23 + } 24 + 25 + public function getWaitBeforeRetry(PhabricatorWorkerTask $task) { 26 + return max($task->getFailureCount(), 1) * 60; 27 + } 28 + 29 + }
+10 -27
src/applications/feed/worker/FeedPublisherWorker.php
··· 1 1 <?php 2 2 3 - final class FeedPublisherWorker extends PhabricatorWorker { 3 + final class FeedPublisherWorker extends FeedPushWorker { 4 4 5 5 protected function doWork() { 6 - $task_data = $this->getTaskData(); 7 - $chrono_key = $task_data['chrono_key']; 8 - $uri = $task_data['uri']; 6 + $story = $this->loadFeedStory(); 9 7 10 - $story = id(new PhabricatorFeedStoryData()) 11 - ->loadOneWhere('chronologicalKey = %s', $chrono_key); 12 - 13 - if (!$story) { 14 - throw new PhabricatorWorkerPermanentFailureException( 15 - 'Feed story was deleted.' 16 - ); 8 + $uris = PhabricatorEnv::getEnvConfig('feed.http-hooks'); 9 + foreach ($uris as $uri) { 10 + PhabricatorWorker::scheduleTask( 11 + 'FeedPublisherHTTPWorker', 12 + array( 13 + 'key' => $story->getChronologicalKey(), 14 + 'uri' => $uri, 15 + )); 17 16 } 18 17 19 - $data = array( 20 - 'storyID' => $story->getID(), 21 - 'storyType' => $story->getStoryType(), 22 - 'storyData' => $story->getStoryData(), 23 - 'storyAuthorPHID' => $story->getAuthorPHID(), 24 - 'epoch' => $story->getEpoch(), 25 - ); 26 - 27 - id(new HTTPFuture($uri, $data)) 28 - ->setMethod('POST') 29 - ->setTimeout(30) 30 - ->resolvex(); 31 - 32 18 } 33 19 34 - public function getWaitBeforeRetry(PhabricatorWorkerTask $task) { 35 - return max($task->getFailureCount(), 1) * 60; 36 - } 37 20 38 21 }
+22
src/applications/feed/worker/FeedPushWorker.php
··· 1 + <?php 2 + 3 + abstract class FeedPushWorker extends PhabricatorWorker { 4 + 5 + protected function loadFeedStory() { 6 + $task_data = $this->getTaskData(); 7 + $key = $task_data['key']; 8 + 9 + $story = id(new PhabricatorFeedQuery()) 10 + ->setViewer(PhabricatorUser::getOmnipotentUser()) 11 + ->withChronologicalKeys(array($key)) 12 + ->executeOne(); 13 + 14 + if (!$story) { 15 + throw new PhabricatorWorkerPermanentFailureException( 16 + 'Feed story does not exist..'); 17 + } 18 + 19 + return $story; 20 + } 21 + 22 + }
+19 -4
src/infrastructure/daemon/workers/PhabricatorWorker.php
··· 8 8 abstract class PhabricatorWorker { 9 9 10 10 private $data; 11 + private static $runAllTasksInProcess = false; 11 12 12 13 13 14 /* -( Configuring Retries and Failures )----------------------------------- */ ··· 85 86 } 86 87 87 88 final public static function scheduleTask($task_class, $data) { 88 - return id(new PhabricatorWorkerActiveTask()) 89 - ->setTaskClass($task_class) 90 - ->setData($data) 91 - ->save(); 89 + if (self::$runAllTasksInProcess) { 90 + $worker = newv($task_class, array($data)); 91 + $worker->doWork(); 92 + } else { 93 + return id(new PhabricatorWorkerActiveTask()) 94 + ->setTaskClass($task_class) 95 + ->setData($data) 96 + ->save(); 97 + } 92 98 } 93 99 94 100 ··· 152 158 public function renderForDisplay() { 153 159 $data = PhutilReadableSerializer::printableValue($this->data); 154 160 return phutil_tag('pre', array(), $data); 161 + } 162 + 163 + /** 164 + * Set this flag to execute scheduled tasks synchronously, in the same 165 + * process. This is useful for debugging, and otherwise dramatically worse 166 + * in every way imaginable. 167 + */ 168 + public static function setRunAllTasksInProcess($all) { 169 + self::$runAllTasksInProcess = $all; 155 170 } 156 171 157 172 }