Select the types of activity you want to include in your feed.
@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
···11+CREATE TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlintmessage (
22+ id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
33+ buildTargetPHID VARBINARY(64) NOT NULL,
44+ path LONGTEXT NOT NULL,
55+ line INT UNSIGNED,
66+ characterOffset INT UNSIGNED,
77+ code VARCHAR(32) COLLATE {$COLLATE_TEXT} NOT NULL,
88+ severity VARCHAR(32) COLLATE {$COLLATE_TEXT} NOT NULL,
99+ name VARCHAR(255) COLLATE {$COLLATE_TEXT} NOT NULL,
1010+ properties LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL,
1111+ dateCreated INT UNSIGNED NOT NULL,
1212+ dateModified INT UNSIGNED NOT NULL,
1313+ KEY `key_target` (buildTargetPHID)
1414+) ENGINE=INNODB, COLLATE {$COLLATE_TEXT};
···11+<?php
22+33+final class DifferentialDiffTransactionQuery
44+ extends PhabricatorApplicationTransactionQuery {
55+66+ public function getTemplateApplicationTransaction() {
77+ return new DifferentialDiffTransaction();
88+ }
99+1010+}
···11+<?php
22+33+final class HarbormasterTargetEngine extends Phobject {
44+55+ private $viewer;
66+ private $object;
77+ private $autoTargetKeys;
88+99+ public function setViewer($viewer) {
1010+ $this->viewer = $viewer;
1111+ return $this;
1212+ }
1313+1414+ public function getViewer() {
1515+ return $this->viewer;
1616+ }
1717+1818+ public function setObject(HarbormasterBuildableInterface $object) {
1919+ $this->object = $object;
2020+ return $this;
2121+ }
2222+2323+ public function getObject() {
2424+ return $this->object;
2525+ }
2626+2727+ public function setAutoTargetKeys(array $auto_keys) {
2828+ $this->autoTargetKeys = $auto_keys;
2929+ return $this;
3030+ }
3131+3232+ public function getAutoTargetKeys() {
3333+ return $this->autoTargetKeys;
3434+ }
3535+3636+ public function buildTargets() {
3737+ $object = $this->getObject();
3838+ $viewer = $this->getViewer();
3939+4040+ $step_map = $this->generateBuildStepMap($this->getAutoTargetKeys());
4141+4242+ $buildable = HarbormasterBuildable::createOrLoadExisting(
4343+ $viewer,
4444+ $object->getHarbormasterBuildablePHID(),
4545+ $object->getHarbormasterContainerPHID());
4646+4747+ $target_map = $this->generateBuildTargetMap($buildable, $step_map);
4848+4949+ return $target_map;
5050+ }
5151+5252+5353+ /**
5454+ * Get a map of the @{class:HarbormasterBuildStep} objects for a list of
5555+ * autotarget keys.
5656+ *
5757+ * This method creates the steps if they do not yet exist.
5858+ *
5959+ * @param list<string> Autotarget keys, like `"core.arc.lint"`.
6060+ * @return map<string, object> Map of keys to step objects.
6161+ */
6262+ private function generateBuildStepMap(array $autotargets) {
6363+ $viewer = $this->getViewer();
6464+6565+ $autosteps = $this->getAutosteps($autotargets);
6666+ $autosteps = mgroup($autosteps, 'getBuildStepAutotargetPlanKey');
6767+6868+ $plans = id(new HarbormasterBuildPlanQuery())
6969+ ->setViewer($viewer)
7070+ ->withPlanAutoKeys(array_keys($autosteps))
7171+ ->needBuildSteps(true)
7272+ ->execute();
7373+ $plans = mpull($plans, null, 'getPlanAutoKey');
7474+7575+ // NOTE: When creating the plan and steps, we save the autokeys as the
7676+ // names. These won't actually be shown in the UI, but make the data more
7777+ // consistent for secondary consumers like typeaheads.
7878+7979+ $step_map = array();
8080+ foreach ($autosteps as $plan_key => $steps) {
8181+ $plan = idx($plans, $plan_key);
8282+ if (!$plan) {
8383+ $plan = HarbormasterBuildPlan::initializeNewBuildPlan($viewer)
8484+ ->setName($plan_key)
8585+ ->setPlanAutoKey($plan_key);
8686+ }
8787+8888+ $current = $plan->getBuildSteps();
8989+ $current = mpull($current, null, 'getStepAutoKey');
9090+ $new_steps = array();
9191+9292+ foreach ($steps as $step_key => $step) {
9393+ if (isset($current[$step_key])) {
9494+ $step_map[$step_key] = $current[$step_key];
9595+ continue;
9696+ }
9797+9898+ $new_step = HarbormasterBuildStep::initializeNewStep($viewer)
9999+ ->setName($step_key)
100100+ ->setClassName(get_class($step))
101101+ ->setStepAutoKey($step_key);
102102+103103+ $new_steps[$step_key] = $new_step;
104104+ }
105105+106106+ if ($new_steps) {
107107+ $plan->openTransaction();
108108+ if (!$plan->getPHID()) {
109109+ $plan->save();
110110+ }
111111+ foreach ($new_steps as $step_key => $step) {
112112+ $step->setBuildPlanPHID($plan->getPHID());
113113+ $step->save();
114114+115115+ $step->attachBuildPlan($plan);
116116+ $step_map[$step_key] = $step;
117117+ }
118118+ $plan->saveTransaction();
119119+ }
120120+ }
121121+122122+ return $step_map;
123123+ }
124124+125125+126126+ /**
127127+ * Get all of the @{class:HarbormasterBuildStepImplementation} objects for
128128+ * a list of autotarget keys.
129129+ *
130130+ * @param list<string> Autotarget keys, like `"core.arc.lint"`.
131131+ * @return map<string, object> Map of keys to implementations.
132132+ */
133133+ private function getAutosteps(array $autotargets) {
134134+ $all_steps = HarbormasterBuildStepImplementation::getImplementations();
135135+ $all_steps = mpull($all_steps, null, 'getBuildStepAutotargetStepKey');
136136+137137+ // Make sure all the targets really exist.
138138+ foreach ($autotargets as $autotarget) {
139139+ if (empty($all_steps[$autotarget])) {
140140+ throw new Exception(
141141+ pht(
142142+ 'No build step provides autotarget "%s"!',
143143+ $autotarget));
144144+ }
145145+ }
146146+147147+ return array_select_keys($all_steps, $autotargets);
148148+ }
149149+150150+151151+ /**
152152+ * Get a list of @{class:HarbormasterBuildTarget} objects for a list of
153153+ * autotarget keys.
154154+ *
155155+ * If some targets or builds do not exist, they are created.
156156+ *
157157+ * @param HarbormasterBuildable A buildable.
158158+ * @param map<string, object> Map of keys to steps.
159159+ * @return map<string, object> Map of keys to targets.
160160+ */
161161+ private function generateBuildTargetMap(
162162+ HarbormasterBuildable $buildable,
163163+ array $step_map) {
164164+165165+ $viewer = $this->getViewer();
166166+ $plan_map = mgroup($step_map, 'getBuildPlanPHID');
167167+168168+ $builds = id(new HarbormasterBuildQuery())
169169+ ->setViewer($viewer)
170170+ ->withBuildablePHIDs(array($buildable->getPHID()))
171171+ ->withBuildPlanPHIDs(array_keys($plan_map))
172172+ ->needBuildTargets(true)
173173+ ->execute();
174174+175175+ $autobuilds = array();
176176+ foreach ($builds as $build) {
177177+ $plan_key = $build->getBuildPlan()->getPlanAutoKey();
178178+ $autobuilds[$plan_key] = $build;
179179+ }
180180+181181+ $new_builds = array();
182182+ foreach ($plan_map as $plan_phid => $steps) {
183183+ $plan = head($steps)->getBuildPlan();
184184+ $plan_key = $plan->getPlanAutoKey();
185185+186186+ $build = idx($autobuilds, $plan_key);
187187+ if ($build) {
188188+ // We already have a build for this set of targets, so we don't need
189189+ // to do any work. (It's possible the build is an older build that
190190+ // doesn't have all of the right targets if new autotargets were
191191+ // recently introduced, but we don't currently try to construct them.)
192192+ continue;
193193+ }
194194+195195+ // NOTE: Normally, `applyPlan()` does not actually generate targets.
196196+ // We need to apply the plan in-process to perform target generation.
197197+ // This is fine as long as autotargets are empty containers that don't
198198+ // do any work, which they always should be.
199199+200200+ PhabricatorWorker::setRunAllTasksInProcess(true);
201201+ try {
202202+203203+ // NOTE: We might race another process here to create the same build
204204+ // with the same `planAutoKey`. The database will prevent this and
205205+ // using autotargets only currently makes sense if you just created the
206206+ // resource and "own" it, so we don't try to handle this, but may need
207207+ // to be more careful here if use of autotargets expands.
208208+209209+ $build = $buildable->applyPlan($plan);
210210+ PhabricatorWorker::setRunAllTasksInProcess(false);
211211+ } catch (Exception $ex) {
212212+ PhabricatorWorker::setRunAllTasksInProcess(false);
213213+ throw $ex;
214214+ }
215215+216216+ $new_builds[] = $build;
217217+ }
218218+219219+ if ($new_builds) {
220220+ $all_targets = id(new HarbormasterBuildTargetQuery())
221221+ ->setViewer($viewer)
222222+ ->withBuildPHIDs(mpull($new_builds, 'getPHID'))
223223+ ->execute();
224224+ } else {
225225+ $all_targets = array();
226226+ }
227227+228228+ foreach ($builds as $build) {
229229+ foreach ($build->getBuildTargets() as $target) {
230230+ $all_targets[] = $target;
231231+ }
232232+ }
233233+234234+ $target_map = array();
235235+ foreach ($all_targets as $target) {
236236+ $target_key = $target
237237+ ->getImplementation()
238238+ ->getBuildStepAutotargetStepKey();
239239+ if (!$target_key) {
240240+ continue;
241241+ }
242242+ $target_map[$target_key] = $target;
243243+ }
244244+245245+ $target_map = array_select_keys($target_map, array_keys($step_map));
246246+247247+ return $target_map;
248248+ }
249249+250250+251251+}
···11+@title Troubleshooting Repository Imports
22+@group fieldmanual
33+44+Guide to the troubleshooting repositories which import incompletely.
55+66+Overview
77+========
88+99+When you first import an external source code repository (or push new commits to
1010+a hosted repository), Phabricator imports those commits in the background.
1111+1212+While a repository is initially importing, some features won't work. While
1313+individual commits are importing, some of their metadata won't be available in
1414+the web UI.
1515+1616+Sometimes, the import process may hang or fail to complete. This document can
1717+help you understand the import process and troubleshoot problems with it.
1818+1919+2020+Understanding the Import Pipeline
2121+=================================
2222+2323+Phabricator first performs commit discovery on repositories. This examines
2424+a repository and identifies all the commits in it at a very shallow level,
2525+then creates stub objects for them. These stub objects primarily serve to
2626+assign various internal IDs to each commit.
2727+2828+Commit discovery occurs in the update phase, and you can learn more about
2929+updates in @{article:Diffusion User Guide: Repository Updates}.
3030+3131+After commits are discovered, background tasks are queued to actually import
3232+commits. These tasks do things like look at commit messages, trigger mentions
3333+and autoclose rules, cache changes, trigger Herald, publish feed stories and
3434+email, and apply Owners rules. You can learn more about some of these steps in
3535+@{article:Diffusion User Guide: Autoclose}.
3636+3737+Specifically, the import pipeline has four steps:
3838+3939+ - **Message**: Parses the commit message and author metadata.
4040+ - **Change**: Caches the paths the commit affected.
4141+ - **Owners**: Runs Owners rules.
4242+ - **Herald**: Runs Herald rules and publishes notifications.
4343+4444+These steps run in sequence for each commit, but all discovered commits import
4545+in parallel.
4646+4747+4848+Identifying Missing Steps
4949+=========================
5050+5151+There are a few major pieces of information you can look at to understand where
5252+the import process is stuck.
5353+5454+First, to identify which commits have missing import steps, run this command:
5555+5656+```
5757+phabricator/ $ ./bin/repository importing rXYZ
5858+```
5959+6060+That will show what work remains to be done. Each line shows a commit which
6161+is discovered but not imported, and the import steps that are remaining for
6262+that commit. Generally, the commit is stuck on the first step in the list.
6363+6464+Second, load the Daemon Console (at `/daemon/` in the web UI). This will show
6565+what work is currently being done and waiting to complete. The most important
6666+sections are "Queued Tasks" (work waiting in queue) and "Leased Tasks"
6767+(work currently being done).
6868+6969+Third, run this command to look at the daemon logs:
7070+7171+```
7272+phabricator/ $ ./bin/phd log
7373+```
7474+7575+This can show you any errors the daemons have encountered recently.
7676+7777+The next sections will walk through how to use this information to understand
7878+and resolve the issue.
7979+8080+8181+Handling Permanent Failures
8282+===========================
8383+8484+Some commits can not be imported, which will permanently stop a repository from
8585+fully importing. These are rare, but can be caused by unusual data in a
8686+repository, version peculiarities, or bugs in the importer.
8787+8888+Permanent failures usually look like a small number of commits stuck on the
8989+"Message" or "Change" steps in the output of `repository importing`. If you
9090+have a larger number of commits, it is less likely that there are any permanent
9191+problems.
9292+9393+In the Daemon console, permanent failures usually look like a small number of
9494+tasks in "Leased Tasks" with a large failure count. These tasks are retrying
9595+until they succeed, but a bug is permanently preventing them from succeeding,
9696+so they'll rack up a large number of retries over time.
9797+9898+In the daemon log, these commits usually emit specific errors showing why
9999+they're failing to import.
100100+101101+These failures are the easiest to identify and understand, and can often be
102102+resolved quickly. Choose some failing commit from the output of `bin/repository
103103+importing` and use this command to re-run any missing steps manually in the
104104+foreground:
105105+106106+```
107107+phabricator/ $ ./bin/repository reparse --importing --trace rXYZabcdef012...
108108+```
109109+110110+This command is always safe to run, no matter what the actual root cause of
111111+the problem is.
112112+113113+If this fails with an error, you've likely identified a problem with
114114+Phabricator. Collect as much information as you can about what makes the commit
115115+special and file a bug in the upstream by following the instructions in
116116+@{article:Contributing Bug Reports}.
117117+118118+If the commit imports cleanly, this is more likely to be caused by some other
119119+issue.
120120+121121+122122+Handling Temporary Failures
123123+===========================
124124+125125+Some commits may temporarily fail to import: perhaps the network or services
126126+may have briefly been down, or some configuration wasn't quite right, or the
127127+daemons were killed halfway through the work.
128128+129129+These commits will retry eventually and usually succeed, but some of the retry
130130+time limits are very conserative (up to 24 hours) and you might not want to
131131+wait that long.
132132+133133+In the Daemon console, temporarily failures usually look like tasks in the
134134+"Leased Tasks" column with a large "Expires" value but a low "Failures" count
135135+(usually 0 or 1). The "Expires" column is showing how long Phabricator is
136136+waiting to retry these tasks.
137137+138138+In the daemon log, these temporary failures might have created log entries, but
139139+might also not have. For example, if the failure was rooted in a network issue
140140+that probably will create a log entry, but if the faiulre was rooted in the
141141+daemons being abruptly killed that may not create a log entry.
142142+143143+You can follow the instructions from "Handling Permanent Failures" above to
144144+reparse steps individually to look for an error that represents a root cause,
145145+but sometimes this can happen because of some transient issue which won't be
146146+identifiable.
147147+148148+The easiest way to fix this is to restart the daemons. When you restart
149149+daemons, all task leases are immediately expired, so any tasks waiting for a
150150+long time will run right away:
151151+152152+```
153153+phabricator/ $ ./bin/phd restart
154154+```
155155+156156+This command is always safe to run, no matter what the actual root cause of
157157+the problem is.
158158+159159+After restarting the daemons, any pending tasks should be able to retry
160160+immediately.
161161+162162+For more information on managing the daemons, see
163163+@{article:Managing Daemons with phd}.
164164+165165+166166+Forced Parsing
167167+==============
168168+169169+In rare cases, the actual tasks may be lost from the task queue. Usually, they
170170+have been stolen by gremlins or spritied away by ghosts, or someone may have
171171+been too ambitious with running manual SQL commands and deleted a bunch of
172172+extra things they shouldn't have.
173173+174174+There is no normal set of conditions under which this should occur, but you can
175175+force Phabricator to re-queue the tasks to recover from it if it does occur.
176176+177177+This will look like missing steps in `repository importing`, but nothing in the
178178+"Queued Tasks" or "Leased Tasks" sections of the daemon console. The daemon
179179+logs will also be empty, since the tasks have vanished.
180180+181181+To re-queue parse tasks for a repository, run this command, which will queue
182182+up all of the missing work in `repository importing`:
183183+184184+```
185185+phabricator/ $ ./bin/repository reparse --importing --all rXYZ
186186+```
187187+188188+This command may cause duplicate work to occur if you have misdiagnosed the
189189+problem and the tasks aren't actually lost. For example, it could queue a
190190+second task to perform publishing, which could cause Phabricator to send a
191191+second copy of email about the commit. Other than that, it is safe to run even
192192+if this isn't the problem.
193193+194194+After running this command, you should see tasks in "Queued Tasks" and "Leased
195195+Tasks" in the console which correspond to the commits in `repository
196196+importing`, and progress should resume.
197197+198198+199199+Forced Imports
200200+==============
201201+202202+In some cases, you might want to force a repository to be flagged as imported
203203+even though the import isn't complete. The most common and reasonable case
204204+where you might want to do this is if you've identified a permanent failure
205205+with a small number of commits (maybe just one) and reported it upstream, and
206206+are waiting for a fix. You might want to start using the repository immediately,
207207+even if a few things can't import yet.
208208+209209+You should be cautious about doing this. The "importing" flag controls
210210+publishing of notifications and email, so if you flag a repository as imported
211211+but it still has a lot of work queued, it may send an enormous amount of email
212212+as that work completes.
213213+214214+To mark a repository as imported even though it really isn't, run this
215215+command:
216216+217217+```
218218+phabricator/ $ ./bin/repository mark-imported rXYZ
219219+```
220220+221221+If you do this by mistake, you can reverse it later by using the
222222+`--mark-not-imported` flag.
223223+224224+225225+General Tips
226226+============
227227+228228+Broadly, `bin/repository` contains several useful debugging commands which
229229+let you figure out where failures are occuring. You can add the `--trace` flag
230230+to any command to get more details about what it is doing. For any command,
231231+you can use `help` to learn more about what it does and which flag it takes:
232232+233233+```
234234+phabricator/ $ bin/repository help <command>
235235+```
236236+237237+In particular, you can use flags with the the `repository reparse` command to
238238+manually run parse steps in the foreground, including re-running steps and
239239+running steps out of order.
240240+241241+242242+Next Steps
243243+==========
244244+245245+Continue by:
246246+247247+ - returning to the @{article:Diffusion User Guide}.
+18-4
src/docs/user/userguide/diffusion.diviner
···94949595= Next Steps =
96969797- - Learn about creating a symbol index at
9797+Continue by:
9898+9999+ - learning how to creating a symbol index at
98100 @{article:Diffusion User Guide: Symbol Indexes}; or
9999- - set up repository hosting with
101101+ - setting up repository hosting with
100102 @{article:Diffusion User Guide: Repository Hosting}; or
101101- - understand daemons in detail with @{article:Managing Daemons with phd}; or
102102- - give us feedback at @{article:Give Feedback! Get Support!}.
103103+ - managing repository hooks with
104104+ @{article:Diffusion User Guide: Commit Hooks}; or
105105+ - understanding daemons in more detail with
106106+ @{article:Managing Daemons with phd}.
107107+108108+If you're having trouble getting things working, these topic guides may be
109109+helpful:
110110+111111+ - get details about automatically closing tasks and revisions in response
112112+ to commits in @{article:Diffusion User Guide: Autoclose}; or
113113+ - understand how Phabricator updates repositories with
114114+ @{article:Diffusion User Guide: Repository Updates}; or
115115+ - fix issues with repository imports with
116116+ @{article:Troubleshooting Repository Imports}.
···5858 no containing branch was configured to autoclose.
5959 - //Field Not Present// This commit was processed before we implemented
6060 this diagnostic feature, and no information is available.
6161+6262+Next Steps
6363+==========
6464+6565+Continue by:
6666+6767+ - troubleshooting in greater depth with
6868+ @{article:Troubleshooting Repository Imports}.
+8
src/docs/user/userguide/diffusion_updates.diviner
···113113114114To catch potential issues with permissions, run this command as the same user
115115that the daemons run as.
116116+117117+Next Steps
118118+==========
119119+120120+Continue by:
121121+122122+ - troubleshooting in greater depth with
123123+ @{article:Troubleshooting Repository Imports}.
+40-1
src/view/phui/PHUIHeaderView.php
···362362 // NOTE: We'll do this even if the viewer has access to only one space, and
363363 // show them information about the existence of spaces if they click
364364 // through.
365365+ $use_space_policy = false;
365366 if ($object instanceof PhabricatorSpacesInterface) {
366367 $space_phid = PhabricatorSpacesNamespaceQuery::getObjectSpacePHID(
367368 $object);
···376377 if ($space_policy) {
377378 if ($space_policy->isStrongerThan($policy)) {
378379 $policy = $space_policy;
380380+ $use_space_policy = true;
379381 }
380382 }
381383 }
382384 }
383385386386+ $container_classes = array();
387387+ $container_classes[] = 'policy-header-callout';
384388 $phid = $object->getPHID();
385389390390+ // If we're going to show the object policy, try to determine if the object
391391+ // policy differs from the default policy. If it does, we'll call it out
392392+ // as changed.
393393+ if (!$use_space_policy) {
394394+ $default_policy = PhabricatorPolicyQuery::getDefaultPolicyForObject(
395395+ $viewer,
396396+ $object,
397397+ $view_capability);
398398+ if ($default_policy) {
399399+ if ($default_policy->getPHID() != $policy->getPHID()) {
400400+ $container_classes[] = 'policy-adjusted';
401401+ if ($default_policy->isStrongerThan($policy)) {
402402+ // The policy has strictly been weakened. For example, the
403403+ // default might be "All Users" and the current policy is "Public".
404404+ $container_classes[] = 'policy-adjusted-weaker';
405405+ } else if ($policy->isStrongerThan($default_policy)) {
406406+ // The policy has strictly been strengthened, and is now more
407407+ // restrictive than the default. For example, "All Users" has
408408+ // been replaced with "No One".
409409+ $container_classes[] = 'policy-adjusted-stronger';
410410+ } else {
411411+ // The policy has been adjusted but not strictly strengthened
412412+ // or weakened. For example, "Members of X" has been replaced with
413413+ // "Members of Y".
414414+ $container_classes[] = 'policy-adjusted-different';
415415+ }
416416+ }
417417+ }
418418+ }
419419+386420 $icon = id(new PHUIIconView())
387421 ->setIconFont($policy->getIcon().' bluegrey');
388422···395429 ),
396430 $policy->getShortName());
397431398398- return array($icon, $link);
432432+ return phutil_tag(
433433+ 'span',
434434+ array(
435435+ 'class' => implode(' ', $container_classes),
436436+ ),
437437+ array($icon, $link));
399438 }
400439401440}