@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Simplify construction and execution of Differential queries for "responsible" users
Summary:
Currently, when querying for responsible users (revisions where a given user is either the author or a reviewer), we do this:
- If the query passes a bunch of hard-coded special cases, use a special hard-coded UNION.
- Otherwise, use a very complicated JOIN/WHERE clause.
This is bad for several reasons:
- Tons and tons of hard-coding and special casing.
- The JOIN/WHERE clause performs very poorly for large datasets.
- (As a material consequence, the homepage issues a responsible query which barely misses the hard-coded special cases and goes down the slow path.)
Instead, //always// use the UNION strategy to execute a "responsible" query. Specifically, if we have responsible PHIDs, temporarily add them to the author list and build a normal query, then repeat for reviewers, then UNION any clauses we built.
Fixes T3377. Ref T603. Ref T2625. Depends on D6342.
There's various folklore about UNION ALL / UNION DISTINCT performance. UNION DISTINCT is simpler here and the number of rows is small, although we could use UNION ALL in the form:
SELECT * FROM ((SELECT ...) UNION ALL (SELECT ...) ORDER) GROUP LIMIT
...if we find that there's some performance benefit at some point.
Test Plan: Used DarkConsole to examine queries. Viewed home page and Differential dashboard/authors/subscribers.
Reviewers: btrahan
Reviewed By: btrahan
CC: aran
Maniphest Tasks: T603, T2625, T3377
Differential Revision: https://secure.phabricator.com/D6343
+70
-93
+70
-93
src/applications/differential/query/DifferentialRevisionQuery.php
+70
-93
src/applications/differential/query/DifferentialRevisionQuery.php
···
376
376
$table = new DifferentialRevision();
377
377
$conn_r = $table->establishConnection('r');
378
378
379
-
if ($this->shouldUseResponsibleFastPath()) {
380
-
$data = $this->loadDataUsingResponsibleFastPath();
381
-
} else {
382
-
$data = $this->loadData();
383
-
}
379
+
$data = $this->loadData();
384
380
385
381
$revisions = $table->loadAllFromArray($data);
386
382
···
417
413
return head($this->execute());
418
414
}
419
415
420
-
421
-
/**
422
-
* Determine if we should execute an optimized, fast-path query to fetch
423
-
* open revisions for one responsible user. This is used by the Differential
424
-
* dashboard and much faster when executed as a UNION ALL than with JOIN
425
-
* and WHERE, which is why we special case it.
426
-
*/
427
-
private function shouldUseResponsibleFastPath() {
428
-
if ((count($this->responsibles) == 1) &&
429
-
($this->status == self::STATUS_OPEN) &&
430
-
($this->order == self::ORDER_MODIFIED) &&
431
-
!$this->offset &&
432
-
!$this->limit &&
433
-
!$this->subscribers &&
434
-
!$this->reviewers &&
435
-
!$this->ccs &&
436
-
!$this->authors &&
437
-
!$this->revIDs &&
438
-
!$this->commitHashes &&
439
-
!$this->phids &&
440
-
!$this->branches &&
441
-
!$this->arcanistProjectPHIDs) {
442
-
return true;
443
-
}
444
-
return false;
445
-
}
446
-
447
-
448
-
private function loadDataUsingResponsibleFastPath() {
449
-
$table = new DifferentialRevision();
450
-
$conn_r = $table->establishConnection('r');
451
-
452
-
$responsible_phid = reset($this->responsibles);
453
-
$open_statuses = array(
454
-
ArcanistDifferentialRevisionStatus::NEEDS_REVIEW,
455
-
ArcanistDifferentialRevisionStatus::NEEDS_REVISION,
456
-
ArcanistDifferentialRevisionStatus::ACCEPTED,
457
-
);
458
-
459
-
return queryfx_all(
460
-
$conn_r,
461
-
'SELECT * FROM %T WHERE authorPHID = %s AND status IN (%Ld)
462
-
UNION ALL
463
-
SELECT r.* FROM %T r JOIN %T rel
464
-
ON rel.revisionID = r.id
465
-
AND rel.relation = %s
466
-
AND rel.objectPHID = %s
467
-
WHERE r.status IN (%Ld) ORDER BY dateModified DESC',
468
-
$table->getTableName(),
469
-
$responsible_phid,
470
-
$open_statuses,
471
-
472
-
$table->getTableName(),
473
-
DifferentialRevision::RELATIONSHIP_TABLE,
474
-
DifferentialRevision::RELATION_REVIEWER,
475
-
$responsible_phid,
476
-
$open_statuses);
477
-
}
478
-
479
416
private function loadData() {
480
417
$table = new DifferentialRevision();
481
418
$conn_r = $table->establishConnection('r');
···
506
443
}
507
444
}
508
445
446
+
$selects = array();
447
+
448
+
// NOTE: If the query includes "responsiblePHIDs", we execute it as a
449
+
// UNION of revisions they own and revisions they're reviewing. This has
450
+
// much better performance than doing it with JOIN/WHERE.
451
+
if ($this->responsibles) {
452
+
$basic_authors = $this->authors;
453
+
$basic_reviewers = $this->reviewers;
454
+
try {
455
+
// Build the query where the responsible users are authors.
456
+
$this->authors = array_merge($basic_authors, $this->responsibles);
457
+
$this->reviewers = $basic_reviewers;
458
+
$selects[] = $this->buildSelectStatement($conn_r);
459
+
460
+
// Build the query where the responsible users are reviewers.
461
+
$this->authors = $basic_authors;
462
+
$this->reviewers = array_merge($basic_reviewers, $this->responsibles);
463
+
$selects[] = $this->buildSelectStatement($conn_r);
464
+
465
+
// Put everything back like it was.
466
+
$this->authors = $basic_authors;
467
+
$this->reviewers = $basic_reviewers;
468
+
} catch (Exception $ex) {
469
+
$this->authors = $basic_authors;
470
+
$this->reviewers = $basic_reviewers;
471
+
throw $ex;
472
+
}
473
+
} else {
474
+
$selects[] = $this->buildSelectStatement($conn_r);
475
+
}
476
+
477
+
if (count($selects) > 1) {
478
+
$query = qsprintf(
479
+
$conn_r,
480
+
'%Q %Q %Q',
481
+
implode(' UNION DISTINCT ', $selects),
482
+
$this->buildOrderByClause($conn_r, $is_global = true),
483
+
$this->buildLimitClause($conn_r));
484
+
} else {
485
+
$query = head($selects);
486
+
}
487
+
488
+
return queryfx_all($conn_r, '%Q', $query);
489
+
}
490
+
491
+
private function buildSelectStatement(AphrontDatabaseConnection $conn_r) {
492
+
$table = new DifferentialRevision();
493
+
509
494
$select = qsprintf(
510
495
$conn_r,
511
496
'SELECT r.* FROM %T r',
···
515
500
$where = $this->buildWhereClause($conn_r);
516
501
$group_by = $this->buildGroupByClause($conn_r);
517
502
$order_by = $this->buildOrderByClause($conn_r);
503
+
$limit = $this->buildLimitClause($conn_r);
518
504
505
+
return qsprintf(
506
+
$conn_r,
507
+
'(%Q %Q %Q %Q %Q %Q)',
508
+
$select,
509
+
$joins,
510
+
$where,
511
+
$group_by,
512
+
$order_by,
513
+
$limit);
514
+
}
515
+
516
+
private function buildLimitClause(AphrontDatabaseConnection $conn_r) {
519
517
$limit = '';
520
518
if ($this->offset || $this->limit) {
521
519
$limit = qsprintf(
···
525
523
$this->limit);
526
524
}
527
525
528
-
return queryfx_all(
529
-
$conn_r,
530
-
'%Q %Q %Q %Q %Q %Q',
531
-
$select,
532
-
$joins,
533
-
$where,
534
-
$group_by,
535
-
$order_by,
536
-
$limit);
526
+
return $limit;
537
527
}
538
-
539
528
540
529
/* -( Internals )---------------------------------------------------------- */
541
530
···
596
585
$this->subscribers);
597
586
}
598
587
599
-
if ($this->responsibles) {
600
-
$joins[] = qsprintf(
601
-
$conn_r,
602
-
'LEFT JOIN %T responsibles_rel ON responsibles_rel.revisionID = r.id '.
603
-
'AND responsibles_rel.relation = %s '.
604
-
'AND responsibles_rel.objectPHID in (%Ls)',
605
-
DifferentialRevision::RELATIONSHIP_TABLE,
606
-
DifferentialRevision::RELATION_REVIEWER,
607
-
$this->responsibles);
608
-
}
609
-
610
588
$joins = implode(' ', $joins);
611
589
612
590
return $joins;
···
673
651
$conn_r,
674
652
'r.phid IN (%Ls)',
675
653
$this->phids);
676
-
}
677
-
678
-
if ($this->responsibles) {
679
-
$where[] = qsprintf(
680
-
$conn_r,
681
-
'(responsibles_rel.objectPHID IS NOT NULL OR r.authorPHID IN (%Ls))',
682
-
$this->responsibles);
683
654
}
684
655
685
656
if ($this->branches) {
···
786
757
/**
787
758
* @task internal
788
759
*/
789
-
private function buildOrderByClause($conn_r) {
760
+
private function buildOrderByClause($conn_r, $is_global = false) {
790
761
switch ($this->order) {
791
762
case self::ORDER_MODIFIED:
763
+
if ($is_global) {
764
+
return 'ORDER BY dateModified DESC';
765
+
}
792
766
return 'ORDER BY r.dateModified DESC';
793
767
case self::ORDER_CREATED:
768
+
if ($is_global) {
769
+
return 'ORDER BY dateCreated DESC';
770
+
}
794
771
return 'ORDER BY r.dateCreated DESC';
795
772
case self::ORDER_PATH_MODIFIED:
796
773
if (!$this->pathIDs) {