@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Queue large ICS files for background import
Summary: Ref T11801. When a file is larger than 512KB, queue it for background import instead of trying to do it in the foreground, sinc we risk hitting `max_execution_time`.
Test Plan: {F1906943}
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T11801
Differential Revision: https://secure.phabricator.com/D16805
+108
-9
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
2145
2145
'PhabricatorCalendarImportOrphanLogType' => 'applications/calendar/importlog/PhabricatorCalendarImportOrphanLogType.php',
2146
2146
'PhabricatorCalendarImportPHIDType' => 'applications/calendar/phid/PhabricatorCalendarImportPHIDType.php',
2147
2147
'PhabricatorCalendarImportQuery' => 'applications/calendar/query/PhabricatorCalendarImportQuery.php',
2148
+
'PhabricatorCalendarImportQueueLogType' => 'applications/calendar/importlog/PhabricatorCalendarImportQueueLogType.php',
2148
2149
'PhabricatorCalendarImportReloadController' => 'applications/calendar/controller/PhabricatorCalendarImportReloadController.php',
2149
2150
'PhabricatorCalendarImportReloadTransaction' => 'applications/calendar/xaction/PhabricatorCalendarImportReloadTransaction.php',
2150
2151
'PhabricatorCalendarImportReloadWorker' => 'applications/calendar/worker/PhabricatorCalendarImportReloadWorker.php',
···
7012
7013
'PhabricatorCalendarImportOrphanLogType' => 'PhabricatorCalendarImportLogType',
7013
7014
'PhabricatorCalendarImportPHIDType' => 'PhabricatorPHIDType',
7014
7015
'PhabricatorCalendarImportQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
7016
+
'PhabricatorCalendarImportQueueLogType' => 'PhabricatorCalendarImportLogType',
7015
7017
'PhabricatorCalendarImportReloadController' => 'PhabricatorCalendarController',
7016
7018
'PhabricatorCalendarImportReloadTransaction' => 'PhabricatorCalendarImportTransactionType',
7017
7019
'PhabricatorCalendarImportReloadWorker' => 'PhabricatorWorker',
+2
-1
src/applications/calendar/editor/PhabricatorCalendarImportEditor.php
+2
-1
src/applications/calendar/editor/PhabricatorCalendarImportEditor.php
···
54
54
55
55
if ($should_reload) {
56
56
$import_engine = $object->getEngine();
57
-
$import_engine->importEventsFromSource($actor, $object);
57
+
$import_engine->importEventsFromSource($actor, $object, true);
58
58
}
59
59
60
60
if ($should_trigger) {
···
107
107
'class' => 'PhabricatorCalendarImportReloadWorker',
108
108
'data' => array(
109
109
'importPHID' => $object->getPHID(),
110
+
'via' => PhabricatorCalendarImportReloadWorker::VIA_TRIGGER,
110
111
),
111
112
'options' => array(
112
113
'objectPHID' => $object->getPHID(),
+6
-2
src/applications/calendar/import/PhabricatorCalendarICSFileImportEngine.php
+6
-2
src/applications/calendar/import/PhabricatorCalendarICSFileImportEngine.php
···
65
65
66
66
public function importEventsFromSource(
67
67
PhabricatorUser $viewer,
68
-
PhabricatorCalendarImport $import) {
68
+
PhabricatorCalendarImport $import,
69
+
$should_queue) {
69
70
70
71
$phid_key = PhabricatorCalendarImportICSFileTransaction::PARAMKEY_FILE;
71
72
$file_phid = $import->getParameter($phid_key);
···
83
84
84
85
$data = $file->loadFileData();
85
86
87
+
if ($should_queue && $this->shouldQueueDataImport($data)) {
88
+
return $this->queueDataImport($import, $data);
89
+
}
90
+
86
91
return $this->importICSData($viewer, $import, $data);
87
92
}
88
-
89
93
90
94
public function canDisable(
91
95
PhabricatorUser $viewer,
+6
-1
src/applications/calendar/import/PhabricatorCalendarICSURIImportEngine.php
+6
-1
src/applications/calendar/import/PhabricatorCalendarICSURIImportEngine.php
···
77
77
78
78
public function importEventsFromSource(
79
79
PhabricatorUser $viewer,
80
-
PhabricatorCalendarImport $import) {
80
+
PhabricatorCalendarImport $import,
81
+
$should_queue) {
81
82
82
83
$uri_key = PhabricatorCalendarImportICSURITransaction::PARAMKEY_URI;
83
84
$uri = $import->getParameter($uri_key);
···
102
103
));
103
104
104
105
$data = $file->loadFileData();
106
+
107
+
if ($should_queue && $this->shouldQueueDataImport($data)) {
108
+
return $this->queueDataImport($import, $data);
109
+
}
105
110
106
111
return $this->importICSData($viewer, $import, $data);
107
112
}
+35
-2
src/applications/calendar/import/PhabricatorCalendarImportEngine.php
+35
-2
src/applications/calendar/import/PhabricatorCalendarImportEngine.php
···
3
3
abstract class PhabricatorCalendarImportEngine
4
4
extends Phobject {
5
5
6
+
const QUEUE_BYTE_LIMIT = 524288;
7
+
6
8
final public function getImportEngineType() {
7
9
return $this->getPhobjectClassConstant('ENGINETYPE', 64);
8
10
}
9
-
10
11
11
12
abstract public function getImportEngineName();
12
13
abstract public function getImportEngineTypeName();
···
27
28
28
29
abstract public function importEventsFromSource(
29
30
PhabricatorUser $viewer,
30
-
PhabricatorCalendarImport $import);
31
+
PhabricatorCalendarImport $import,
32
+
$should_queue);
31
33
32
34
abstract public function canDisable(
33
35
PhabricatorUser $viewer,
···
567
569
568
570
return (bool)$any_event;
569
571
}
572
+
573
+
final protected function shouldQueueDataImport($data) {
574
+
return (strlen($data) > self::QUEUE_BYTE_LIMIT);
575
+
}
576
+
577
+
final protected function queueDataImport(
578
+
PhabricatorCalendarImport $import,
579
+
$data) {
580
+
581
+
$import->newLogMessage(
582
+
PhabricatorCalendarImportQueueLogType::LOGTYPE,
583
+
array(
584
+
'data.size' => strlen($data),
585
+
'data.limit' => self::QUEUE_BYTE_LIMIT,
586
+
));
587
+
588
+
// When we queue on this pathway, we're queueing in response to an explicit
589
+
// user action (like uploading a big `.ics` file), so we queue at normal
590
+
// priority instead of bulk/import priority.
591
+
592
+
PhabricatorWorker::scheduleTask(
593
+
'PhabricatorCalendarImportReloadWorker',
594
+
array(
595
+
'importPHID' => $import->getPHID(),
596
+
'via' => PhabricatorCalendarImportReloadWorker::VIA_BACKGROUND,
597
+
),
598
+
array(
599
+
'objectPHID' => $import->getPHID(),
600
+
));
601
+
}
602
+
570
603
571
604
}
+40
src/applications/calendar/importlog/PhabricatorCalendarImportQueueLogType.php
+40
src/applications/calendar/importlog/PhabricatorCalendarImportQueueLogType.php
···
1
+
<?php
2
+
3
+
final class PhabricatorCalendarImportQueueLogType
4
+
extends PhabricatorCalendarImportLogType {
5
+
6
+
const LOGTYPE = 'queue';
7
+
8
+
public function getDisplayType(
9
+
PhabricatorUser $viewer,
10
+
PhabricatorCalendarImportLog $log) {
11
+
return pht('Queued');
12
+
}
13
+
14
+
public function getDisplayDescription(
15
+
PhabricatorUser $viewer,
16
+
PhabricatorCalendarImportLog $log) {
17
+
18
+
$size = $log->getParameter('data.size');
19
+
$limit = $log->getParameter('data.limit');
20
+
21
+
return pht(
22
+
'Queued for background import: data size (%s) exceeds limit for '.
23
+
'immediate processing (%s).',
24
+
phutil_format_bytes($size),
25
+
phutil_format_bytes($limit));
26
+
}
27
+
28
+
public function getDisplayIcon(
29
+
PhabricatorUser $viewer,
30
+
PhabricatorCalendarImportLog $log) {
31
+
return 'fa-sort-amount-desc';
32
+
}
33
+
34
+
public function getDisplayColor(
35
+
PhabricatorUser $viewer,
36
+
PhabricatorCalendarImportLog $log) {
37
+
return 'blue';
38
+
}
39
+
40
+
}
+9
-1
src/applications/calendar/importlog/PhabricatorCalendarImportTriggerLogType.php
+9
-1
src/applications/calendar/importlog/PhabricatorCalendarImportTriggerLogType.php
···
14
14
public function getDisplayDescription(
15
15
PhabricatorUser $viewer,
16
16
PhabricatorCalendarImportLog $log) {
17
-
return pht('Triggered a periodic update.');
17
+
18
+
$via = $log->getParameter('via');
19
+
switch ($via) {
20
+
case PhabricatorCalendarImportReloadWorker::VIA_BACKGROUND:
21
+
return pht('Started background processing.');
22
+
case PhabricatorCalendarImportReloadWorker::VIA_TRIGGER:
23
+
default:
24
+
return pht('Triggered a periodic update.');
25
+
}
18
26
}
19
27
20
28
public function getDisplayIcon(
+8
-2
src/applications/calendar/worker/PhabricatorCalendarImportReloadWorker.php
+8
-2
src/applications/calendar/worker/PhabricatorCalendarImportReloadWorker.php
···
2
2
3
3
final class PhabricatorCalendarImportReloadWorker extends PhabricatorWorker {
4
4
5
+
const VIA_TRIGGER = 'trigger';
6
+
const VIA_BACKGROUND = 'background';
7
+
5
8
protected function doWork() {
6
9
$import = $this->loadImport();
7
10
$viewer = PhabricatorUser::getOmnipotentUser();
···
18
21
19
22
$import_engine = $import->getEngine();
20
23
24
+
$data = $this->getTaskData();
21
25
$import->newLogMessage(
22
26
PhabricatorCalendarImportTriggerLogType::LOGTYPE,
23
-
array());
27
+
array(
28
+
'via' => idx($data, 'via', self::VIA_TRIGGER),
29
+
));
24
30
25
-
$import_engine->importEventsFromSource($author, $import);
31
+
$import_engine->importEventsFromSource($author, $import, false);
26
32
}
27
33
28
34
private function loadImport() {