@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add a Nuance GitHub repository source and basic polling
Summary: Ref T10537. Ref T10538. This calls GitHub, sorta?
Test Plan:
```
$ ./bin/nuance import --source poem
<cursor:events.repository> Polling GitHub Repository API endpoint "/repos/epriestley/poems/events".
<cursor:events.repository> This key has 4,988 remaining API request(s), limit resets in 1,871 second(s).
<cursor:events.repository> ETag for this request was ""4abdd3d66ad5ca38f5117b094e76f4ba"".
array(4) {
[0]=>
array(7) {
["id"]=>
string(10) "3733510485"
...
```
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T10537, T10538
Differential Revision: https://secure.phabricator.com/D15439
+325
-30
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
1421
1421
'NuanceController' => 'applications/nuance/controller/NuanceController.php',
1422
1422
'NuanceCreateItemConduitAPIMethod' => 'applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php',
1423
1423
'NuanceDAO' => 'applications/nuance/storage/NuanceDAO.php',
1424
+
'NuanceGitHubRepositoryImportCursor' => 'applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php',
1425
+
'NuanceGitHubRepositorySourceDefinition' => 'applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php',
1424
1426
'NuanceImportCursor' => 'applications/nuance/cursor/NuanceImportCursor.php',
1425
1427
'NuanceImportCursorData' => 'applications/nuance/storage/NuanceImportCursorData.php',
1426
1428
'NuanceImportCursorDataQuery' => 'applications/nuance/query/NuanceImportCursorDataQuery.php',
···
5668
5670
'NuanceController' => 'PhabricatorController',
5669
5671
'NuanceCreateItemConduitAPIMethod' => 'NuanceConduitAPIMethod',
5670
5672
'NuanceDAO' => 'PhabricatorLiskDAO',
5673
+
'NuanceGitHubRepositoryImportCursor' => 'NuanceImportCursor',
5674
+
'NuanceGitHubRepositorySourceDefinition' => 'NuanceSourceDefinition',
5671
5675
'NuanceImportCursor' => 'Phobject',
5672
5676
'NuanceImportCursorData' => 'NuanceDAO',
5673
5677
'NuanceImportCursorDataQuery' => 'NuanceQuery',
+114
src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php
+114
src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php
···
1
+
<?php
2
+
3
+
final class NuanceGitHubRepositoryImportCursor
4
+
extends NuanceImportCursor {
5
+
6
+
const CURSORTYPE = 'github.repository';
7
+
8
+
protected function shouldPullDataFromSource() {
9
+
$now = PhabricatorTime::getNow();
10
+
11
+
// Respect GitHub's poll interval header. If we made a request recently,
12
+
// don't make another one until we've waited long enough.
13
+
$ttl = $this->getCursorProperty('github.poll.ttl');
14
+
if ($ttl && ($ttl >= $now)) {
15
+
$this->logInfo(
16
+
pht(
17
+
'Respecting "%s": waiting for %s second(s) to poll GitHub.',
18
+
'X-Poll-Interval',
19
+
new PhutilNumber(1 + ($ttl - $now))));
20
+
21
+
return false;
22
+
}
23
+
24
+
// Respect GitHub's API rate limiting. If we've exceeded the rate limit,
25
+
// wait until it resets to try again.
26
+
$limit = $this->getCursorProperty('github.limit.ttl');
27
+
if ($limit && ($limit >= $now)) {
28
+
$this->logInfo(
29
+
pht(
30
+
'Respecting "%s": waiting for %s second(s) to poll GitHub.',
31
+
'X-RateLimit-Reset',
32
+
new PhutilNumber(1 + ($limit - $now))));
33
+
return false;
34
+
}
35
+
36
+
return true;
37
+
}
38
+
39
+
protected function pullDataFromSource() {
40
+
$source = $this->getSource();
41
+
42
+
$user = $source->getSourceProperty('github.user');
43
+
$repository = $source->getSourceProperty('github.repository');
44
+
$api_token = $source->getSourceProperty('github.token');
45
+
46
+
$uri = "/repos/{$user}/{$repository}/events";
47
+
$data = array();
48
+
49
+
$future = id(new PhutilGitHubFuture())
50
+
->setAccessToken($api_token)
51
+
->setRawGitHubQuery($uri, $data);
52
+
53
+
$etag = $this->getCursorProperty('github.poll.etag');
54
+
if ($etag) {
55
+
$future->addHeader('If-None-Match', $etag);
56
+
}
57
+
58
+
$this->logInfo(
59
+
pht(
60
+
'Polling GitHub Repository API endpoint "%s".',
61
+
$uri));
62
+
$response = $future->resolve();
63
+
64
+
// Do this first: if we hit the rate limit, we get a response but the
65
+
// body isn't valid.
66
+
$this->updateRateLimits($response);
67
+
68
+
// This means we hit a rate limit or a "Not Modified" because of the "ETag"
69
+
// header. In either case, we should bail out.
70
+
if ($response->getStatus()->isError()) {
71
+
// TODO: Save cursor data!
72
+
return false;
73
+
}
74
+
75
+
$this->updateETag($response);
76
+
77
+
var_dump($response->getBody());
78
+
}
79
+
80
+
private function updateRateLimits(PhutilGitHubResponse $response) {
81
+
$remaining = $response->getHeaderValue('X-RateLimit-Remaining');
82
+
$limit_reset = $response->getHeaderValue('X-RateLimit-Reset');
83
+
$now = PhabricatorTime::getNow();
84
+
85
+
$limit_ttl = null;
86
+
if (strlen($remaining)) {
87
+
$remaining = (int)$remaining;
88
+
if (!$remaining) {
89
+
$limit_ttl = (int)$limit_reset;
90
+
}
91
+
}
92
+
93
+
$this->setCursorProperty('github.limit.ttl', $limit_ttl);
94
+
95
+
$this->logInfo(
96
+
pht(
97
+
'This key has %s remaining API request(s), '.
98
+
'limit resets in %s second(s).',
99
+
new PhutilNumber($remaining),
100
+
new PhutilNumber($limit_reset - $now)));
101
+
}
102
+
103
+
private function updateETag(PhutilGitHubResponse $response) {
104
+
$etag = $response->getHeaderValue('ETag');
105
+
106
+
$this->setCursorProperty('github.poll.etag', $etag);
107
+
108
+
$this->logInfo(
109
+
pht(
110
+
'ETag for this request was "%s".',
111
+
$etag));
112
+
}
113
+
114
+
}
+90
-2
src/applications/nuance/cursor/NuanceImportCursor.php
+90
-2
src/applications/nuance/cursor/NuanceImportCursor.php
···
2
2
3
3
abstract class NuanceImportCursor extends Phobject {
4
4
5
+
private $cursorData;
6
+
private $cursorKey;
7
+
private $source;
8
+
9
+
abstract protected function shouldPullDataFromSource();
10
+
abstract protected function pullDataFromSource();
11
+
12
+
final public function getCursorType() {
13
+
return $this->getPhobjectClassConstant('CURSORTYPE', 32);
14
+
}
15
+
16
+
public function setCursorData(NuanceImportCursorData $cursor_data) {
17
+
$this->cursorData = $cursor_data;
18
+
return $this;
19
+
}
20
+
21
+
public function getCursorData() {
22
+
return $this->cursorData;
23
+
}
24
+
25
+
public function setSource($source) {
26
+
$this->source = $source;
27
+
return $this;
28
+
}
29
+
30
+
public function getSource() {
31
+
return $this->source;
32
+
}
33
+
34
+
public function setCursorKey($cursor_key) {
35
+
$this->cursorKey = $cursor_key;
36
+
return $this;
37
+
}
38
+
39
+
public function getCursorKey() {
40
+
return $this->cursorKey;
41
+
}
42
+
5
43
final public function importFromSource() {
6
-
// TODO: Perhaps, do something.
7
-
return false;
44
+
if (!$this->shouldPullDataFromSource()) {
45
+
return false;
46
+
}
47
+
48
+
$source = $this->getSource();
49
+
$key = $this->getCursorKey();
50
+
51
+
$parts = array(
52
+
'nsc',
53
+
$source->getID(),
54
+
PhabricatorHash::digestToLength($key, 20),
55
+
);
56
+
$lock_name = implode('.', $parts);
57
+
58
+
$lock = PhabricatorGlobalLock::newLock($lock_name);
59
+
$lock->lock(1);
60
+
61
+
try {
62
+
$more_data = $this->pullDataFromSource();
63
+
} catch (Exception $ex) {
64
+
$lock->unlock();
65
+
throw $ex;
66
+
}
67
+
68
+
$lock->unlock();
69
+
70
+
return $more_data;
71
+
}
72
+
73
+
final public function newEmptyCursorData(NuanceSource $source) {
74
+
return id(new NuanceImportCursorData())
75
+
->setCursorKey($this->getCursorKey())
76
+
->setCursorType($this->getCursorType())
77
+
->setSourcePHID($source->getPHID());
78
+
}
79
+
80
+
final protected function logInfo($message) {
81
+
echo tsprintf(
82
+
"<cursor:%s> %s\n",
83
+
$this->getCursorKey(),
84
+
$message);
85
+
86
+
return $this;
87
+
}
88
+
89
+
final protected function getCursorProperty($key, $default = null) {
90
+
return $this->getCursorData()->getCursorProperty($key, $default);
91
+
}
92
+
93
+
final protected function setCursorProperty($key, $value) {
94
+
$this->getCursorData()->setCursorProperty($key, $value);
95
+
return $this;
8
96
}
9
97
10
98
}
+3
-3
src/applications/nuance/management/NuanceManagementImportWorkflow.php
+3
-3
src/applications/nuance/management/NuanceManagementImportWorkflow.php
+29
src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php
+29
src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php
···
1
+
<?php
2
+
3
+
final class NuanceGitHubRepositorySourceDefinition
4
+
extends NuanceSourceDefinition {
5
+
6
+
public function getName() {
7
+
return pht('GitHub Repository');
8
+
}
9
+
10
+
public function getSourceDescription() {
11
+
return pht('Import issues and pull requests from a GitHub repository.');
12
+
}
13
+
14
+
public function getSourceTypeConstant() {
15
+
return 'github.repository';
16
+
}
17
+
18
+
public function hasImportCursors() {
19
+
return true;
20
+
}
21
+
22
+
protected function newImportCursors() {
23
+
return array(
24
+
id(new NuanceGitHubRepositoryImportCursor())
25
+
->setCursorKey('events.repository'),
26
+
);
27
+
}
28
+
29
+
}
-9
src/applications/nuance/source/NuancePhabricatorFormSourceDefinition.php
-9
src/applications/nuance/source/NuancePhabricatorFormSourceDefinition.php
···
26
26
return $actions;
27
27
}
28
28
29
-
public function updateItems() {
30
-
return null;
31
-
}
32
-
33
-
public function renderView() {}
34
-
35
-
public function renderListView() {}
36
-
37
-
38
29
public function handleActionRequest(AphrontRequest $request) {
39
30
$viewer = $request->getViewer();
40
31
+66
-15
src/applications/nuance/source/NuanceSourceDefinition.php
+66
-15
src/applications/nuance/source/NuanceSourceDefinition.php
···
53
53
pht('This source has no input cursors.'));
54
54
}
55
55
56
-
return $this->newImportCursors();
56
+
$source = $this->getSource();
57
+
$cursors = $this->newImportCursors();
58
+
59
+
$data = id(new NuanceImportCursorDataQuery())
60
+
->setViewer(PhabricatorUser::getOmnipotentUser())
61
+
->withSourcePHIDs(array($source->getPHID()))
62
+
->execute();
63
+
$data = mpull($data, 'getCursorKey');
64
+
65
+
$map = array();
66
+
foreach ($cursors as $cursor) {
67
+
if (!($cursor instanceof NuanceImportCursor)) {
68
+
throw new Exception(
69
+
pht(
70
+
'Source "%s" (of class "%s") returned an invalid value from '.
71
+
'method "%s": all values must be objects of class "%s".',
72
+
$this->getName(),
73
+
get_class($this),
74
+
'newImportCursors()',
75
+
'NuanceImportCursor'));
76
+
}
77
+
78
+
$key = $cursor->getCursorKey();
79
+
if (!strlen($key)) {
80
+
throw new Exception(
81
+
pht(
82
+
'Source "%s" (of class "%s") returned an import cursor with '.
83
+
'a missing key from "%s". Each cursor must have a unique, '.
84
+
'nonempty key.',
85
+
$this->getName(),
86
+
get_class($this),
87
+
'newImportCursors()'));
88
+
}
89
+
90
+
$other = idx($map, $key);
91
+
if ($other) {
92
+
throw new Exception(
93
+
pht(
94
+
'Source "%s" (of class "%s") returned two cursors from method '.
95
+
'"%s" with the same key ("%s"). Each cursor must have a unique '.
96
+
'key.',
97
+
$this->getName(),
98
+
get_class($this),
99
+
'newImportCursors()',
100
+
$key));
101
+
}
102
+
103
+
$map[$key] = $cursor;
104
+
105
+
$cursor->setSource($source);
106
+
107
+
$cursor_data = idx($data, $key);
108
+
if (!$cursor_data) {
109
+
$cursor_data = $cursor->newEmptyCursorData($source);
110
+
}
111
+
112
+
$cursor->setCursorData($cursor_data);
113
+
}
114
+
115
+
return $cursors;
57
116
}
58
117
59
118
protected function newImportCursors() {
···
79
138
*/
80
139
abstract public function getSourceTypeConstant();
81
140
82
-
/**
83
-
* Code to create and update @{class:NuanceItem}s and
84
-
* @{class:NuanceRequestor}s via daemons goes here.
85
-
*
86
-
* If that does not make sense for the @{class:NuanceSource} you are
87
-
* defining, simply return null. For example,
88
-
* @{class:NuancePhabricatorFormSourceDefinition} since these are one-way
89
-
* contact forms.
90
-
*/
91
-
abstract public function updateItems();
92
-
93
-
abstract public function renderView();
94
-
95
-
abstract public function renderListView();
141
+
public function renderView() {
142
+
return null;
143
+
}
96
144
145
+
public function renderListView() {
146
+
return null;
147
+
}
97
148
98
149
protected function newItemFromProperties(
99
150
NuanceRequestor $requestor,
+9
src/applications/nuance/storage/NuanceImportCursorData.php
+9
src/applications/nuance/storage/NuanceImportCursorData.php
···
32
32
NuanceImportCursorPHIDType::TYPECONST);
33
33
}
34
34
35
+
public function getCursorProperty($key, $default = null) {
36
+
return idx($this->properties, $key, $default);
37
+
}
38
+
39
+
public function setCursorProperty($key, $value) {
40
+
$this->properties[$key] = $value;
41
+
return $this;
42
+
}
43
+
35
44
}
+10
-1
src/applications/nuance/storage/NuanceSource.php
+10
-1
src/applications/nuance/storage/NuanceSource.php
···
8
8
9
9
protected $name;
10
10
protected $type;
11
-
protected $data;
11
+
protected $data = array();
12
12
protected $mailKey;
13
13
protected $viewPolicy;
14
14
protected $editPolicy;
···
79
79
80
80
public function attachDefinition(NuanceSourceDefinition $definition) {
81
81
$this->definition = $definition;
82
+
return $this;
83
+
}
84
+
85
+
public function getSourceProperty($key, $default = null) {
86
+
return idx($this->data, $key, $default);
87
+
}
88
+
89
+
public function setSourceProperty($key, $value) {
90
+
$this->data[$key] = $value;
82
91
return $this;
83
92
}
84
93