@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Port Diviner Core to Phabricator
Summary:
This implements most/all of the difficult parts of Diviner on top of Phabricator instead of as standalone components. See T988. In particular, here are the things I want to fix:
**Performance** The Diviner parser works in two stages. The first stage breaks source files into "Atoms". The second stage renders atoms into a display format (e.g., HTML). Diviner currently has a good caching story on the first step of the pipeline, but zero caching in the second step. This means it's very slow, even for a fairly small project like Phabricator. We must re-render every piece of documentation every time, instead of only changed documentation. Most of this diff concerns itself with addressing this problem. There's a fairly large explanatory comment about it, but the trickiest part is that when an atom changes, other atoms (defined in other places) may also change -- for example, if `class B extends A`, editing A should dirty B, even if B is in an entirely different file. We perform analysis in two stages to propagate these changes: first detecting direct changes, then detecting indirect changes. This isn't completely implemented -- we need to propagate 'extends' through more levels -- but I believe it's structurally correct and good enough until we actually document classes.
**Inheritance** Diviner currently has a very weak story on inheritance. I want to inherit a lot more metas/docs. If an interface documents a method, we should just pull that documentation in to every implementation by default (implementations can still override it if they want). It can be shown in grey or something, but it should be desirable and correct to omit documentation of a method implementation when you are implementing a parent. Similarly, I want to pull in inherited methods and @tasks and such. This diff sets up for that, by formalizing "extends" relationships between atoms.
**Overspecialization** Diviner currently specializes atoms (FileAtom, FunctionAtom, ClassAtom, etc.). This is pretty much not useful, because Atomizers (which produce the atoms) need to be highly specialized, and Renderers/Publishers (which consume the atoms) also need to be highly specialized. Nothing interesting actually lives in the atom specializations, and we don't benefit from having them -- it just costs us generality in storage/caches for them. In the new code, I've used a single Atom class to represent any type of atom.
**URIs** We have fairly hideous URIs right now, which are very cumbersome For in-app doc links, I want to provide nice URIs ("/h/notfications" or similar) which are stable redirects, and probably add remarkup for it: !{notifications} or similar. This diff isn't related to that since it's too premature.
**Search** Once we have a database generation target, we can index the documentation.
**Design** Chad has some nice mocks.
Test Plan: Ran `bin/diviner generate`, `bin/diviner generate --clean`. Saw appropriate graph propagation after edits. This diff doesn't do anything very useful yet.
Reviewers: btrahan, vrana
Reviewed By: btrahan
CC: aran
Maniphest Tasks: T988
Differential Revision: https://secure.phabricator.com/D4340
+1260
+1
bin/diviner
+1
bin/diviner
···
1
+
../scripts/diviner/diviner.php
+22
scripts/diviner/diviner.php
+22
scripts/diviner/diviner.php
···
1
+
#!/usr/bin/env php
2
+
<?php
3
+
4
+
$root = dirname(dirname(dirname(__FILE__)));
5
+
require_once $root.'/scripts/__init_script__.php';
6
+
7
+
$args = new PhutilArgumentParser($argv);
8
+
9
+
$args->setTagline('documentation generator');
10
+
$args->setSynopsis(<<<EOHELP
11
+
**diviner** __command__ [__options__]
12
+
Generate documentation.
13
+
EOHELP
14
+
);
15
+
$args->parseStandardArguments();
16
+
17
+
$args->parseWorkflows(
18
+
array(
19
+
new DivinerGenerateWorkflow(),
20
+
new DivinerAtomizeWorkflow(),
21
+
new PhutilHelpArgumentWorkflow(),
22
+
));
+14
src/__phutil_library_map__.php
+14
src/__phutil_library_map__.php
···
422
422
'DiffusionTagListView' => 'applications/diffusion/view/DiffusionTagListView.php',
423
423
'DiffusionURITestCase' => 'applications/diffusion/request/__tests__/DiffusionURITestCase.php',
424
424
'DiffusionView' => 'applications/diffusion/view/DiffusionView.php',
425
+
'DivinerArticleAtomizer' => 'applications/diviner/atomizer/DivinerArticleAtomizer.php',
426
+
'DivinerAtom' => 'applications/diviner/atom/DivinerAtom.php',
427
+
'DivinerAtomCache' => 'applications/diviner/cache/DivinerAtomCache.php',
428
+
'DivinerAtomRef' => 'applications/diviner/atom/DivinerAtomRef.php',
429
+
'DivinerAtomizeWorkflow' => 'applications/diviner/workflow/DivinerAtomizeWorkflow.php',
430
+
'DivinerAtomizer' => 'applications/diviner/atomizer/DivinerAtomizer.php',
431
+
'DivinerFileAtomizer' => 'applications/diviner/atomizer/DivinerFileAtomizer.php',
432
+
'DivinerGenerateWorkflow' => 'applications/diviner/workflow/DivinerGenerateWorkflow.php',
425
433
'DivinerListController' => 'applications/diviner/controller/DivinerListController.php',
434
+
'DivinerWorkflow' => 'applications/diviner/workflow/DivinerWorkflow.php',
426
435
'DrydockAllocatorWorker' => 'applications/drydock/worker/DrydockAllocatorWorker.php',
427
436
'DrydockApacheWebrootInterface' => 'applications/drydock/interface/webroot/DrydockApacheWebrootInterface.php',
428
437
'DrydockBlueprint' => 'applications/drydock/blueprint/DrydockBlueprint.php',
···
1780
1789
'DiffusionTagListView' => 'DiffusionView',
1781
1790
'DiffusionURITestCase' => 'ArcanistPhutilTestCase',
1782
1791
'DiffusionView' => 'AphrontView',
1792
+
'DivinerArticleAtomizer' => 'DivinerAtomizer',
1793
+
'DivinerAtomizeWorkflow' => 'DivinerWorkflow',
1794
+
'DivinerFileAtomizer' => 'DivinerAtomizer',
1795
+
'DivinerGenerateWorkflow' => 'DivinerWorkflow',
1783
1796
'DivinerListController' => 'PhabricatorController',
1797
+
'DivinerWorkflow' => 'PhutilArgumentWorkflow',
1784
1798
'DrydockAllocatorWorker' => 'PhabricatorWorker',
1785
1799
'DrydockApacheWebrootInterface' => 'DrydockWebrootInterface',
1786
1800
'DrydockCommandInterface' => 'DrydockInterface',
+289
src/applications/diviner/atom/DivinerAtom.php
+289
src/applications/diviner/atom/DivinerAtom.php
···
1
+
<?php
2
+
3
+
final class DivinerAtom {
4
+
5
+
const TYPE_FILE = 'file';
6
+
const TYPE_ARTICLE = 'article';
7
+
8
+
private $type;
9
+
private $name;
10
+
private $file;
11
+
private $line;
12
+
private $hash;
13
+
private $contentRaw;
14
+
private $length;
15
+
private $language;
16
+
private $docblockRaw;
17
+
private $docblockText;
18
+
private $docblockMeta;
19
+
private $warnings = array();
20
+
private $parentHash;
21
+
private $childHashes = array();
22
+
private $context;
23
+
private $extends = array();
24
+
private $links = array();
25
+
private $project;
26
+
27
+
public function setProject($project) {
28
+
$this->project = $project;
29
+
return $this;
30
+
}
31
+
32
+
public function getProject() {
33
+
return $this->project;
34
+
}
35
+
36
+
public function setContext($context) {
37
+
$this->context = $context;
38
+
return $this;
39
+
}
40
+
41
+
public function getContext() {
42
+
return $this->context;
43
+
}
44
+
45
+
public static function getAtomSerializationVersion() {
46
+
return 1;
47
+
}
48
+
49
+
public function addWarning($warning) {
50
+
$this->warnings[] = $warning;
51
+
return $this;
52
+
}
53
+
54
+
public function getWarnings() {
55
+
return $this->warnings;
56
+
}
57
+
58
+
public function setDocblockRaw($docblock_raw) {
59
+
$this->docblockRaw = $docblock_raw;
60
+
61
+
$parser = new PhutilDocblockParser();
62
+
list($text, $meta) = $parser->parse($docblock_raw);
63
+
$this->docblockText = $text;
64
+
$this->docblockMeta = $meta;
65
+
66
+
return $this;
67
+
}
68
+
69
+
public function getDocblockRaw() {
70
+
return $this->docblockRaw;
71
+
}
72
+
73
+
public function getDocblockText() {
74
+
if ($this->docblockText === null) {
75
+
throw new Exception("Call setDocblockRaw() before getDocblockText()!");
76
+
}
77
+
return $this->docblockText;
78
+
}
79
+
80
+
public function getDocblockMeta() {
81
+
if ($this->docblockMeta === null) {
82
+
throw new Exception("Call setDocblockRaw() before getDocblockMeta()!");
83
+
}
84
+
return $this->docblockMeta;
85
+
}
86
+
87
+
public function setType($type) {
88
+
$this->type = $type;
89
+
return $this;
90
+
}
91
+
92
+
public function getType() {
93
+
return $this->type;
94
+
}
95
+
96
+
public function setName($name) {
97
+
$this->name = $name;
98
+
return $this;
99
+
}
100
+
101
+
public function getName() {
102
+
return $this->name;
103
+
}
104
+
105
+
public function setFile($file) {
106
+
$this->file = $file;
107
+
return $this;
108
+
}
109
+
110
+
public function getFile() {
111
+
return $this->file;
112
+
}
113
+
114
+
public function setLine($line) {
115
+
$this->line = $line;
116
+
return $this;
117
+
}
118
+
119
+
public function getLine() {
120
+
return $this->line;
121
+
}
122
+
123
+
public function setContentRaw($content_raw) {
124
+
$this->contentRaw = $content_raw;
125
+
return $this;
126
+
}
127
+
128
+
public function getContentRaw() {
129
+
return $this->contentRaw;
130
+
}
131
+
132
+
public function setHash($hash) {
133
+
$this->hash = $hash;
134
+
return $this;
135
+
}
136
+
137
+
public function addLink(DivinerAtomRef $ref) {
138
+
$this->links[] = $ref;
139
+
return $this;
140
+
}
141
+
142
+
public function addExtends(DivinerAtomRef $ref) {
143
+
$this->extends[] = $ref;
144
+
return $this;
145
+
}
146
+
147
+
public function getLinkDictionaries() {
148
+
return mpull($this->links, 'toDictionary');
149
+
}
150
+
151
+
public function getExtendsDictionaries() {
152
+
return mpull($this->extends, 'toDictionary');
153
+
}
154
+
155
+
public function getHash() {
156
+
if ($this->hash) {
157
+
return $this->hash;
158
+
}
159
+
160
+
$parts = array(
161
+
$this->getType(),
162
+
$this->getName(),
163
+
$this->getFile(),
164
+
$this->getLine(),
165
+
$this->getLength(),
166
+
$this->getLanguage(),
167
+
$this->getContentRaw(),
168
+
$this->getDocblockRaw(),
169
+
mpull($this->extends, 'toHash'),
170
+
mpull($this->links, 'toHash'),
171
+
);
172
+
173
+
return md5(serialize($parts)).'N';
174
+
}
175
+
176
+
public function setLength($length) {
177
+
$this->length = $length;
178
+
return $this;
179
+
}
180
+
181
+
public function getLength() {
182
+
return $this->length;
183
+
}
184
+
185
+
public function setLanguage($language) {
186
+
$this->language = $language;
187
+
return $this;
188
+
}
189
+
190
+
public function getLanguage() {
191
+
return $this->language;
192
+
}
193
+
194
+
public function addChildHash($child_hash) {
195
+
$this->childHashes[] = $child_hash;
196
+
return $this;
197
+
}
198
+
199
+
public function getChildHashes() {
200
+
return $this->childHashes;
201
+
}
202
+
203
+
public function setParentHash($parent_hash) {
204
+
if ($this->parentHash) {
205
+
throw new Exception("Atom already has a parent!");
206
+
}
207
+
$this->parentHash = $parent_hash;
208
+
return $this;
209
+
}
210
+
211
+
public function getParentHash() {
212
+
return $this->parentHash;
213
+
}
214
+
215
+
public function addChild(DivinerAtom $atom) {
216
+
$atom->setParentHash($this->getHash());
217
+
$this->addChildHash($atom->getHash());
218
+
return $this;
219
+
}
220
+
221
+
public function getURI() {
222
+
$parts = array();
223
+
$parts[] = phutil_escape_uri_path_component($this->getType());
224
+
if ($this->getContext()) {
225
+
$parts[] = phutil_escape_uri_path_component($this->getContext());
226
+
}
227
+
$parts[] = phutil_escape_uri_path_component($this->getName());
228
+
$parts[] = null;
229
+
return implode('/', $parts);
230
+
}
231
+
232
+
233
+
public function toDictionary() {
234
+
// NOTE: If you change this format, bump the format version in
235
+
// getAtomSerializationVersion().
236
+
237
+
return array(
238
+
'type' => $this->getType(),
239
+
'name' => $this->getName(),
240
+
'file' => $this->getFile(),
241
+
'line' => $this->getLine(),
242
+
'hash' => $this->getHash(),
243
+
'uri' => $this->getURI(),
244
+
'length' => $this->getLength(),
245
+
'context' => $this->getContext(),
246
+
'language' => $this->getLanguage(),
247
+
'docblockRaw' => $this->getDocblockRaw(),
248
+
'warnings' => $this->getWarnings(),
249
+
'parentHash' => $this->getParentHash(),
250
+
'childHashes' => $this->getChildHashes(),
251
+
'extends' => $this->getExtendsDictionaries(),
252
+
'links' => $this->getLinkDictionaries(),
253
+
'ref' => $this->getRef()->toDictionary(),
254
+
);
255
+
}
256
+
257
+
public function getRef() {
258
+
return id(new DivinerAtomRef())
259
+
->setProject($this->getProject())
260
+
->setContext($this->getContext())
261
+
->setType($this->getType())
262
+
->setName($this->getName());
263
+
}
264
+
265
+
public static function newFromDictionary(array $dictionary) {
266
+
$atom = id(new DivinerAtom())
267
+
->setType(idx($dictionary, 'type'))
268
+
->setName(idx($dictionary, 'name'))
269
+
->setFile(idx($dictionary, 'file'))
270
+
->setLine(idx($dictionary, 'line'))
271
+
->setHash(idx($dictionary, 'hash'))
272
+
->setLength(idx($dictionary, 'length'))
273
+
->setContext(idx($dictionary, 'context'))
274
+
->setLanguage(idx($dictionary, 'language'))
275
+
->setParentHash(idx($dictionary, 'parentHash'))
276
+
->setDocblockRaw(idx($dictionary, 'docblockRaw'));
277
+
278
+
foreach (idx($dictionary, 'warnings', array()) as $warning) {
279
+
$atom->addWarning($warning);
280
+
}
281
+
282
+
foreach (idx($dictionary, 'childHashes', array()) as $child) {
283
+
$atom->addChildHash($child);
284
+
}
285
+
286
+
return $atom;
287
+
}
288
+
289
+
}
+69
src/applications/diviner/atom/DivinerAtomRef.php
+69
src/applications/diviner/atom/DivinerAtomRef.php
···
1
+
<?php
2
+
3
+
final class DivinerAtomRef {
4
+
5
+
private $project;
6
+
private $context;
7
+
private $type;
8
+
private $name;
9
+
10
+
public function setName($name) {
11
+
$this->name = $name;
12
+
return $this;
13
+
}
14
+
15
+
public function getName() {
16
+
return $this->name;
17
+
}
18
+
19
+
public function setType($type) {
20
+
$this->type = $type;
21
+
return $this;
22
+
}
23
+
24
+
public function getType() {
25
+
return $this->type;
26
+
}
27
+
28
+
public function setContext($context) {
29
+
$this->context = $context;
30
+
return $this;
31
+
}
32
+
33
+
public function getContext() {
34
+
return $this->context;
35
+
}
36
+
37
+
public function setProject($project) {
38
+
$this->project = $project;
39
+
return $this;
40
+
}
41
+
42
+
public function getProject() {
43
+
return $this->project;
44
+
}
45
+
46
+
public function toDictionary() {
47
+
return array(
48
+
'project' => $this->getProject(),
49
+
'context' => $this->getContext(),
50
+
'type' => $this->getType(),
51
+
'name' => $this->getName(),
52
+
);
53
+
}
54
+
55
+
public function toHash() {
56
+
$dict = $this->toDictionary();
57
+
ksort($dict);
58
+
return md5(serialize($dict)).'S';
59
+
}
60
+
61
+
public static function newFromDictionary(array $dict) {
62
+
$obj = new DivinerAtomRef();
63
+
$obj->project = idx($dict, 'project');
64
+
$obj->context = idx($dict, 'context');
65
+
$obj->type = idx($dict, 'type');
66
+
$obj->name = idx($dict, 'name');
67
+
return $obj;
68
+
}
69
+
}
+25
src/applications/diviner/atomizer/DivinerArticleAtomizer.php
+25
src/applications/diviner/atomizer/DivinerArticleAtomizer.php
···
1
+
<?php
2
+
3
+
final class DivinerArticleAtomizer extends DivinerAtomizer {
4
+
5
+
public function atomize($file_name, $file_data) {
6
+
$atom = $this->newAtom(DivinerAtom::TYPE_ARTICLE)
7
+
->setLine(1)
8
+
->setLength(count(explode("\n", $file_data)))
9
+
->setLanguage('human');
10
+
11
+
$block = "/**\n".str_replace("\n", "\n * ", $file_data)."\n */";
12
+
$atom->setDocblockRaw($block);
13
+
14
+
$meta = $atom->getDocblockMeta();
15
+
$title = idx($meta, 'title');
16
+
if (!strlen($title)) {
17
+
$title = 'Untitled Article "'.basename($file_name).'"';
18
+
$atom->addWarning("Article has no @title!");
19
+
}
20
+
$atom->setName($title);
21
+
22
+
return array($atom);
23
+
}
24
+
25
+
}
+45
src/applications/diviner/atomizer/DivinerAtomizer.php
+45
src/applications/diviner/atomizer/DivinerAtomizer.php
···
1
+
<?php
2
+
3
+
/**
4
+
* Generate @{class:DivinerAtom}s from source code.
5
+
*/
6
+
abstract class DivinerAtomizer {
7
+
8
+
private $project;
9
+
10
+
/**
11
+
* If you make a significant change to an atomizer, you can bump this
12
+
* version to drop all the old atom caches.
13
+
*/
14
+
public static function getAtomizerVersion() {
15
+
return 1;
16
+
}
17
+
18
+
abstract public function atomize($file_name, $file_data);
19
+
20
+
final public function setProject($project) {
21
+
$this->project = $project;
22
+
return $this;
23
+
}
24
+
25
+
final public function getProject() {
26
+
return $this->project;
27
+
}
28
+
29
+
protected function newAtom($type) {
30
+
return id(new DivinerAtom())
31
+
->setProject($this->getProject())
32
+
->setType($type);
33
+
}
34
+
35
+
protected function newRef($type, $name, $project = null, $context = null) {
36
+
$project = coalesce($project, $this->getProject());
37
+
38
+
return id(new DivinerAtomRef())
39
+
->setProject($project)
40
+
->setContext($context)
41
+
->setType($type)
42
+
->setName($name);
43
+
}
44
+
45
+
}
+14
src/applications/diviner/atomizer/DivinerFileAtomizer.php
+14
src/applications/diviner/atomizer/DivinerFileAtomizer.php
···
1
+
<?php
2
+
3
+
final class DivinerFileAtomizer extends DivinerAtomizer {
4
+
5
+
public function atomize($file_name, $file_data) {
6
+
$atom = $this->newAtom(DivinerAtom::TYPE_FILE)
7
+
->setName($file_name)
8
+
->setFile($file_name)
9
+
->setContentRaw($file_data);
10
+
11
+
return array($atom);
12
+
}
13
+
14
+
}
+261
src/applications/diviner/cache/DivinerAtomCache.php
+261
src/applications/diviner/cache/DivinerAtomCache.php
···
1
+
<?php
2
+
3
+
final class DivinerAtomCache {
4
+
5
+
private $cache;
6
+
7
+
private $fileHashMap;
8
+
private $atomMap;
9
+
private $symbolMap;
10
+
private $edgeSrcMap;
11
+
private $edgeDstMap;
12
+
private $graphMap;
13
+
14
+
private $atoms = array();
15
+
private $writeAtoms = array();
16
+
17
+
public function __construct($cache_directory) {
18
+
$dir_cache = id(new PhutilKeyValueCacheDirectory())
19
+
->setCacheDirectory($cache_directory);
20
+
$profiled_cache = id(new PhutilKeyValueCacheProfiler($dir_cache))
21
+
->setProfiler(PhutilServiceProfiler::getInstance())
22
+
->setName('diviner-atom-cache');
23
+
$this->cache = $profiled_cache;
24
+
}
25
+
26
+
private function getCache() {
27
+
return $this->cache;
28
+
}
29
+
30
+
public function delete() {
31
+
$this->getCache()->destroyCache();
32
+
$this->fileHashMap = null;
33
+
$this->atomMap = null;
34
+
$this->atoms = array();
35
+
36
+
return $this;
37
+
}
38
+
39
+
/**
40
+
* Convert a long-form hash key like `ccbbaaaaaaaaaaaaaaaaaaaaaaaaaaaaN` into
41
+
* a shortened directory form, like `cc/bb/aaaaaaaaN`. In conjunction with
42
+
* @{class:PhutilKeyValueCacheDirectory}, this gives us nice directories
43
+
* inside .divinercache instead of a million hash files with huge names at
44
+
* top level.
45
+
*/
46
+
private function getHashKey($hash) {
47
+
return implode(
48
+
'/',
49
+
array(
50
+
substr($hash, 0, 2),
51
+
substr($hash, 2, 2),
52
+
substr($hash, 4, 8),
53
+
));
54
+
}
55
+
56
+
57
+
/* -( File Hash Map )------------------------------------------------------ */
58
+
59
+
60
+
public function getFileHashMap() {
61
+
if ($this->fileHashMap === null) {
62
+
$this->fileHashMap = $this->getCache()->getKey('file', array());
63
+
}
64
+
return $this->fileHashMap;
65
+
}
66
+
67
+
public function addFileHash($file_hash, $atom_hash) {
68
+
$this->getFileHashMap();
69
+
$this->fileHashMap[$file_hash] = $atom_hash;
70
+
return $this;
71
+
}
72
+
73
+
public function fileHashExists($file_hash) {
74
+
$map = $this->getFileHashMap();
75
+
return isset($map[$file_hash]);
76
+
}
77
+
78
+
public function deleteFileHash($file_hash) {
79
+
if ($this->fileHashExists($file_hash)) {
80
+
$map = $this->getFileHashMap();
81
+
$atom_hash = $map[$file_hash];
82
+
unset($this->fileHashMap[$file_hash]);
83
+
84
+
$this->deleteAtomHash($atom_hash);
85
+
}
86
+
87
+
return $this;
88
+
}
89
+
90
+
91
+
/* -( Atom Map )----------------------------------------------------------- */
92
+
93
+
94
+
public function getAtomMap() {
95
+
if ($this->atomMap === null) {
96
+
$this->atomMap = $this->getCache()->getKey('atom', array());
97
+
}
98
+
return $this->atomMap;
99
+
}
100
+
101
+
public function getAtom($atom_hash) {
102
+
if (!array_key_exists($atom_hash, $this->atoms)) {
103
+
$key = 'atom/'.$this->getHashKey($atom_hash);
104
+
$this->atoms[$atom_hash] = $this->getCache()->getKey($key);
105
+
}
106
+
return $this->atoms[$atom_hash];
107
+
}
108
+
109
+
public function addAtom(array $atom) {
110
+
$hash = $atom['hash'];
111
+
$this->atoms[$hash] = $atom;
112
+
113
+
$this->getAtomMap();
114
+
$this->atomMap[$hash] = true;
115
+
116
+
$this->writeAtoms['atom/'.$this->getHashKey($hash)] = $atom;
117
+
118
+
return $this;
119
+
}
120
+
121
+
public function deleteAtomHash($atom_hash) {
122
+
$atom = $this->getAtom($atom_hash);
123
+
if ($atom) {
124
+
foreach ($atom['childHashes'] as $child_hash) {
125
+
$this->deleteAtomHash($child_hash);
126
+
}
127
+
}
128
+
129
+
$this->getAtomMap();
130
+
unset($this->atomMap[$atom_hash]);
131
+
unset($this->writeAtoms[$atom_hash]);
132
+
133
+
$this->getCache()->deleteKey('atom/'.$this->getHashKey($atom_hash));
134
+
135
+
return $this;
136
+
}
137
+
138
+
public function saveAtoms() {
139
+
$this->getCache()->setKeys(
140
+
array(
141
+
'file' => $this->getFileHashMap(),
142
+
'atom' => $this->getAtomMap(),
143
+
) + $this->writeAtoms);
144
+
$this->writeAtoms = array();
145
+
return $this;
146
+
}
147
+
148
+
149
+
/* -( Symbol Hash Map )---------------------------------------------------- */
150
+
151
+
152
+
public function getSymbolMap() {
153
+
if ($this->symbolMap === null) {
154
+
$this->symbolMap = $this->getCache()->getKey('symbol', array());
155
+
}
156
+
return $this->symbolMap;
157
+
}
158
+
159
+
public function addSymbol($atom_hash, $symbol_hash) {
160
+
$this->getSymbolMap();
161
+
$this->symbolMap[$atom_hash] = $symbol_hash;
162
+
return $this;
163
+
}
164
+
165
+
public function deleteSymbol($atom_hash) {
166
+
$this->getSymbolMap();
167
+
unset($this->symbolMap[$atom_hash]);
168
+
169
+
return $this;
170
+
}
171
+
172
+
public function saveSymbols() {
173
+
$this->getCache()->setKeys(
174
+
array(
175
+
'symbol' => $this->getSymbolMap(),
176
+
));
177
+
return $this;
178
+
}
179
+
180
+
/* -( Edge Map )----------------------------------------------------------- */
181
+
182
+
183
+
public function getEdgeMap() {
184
+
if ($this->edgeDstMap === null) {
185
+
$this->edgeDstMap = $this->getCache()->getKey('edge', array());
186
+
$this->edgeSrcMap = array();
187
+
foreach ($this->edgeDstMap as $dst => $srcs) {
188
+
foreach ($srcs as $src => $ignored) {
189
+
$this->edgeSrcMap[$src][$dst] = true;
190
+
}
191
+
}
192
+
}
193
+
return $this->edgeDstMap;
194
+
}
195
+
196
+
public function getEdgesWithDestination($symbol_hash) {
197
+
$this->getEdgeMap();
198
+
return array_keys(idx($this->edgeDstMap, $symbol_hash, array()));
199
+
}
200
+
201
+
public function addEdges($node_hash, array $symbol_hash_list) {
202
+
$this->getEdgeMap();
203
+
$this->edgeSrcMap[$node_hash] = array_fill_keys($symbol_hash_list, true);
204
+
foreach ($symbol_hash_list as $symbol_hash) {
205
+
$this->edgeDstMap[$symbol_hash][$node_hash] = true;
206
+
}
207
+
return $this;
208
+
}
209
+
210
+
public function deleteEdges($node_hash) {
211
+
$this->getEdgeMap();
212
+
foreach (idx($this->edgeSrcMap, $node_hash, array()) as $dst => $ignored) {
213
+
unset($this->edgeDstMap[$dst][$node_hash]);
214
+
if (empty($this->edgeDstMap[$dst])) {
215
+
unset($this->edgeDstMap[$dst]);
216
+
}
217
+
}
218
+
unset($this->edgeSrcMap[$node_hash]);
219
+
return $this;
220
+
}
221
+
222
+
public function saveEdges() {
223
+
$this->getCache()->setKeys(
224
+
array(
225
+
'edge' => $this->getEdgeMap(),
226
+
));
227
+
return $this;
228
+
}
229
+
230
+
231
+
/* -( Graph Map )---------------------------------------------------------- */
232
+
233
+
234
+
public function getGraphMap() {
235
+
if ($this->graphMap === null) {
236
+
$this->graphMap = $this->getCache()->getKey('graph', array());
237
+
}
238
+
return $this->graphMap;
239
+
}
240
+
241
+
public function deleteGraph($node_hash) {
242
+
$this->getGraphMap();
243
+
unset($this->graphMap[$node_hash]);
244
+
return $this;
245
+
}
246
+
247
+
public function addGraph($node_hash, $graph_hash) {
248
+
$this->getGraphMap();
249
+
$this->graphMap[$node_hash] = $graph_hash;
250
+
return $this;
251
+
}
252
+
253
+
public function saveGraph() {
254
+
$this->getCache()->setKeys(
255
+
array(
256
+
'graph' => $this->getGraphMap(),
257
+
));
258
+
return $this;
259
+
}
260
+
261
+
}
+103
src/applications/diviner/workflow/DivinerAtomizeWorkflow.php
+103
src/applications/diviner/workflow/DivinerAtomizeWorkflow.php
···
1
+
<?php
2
+
3
+
final class DivinerAtomizeWorkflow extends DivinerWorkflow {
4
+
5
+
public function didConstruct() {
6
+
$this
7
+
->setName('atomize')
8
+
->setSynopsis(pht('Build atoms from source.'))
9
+
->setArguments(
10
+
array(
11
+
array(
12
+
'name' => 'atomizer',
13
+
'param' => 'class',
14
+
'help' => 'Specify a subclass of DivinerAtomizer.',
15
+
),
16
+
array(
17
+
'name' => 'files',
18
+
'wildcard' => true,
19
+
),
20
+
array(
21
+
'name' => 'ugly',
22
+
'help' => 'Produce ugly (but faster) output.',
23
+
),
24
+
));
25
+
}
26
+
27
+
public function execute(PhutilArgumentParser $args) {
28
+
$console = PhutilConsole::getConsole();
29
+
30
+
$atomizer_class = $args->getArg('atomizer');
31
+
if (!$atomizer_class) {
32
+
throw new Exception("Specify an atomizer class with --atomizer.");
33
+
}
34
+
35
+
$symbols = id(new PhutilSymbolLoader())
36
+
->setName($atomizer_class)
37
+
->setConcreteOnly(true)
38
+
->setAncestorClass('DivinerAtomizer')
39
+
->selectAndLoadSymbols();
40
+
if (!$symbols) {
41
+
throw new Exception(
42
+
"Atomizer class '{$atomizer_class}' must be a concrete subclass of ".
43
+
"DivinerAtomizer.");
44
+
}
45
+
46
+
$atomizer = newv($atomizer_class, array());
47
+
48
+
$files = $args->getArg('files');
49
+
if (!$files) {
50
+
throw new Exception("Specify one or more files to atomize.");
51
+
}
52
+
53
+
$file_atomizer = new DivinerFileAtomizer();
54
+
55
+
$all_atoms = array();
56
+
foreach ($files as $file) {
57
+
$data = Filesystem::readFile($file);
58
+
59
+
if (!$this->shouldAtomizeFile($file, $data)) {
60
+
$console->writeLog("Skipping %s...\n", $file);
61
+
continue;
62
+
} else {
63
+
$console->writeLog("Atomizing %s...\n", $file);
64
+
}
65
+
66
+
$file_atoms = $file_atomizer->atomize($file, $data);
67
+
$all_atoms[] = $file_atoms;
68
+
69
+
if (count($file_atoms) !== 1) {
70
+
throw new Exception("Expected exactly one atom from file atomizer.");
71
+
}
72
+
$file_atom = head($file_atoms);
73
+
74
+
$atoms = $atomizer->atomize($file, $data);
75
+
76
+
foreach ($atoms as $atom) {
77
+
$file_atom->addChild($atom);
78
+
}
79
+
80
+
$all_atoms[] = $atoms;
81
+
}
82
+
83
+
$all_atoms = array_mergev($all_atoms);
84
+
$all_atoms = mpull($all_atoms, 'toDictionary');
85
+
$all_atoms = ipull($all_atoms, null, 'hash');
86
+
87
+
if ($args->getArg('ugly')) {
88
+
$json = json_encode($all_atoms);
89
+
} else {
90
+
$json_encoder = new PhutilJSON();
91
+
$json = $json_encoder->encodeFormatted($all_atoms);
92
+
}
93
+
94
+
$console->writeOut('%s', $json);
95
+
96
+
return 0;
97
+
}
98
+
99
+
private function shouldAtomizeFile($file_name, $file_data) {
100
+
return (strpos($file_data, '@'.'undivinable') === false);
101
+
}
102
+
103
+
}
+384
src/applications/diviner/workflow/DivinerGenerateWorkflow.php
+384
src/applications/diviner/workflow/DivinerGenerateWorkflow.php
···
1
+
<?php
2
+
3
+
final class DivinerGenerateWorkflow extends DivinerWorkflow {
4
+
5
+
public function didConstruct() {
6
+
$this
7
+
->setName('generate')
8
+
->setSynopsis(pht('Generate documentation.'))
9
+
->setArguments(
10
+
array(
11
+
array(
12
+
'name' => 'clean',
13
+
'help' => 'Clear the caches before generating documentation.',
14
+
),
15
+
));
16
+
}
17
+
18
+
public function execute(PhutilArgumentParser $args) {
19
+
if ($args->getArg('clean')) {
20
+
$this->log(pht('CLEARING CACHES'));
21
+
$this->getAtomCache()->delete();
22
+
}
23
+
24
+
// The major challenge of documentation generation is one of dependency
25
+
// management. When regenerating documentation, we want to do the smallest
26
+
// amount of work we can, so that regenerating documentation after minor
27
+
// changes is quick.
28
+
//
29
+
// ATOM CACHE
30
+
//
31
+
// In the first stage, we find all the direct changes to source code since
32
+
// the last run. This stage relies on two data structures:
33
+
//
34
+
// - File Hash Map: map<file_hash, node_hash>
35
+
// - Atom Map: map<node_hash, true>
36
+
//
37
+
// First, we hash all the source files in the project to detect any which
38
+
// have changed since the previous run (i.e., their hash is not present in
39
+
// the File Hash Map). If a file's content hash appears in the map, it has
40
+
// not changed, so we don't need to reparse it.
41
+
//
42
+
// We break the contents of each file into "atoms", which represent a unit
43
+
// of source code (like a function, method, class or file). Each atom has a
44
+
// "node hash" based on the content of the atom: if a function definition
45
+
// changes, the node hash of the atom changes too. The primary output of
46
+
// the atom cache is a list of node hashes which exist in the project. This
47
+
// is the Atom Map. The node hash depends only on the definition of the atom
48
+
// and the atomizer implementation. It ends with an "N", for "node".
49
+
//
50
+
// (We need the Atom Map in addition to the File Hash Map because each file
51
+
// may have several atoms in it (e.g., multiple functions, or a class and
52
+
// its methods). The File Hash Map contains an exhaustive list of all atoms
53
+
// with type "file", but not child atoms of those top-level atoms.)
54
+
//
55
+
// GRAPH CACHE
56
+
//
57
+
// We now know which atoms exist, and can compare the Atom Map to some
58
+
// existing cache to figure out what has changed. However, this isn't
59
+
// sufficient to figure out which documentation actually needs to be
60
+
// regnerated, because atoms depend on other atoms. For example, if "B
61
+
// extends A" and the definition for A changes, we need to regenerate the
62
+
// documentation in B. Similarly, if X links to Y and Y changes, we should
63
+
// regenerate X. (In both these cases, the documentation for the connected
64
+
// atom may not acutally change, but in some cases it will, and the extra
65
+
// work we need to do is generally very small compared to the size of the
66
+
// project.)
67
+
//
68
+
// To figure out which other nodes have changed, we compute a "graph hash"
69
+
// for each node. This hash combines the "node hash" with the node hashes
70
+
// of connected nodes. Our primary output is a list of graph hashes, which
71
+
// a documentation generator can use to easily determine what work needs
72
+
// to be done by comparing the list with a list of cached graph hashes,
73
+
// then generating documentation for new hashes and deleting documentation
74
+
// for missing hashes. The graph hash ends with a "G", for "graph".
75
+
//
76
+
// In this stage, we rely on three data structures:
77
+
//
78
+
// - Symbol Map: map<node_hash, symbol_hash>
79
+
// - Edge Map: map<node_hash, list<symbol_hash>>
80
+
// - Graph Map: map<node_hash, graph_hash>
81
+
//
82
+
// Calculating the graph hash requires several steps, because we need to
83
+
// figure out which nodes an atom is attached to. The atom contains symbolic
84
+
// references to other nodes by name (e.g., "extends SomeClass") in the form
85
+
// of DivinerAtomRefs. We can also build a symbolic reference for any atom
86
+
// from the atom itself. Each DivinerAtomRef generates a symbol hash,
87
+
// which ends with an "S", for "symbol".
88
+
//
89
+
// First, we update the symbol map. We remove (and mark dirty) any symbols
90
+
// associated with node hashes which no longer exist (e.g., old/dead nodes).
91
+
// Second, we add (and mark dirty) any symbols associated with new nodes.
92
+
// We also add edges defined by new nodes to the graph.
93
+
//
94
+
// We initialize a list of dirty nodes to the list of new nodes, then
95
+
// find all nodes connected to dirty symbols and add them to the dirty
96
+
// node list. This list now contains every node with a new or changed
97
+
// graph hash.
98
+
//
99
+
// We walk the dirty list and compute the new graph hashes, adding them
100
+
// to the graph hash map. This Graph Map can then be passed to an actual
101
+
// documentation generator, which can compare the graph hashes to a list
102
+
// of already-generated graph hashes and easily assess which documents need
103
+
// to be regenerated and which can be deleted.
104
+
105
+
$this->buildAtomCache();
106
+
$this->buildGraphCache();
107
+
}
108
+
109
+
/* -( Atom Cache )--------------------------------------------------------- */
110
+
111
+
private function buildAtomCache() {
112
+
$this->log(pht('BUILDING ATOM CACHE'));
113
+
114
+
$file_hashes = $this->findFilesInProject();
115
+
116
+
$this->log(pht('Found %d file(s) in project.', count($file_hashes)));
117
+
118
+
$this->deleteDeadAtoms($file_hashes);
119
+
120
+
$atomize = $this->getFilesToAtomize($file_hashes);
121
+
122
+
$this->log(pht('Found %d unatomized, uncached file(s).', count($atomize)));
123
+
124
+
$file_atomizers = $this->getAtomizersForFiles($atomize);
125
+
126
+
$this->log(pht('Found %d file(s) to atomize.', count($file_atomizers)));
127
+
128
+
$futures = $this->buildAtomizerFutures($file_atomizers);
129
+
if ($futures) {
130
+
$this->resolveAtomizerFutures($futures, $file_hashes);
131
+
$this->log(pht("Atomization complete."));
132
+
} else {
133
+
$this->log(pht("Atom cache is up to date, no files to atomize."));
134
+
}
135
+
136
+
$this->log(pht("Writing atom cache."));
137
+
138
+
$this->getAtomCache()->saveAtoms();
139
+
140
+
$this->log(pht("Done."));
141
+
}
142
+
143
+
private function getAtomizersForFiles(array $files) {
144
+
$rules = $this->getRules();
145
+
146
+
$atomizers = array();
147
+
148
+
foreach ($files as $file) {
149
+
foreach ($rules as $rule => $atomizer) {
150
+
$ok = preg_match($rule, $file);
151
+
if ($ok === false) {
152
+
throw new Exception(
153
+
"Rule '{$rule}' is not a valid regular expression.");
154
+
}
155
+
if ($ok) {
156
+
$atomizers[$file] = $atomizer;
157
+
continue;
158
+
}
159
+
}
160
+
}
161
+
162
+
return $atomizers;
163
+
}
164
+
165
+
private function getRules() {
166
+
return $this->getConfig('rules', array()) + array(
167
+
'/\\.diviner$/' => 'DivinerArticleAtomizer',
168
+
);
169
+
}
170
+
171
+
172
+
private function findFilesInProject() {
173
+
$file_hashes = id(new FileFinder($this->getRoot()))
174
+
->excludePath('*/.*')
175
+
->withType('f')
176
+
->setGenerateChecksums(true)
177
+
->find();
178
+
179
+
$version = $this->getDivinerAtomWorldVersion();
180
+
181
+
foreach ($file_hashes as $file => $md5_hash) {
182
+
// We want the hash to change if the file moves or Diviner gets updated,
183
+
// not just if the file content changes. Derive a hash from everything
184
+
// we care about.
185
+
$file_hashes[$file] = md5("{$file}\0{$md5_hash}\0{$version}").'F';
186
+
}
187
+
188
+
return $file_hashes;
189
+
}
190
+
191
+
private function deleteDeadAtoms(array $file_hashes) {
192
+
$atom_cache = $this->getAtomCache();
193
+
194
+
$hash_to_file = array_flip($file_hashes);
195
+
foreach ($atom_cache->getFileHashMap() as $hash => $atom) {
196
+
if (empty($hash_to_file[$hash])) {
197
+
$atom_cache->deleteFileHash($hash);
198
+
}
199
+
}
200
+
}
201
+
202
+
private function getFilesToAtomize(array $file_hashes) {
203
+
$atom_cache = $this->getAtomCache();
204
+
205
+
$atomize = array();
206
+
foreach ($file_hashes as $file => $hash) {
207
+
if (!$atom_cache->fileHashExists($hash)) {
208
+
$atomize[] = $file;
209
+
}
210
+
}
211
+
212
+
return $atomize;
213
+
}
214
+
215
+
private function buildAtomizerFutures(array $file_atomizers) {
216
+
$atomizers = array();
217
+
foreach ($file_atomizers as $file => $atomizer) {
218
+
$atomizers[$atomizer][] = $file;
219
+
}
220
+
221
+
$futures = array();
222
+
foreach ($atomizers as $class => $files) {
223
+
foreach (array_chunk($files, 32) as $chunk) {
224
+
$future = new ExecFuture(
225
+
'%s atomize --atomizer %s -- %Ls',
226
+
dirname(phutil_get_library_root('phabricator')).'/bin/diviner',
227
+
$class,
228
+
$chunk);
229
+
$future->setCWD($this->getRoot());
230
+
231
+
$futures[] = $future;
232
+
}
233
+
}
234
+
235
+
return $futures;
236
+
}
237
+
238
+
private function resolveAtomizerFutures(array $futures, array $file_hashes) {
239
+
assert_instances_of($futures, 'Future');
240
+
241
+
$atom_cache = $this->getAtomCache();
242
+
foreach (Futures($futures)->limit(4) as $key => $future) {
243
+
$atoms = $future->resolveJSON();
244
+
245
+
foreach ($atoms as $atom) {
246
+
if ($atom['type'] == DivinerAtom::TYPE_FILE) {
247
+
$file_hash = $file_hashes[$atom['file']];
248
+
$atom_cache->addFileHash($file_hash, $atom['hash']);
249
+
}
250
+
$atom_cache->addAtom($atom);
251
+
}
252
+
}
253
+
}
254
+
255
+
256
+
/**
257
+
* Get a global version number, which changes whenever any atom or atomizer
258
+
* implementation changes in a way which is not backward-compatible.
259
+
*/
260
+
private function getDivinerAtomWorldVersion() {
261
+
$version = array();
262
+
$version['atom'] = DivinerAtom::getAtomSerializationVersion();
263
+
$version['rules'] = $this->getRules();
264
+
265
+
$atomizers = id(new PhutilSymbolLoader())
266
+
->setAncestorClass('DivinerAtomizer')
267
+
->setConcreteOnly(true)
268
+
->selectAndLoadSymbols();
269
+
270
+
$atomizer_versions = array();
271
+
foreach ($atomizers as $atomizer) {
272
+
$atomizer_versions[$atomizer['name']] = call_user_func(
273
+
array(
274
+
$atomizer['name'],
275
+
'getAtomizerVersion',
276
+
));
277
+
}
278
+
279
+
ksort($atomizer_versions);
280
+
$version['atomizers'] = $atomizer_versions;
281
+
282
+
return md5(serialize($version));
283
+
}
284
+
285
+
286
+
/* -( Graph Cache )-------------------------------------------------------- */
287
+
288
+
289
+
private function buildGraphCache() {
290
+
$this->log(pht('BUILDING GRAPH CACHE'));
291
+
292
+
$atom_cache = $this->getAtomCache();
293
+
$symbol_map = $atom_cache->getSymbolMap();
294
+
$atoms = $atom_cache->getAtomMap();
295
+
296
+
$dirty_symbols = array();
297
+
$dirty_nhashes = array();
298
+
299
+
$del_atoms = array_diff_key($symbol_map, $atoms);
300
+
$this->log(pht('Found %d obsolete atom(s) in graph.', count($del_atoms)));
301
+
foreach ($del_atoms as $nhash => $shash) {
302
+
$atom_cache->deleteSymbol($nhash);
303
+
$dirty_symbols[$shash] = true;
304
+
305
+
$atom_cache->deleteEdges($nhash);
306
+
$atom_cache->deleteGraph($nhash);
307
+
}
308
+
309
+
$new_atoms = array_diff_key($atoms, $symbol_map);
310
+
$this->log(pht('Found %d new atom(s) in graph.', count($new_atoms)));
311
+
foreach ($new_atoms as $nhash => $ignored) {
312
+
$shash = $this->computeSymbolHash($nhash);
313
+
$atom_cache->addSymbol($nhash, $shash);
314
+
$dirty_symbols[$shash] = true;
315
+
316
+
$atom_cache->addEdges(
317
+
$nhash,
318
+
$this->getEdges($nhash));
319
+
320
+
$dirty_nhashes[$nhash] = true;
321
+
}
322
+
323
+
$this->log(pht('Propagating changes through the graph.'));
324
+
325
+
foreach ($dirty_symbols as $symbol => $ignored) {
326
+
foreach ($atom_cache->getEdgesWithDestination($symbol) as $edge) {
327
+
$dirty_nhashes[$edge] = true;
328
+
}
329
+
}
330
+
331
+
$this->log(pht('Found %d affected atoms.', count($dirty_nhashes)));
332
+
333
+
foreach ($dirty_nhashes as $nhash => $ignored) {
334
+
$atom_cache->addGraph($nhash, $this->computeGraphHash($nhash));
335
+
}
336
+
337
+
$this->log(pht('Writing graph cache.'));
338
+
339
+
$atom_cache->saveGraph();
340
+
$atom_cache->saveEdges();
341
+
$atom_cache->saveSymbols();
342
+
343
+
$this->log(pht('Done.'));
344
+
}
345
+
346
+
private function computeSymbolHash($node_hash) {
347
+
$atom_cache = $this->getAtomCache();
348
+
$atom = $atom_cache->getAtom($node_hash);
349
+
350
+
$ref = DivinerAtomRef::newFromDictionary($atom['ref']);
351
+
return $ref->toHash();
352
+
}
353
+
354
+
private function getEdges($node_hash) {
355
+
$atom_cache = $this->getAtomCache();
356
+
$atom = $atom_cache->getAtom($node_hash);
357
+
358
+
$refs = array();
359
+
foreach (array_merge($atom['extends'], $atom['links']) as $ref_dict) {
360
+
$ref = DivinerAtomRef::newFromDictionary($ref_dict);
361
+
if ($ref->getProject() == $atom['project']) {
362
+
$refs[$ref->toHash()] = true;
363
+
}
364
+
}
365
+
366
+
return array_keys($refs);
367
+
}
368
+
369
+
private function computeGraphHash($node_hash) {
370
+
$atom_cache = $this->getAtomCache();
371
+
$atom = $atom_cache->getAtom($node_hash);
372
+
373
+
$edges = $this->getEdges($node_hash);
374
+
sort($edges);
375
+
376
+
$inputs = array(
377
+
'atomHash' => $atom['hash'],
378
+
'edges' => $edges,
379
+
);
380
+
381
+
return md5(serialize($inputs)).'G';
382
+
}
383
+
384
+
}
+33
src/applications/diviner/workflow/DivinerWorkflow.php
+33
src/applications/diviner/workflow/DivinerWorkflow.php
···
1
+
<?php
2
+
3
+
abstract class DivinerWorkflow extends PhutilArgumentWorkflow {
4
+
5
+
private $atomCache;
6
+
7
+
public function isExecutable() {
8
+
return true;
9
+
}
10
+
11
+
protected function getRoot() {
12
+
return getcwd();
13
+
}
14
+
15
+
protected function getConfig($key, $default = null) {
16
+
return $default;
17
+
}
18
+
19
+
protected function getAtomCache() {
20
+
if (!$this->atomCache) {
21
+
$cache_directory = $this->getRoot().'/.divinercache';
22
+
$this->atomCache = new DivinerAtomCache($cache_directory);
23
+
}
24
+
return $this->atomCache;
25
+
}
26
+
27
+
protected function log($message) {
28
+
$console = PhutilConsole::getConsole();
29
+
$console->getServer()->setEnableLog(true);
30
+
$console->writeLog($message."\n");
31
+
}
32
+
33
+
}