@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
diviner: Atomize using PHP-Parser
Summary:
This allows Diviner to support atomizing PHP source files that use
features of newer versions of PHP.
Ref T16289
Test Plan:
* (Optional) Edit source files to use new PHP features (enums, union types)
* Run `./bin/diviner generate --clean`
* Look at the generated documention on http://phorge.localhost/book/dev/ of the file modified
Reviewers: O1 Blessed Committers, aklapper
Reviewed By: O1 Blessed Committers, aklapper
Subscribers: aklapper, tobiaswiese, valerio.bozzolan, Matthew, Cigaryno
Maniphest Tasks: T16289
Differential Revision: https://we.phorge.it/D26755
+159
-113
+12
src/applications/diviner/atom/DivinerAtom.php
+12
src/applications/diviner/atom/DivinerAtom.php
···
4
4
5
5
const TYPE_ARTICLE = 'article';
6
6
const TYPE_CLASS = 'class';
7
+
const TYPE_ENUM = 'enum';
7
8
const TYPE_FILE = 'file';
8
9
const TYPE_FUNCTION = 'function';
9
10
const TYPE_INTERFACE = 'interface';
10
11
const TYPE_METHOD = 'method';
12
+
const TYPE_TRAIT = 'trait';
11
13
12
14
private $type;
13
15
private $name;
···
396
398
return pht('This interface is not documented.');
397
399
case self::TYPE_METHOD:
398
400
return pht('This method is not documented.');
401
+
case self::TYPE_TRAIT:
402
+
return pht('This trait is not documented.');
403
+
case self::TYPE_ENUM:
404
+
return pht('This enum is not documented.');
399
405
default:
400
406
phlog(pht("Need translation for '%s'.", $type));
401
407
return pht('This %s is not documented.', $type);
···
410
416
self::TYPE_FUNCTION,
411
417
self::TYPE_INTERFACE,
412
418
self::TYPE_METHOD,
419
+
self::TYPE_TRAIT,
420
+
self::TYPE_ENUM,
413
421
);
414
422
}
415
423
···
425
433
return pht('Function');
426
434
case self::TYPE_INTERFACE:
427
435
return pht('Interface');
436
+
case self::TYPE_TRAIT:
437
+
return pht('Trait');
438
+
case self::TYPE_ENUM:
439
+
return pht('Enum');
428
440
case self::TYPE_METHOD:
429
441
return pht('Method');
430
442
default:
+147
-113
src/applications/diviner/atomizer/DivinerPHPAtomizer.php
+147
-113
src/applications/diviner/atomizer/DivinerPHPAtomizer.php
···
1
1
<?php
2
2
3
+
/**
4
+
* @phutil-external-symbol class PhpParser\Node
5
+
* @phutil-external-symbol class PhpParser\NodeTraverser
6
+
* @phutil-external-symbol class PhpParser\Node\FunctionLike
7
+
* @phutil-external-symbol class PhpParser\NodeVisitor\FindingVisitor
8
+
* @phutil-external-symbol class PhpParser\NodeVisitor\NameResolver
9
+
* @phutil-external-symbol class PhpParser\Node\Stmt\Class_
10
+
* @phutil-external-symbol class PhpParser\Node\Stmt\ClassLike
11
+
* @phutil-external-symbol class PhpParser\Node\Stmt\Enum_
12
+
* @phutil-external-symbol class PhpParser\Node\Stmt\Function_
13
+
* @phutil-external-symbol class PhpParser\Node\Stmt\Interface_
14
+
* @phutil-external-symbol class PhpParser\Node\Stmt\Trait_
15
+
* @phutil-external-symbol class PhpParser\PrettyPrinter\Standard
16
+
*/
3
17
final class DivinerPHPAtomizer extends DivinerAtomizer {
4
18
5
19
protected function newAtom($type) {
···
7
21
}
8
22
9
23
protected function executeAtomize($file_name, $file_data) {
10
-
$future = PhutilXHPASTBinary::getParserFuture($file_data);
11
-
$tree = XHPASTTree::newFromDataAndResolvedExecFuture(
12
-
$file_data,
13
-
$future->resolve());
24
+
$parser = PhutilPHPParserLibrary::getParser();
25
+
$ast = $parser->parse($file_data);
14
26
15
-
$atoms = array();
16
-
$root = $tree->getRootNode();
27
+
$classlike_finder = new PhpParser\NodeVisitor\FindingVisitor(
28
+
function ($node) {
29
+
return $node instanceof PhpParser\Node\Stmt\ClassLike;
30
+
});
31
+
$function_finder = new PhpParser\NodeVisitor\FindingVisitor(
32
+
function ($node) {
33
+
return $node instanceof PhpParser\Node\Stmt\Function_;
34
+
});
17
35
18
-
$func_decl = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION');
19
-
foreach ($func_decl as $func) {
20
-
$name = $func->getChildByIndex(2);
36
+
$namespace_resolver = new PhpParser\NodeVisitor\NameResolver();
37
+
$traverser = new PhpParser\NodeTraverser();
38
+
$traverser->addVisitor($namespace_resolver);
39
+
$traverser->addVisitor($classlike_finder);
40
+
$traverser->addVisitor($function_finder);
41
+
$traverser->traverse($ast);
21
42
22
-
// Don't atomize closures
23
-
if ($name->getTypeName() === 'n_EMPTY') {
24
-
continue;
25
-
}
43
+
$atoms = array();
26
44
45
+
foreach ($function_finder->getFoundNodes() as $func) {
27
46
$atom = $this->newAtom(DivinerAtom::TYPE_FUNCTION)
28
-
->setName($name->getConcreteString())
29
-
->setLine($func->getLineNumber())
47
+
->setName($func->namespacedName->toString())
48
+
->setLine($func->getStartLine())
30
49
->setFile($file_name);
31
50
32
51
$this->findAtomDocblock($atom, $func);
···
37
56
}
38
57
39
58
$class_types = array(
40
-
DivinerAtom::TYPE_CLASS => 'n_CLASS_DECLARATION',
41
-
DivinerAtom::TYPE_INTERFACE => 'n_INTERFACE_DECLARATION',
59
+
PhpParser\Node\Stmt\Class_::class => DivinerAtom::TYPE_CLASS,
60
+
PhpParser\Node\Stmt\Interface_::class => DivinerAtom::TYPE_INTERFACE,
61
+
PhpParser\Node\Stmt\Trait_::class => DivinerAtom::TYPE_TRAIT,
62
+
PhpParser\Node\Stmt\Enum_::class => DivinerAtom::TYPE_ENUM,
42
63
);
43
-
foreach ($class_types as $atom_type => $node_type) {
44
-
$class_decls = $root->selectDescendantsOfType($node_type);
45
64
46
-
foreach ($class_decls as $class) {
47
-
$name = $class->getChildByIndex(1, 'n_CLASS_NAME');
65
+
foreach ($classlike_finder->getFoundNodes() as $class) {
66
+
$atom_type = $class_types[get_class($class)];
48
67
49
-
$atom = $this->newAtom($atom_type)
50
-
->setName($name->getConcreteString())
51
-
->setFile($file_name)
52
-
->setLine($class->getLineNumber());
68
+
// Don't analyze anonymous classes.
69
+
if (!$class->name) {
70
+
continue;
71
+
}
72
+
73
+
$atom = $this->newAtom($atom_type)
74
+
->setName($class->namespacedName->toString())
75
+
->setFile($file_name)
76
+
->setLine($class->getStartLine());
53
77
54
-
// This parses `final` and `abstract`.
55
-
$attributes = $class->getChildByIndex(0, 'n_CLASS_ATTRIBUTES');
56
-
foreach ($attributes->selectDescendantsOfType('n_STRING') as $attr) {
57
-
$atom->setProperty($attr->getConcreteString(), true);
78
+
if ($class instanceof PhpParser\Node\Stmt\Class_) {
79
+
if ($class->isAbstract()) {
80
+
$atom->setProperty('abstract', true);
81
+
} else if ($class->isFinal()) {
82
+
$atom->setProperty('final', true);
83
+
} else if ($class->isReadonly()) {
84
+
$atom->setProperty('readonly', true);
58
85
}
59
86
60
-
// If this exists, it is `n_EXTENDS_LIST`.
61
-
$extends = $class->getChildByIndex(2);
62
-
$extends_class = $extends->selectDescendantsOfType('n_CLASS_NAME');
63
-
foreach ($extends_class as $parent_class) {
87
+
if ($class->extends) {
64
88
$atom->addExtends(
65
89
$this->newRef(
66
90
DivinerAtom::TYPE_CLASS,
67
-
$parent_class->getConcreteString()));
91
+
$class->extends->toString()));
68
92
}
69
93
70
-
// If this exists, it is `n_IMPLEMENTS_LIST`.
71
-
$implements = $class->getChildByIndex(3);
72
-
$iface_names = $implements->selectDescendantsOfType('n_CLASS_NAME');
73
-
foreach ($iface_names as $iface_name) {
94
+
foreach ($class->implements as $implement) {
95
+
$atom->addExtends(
96
+
$this->newRef(
97
+
DivinerAtom::TYPE_INTERFACE,
98
+
$implement->toString()));
99
+
}
100
+
} else if ($class instanceof PhpParser\Node\Stmt\Interface_) {
101
+
foreach ($class->extends as $extend) {
74
102
$atom->addExtends(
75
103
$this->newRef(
76
104
DivinerAtom::TYPE_INTERFACE,
77
-
$iface_name->getConcreteString()));
105
+
$extend->toString()));
78
106
}
79
-
80
-
$this->findAtomDocblock($atom, $class);
107
+
} else if ($class instanceof PhpParser\Node\Stmt\Enum_) {
108
+
foreach ($class->implements as $implement) {
109
+
$atom->addExtends(
110
+
$this->newRef(
111
+
DivinerAtom::TYPE_INTERFACE,
112
+
$implement->toString()));
113
+
}
114
+
}
81
115
82
-
$methods = $class->selectDescendantsOfType('n_METHOD_DECLARATION');
83
-
foreach ($methods as $method) {
84
-
$matom = $this->newAtom(DivinerAtom::TYPE_METHOD);
116
+
$this->findAtomDocblock($atom, $class);
85
117
86
-
$this->findAtomDocblock($matom, $method);
118
+
foreach ($class->getMethods() as $method) {
119
+
$matom = $this->newAtom(DivinerAtom::TYPE_METHOD)
120
+
->setName($method->name->toString())
121
+
->setLine($method->getStartLine())
122
+
->setFile($file_name);
87
123
88
-
$attribute_list = $method->getChildByIndex(0);
89
-
$attributes = $attribute_list->selectDescendantsOfType('n_STRING');
90
-
if ($attributes) {
91
-
foreach ($attributes as $attribute) {
92
-
$attr = strtolower($attribute->getConcreteString());
93
-
switch ($attr) {
94
-
case 'final':
95
-
case 'abstract':
96
-
case 'static':
97
-
$matom->setProperty($attr, true);
98
-
break;
99
-
case 'public':
100
-
case 'protected':
101
-
case 'private':
102
-
$matom->setProperty('access', $attr);
103
-
break;
104
-
}
105
-
}
106
-
} else {
107
-
$matom->setProperty('access', 'public');
108
-
}
124
+
$this->findAtomDocblock($matom, $method);
109
125
110
-
$this->parseParams($matom, $method);
126
+
if ($method->isFinal()) {
127
+
$matom->setProperty('final', true);
128
+
}
111
129
112
-
$matom->setName($method->getChildByIndex(2)->getConcreteString());
113
-
$matom->setLine($method->getLineNumber());
114
-
$matom->setFile($file_name);
130
+
if ($method->isAbstract()) {
131
+
$matom->setProperty('abstract', true);
132
+
}
115
133
116
-
$this->parseReturnType($matom, $method);
117
-
$atom->addChild($matom);
134
+
if ($method->isStatic()) {
135
+
$matom->setProperty('static', true);
136
+
}
118
137
119
-
$atoms[] = $matom;
138
+
if ($method->isPrivate()) {
139
+
$matom->setProperty('access', 'private');
140
+
} else if ($method->isProtected()) {
141
+
$matom->setProperty('access', 'protected');
142
+
} else {
143
+
$matom->setProperty('access', 'public');
120
144
}
121
145
122
-
$atoms[] = $atom;
146
+
$this->parseParams($matom, $method);
147
+
148
+
$this->parseReturnType($matom, $method);
149
+
$atom->addChild($matom);
150
+
151
+
$atoms[] = $matom;
123
152
}
153
+
154
+
$atoms[] = $atom;
124
155
}
125
156
126
157
return $atoms;
127
158
}
128
159
129
-
private function parseParams(DivinerAtom $atom, AASTNode $func) {
130
-
$params = $func
131
-
->getChildOfType(3, 'n_DECLARATION_PARAMETER_LIST')
132
-
->selectDescendantsOfType('n_DECLARATION_PARAMETER');
160
+
private function parseParams(
161
+
DivinerAtom $atom,
162
+
PhpParser\Node\FunctionLike $func) {
163
+
164
+
$params = $func->getParams();
133
165
134
166
$param_spec = array();
135
167
···
158
190
}
159
191
160
192
foreach ($params as $param) {
161
-
$name = $param->getChildByIndex(1)->getConcreteString();
193
+
$name = '$'.$param->var->name;
162
194
$dict = array(
163
-
'type' => $param->getChildByIndex(0)->getConcreteString(),
164
-
'default' => $param->getChildByIndex(2)->getConcreteString(),
195
+
'type' => $this->stringify($param->type),
196
+
'default' => $this->stringify($param->default),
165
197
);
166
198
167
199
if ($docs) {
···
190
222
$atom->setProperty('parameters', $param_spec);
191
223
}
192
224
193
-
private function findAtomDocblock(DivinerAtom $atom, XHPASTNode $node) {
194
-
$token = $node->getDocblockToken();
195
-
if ($token) {
196
-
$atom->setDocblockRaw($token->getValue());
197
-
return true;
225
+
private function findAtomDocblock(DivinerAtom $atom, PhpParser\Node $node) {
226
+
$doc_comment = $node->getDocComment();
227
+
228
+
if ($doc_comment) {
229
+
$atom->setDocblockRaw($doc_comment->getText());
198
230
} else {
199
-
$tokens = $node->getTokens();
200
-
if ($tokens) {
201
-
$prev = head($tokens);
202
-
while ($prev = $prev->getPrevToken()) {
203
-
if ($prev->isAnyWhitespace()) {
204
-
continue;
205
-
}
206
-
break;
207
-
}
231
+
$comments = $node->getComments();
208
232
209
-
if ($prev && $prev->isComment()) {
210
-
$value = $prev->getValue();
211
-
$matches = null;
212
-
if (preg_match('/@(return|param|task|author)/', $value, $matches)) {
213
-
$atom->addWarning(
214
-
pht(
215
-
'Atom "%s" is preceded by a comment containing `%s`, but '.
216
-
'the comment is not a documentation comment. Documentation '.
217
-
'comments must begin with `%s`, followed by a newline. Did '.
218
-
'you mean to use a documentation comment? (As the comment is '.
219
-
'not a documentation comment, it will be ignored.)',
220
-
$atom->getName(),
221
-
'@'.$matches[1],
222
-
'/**'));
223
-
}
233
+
foreach ($comments as $comment) {
234
+
$value = $comment->getText();
235
+
$matches = null;
236
+
if (preg_match('/@(return|param|task|author)/', $value, $matches)) {
237
+
$atom->addWarning(
238
+
pht(
239
+
'Atom "%s" is preceded by a comment containing `%s`, but '.
240
+
'the comment is not a documentation comment. Documentation '.
241
+
'comments must begin with `%s`, followed by a newline. Did '.
242
+
'you mean to use a documentation comment? (As the comment is '.
243
+
'not a documentation comment, it will be ignored.)',
244
+
$atom->getName(),
245
+
'@'.$matches[1],
246
+
'/**'));
224
247
}
225
248
}
226
249
227
250
$atom->setDocblockRaw('');
228
-
return false;
229
251
}
230
252
}
231
253
···
263
285
return $dict;
264
286
}
265
287
266
-
private function parseReturnType(DivinerAtom $atom, XHPASTNode $decl) {
288
+
private function parseReturnType(
289
+
DivinerAtom $atom,
290
+
PhpParser\Node\FunctionLike $decl) {
291
+
267
292
$return_spec = array();
268
293
269
294
$metadata = $atom->getDocblockMeta();
···
314
339
$type = $split[0];
315
340
}
316
341
317
-
if ($decl->getChildByIndex(1)->getTypeName() == 'n_REFERENCE') {
342
+
if ($decl->returnsByRef()) {
318
343
$type = $type.' &';
319
344
}
320
345
···
333
358
}
334
359
335
360
$atom->setProperty('return', $return_spec);
361
+
}
362
+
363
+
private function stringify(?PhpParser\Node $node) {
364
+
if (!$node) {
365
+
return '';
366
+
}
367
+
368
+
return id(new PhpParser\PrettyPrinter\Standard())
369
+
->prettyPrint(array($node));
336
370
}
337
371
338
372
}