Update symbol generation scripts

Summary:
Scripts now return scoped symbols -- in particular, PHP class constants, fields, and methods. ctags gives some for other languages.

(Turns out XHPAST doesn't support traits. But no one uses traits anyway so it's probably fine.)

I couldn't find a list of the context types ctags uses (class/struct/union/enum/maybe others?), so the context code just ignores that. Also, it uses a blacklist for the symbol type instead of a whitelist because there are a ton, they vary by language, and I didn't want to unintentionally exclude anything (P480).

Test Plan: Scrape symbols from arcanist and phabricator. Upload them to sandbox. Search for things.

Reviewers: epriestley

Reviewed By: epriestley

CC: nh, aran, Korvin

Maniphest Tasks: T1602

Differential Revision: https://secure.phabricator.com/D3202

Alan Huang 14 years ago fd3ba984 8fbe6347

+65 -13

2 changed files

expand all

scripts

symbols

generate_ctags_symbols.php

generate_php_symbols.php

+16 -11

scripts/symbols/generate_ctags_symbols.php

··· 68 68 continue; 69 69 } 70 70 71 + // default $context to empty 72 + $extension_fields[] = ''; 71 73 list($token, $file_path, $line_num) = $tag_info; 72 - list($type, $language) = $extension_fields; 74 + list($type, $language, $context) = $extension_fields; 73 75 74 76 // strip "language:" 75 77 $language = substr($language, 9); ··· 82 84 $language = str_ireplace("c++", "cpp", $language); 83 85 $language = str_ireplace("c#", "csharp", $language); 84 86 85 - switch ($type) { 86 - case 'class': 87 - print_symbol($file_path, $line_num, 'class', $token, $language); 88 - break; 89 - case 'function': 90 - print_symbol($file_path, $line_num, 'function', $token, $language); 91 - break; 92 - default: 87 + // Ruby has "singleton method", for example 88 + $type = substr(str_replace(' ', '_', $type), 0, 12); 89 + // class:foo, struct:foo, union:foo, enum:foo, ... 90 + $context = last(explode(':', $context, 2)); 91 + 92 + $ignore = array( 93 + 'variable' => true, 94 + ); 95 + if (empty($ignore[$type])) { 96 + print_symbol($file_path, $line_num, $type, $token, $context, $language); 93 97 } 94 98 } 95 99 } 96 100 97 101 function ctags_get_parser_future($file_path) { 98 - $future = new ExecFuture('ctags -n --fields=Kl -o - %s', 102 + $future = new ExecFuture('ctags -n --fields=Kls -o - %s', 99 103 $file_path); 100 104 return $future; 101 105 } ··· 111 115 return true; 112 116 } 113 117 114 - function print_symbol($file, $line_num, $type, $token, $language) { 118 + function print_symbol($file, $line_num, $type, $token, $context, $language) { 115 119 // get rid of relative path 116 120 $file = explode('/', $file); 117 121 if ($file[0] == '.' || $file[0] == "..") { ··· 120 124 $file = '/' . implode('/', $file); 121 125 122 126 $parts = array( 127 + $context, 123 128 $token, 124 129 $type, 125 130 strtolower($language),

+49 -2

scripts/symbols/generate_php_symbols.php

··· 45 45 $future->resolve()); 46 46 47 47 $root = $tree->getRootNode(); 48 + $scopes = array(); 48 49 49 50 $functions = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION'); 50 51 foreach ($functions as $function) { ··· 56 57 foreach ($classes as $class) { 57 58 $class_name = $class->getChildByIndex(1); 58 59 print_symbol($file, 'class', $class_name); 60 + $scopes[] = array($class, $class_name); 59 61 } 60 62 61 63 $interfaces = $root->selectDescendantsOfType('n_INTERFACE_DECLARATION'); 62 64 foreach ($interfaces as $interface) { 63 65 $interface_name = $interface->getChildByIndex(1); 64 66 print_symbol($file, 'interface', $interface_name); 67 + $scopes[] = array($interface, $interface_name); 68 + } 69 + 70 + $constants = $root->selectDescendantsOfType('n_CONSTANT_DECLARATION_LIST'); 71 + foreach ($constants as $constant_list) { 72 + foreach ($constant_list->getChildren() as $constant) { 73 + $constant_name = $constant->getChildByIndex(0); 74 + print_symbol($file, 'constant', $constant_name); 75 + } 76 + } 77 + 78 + foreach ($scopes as $scope) { 79 + // this prints duplicate symbols in the case of nested classes 80 + // luckily, PHP doesn't allow those 81 + list($class, $class_name) = $scope; 82 + 83 + $consts = $class->selectDescendantsOfType( 84 + 'n_CLASS_CONSTANT_DECLARATION_LIST'); 85 + foreach ($consts as $const_list) { 86 + foreach ($const_list->getChildren() as $const) { 87 + $const_name = $const->getChildByIndex(0); 88 + print_symbol($file, 'class_const', $const_name, $class_name); 89 + } 90 + } 91 + 92 + $members = $class->selectDescendantsOfType( 93 + 'n_CLASS_MEMBER_DECLARATION_LIST'); 94 + foreach ($members as $member_list) { 95 + foreach ($member_list->getChildren() as $member) { 96 + if ($member->getTypeName() == 'n_CLASS_MEMBER_MODIFIER_LIST') { 97 + continue; 98 + } 99 + $member_name = $member->getChildByIndex(0); 100 + print_symbol($file, 'member', $member_name, $class_name); 101 + } 102 + } 103 + 104 + $methods = $class->selectDescendantsOfType('n_METHOD_DECLARATION'); 105 + foreach ($methods as $method) { 106 + $method_name = $method->getChildByIndex(2); 107 + print_symbol($file, 'method', $method_name, $class_name); 108 + } 65 109 } 66 110 } 67 111 68 - function print_symbol($file, $type, $token) { 112 + function print_symbol($file, $type, $token, $context=null) { 69 113 $parts = array( 70 - $token->getConcreteString(), 114 + $context ? $context->getConcreteString() : '', 115 + // variable tokens are `$name`, not just `name`, so strip the $ off of 116 + // class field names 117 + ltrim($token->getConcreteString(), '$'), 71 118 $type, 72 119 'php', 73 120 $token->getLineNumber(),

Configure Feed

Configure Feed