When rendering Jupyter notebook diffs, split code inputs into individual blocks

+21 -21

resources/celerity/map.php

··· 9 9 'names' => array( 10 10 'conpherence.pkg.css' => '3c8a0668', 11 11 'conpherence.pkg.js' => '020aebcf', 12 - 'core.pkg.css' => '6a8c9533', 12 + 'core.pkg.css' => '19ec9519', 13 13 'core.pkg.js' => '6e5c894f', 14 - 'differential.pkg.css' => 'abd2c0d8', 15 - 'differential.pkg.js' => '68fa36fc', 14 + 'differential.pkg.css' => '607c84be', 15 + 'differential.pkg.js' => 'a0212a0b', 16 16 'diffusion.pkg.css' => '42c75c37', 17 17 'diffusion.pkg.js' => 'a98c0bf7', 18 18 'maniphest.pkg.css' => '35995d6d', ··· 61 61 'rsrc/css/application/dashboard/dashboard.css' => '5a205b9d', 62 62 'rsrc/css/application/diff/inline-comment-summary.css' => '81eb368d', 63 63 'rsrc/css/application/differential/add-comment.css' => '7e5900d9', 64 - 'rsrc/css/application/differential/changeset-view.css' => '5dda5e53', 64 + 'rsrc/css/application/differential/changeset-view.css' => '489b6995', 65 65 'rsrc/css/application/differential/core.css' => '7300a73e', 66 66 'rsrc/css/application/differential/phui-inline-comment.css' => '48acce5b', 67 67 'rsrc/css/application/differential/revision-comment.css' => '7dbc8d1d', ··· 113 113 'rsrc/css/application/uiexample/example.css' => 'b4795059', 114 114 'rsrc/css/core/core.css' => '1b29ed61', 115 115 'rsrc/css/core/remarkup.css' => 'f06cc20e', 116 - 'rsrc/css/core/syntax.css' => '4234f572', 116 + 'rsrc/css/core/syntax.css' => '220b85f9', 117 117 'rsrc/css/core/z-index.css' => '99c0f5eb', 118 118 'rsrc/css/diviner/diviner-shared.css' => '4bd263b0', 119 119 'rsrc/css/font/font-awesome.css' => '3883938a', ··· 169 169 'rsrc/css/phui/phui-pager.css' => 'd022c7ad', 170 170 'rsrc/css/phui/phui-pinboard-view.css' => '1f08f5d8', 171 171 'rsrc/css/phui/phui-policy-section-view.css' => '139fdc64', 172 - 'rsrc/css/phui/phui-property-list-view.css' => '34180764', 172 + 'rsrc/css/phui/phui-property-list-view.css' => 'ad841f1c', 173 173 'rsrc/css/phui/phui-remarkup-preview.css' => '91767007', 174 174 'rsrc/css/phui/phui-segment-bar-view.css' => '5166b370', 175 175 'rsrc/css/phui/phui-spacing.css' => 'b05cadc3', ··· 377 377 'rsrc/js/application/dashboard/behavior-dashboard-query-panel-select.js' => '1e413dc9', 378 378 'rsrc/js/application/dashboard/behavior-dashboard-tab-panel.js' => '0116d3e8', 379 379 'rsrc/js/application/diff/DiffChangeset.js' => 'a31ffc00', 380 - 'rsrc/js/application/diff/DiffChangesetList.js' => '22f6bb51', 380 + 'rsrc/js/application/diff/DiffChangesetList.js' => '0f5c016d', 381 381 'rsrc/js/application/diff/DiffInline.js' => 'a4a14a94', 382 382 'rsrc/js/application/diff/behavior-preview-link.js' => 'f51e9c17', 383 383 'rsrc/js/application/differential/behavior-diff-radios.js' => '925fe8cd', ··· 554 554 'conpherence-thread-manager' => 'aec8e38c', 555 555 'conpherence-transaction-css' => '3a3f5e7e', 556 556 'd3' => '9d068042', 557 - 'differential-changeset-view-css' => '5dda5e53', 557 + 'differential-changeset-view-css' => '489b6995', 558 558 'differential-core-view-css' => '7300a73e', 559 559 'differential-revision-add-comment-css' => '7e5900d9', 560 560 'differential-revision-comment-css' => '7dbc8d1d', ··· 774 774 'phabricator-darkmessage' => '26cd4b73', 775 775 'phabricator-dashboard-css' => '5a205b9d', 776 776 'phabricator-diff-changeset' => 'a31ffc00', 777 - 'phabricator-diff-changeset-list' => '22f6bb51', 777 + 'phabricator-diff-changeset-list' => '0f5c016d', 778 778 'phabricator-diff-inline' => 'a4a14a94', 779 779 'phabricator-drag-and-drop-file-upload' => '4370900d', 780 780 'phabricator-draggable-list' => 'c9ad6f70', ··· 865 865 'phui-pager-css' => 'd022c7ad', 866 866 'phui-pinboard-view-css' => '1f08f5d8', 867 867 'phui-policy-section-view-css' => '139fdc64', 868 - 'phui-property-list-view-css' => '34180764', 868 + 'phui-property-list-view-css' => 'ad841f1c', 869 869 'phui-remarkup-preview-css' => '91767007', 870 870 'phui-segment-bar-view-css' => '5166b370', 871 871 'phui-spacing-css' => 'b05cadc3', ··· 900 900 'sprite-login-css' => '18b368a6', 901 901 'sprite-tokens-css' => 'f1896dc5', 902 902 'syntax-default-css' => '055fc231', 903 - 'syntax-highlighting-css' => '4234f572', 903 + 'syntax-highlighting-css' => '220b85f9', 904 904 'tokens-css' => 'ce5a50bd', 905 905 'trigger-rule' => '41b7b4f6', 906 906 'trigger-rule-control' => '5faf27b9', ··· 1005 1005 'javelin-workflow', 1006 1006 'phuix-icon-view', 1007 1007 ), 1008 + '0f5c016d' => array( 1009 + 'javelin-install', 1010 + 'phuix-button-view', 1011 + ), 1008 1012 '111bfd2d' => array( 1009 1013 'javelin-install', 1010 1014 ), ··· 1065 1069 'javelin-behavior', 1066 1070 'javelin-request', 1067 1071 ), 1072 + '220b85f9' => array( 1073 + 'syntax-default-css', 1074 + ), 1068 1075 '225bbb98' => array( 1069 1076 'javelin-install', 1070 1077 'javelin-reactor', ··· 1075 1082 'javelin-typeahead-source', 1076 1083 'javelin-util', 1077 1084 ), 1078 - '22f6bb51' => array( 1079 - 'javelin-install', 1080 - 'phuix-button-view', 1081 - ), 1082 1085 23387297 => array( 1083 1086 'javelin-install', 1084 1087 'javelin-util', ··· 1260 1263 'javelin-behavior', 1261 1264 'javelin-uri', 1262 1265 ), 1263 - '4234f572' => array( 1264 - 'syntax-default-css', 1265 - ), 1266 1266 '4370900d' => array( 1267 1267 'javelin-install', 1268 1268 'javelin-util', ··· 1303 1303 'javelin-dom', 1304 1304 'phabricator-draggable-list', 1305 1305 ), 1306 + '489b6995' => array( 1307 + 'phui-inline-comment-view-css', 1308 + ), 1306 1309 '48fe33d0' => array( 1307 1310 'javelin-behavior', 1308 1311 'javelin-dom', ··· 1456 1459 'javelin-stratcom', 1457 1460 'javelin-dom', 1458 1461 'phuix-dropdown-menu', 1459 - ), 1460 - '5dda5e53' => array( 1461 - 'phui-inline-comment-view-css', 1462 1462 ), 1463 1463 '5faf27b9' => array( 1464 1464 'phuix-form-control-view',

+6

src/applications/differential/render/DifferentialChangesetTwoUpRenderer.php

··· 387 387 $old_classes[] = 'old-full'; 388 388 } 389 389 390 + $old_classes[] = 'diff-flush'; 391 + 390 392 $old_classes = implode(' ', $old_classes); 391 393 } else { 392 394 $old_content = null; ··· 404 406 $new_classes[] = 'new-full'; 405 407 } 406 408 409 + $new_classes[] = 'diff-flush'; 410 + 407 411 $new_classes = implode(' ', $new_classes); 408 412 } else { 409 413 $new_content = null; ··· 451 455 'td', 452 456 array( 453 457 'class' => $old_classes, 458 + 'data-copy-mode' => 'copy-l', 454 459 ), 455 460 $old_content); 456 461 ··· 479 484 array( 480 485 'class' => $new_classes, 481 486 'colspan' => '2', 487 + 'data-copy-mode' => 'copy-r', 482 488 ), 483 489 $new_content); 484 490

+210 -44

src/applications/files/document/PhabricatorJupyterDocumentEngine.php

··· 57 57 $viewer = $this->getViewer(); 58 58 $content = $ref->loadData(); 59 59 60 - $data = phutil_json_decode($content); 61 - $cells = idx($data, 'cells'); 62 - if (!is_array($cells)) { 63 - throw new Exception('Missing "cells".'); 64 - } 60 + $cells = $this->newCells($content, true); 65 61 66 62 $idx = 1; 67 63 $blocks = array(); ··· 82 78 ), 83 79 $notebook_table); 84 80 81 + // When the cell is a source code line, we can hash just the raw 82 + // input rather than all the cell metadata. 83 + 84 + switch (idx($cell, 'cell_type')) { 85 + case 'code/line': 86 + $hash_input = $cell['raw']; 87 + break; 88 + default: 89 + $hash_input = serialize($cell); 90 + break; 91 + } 92 + 85 93 $hash = PhabricatorHash::digestWithNamedKey( 86 - serialize($cell), 94 + $hash_input, 87 95 'document-engine.content-digest'); 88 96 89 97 $blocks[] = id(new PhabricatorDocumentEngineBlock()) ··· 102 110 $content = $ref->loadData(); 103 111 104 112 try { 113 + $cells = $this->newCells($content, false); 114 + } catch (Exception $ex) { 115 + return $this->newMessage($ex->getMessage()); 116 + } 117 + 118 + $rows = array(); 119 + foreach ($cells as $cell) { 120 + $rows[] = $this->renderJupyterCell($viewer, $cell); 121 + } 122 + 123 + $notebook_table = phutil_tag( 124 + 'table', 125 + array( 126 + 'class' => 'jupyter-notebook', 127 + ), 128 + $rows); 129 + 130 + $container = phutil_tag( 131 + 'div', 132 + array( 133 + 'class' => 'document-engine-jupyter', 134 + ), 135 + $notebook_table); 136 + 137 + return $container; 138 + } 139 + 140 + private function newCells($content, $for_diff) { 141 + try { 105 142 $data = phutil_json_decode($content); 106 143 } catch (PhutilJSONParserException $ex) { 107 - return $this->newMessage( 144 + throw new Exception( 108 145 pht( 109 146 'This is not a valid JSON document and can not be rendered as '. 110 147 'a Jupyter notebook: %s.', ··· 112 149 } 113 150 114 151 if (!is_array($data)) { 115 - return $this->newMessage( 152 + throw new Exception( 116 153 pht( 117 154 'This document does not encode a valid JSON object and can not '. 118 155 'be rendered as a Jupyter notebook.')); ··· 121 158 122 159 $nbformat = idx($data, 'nbformat'); 123 160 if (!strlen($nbformat)) { 124 - return $this->newMessage( 161 + throw new Exception( 125 162 pht( 126 163 'This document is missing an "nbformat" field. Jupyter notebooks '. 127 164 'must have this field.')); 128 165 } 129 166 130 167 if ($nbformat !== 4) { 131 - return $this->newMessage( 168 + throw new Exception( 132 169 pht( 133 170 'This Jupyter notebook uses an unsupported version of the file '. 134 171 'format (found version %s, expected version 4).', ··· 137 174 138 175 $cells = idx($data, 'cells'); 139 176 if (!is_array($cells)) { 140 - return $this->newMessage( 177 + throw new Exception( 141 178 pht( 142 179 'This Jupyter notebook does not specify a list of "cells".')); 143 180 } 144 181 145 182 if (!$cells) { 146 - return $this->newMessage( 183 + throw new Exception( 147 184 pht( 148 185 'This Jupyter notebook does not specify any notebook cells.')); 149 186 } 150 187 151 - $rows = array(); 188 + if (!$for_diff) { 189 + return $cells; 190 + } 191 + 192 + // If we're extracting cells to build a diff view, split code cells into 193 + // individual lines and individual outputs. We want users to be able to 194 + // add inline comments to each line and each output block. 195 + 196 + $results = array(); 152 197 foreach ($cells as $cell) { 153 - $rows[] = $this->renderJupyterCell($viewer, $cell); 154 - } 198 + $cell_type = idx($cell, 'cell_type'); 155 199 156 - $notebook_table = phutil_tag( 157 - 'table', 158 - array( 159 - 'class' => 'jupyter-notebook', 160 - ), 161 - $rows); 200 + if ($cell_type !== 'code') { 201 + $results[] = $cell; 202 + continue; 203 + } 204 + 205 + $label = $this->newCellLabel($cell); 206 + 207 + $lines = idx($cell, 'source'); 208 + if (!is_array($lines)) { 209 + $lines = array(); 210 + } 211 + 212 + $content = $this->highlightLines($lines); 213 + 214 + $count = count($lines); 215 + for ($ii = 0; $ii < $count; $ii++) { 216 + $is_head = ($ii === 0); 217 + $is_last = ($ii === ($count - 1)); 218 + 219 + if ($is_head) { 220 + $line_label = $label; 221 + } else { 222 + $line_label = null; 223 + } 162 224 163 - $container = phutil_tag( 164 - 'div', 165 - array( 166 - 'class' => 'document-engine-jupyter', 167 - ), 168 - $notebook_table); 225 + $results[] = array( 226 + 'cell_type' => 'code/line', 227 + 'label' => $line_label, 228 + 'raw' => $lines[$ii], 229 + 'display' => idx($content, $ii), 230 + 'head' => $is_head, 231 + 'last' => $is_last, 232 + ); 233 + } 169 234 170 - return $container; 235 + $outputs = array(); 236 + $output_list = idx($cell, 'outputs'); 237 + if (is_array($output_list)) { 238 + foreach ($output_list as $output) { 239 + $results[] = array( 240 + 'cell_type' => 'code/output', 241 + 'output' => $output, 242 + ); 243 + } 244 + } 245 + } 246 + 247 + return $results; 171 248 } 249 + 172 250 173 251 private function renderJupyterCell( 174 252 PhabricatorUser $viewer, ··· 177 255 list($label, $content) = $this->renderJupyterCellContent($viewer, $cell); 178 256 179 257 $label_cell = phutil_tag( 180 - 'th', 181 - array(), 258 + 'td', 259 + array( 260 + 'class' => 'jupyter-label', 261 + ), 182 262 $label); 183 263 264 + $classes = null; 265 + switch (idx($cell, 'cell_type')) { 266 + case 'code/line': 267 + $classes = 'jupyter-cell-flush'; 268 + break; 269 + } 270 + 184 271 $content_cell = phutil_tag( 185 272 'td', 186 - array(), 273 + array( 274 + 'class' => $classes, 275 + ), 187 276 $content); 188 277 189 278 return phutil_tag( ··· 204 293 case 'markdown': 205 294 return $this->newMarkdownCell($cell); 206 295 case 'code': 207 - return $this->newCodeCell($cell); 296 + return $this->newCodeCell($cell); 297 + case 'code/line': 298 + return $this->newCodeLineCell($cell); 299 + case 'code/output': 300 + return $this->newCodeOutputCell($cell); 208 301 } 209 302 210 303 return $this->newRawCell(id(new PhutilJSON())->encodeFormatted($cell)); ··· 243 336 } 244 337 245 338 private function newCodeCell(array $cell) { 246 - $execution_count = idx($cell, 'execution_count'); 247 - if ($execution_count) { 248 - $label = 'In ['.$execution_count.']:'; 249 - } else { 250 - $label = null; 251 - } 339 + $label = $this->newCellLabel($cell); 252 340 253 341 $content = idx($cell, 'source'); 254 342 if (!is_array($content)) { 255 343 $content = array(); 256 344 } 257 345 258 - $content = implode('', $content); 259 - 260 - $content = PhabricatorSyntaxHighlighter::highlightWithLanguage( 261 - 'py', 262 - $content); 346 + $content = $this->highlightLines($content); 263 347 264 348 $outputs = array(); 265 349 $output_list = idx($cell, 'outputs'); ··· 275 359 phutil_tag( 276 360 'div', 277 361 array( 278 - 'class' => 'jupyter-cell-code PhabricatorMonospaced remarkup-code', 362 + 'class' => 363 + 'jupyter-cell-code jupyter-cell-code-block '. 364 + 'PhabricatorMonospaced remarkup-code', 279 365 ), 280 366 array( 281 367 $content, ··· 285 371 ); 286 372 } 287 373 374 + private function newCodeLineCell(array $cell) { 375 + $classes = array(); 376 + $classes[] = 'PhabricatorMonospaced'; 377 + $classes[] = 'remarkup-code'; 378 + $classes[] = 'jupyter-cell-code'; 379 + $classes[] = 'jupyter-cell-code-line'; 380 + 381 + if ($cell['head']) { 382 + $classes[] = 'jupyter-cell-code-head'; 383 + } 384 + 385 + if ($cell['last']) { 386 + $classes[] = 'jupyter-cell-code-last'; 387 + } 388 + 389 + $classes = implode(' ', $classes); 390 + 391 + return array( 392 + $cell['label'], 393 + array( 394 + phutil_tag( 395 + 'div', 396 + array( 397 + 'class' => $classes, 398 + ), 399 + array( 400 + $cell['display'], 401 + )), 402 + ), 403 + ); 404 + } 405 + 406 + private function newCodeOutputCell(array $cell) { 407 + return array( 408 + null, 409 + $this->newOutput($cell['output']), 410 + ); 411 + } 412 + 288 413 private function newOutput(array $output) { 289 414 if (!is_array($output)) { 290 415 return pht('<Invalid Output>'); ··· 369 494 'class' => implode(' ', $classes), 370 495 ), 371 496 $content); 497 + } 498 + 499 + private function newCellLabel(array $cell) { 500 + $execution_count = idx($cell, 'execution_count'); 501 + if ($execution_count) { 502 + $label = 'In ['.$execution_count.']:'; 503 + } else { 504 + $label = null; 505 + } 506 + 507 + return $label; 508 + } 509 + 510 + private function highlightLines(array $lines) { 511 + $head = head($lines); 512 + $matches = null; 513 + if (preg_match('/^%%(.*)$/', $head, $matches)) { 514 + $restore = array_shift($lines); 515 + $lang = $matches[1]; 516 + } else { 517 + $restore = null; 518 + $lang = 'py'; 519 + } 520 + 521 + $content = PhabricatorSyntaxHighlighter::highlightWithLanguage( 522 + $lang, 523 + implode('', $lines)); 524 + $content = phutil_split_lines($content); 525 + 526 + if ($restore !== null) { 527 + $language_tag = phutil_tag( 528 + 'span', 529 + array( 530 + 'class' => 'language-tag', 531 + ), 532 + $restore); 533 + 534 + array_unshift($content, $language_tag); 535 + } 536 + 537 + return $content; 372 538 } 373 539 374 540 }

+5

webroot/rsrc/css/application/differential/changeset-view.css

··· 63 63 padding: 1px 8px; 64 64 } 65 65 66 + .differential-diff td.diff-flush { 67 + padding-top: 0; 68 + padding-bottom: 0; 69 + } 70 + 66 71 .device .differential-diff td { 67 72 padding: 1px 4px; 68 73 }

+4

webroot/rsrc/css/core/syntax.css

··· 6 6 color: #aa0066; 7 7 } 8 8 9 + .remarkup-code .language-tag { 10 + color: {$lightgreytext}; 11 + } 12 + 9 13 .remarkup-code td > span { 10 14 display: inline; 11 15 word-break: break-all;

+39 -5

webroot/rsrc/css/phui/phui-property-list-view.css

··· 298 298 299 299 .jupyter-cell-code { 300 300 white-space: pre-wrap; 301 + word-break: break-word; 301 302 background: {$lightgreybackground}; 303 + border-radius: 2px; 304 + border-color: {$lightgreyborder}; 305 + border-style: solid; 306 + } 307 + 308 + .jupyter-cell-code-block { 302 309 padding: 8px; 303 - border: 1px solid {$lightgreyborder}; 304 - border-radius: 2px; 310 + border-width: 1px; 305 311 } 306 312 307 - .jupyter-notebook > tbody > tr > th, 313 + .jupyter-cell-code-line { 314 + padding: 2px 8px; 315 + border-width: 0 1px; 316 + } 317 + 318 + .jupyter-cell-code-head { 319 + border-top-width: 1px; 320 + margin-top: 4px; 321 + padding-top: 8px; 322 + } 323 + 324 + .jupyter-cell-code-last { 325 + border-bottom-width: 1px; 326 + margin-bottom: 4px; 327 + padding-bottom: 8px; 328 + } 329 + 308 330 .jupyter-notebook > tbody > tr > td { 309 331 padding: 8px; 310 332 } 311 333 312 - .jupyter-notebook > tbody > tr > th { 334 + .jupyter-notebook > tbody > tr > td.jupyter-cell-flush { 335 + padding-top: 0; 336 + padding-bottom: 0; 337 + } 338 + 339 + .jupyter-notebook, 340 + .jupyter-notebook > tbody > tr > td { 341 + width: 100%; 342 + } 343 + 344 + .jupyter-notebook > tbody > tr > td.jupyter-label { 313 345 white-space: nowrap; 314 346 text-align: right; 315 - min-width: 48px; 347 + min-width: 56px; 316 348 font-weight: bold; 349 + width: auto; 350 + padding: 8px 8px 0; 317 351 } 318 352 319 353 .jupyter-output {

+14 -29

webroot/rsrc/js/application/diff/DiffChangesetList.js

··· 1195 1195 bot = tmp; 1196 1196 } 1197 1197 1198 - // Find the leftmost cell that we're going to highlight: this is the next 1199 - // <td /> in the row that does not have a "data-n" (line number) 1200 - // attribute. In 2up views, it should be directly adjacent. In 1201 - // 1up views, we may have to skip over the other line number column. 1202 - var l = top; 1203 - while (l.nextSibling && l.getAttribute('data-n')) { 1204 - l = l.nextSibling; 1198 + // Find the leftmost cell that we're going to highlight. This is the 1199 + // next sibling with a "data-copy-mode" attribute, which is a marker 1200 + // for the cell with actual content in it. 1201 + var content_cell = top; 1202 + while (content_cell && !content_cell.getAttribute('data-copy-mode')) { 1203 + content_cell = content_cell.nextSibling; 1205 1204 } 1206 1205 1207 - // Find the rightmost cell that we're going to highlight: this is the 1208 - // farthest consecutive, adjacent <td /> in the row that does not have 1209 - // a "data-n" (line number) attribute. Sometimes the left and right nodes 1210 - // are the same (left side of 2up view); sometimes we're going to 1211 - // highlight several nodes (copy + code + coverage). 1212 - var r = l; 1213 - while (true) { 1214 - // No more cells in the row, so we can't keep expanding. 1215 - if (!r.nextSibling) { 1216 - break; 1217 - } 1218 - 1219 - if (r.nextSibling.getAttribute('data-n')) { 1220 - break; 1221 - } 1222 - 1223 - r = r.nextSibling; 1206 + // If we didn't find a cell to highlight, don't highlight anything. 1207 + if (!content_cell) { 1208 + return; 1224 1209 } 1225 1210 1226 - var pos = JX.$V(l) 1227 - .add(JX.Vector.getAggregateScrollForNode(l)); 1211 + var pos = JX.$V(content_cell) 1212 + .add(JX.Vector.getAggregateScrollForNode(content_cell)); 1228 1213 1229 - var dim = JX.$V(r) 1230 - .add(JX.Vector.getAggregateScrollForNode(r)) 1214 + var dim = JX.$V(content_cell) 1215 + .add(JX.Vector.getAggregateScrollForNode(content_cell)) 1231 1216 .add(-pos.x, -pos.y) 1232 - .add(JX.Vector.getDim(r)); 1217 + .add(JX.Vector.getDim(content_cell)); 1233 1218 1234 1219 var bpos = JX.$V(bot) 1235 1220 .add(JX.Vector.getAggregateScrollForNode(bot));

Configure Feed

Configure Feed