@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add a very rough, proof-of-concept Jupyter notebook document engine
Summary:
Depends on D19252. Ref T13105. This very roughly renders Jupyter notebooks.
It's probably better than showing the raw JSON, but not by much.
Test Plan:
- Viewed various notebooks with various cell types, including markdown, code, stdout, stderr, images, HTML, and Javascript.
- HTML and Javascript are not live-fired since they're wildly dangerous.
Maniphest Tasks: T13105
Differential Revision: https://secure.phabricator.com/D19253
+366
-3
+3
-3
resources/celerity/map.php
+3
-3
resources/celerity/map.php
···
9
9
'names' => array(
10
10
'conpherence.pkg.css' => 'e68cf1fa',
11
11
'conpherence.pkg.js' => '15191c65',
12
-
'core.pkg.css' => '2d73b2f3',
12
+
'core.pkg.css' => '7daac340',
13
13
'core.pkg.js' => 'b9b4a943',
14
14
'differential.pkg.css' => '113e692c',
15
15
'differential.pkg.js' => 'f6d809c0',
···
168
168
'rsrc/css/phui/phui-object-box.css' => '9cff003c',
169
169
'rsrc/css/phui/phui-pager.css' => 'edcbc226',
170
170
'rsrc/css/phui/phui-pinboard-view.css' => '2495140e',
171
-
'rsrc/css/phui/phui-property-list-view.css' => '47018d3c',
171
+
'rsrc/css/phui/phui-property-list-view.css' => '871f6815',
172
172
'rsrc/css/phui/phui-remarkup-preview.css' => '54a34863',
173
173
'rsrc/css/phui/phui-segment-bar-view.css' => 'b1d1b892',
174
174
'rsrc/css/phui/phui-spacing.css' => '042804d6',
···
850
850
'phui-oi-simple-ui-css' => 'a8beebea',
851
851
'phui-pager-css' => 'edcbc226',
852
852
'phui-pinboard-view-css' => '2495140e',
853
-
'phui-property-list-view-css' => '47018d3c',
853
+
'phui-property-list-view-css' => '871f6815',
854
854
'phui-remarkup-preview-css' => '54a34863',
855
855
'phui-segment-bar-view-css' => 'b1d1b892',
856
856
'phui-spacing-css' => '042804d6',
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
3190
3190
'PhabricatorJSONExportFormat' => 'infrastructure/export/format/PhabricatorJSONExportFormat.php',
3191
3191
'PhabricatorJavelinLinter' => 'infrastructure/lint/linter/PhabricatorJavelinLinter.php',
3192
3192
'PhabricatorJiraIssueHasObjectEdgeType' => 'applications/doorkeeper/edge/PhabricatorJiraIssueHasObjectEdgeType.php',
3193
+
'PhabricatorJupyterDocumentEngine' => 'applications/files/document/PhabricatorJupyterDocumentEngine.php',
3193
3194
'PhabricatorKeyValueDatabaseCache' => 'applications/cache/PhabricatorKeyValueDatabaseCache.php',
3194
3195
'PhabricatorKeyValueSerializingCacheProxy' => 'applications/cache/PhabricatorKeyValueSerializingCacheProxy.php',
3195
3196
'PhabricatorKeyboardRemarkupRule' => 'infrastructure/markup/rule/PhabricatorKeyboardRemarkupRule.php',
···
8800
8801
'PhabricatorJSONExportFormat' => 'PhabricatorExportFormat',
8801
8802
'PhabricatorJavelinLinter' => 'ArcanistLinter',
8802
8803
'PhabricatorJiraIssueHasObjectEdgeType' => 'PhabricatorEdgeType',
8804
+
'PhabricatorJupyterDocumentEngine' => 'PhabricatorDocumentEngine',
8803
8805
'PhabricatorKeyValueDatabaseCache' => 'PhutilKeyValueCache',
8804
8806
'PhabricatorKeyValueSerializingCacheProxy' => 'PhutilKeyValueCacheProxy',
8805
8807
'PhabricatorKeyboardRemarkupRule' => 'PhutilRemarkupRule',
+9
src/applications/files/document/PhabricatorDocumentRef.php
+9
src/applications/files/document/PhabricatorDocumentRef.php
···
110
110
return (strpos($snippet, "\0") === false);
111
111
}
112
112
113
+
public function isProbablyJSON() {
114
+
if (!$this->isProbablyText()) {
115
+
return false;
116
+
}
117
+
118
+
$snippet = $this->getSnippet();
119
+
return phutil_is_utf8($snippet);
120
+
}
121
+
113
122
public function getSnippet() {
114
123
if ($this->snippet === null) {
115
124
$this->snippet = $this->loadData(null, (1024 * 1024 * 1));
+305
src/applications/files/document/PhabricatorJupyterDocumentEngine.php
+305
src/applications/files/document/PhabricatorJupyterDocumentEngine.php
···
1
+
<?php
2
+
3
+
final class PhabricatorJupyterDocumentEngine
4
+
extends PhabricatorDocumentEngine {
5
+
6
+
const ENGINEKEY = 'jupyter';
7
+
8
+
public function getViewAsLabel(PhabricatorDocumentRef $ref) {
9
+
return pht('View as Jupyter Notebook');
10
+
}
11
+
12
+
protected function getDocumentIconIcon(PhabricatorDocumentRef $ref) {
13
+
return 'fa-sun-o';
14
+
}
15
+
16
+
protected function getContentScore(PhabricatorDocumentRef $ref) {
17
+
$name = $ref->getName();
18
+
19
+
if (preg_match('/\\.ipynb\z/i', $name)) {
20
+
return 2000;
21
+
}
22
+
23
+
return 500;
24
+
}
25
+
26
+
protected function canRenderDocumentType(PhabricatorDocumentRef $ref) {
27
+
return $ref->isProbablyJSON();
28
+
}
29
+
30
+
protected function newDocumentContent(PhabricatorDocumentRef $ref) {
31
+
$viewer = $this->getViewer();
32
+
$content = $ref->loadData();
33
+
34
+
try {
35
+
$data = phutil_json_decode($content);
36
+
} catch (PhutilJSONParserException $ex) {
37
+
return $this->newMessage(
38
+
pht(
39
+
'This is not a valid JSON document and can not be rendered as '.
40
+
'a Jupyter notebook: %s.',
41
+
$ex->getMessage()));
42
+
}
43
+
44
+
if (!is_array($data)) {
45
+
return $this->newMessage(
46
+
pht(
47
+
'This document does not encode a valid JSON object and can not '.
48
+
'be rendered as a Jupyter notebook.'));
49
+
}
50
+
51
+
52
+
$nbformat = idx($data, 'nbformat');
53
+
if (!strlen($nbformat)) {
54
+
return $this->newMessage(
55
+
pht(
56
+
'This document is missing an "nbformat" field. Jupyter notebooks '.
57
+
'must have this field.'));
58
+
}
59
+
60
+
if ($nbformat !== 4) {
61
+
return $this->newMessage(
62
+
pht(
63
+
'This Jupyter notebook uses an unsupported version of the file '.
64
+
'format (found version %s, expected version 4).',
65
+
$nbformat));
66
+
}
67
+
68
+
$cells = idx($data, 'cells');
69
+
if (!is_array($cells)) {
70
+
return $this->newMessage(
71
+
pht(
72
+
'This Jupyter notebook does not specify a list of "cells".'));
73
+
}
74
+
75
+
if (!$cells) {
76
+
return $this->newMessage(
77
+
pht(
78
+
'This Jupyter notebook does not specify any notebook cells.'));
79
+
}
80
+
81
+
$rows = array();
82
+
foreach ($cells as $cell) {
83
+
$rows[] = $this->renderJupyterCell($viewer, $cell);
84
+
}
85
+
86
+
$notebook_table = phutil_tag(
87
+
'table',
88
+
array(
89
+
'class' => 'jupyter-notebook',
90
+
),
91
+
$rows);
92
+
93
+
$container = phutil_tag(
94
+
'div',
95
+
array(
96
+
'class' => 'document-engine-jupyter',
97
+
),
98
+
$notebook_table);
99
+
100
+
return $container;
101
+
}
102
+
103
+
private function renderJupyterCell(
104
+
PhabricatorUser $viewer,
105
+
array $cell) {
106
+
107
+
list($label, $content) = $this->renderJupyterCellContent($viewer, $cell);
108
+
109
+
$label_cell = phutil_tag(
110
+
'th',
111
+
array(),
112
+
$label);
113
+
114
+
$content_cell = phutil_tag(
115
+
'td',
116
+
array(),
117
+
$content);
118
+
119
+
return phutil_tag(
120
+
'tr',
121
+
array(),
122
+
array(
123
+
$label_cell,
124
+
$content_cell,
125
+
));
126
+
}
127
+
128
+
private function renderJupyterCellContent(
129
+
PhabricatorUser $viewer,
130
+
array $cell) {
131
+
132
+
$cell_type = idx($cell, 'cell_type');
133
+
switch ($cell_type) {
134
+
case 'markdown':
135
+
return $this->newMarkdownCell($cell);
136
+
case 'code':
137
+
return $this->newCodeCell($cell);
138
+
}
139
+
140
+
return $this->newRawCell(id(new PhutilJSON())->encodeFormatted($cell));
141
+
}
142
+
143
+
private function newRawCell($content) {
144
+
return array(
145
+
null,
146
+
phutil_tag(
147
+
'div',
148
+
array(
149
+
'class' => 'jupyter-cell-raw PhabricatorMonospaced',
150
+
),
151
+
$content),
152
+
);
153
+
}
154
+
155
+
private function newMarkdownCell(array $cell) {
156
+
$content = idx($cell, 'source');
157
+
if (!is_array($content)) {
158
+
$content = array();
159
+
}
160
+
161
+
$content = implode('', $content);
162
+
$content = phutil_escape_html_newlines($content);
163
+
164
+
return array(
165
+
null,
166
+
phutil_tag(
167
+
'div',
168
+
array(
169
+
'class' => 'jupyter-cell-markdown',
170
+
),
171
+
$content),
172
+
);
173
+
}
174
+
175
+
private function newCodeCell(array $cell) {
176
+
$execution_count = idx($cell, 'execution_count');
177
+
if ($execution_count) {
178
+
$label = 'In ['.$execution_count.']:';
179
+
} else {
180
+
$label = null;
181
+
}
182
+
183
+
$content = idx($cell, 'source');
184
+
if (!is_array($content)) {
185
+
$content = array();
186
+
}
187
+
188
+
$content = implode('', $content);
189
+
190
+
$content = PhabricatorSyntaxHighlighter::highlightWithLanguage(
191
+
'python',
192
+
$content);
193
+
194
+
$outputs = array();
195
+
$output_list = idx($cell, 'outputs');
196
+
if (is_array($output_list)) {
197
+
foreach ($output_list as $output) {
198
+
$outputs[] = $this->newOutput($output);
199
+
}
200
+
}
201
+
202
+
return array(
203
+
$label,
204
+
array(
205
+
phutil_tag(
206
+
'div',
207
+
array(
208
+
'class' => 'jupyter-cell-code PhabricatorMonospaced remarkup-code',
209
+
),
210
+
array(
211
+
$content,
212
+
)),
213
+
$outputs,
214
+
),
215
+
);
216
+
}
217
+
218
+
private function newOutput(array $output) {
219
+
if (!is_array($output)) {
220
+
return pht('<Invalid Output>');
221
+
}
222
+
223
+
$classes = array(
224
+
'jupyter-output',
225
+
'PhabricatorMonospaced',
226
+
);
227
+
228
+
$output_name = idx($output, 'name');
229
+
switch ($output_name) {
230
+
case 'stderr':
231
+
$classes[] = 'jupyter-output-stderr';
232
+
break;
233
+
}
234
+
235
+
$output_type = idx($output, 'output_type');
236
+
switch ($output_type) {
237
+
case 'execute_result':
238
+
case 'display_data':
239
+
$data = idx($output, 'data');
240
+
241
+
$image_formats = array(
242
+
'image/png',
243
+
'image/jpeg',
244
+
'image/jpg',
245
+
'image/gif',
246
+
);
247
+
248
+
foreach ($image_formats as $image_format) {
249
+
if (!isset($data[$image_format])) {
250
+
continue;
251
+
}
252
+
253
+
$raw_data = $data[$image_format];
254
+
if (!is_array($raw_data)) {
255
+
continue;
256
+
}
257
+
258
+
$raw_data = implode('', $raw_data);
259
+
260
+
$content = phutil_tag(
261
+
'img',
262
+
array(
263
+
'src' => 'data:'.$image_format.';base64,'.$raw_data,
264
+
));
265
+
266
+
break 2;
267
+
}
268
+
269
+
if (isset($data['text/html'])) {
270
+
$content = $data['text/html'];
271
+
$classes[] = 'jupyter-output-html';
272
+
break;
273
+
}
274
+
275
+
if (isset($data['application/javascript'])) {
276
+
$content = $data['application/javascript'];
277
+
$classes[] = 'jupyter-output-html';
278
+
break;
279
+
}
280
+
281
+
if (isset($data['text/plain'])) {
282
+
$content = $data['text/plain'];
283
+
break;
284
+
}
285
+
286
+
break;
287
+
case 'stream':
288
+
default:
289
+
$content = idx($output, 'text');
290
+
if (!is_array($content)) {
291
+
$content = array();
292
+
}
293
+
$content = implode('', $content);
294
+
break;
295
+
}
296
+
297
+
return phutil_tag(
298
+
'div',
299
+
array(
300
+
'class' => implode(' ', $classes),
301
+
),
302
+
$content);
303
+
}
304
+
305
+
}
+47
webroot/rsrc/css/phui/phui-property-list-view.css
+47
webroot/rsrc/css/phui/phui-property-list-view.css
···
257
257
.document-engine-pdf .phabricator-remarkup-embed-layout-link {
258
258
text-align: left;
259
259
}
260
+
261
+
.document-engine-jupyter {
262
+
overflow: hidden;
263
+
margin: 20px;
264
+
}
265
+
266
+
.jupyter-cell-raw {
267
+
white-space: pre-wrap;
268
+
background: {$lightgreybackground};
269
+
color: {$greytext};
270
+
padding: 8px;
271
+
}
272
+
273
+
.jupyter-cell-code {
274
+
white-space: pre-wrap;
275
+
background: {$lightgreybackground};
276
+
padding: 8px;
277
+
border: 1px solid {$lightgreyborder};
278
+
border-radius: 2px;
279
+
}
280
+
281
+
.jupyter-notebook > tbody > tr > th,
282
+
.jupyter-notebook > tbody > tr > td {
283
+
padding: 8px;
284
+
}
285
+
286
+
.jupyter-notebook > tbody > tr > th {
287
+
white-space: nowrap;
288
+
text-align: right;
289
+
min-width: 48px;
290
+
font-weight: bold;
291
+
}
292
+
293
+
.jupyter-output {
294
+
margin: 4px 0;
295
+
padding: 8px;
296
+
white-space: pre-wrap;
297
+
word-break: break-all;
298
+
}
299
+
300
+
.jupyter-output-stderr {
301
+
background: {$sh-redbackground};
302
+
}
303
+
304
+
.jupyter-output-html {
305
+
background: {$sh-indigobackground};
306
+
}