Merge pull request #70 from jannis-baum/issue/55-render-jupyter-noteb…

…ooks

Render Jupyter Notebooks

.github/workflows/ci.yaml

@@ -34,15 +34,11 @@ jobs:
  run: |
  yarn
  make linux
- - name: package
- run: |
- cd build/linux
- zip -r vivify-linux.zip *
  - name: upload artifact
  uses: actions/upload-artifact@v4
  with:
  name: vivify-linux
- path: build/linux/vivify-linux.zip
+ path: build/linux/*
 
  build-macos:
  name: Build MacOS
@@ -55,15 +51,11 @@ jobs:
  run: |
  yarn
  make macos
- - name: package
- run: |
- cd build/macos
- zip -r vivify-macos.zip *
  - name: upload artifact
  uses: actions/upload-artifact@v4
  with:
  name: vivify-macos
- path: build/macos/vivify-macos.zip
+ path: build/macos/*
 
  release:
  name: Release

package.json

@@ -3,12 +3,13 @@
  "repository": "https:/jannis-baum/vivify.git",
  "author": "Jannis Baum",
  "scripts": {
- "dev": "VIV_TIMEOUT=0 VIV_PORT=3000 nodemon --exec ts-node ./src/app.ts",
+ "dev": "VIV_TIMEOUT=0 VIV_PORT=3000 NODE_ENV=development nodemon --exec ts-node ./src/app.ts",
  "viv": "VIV_PORT=3000 ./viv",
  "lint": "eslint src static"
  },
  "dependencies": {
  "@viz-js/viz": "^3.1.0",
+ "ansi_up": "^6.0.2",
  "express": "^4.18.2",
  "highlight.js": "^11.8.0",
  "katex": "^0.15.6",
@@ -26,6 +27,7 @@
  "ws": "^8.13.0"
  },
  "devDependencies": {
+ "@jupyterlab/nbformat": "^4.2.3",
  "@types/express": "^4.17.17",
  "@types/markdown-it": "^12.2.3",
  "@types/node": "^20.4.2",

src/parser/ansi.ts

@@ -0,0 +1,27 @@
+import { Renderer } from './parser';
+
+// this entire file is very ugly because ansi_up is only available as module so
+// we have to use a dynamic (async) import
+let _renderAnsi: Renderer | undefined = undefined;
+
+// here we asynchronously import ansi_up
+(async () => {
+ const { AnsiUp } = await (async () => {
+ // this gets even uglier because we can't directly use dynamic imports
+ // with ts-node https:/TypeStrong/ts-node/discussions/1290
+ if (process.env.NODE_ENV === 'development') {
+ const dynamicImport = new Function('specifier', 'return import(specifier)');
+ return await dynamicImport('ansi_up');
+ }
+ // for compilation however we have to directly use dynamic imports
+ return await import('ansi_up');
+ })();
+ const ansiup = new AnsiUp();
+ _renderAnsi = (content: string): string => ansiup.ansi_to_html(content);
+})();
+
+// use identity while ansi_up loads
+const renderAnsi: Renderer = (content: string): string =>
+ _renderAnsi ? _renderAnsi(content) : content;
+
+export default renderAnsi;

src/parser/ipynb.ts

@@ -0,0 +1,117 @@
+import {
+ ICell,
+ ICodeCell,
+ IDisplayData,
+ IError,
+ IMimeBundle,
+ INotebookContent,
+ IOutput,
+ IStream,
+ MultilineString,
+ IExecuteResult,
+} from '@jupyterlab/nbformat';
+import renderAnsi from './ansi';
+import renderMarkdown from './markdown';
+import { Renderer } from './parser';
+
+function joinMultilineString(str: MultilineString): string {
+ return Array.isArray(str) ? str.join('') : str;
+}
+
+function contain(
+ content: MultilineString,
+ classNames: string | string[],
+ tag: string = 'div',
+): string {
+ function prefix(classNames: string | string[]): string {
+ if (Array.isArray(classNames)) {
+ return classNames.map((c) => prefix(c)).join(' ');
+ }
+ return `ipynb-${classNames}`;
+ }
+ classNames = prefix(classNames);
+
+ return `<${tag} class="${classNames}">${joinMultilineString(content)}</${tag}>`;
+}
+
+function escapeHTML(raw: string): string {
+ return raw.replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+
+function executionCount(count: ICell['execution_count']): string {
+ return contain(`[${count?.toString() ?? ' '}]:`, 'execution-count');
+}
+
+const renderNotebook: Renderer = (content: string): string => {
+ const nb = JSON.parse(content) as INotebookContent;
+ const language = nb.metadata.kernelspec?.language ?? nb.metadata.language_info?.name;
+
+ function renderDisplayData(data: IMimeBundle): string {
+ const identity = (content: string) => content;
+ const renderMap: [string, (content: string) => string][] = [
+ ['image/png', (content) => `<img src="data:image/png;base64,${content}" />`],
+ ['image/jpeg', (content) => `<img src="data:image/jpeg;base64,${content}" />`],
+ ['image/svg+xml', identity],
+ ['text/svg+xml', identity],
+ ['text/html', identity],
+ ['text/markdown', renderMarkdown],
+ ['text/plain', (content) => contain(escapeHTML(content), 'output-plain', 'pre')],
+ ];
+ const result = renderMap.find(([mimeType]) => mimeType in data);
+ if (!result) return '';
+ const [format, render] = result;
+ return render(joinMultilineString(data[format] as MultilineString));
+ }
+
+ function renderOutput(output: IOutput): string {
+ switch (output.output_type) {
+ case 'stream':
+ const text = joinMultilineString((output as IStream).text);
+ return contain(
+ renderAnsi(text),
+ `output-stream-${(output as IStream).name}`,
+ 'pre',
+ );
+ case 'error':
+ const traceback = (output as IError).traceback.join('\n');
+ return contain(renderAnsi(traceback), 'output-error', 'pre');
+ case 'display_data': {
+ const displayData = (output as IDisplayData).data;
+ return renderDisplayData(displayData);
+ }
+ case 'execute_result': {
+ const displayData = (output as IExecuteResult).data;
+ return contain(
+ [executionCount(output.execution_count), renderDisplayData(displayData)],
+ 'output-execution-result',
+ );
+ }
+ default:
+ return '';
+ }
+ }
+
+ function renderCell(cell: ICell): string {
+ const source = joinMultilineString(cell.source);
+
+ switch (cell.cell_type) {
+ case 'code':
+ const content = [
+ executionCount(cell.execution_count),
+ renderMarkdown(`\`\`\`${language}\n${source}\n\`\`\``),
+ ];
+ if (Array.isArray(cell.outputs)) {
+ content.push(...(cell as ICodeCell).outputs.map(renderOutput));
+ }
+ return contain(content, 'cell-code');
+ case 'markdown':
+ return contain(renderMarkdown(source), 'cell-markdown');
+ default:
+ // 'raw' cells aren't normally rendered
+ return '';
+ }
+ }
+
+ return nb.cells.map(renderCell).join('\n\n');
+};
+export default renderNotebook;

src/parser/markdown.ts

@@ -0,0 +1,36 @@
+import MarkdownIt from 'markdown-it';
+import anchor from 'markdown-it-anchor';
+import highlight from './highlight';
+import graphviz from './dot';
+import config from './config';
+import { Renderer } from './parser';
+
+const mdit = new MarkdownIt({
+ html: true,
+ highlight: highlight,
+ linkify: true,
+});
+
+mdit.use(anchor, {
+ permalink: anchor.permalink.ariaHidden({
+ placement: 'before',
+ }),
+});
+/* eslint-disable @typescript-eslint/no-var-requires */
+mdit.use(require('markdown-it-emoji'));
+mdit.use(require('markdown-it-task-lists'));
+mdit.use(require('markdown-it-footnote'));
+mdit.use(require('markdown-it-inject-linenumbers'));
+mdit.use(require('markdown-it-texmath'), {
+ engine: require('katex'),
+ delimiters: 'dollars',
+ katexOptions: config.katexOptions,
+});
+mdit.use(require('markdown-it-deflist'));
+/* eslint-enable @typescript-eslint/no-var-requires */
+mdit.use(graphviz);
+
+const renderMarkdown: Renderer = (content: string) => {
+ return mdit.render(content);
+};
+export default renderMarkdown;

src/parser/parser.ts

@@ -1,47 +1,55 @@
+import { Dirent, readdirSync } from 'fs';
 import { homedir } from 'os';
-
-import MarkdownIt from 'markdown-it';
-import anchor from 'markdown-it-anchor';
-import highlight from './highlight';
-import graphviz from './dot';
+import { join as pjoin } from 'path';
+import { pathToURL } from '../utils/path';
 import config from './config';
+import renderNotebook from './ipynb';
+import renderMarkdown from './markdown';
+
+export type Renderer = (content: string) => string;
+
+const pathHeading: Renderer = (path: string) => `# \`${path.replace(homedir(), '~')}\``;
+const wrap = (contentType: string, content: string) =>
+ `<div class="content-${contentType}">${content}</div>`;
+
+const mdExtensions = config.mdExtensions ?? ['markdown', 'md', 'mdown', 'mdwn', 'mkd', 'mkdn'];
+function textRenderer(
+ fileEnding: string | undefined,
+): { render: Renderer; contentType: string } | undefined {
+ if (!fileEnding) return undefined;
+ if (mdExtensions.includes(fileEnding)) {
+ return { render: renderMarkdown, contentType: 'markdown' };
+ }
+ if (fileEnding === 'ipynb') {
+ return { render: renderNotebook, contentType: 'ipynb' };
+ }
+}
 
-const mdit = new MarkdownIt({
- html: true,
- highlight: highlight,
- linkify: true,
-});
-
-mdit.use(anchor, {
- permalink: anchor.permalink.ariaHidden({
- placement: 'before',
- }),
-});
-/* eslint-disable @typescript-eslint/no-var-requires */
-mdit.use(require('markdown-it-emoji'));
-mdit.use(require('markdown-it-task-lists'));
-mdit.use(require('markdown-it-footnote'));
-mdit.use(require('markdown-it-inject-linenumbers'));
-mdit.use(require('markdown-it-texmath'), {
- engine: require('katex'),
- delimiters: 'dollars',
- katexOptions: config.katexOptions,
-});
-mdit.use(require('markdown-it-deflist'));
-/* eslint-enable @typescript-eslint/no-var-requires */
-mdit.use(graphviz);
-
-export const pathHeading = (path: string) => `# \`${path.replace(homedir(), '~')}\``;
-
-export default function parse(src: string, path?: string) {
- let md = src;
-
- const mdExtensions = config.mdExtensions ?? ['markdown', 'md', 'mdown', 'mdwn', 'mkd', 'mkdn'];
-
+export function renderTextFile(content: string, path: string): string {
  const fileEnding = path?.split('.')?.at(-1);
- if (fileEnding && !mdExtensions.includes(fileEnding)) {
- md = `${pathHeading(path!)}\n\n\`\`\`${fileEnding}\n${src}\n\`\`\``;
+ const renderInformation = textRenderer(fileEnding);
+ if (renderInformation === undefined) {
+ return wrap(
+ 'txt',
+ renderMarkdown(`${pathHeading(path!)}\n\n\`\`\`${fileEnding}\n${content}\n\`\`\``),
+ );
  }
+ const { render, contentType } = renderInformation;
+ return wrap(contentType, render(content));
+}
 
- return mdit.render(md);
+const dirListItem = (item: Dirent, path: string) =>
+ `<li class="dir-list-${item.isDirectory() ? 'directory' : 'file'}"><a href="${pathToURL(
+ pjoin(path, item.name),
+ )}">${item.name}</a></li>`;
+
+export function renderDirectory(path: string): string {
+ const list = readdirSync(path, { withFileTypes: true })
+ .sort((a, b) => +b.isDirectory() - +a.isDirectory())
+ .map((item) => dirListItem(item, path))
+ .join('\n');
+ return wrap(
+ 'directory',
+ renderMarkdown(`${pathHeading(path)}\n\n<ul class="dir-list">\n${list}\n</ul>`),
+ );
 }