Fix incorrect output from prints originating from different processes

myst_nb/core/utils.py

@@ -25,8 +25,11 @@ def coalesce_streams(outputs: list[NotebookNode]) -> list[NotebookNode]:
  for output in outputs:
  if output["output_type"] == "stream":
  if output["name"] in streams:
- streams[output["name"]]["text"] += output["text"]
+ out = output["text"].strip()
+ if out:
+ streams[output["name"]]["text"] += f"{out}\n"
  else:
+ output["text"] = output["text"].strip() + "\n"
  new_outputs.append(output)
  streams[output["name"]] = output
  else:

tests/notebooks/merge_streams_parallel.ipynb

@@ -0,0 +1,186 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2024-09-19T21:44:29.809012Z",
+ "iopub.status.busy": "2024-09-19T21:44:29.808809Z",
+ "iopub.status.idle": "2024-09-19T21:44:29.978481Z",
+ "shell.execute_reply": "2024-09-19T21:44:29.977891Z"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "from concurrent.futures import ProcessPoolExecutor\n",
+ "\n",
+ "with ProcessPoolExecutor() as executor:\n",
+ " for i in executor.map(print, [0] * 10):\n",
+ " pass"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+ }

tests/test_render_outputs.py

@@ -1,4 +1,5 @@
 """Tests for rendering code cell outputs."""
+
 import pytest
 
 from myst_nb.core.render import EntryPointError, load_renderer
@@ -103,6 +104,18 @@ def test_merge_streams(sphinx_run, file_regression):
  file_regression.check(doctree.pformat(), extension=".xml", encoding="utf-8")
 
 
+@pytest.mark.sphinx_params(
+ "merge_streams_parallel.ipynb",
+ conf={"nb_execution_mode": "off", "nb_merge_streams": True},
+)
+def test_merge_streams_parallel(sphinx_run, file_regression):
+ """Test configuring multiple concurrent stdout/stderr outputs to be merged."""
+ sphinx_run.build()
+ assert sphinx_run.warnings() == ""
+ doctree = sphinx_run.get_resolved_doctree("merge_streams_parallel")
+ file_regression.check(doctree.pformat(), extension=".xml", encoding="utf-8")
+
+
 @pytest.mark.sphinx_params(
  "metadata_image.ipynb",
  conf={"nb_execution_mode": "off", "nb_cell_metadata_key": "myst"},

tests/test_render_outputs/test_merge_streams_parallel.xml

@@ -0,0 +1,21 @@
+<document source="merge_streams_parallel">
+ <container cell_index="0" cell_metadata="{'execution': {'iopub.execute_input': '2024-09-19T21:44:29.809012Z', 'iopub.status.busy': '2024-09-19T21:44:29.808809Z', 'iopub.status.idle': '2024-09-19T21:44:29.978481Z', 'shell.execute_reply': '2024-09-19T21:44:29.977891Z'}}" classes="cell" exec_count="1" nb_element="cell_code">
+ <container classes="cell_input" nb_element="cell_code_source">
+ <literal_block language="ipython3" linenos="False" xml:space="preserve">
+ from concurrent.futures import ProcessPoolExecutor
+
+ with ProcessPoolExecutor() as executor:
+ for i in executor.map(print, [0] * 10):
+ pass
+ <container classes="cell_output" nb_element="cell_code_output">
+ <literal_block classes="output stream" language="myst-ansi" linenos="False" xml:space="preserve">
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0