V2.70.0 backport to v3

.github/workflows/publish-pypi.yml

@@ -0,0 +1,31 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+ release:
+ types: [published]
+ workflow_dispatch:
+
+jobs:
+ deploy:
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: '3.8'
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install setuptools wheel build twine
+ pip install -r requirements.txt
+ - name: Build and publish
+ env:
+ TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+ run: make pypi

CHANGELOG.md

@@ -128,6 +128,58 @@ Added:
  - `Processor.verify`: handle fileGrp cardinality verification, with default implementation
  - `Processor.setup`: to set up processor before processing, optional
 
+## [2.70.0] - 2024-10-10
+
+Added:
+
+ - `ocrd network client workflow run`: Add `--print-status` flag to periodically print the job status, #1277
+ - Processing Server: `DELETE /mets_server_zombies` to kill any renegade METS servers, #1277
+ - No more zombie METS Server by properly shutting them down, #1284
+ - `OCRD_NETWORK_RABBITMQ_HEARBEAT` to allow overriding the [heartbeat](https://pika.readthedocs.io/en/stable/examples/heartbeat_and_blocked_timeouts.html) behavior of RabbitMQ, #1285
+
+Changed:
+
+ - significantly more detailed logging for the METS Server and Processing Server, #1284
+ - Only import `ocrd_network` in src/ocrd/decorators/__init__.py once needed, #1289
+ - Automate release via GitHub Actions, #1290
+
+Fixed:
+
+ - `ocrd/core-cuda-torch`: Install torchvision as well, #1286
+ - Processing Server: remove shut down METS servers from deployer's cache, #1287
+ - typos, #1274
+
+## [2.69.0] - 2024-09-30
+
+Fixed:
+ - tests: ensure `ocrd_utils.config` gets reset whenever changing it globally
+ - `ocrd.cli.workspace`: consistently pass on `--mets-server-url` and `--backup`
+ - `ocrd.cli.workspace`: make `list-page` work w/ METS Server
+ - `ocrd.cli.validate "tasks"`: pass on `--mets-server-url`
+ - `lib.bash`: fix `errexit` handling
+ - actually apply CLI `--log-filename`, and show in `--help`
+ - adapt to Pillow changes
+ - `ocrd workspace clone`: do pass on `--file-grp` (for download filtering)
+ - `OcrdMetsServer.add_file`: pass on `force` kwarg
+ - `Workspace.reload_mets`: handle ClientSideOcrdMets as well
+ - `OcrdMets.get_physical_pages`: cover `return_divs` w/o `for_fileIds` and `for_pageIds`
+ - `disableLogging`: also re-instate root logger to Python defaults
+ - `OcrdExif`: handle multi-frame TIFFs gracefully in `identify` callout, #1276
+
+Changed:
+ - `run_processor`: be robust if `ocrd_tool` is missing `steps`
+ - `PcGtsType.PageType.id` via `make_xml_id`: replace `/` with `_`
+ - `ClientSideOcrdMets`: use same logger name prefix as METS Server
+ - `Processor.zip_input_files`: when `--page-id` yields empty list, just log instead of raise
+
+Added:
+ - `OcrdPage`: new `PageType.get_ReadingOrderGroups()` to retrieve recursive RO as dict
+ - METS Server: export and delegate `physical_pages`
+ - ocrd.cli.workspace `server`: add subcommands `reload` and `save`
+ - processor CLI: delegate `--resolve-resource`, too
+ - `OcrdConfig.reset_defaults` to reset config variables to their defaults
+ - `ocrd_utils.scale_coordinates` for resizing images
+
 ## [2.68.0] - 2024-08-23
 
 Changed:
@@ -2294,6 +2346,8 @@ Initial Release
 [3.0.0b1]: ../../compare/v3.0.0b1..v3.0.0a2
 [3.0.0a2]: ../../compare/v3.0.0a2..v3.0.0a1
 [3.0.0a1]: ../../compare/v3.0.0a1..v2.67.2
+[2.70.0]: ../../compare/v2.70.0..v2.69.0
+[2.69.0]: ../../compare/v2.69.0..v2.68.0
 [2.68.0]: ../../compare/v2.68.0..v2.67.2
 [2.67.2]: ../../compare/v2.67.2..v2.67.1
 [2.67.1]: ../../compare/v2.67.1..v2.67.0

Dockerfile.cuda-torch

@@ -9,7 +9,5 @@ RUN make deps-torch
 
 WORKDIR /data
 
-RUN rm -fr /build
-
 CMD ["/usr/local/bin/ocrd", "--help"]
 

Makefile

@@ -63,7 +63,7 @@ deps-cuda: CONDA_EXE ?= /usr/local/bin/conda
 deps-cuda: export CONDA_PREFIX ?= /conda
 deps-cuda: PYTHON_PREFIX != $(PYTHON) -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])'
 deps-cuda:
- curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
+ curl --retry 6 -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
  mv bin/micromamba $(CONDA_EXE)
 # Install Conda system-wide (for interactive / login shells)
  echo 'export MAMBA_EXE=$(CONDA_EXE) MAMBA_ROOT_PREFIX=$(CONDA_PREFIX) CONDA_PREFIX=$(CONDA_PREFIX) PATH=$(CONDA_PREFIX)/bin:$$PATH' >> /etc/profile.d/98-conda.sh
@@ -158,7 +158,7 @@ deps-tf2:
  fi
 
 deps-torch:
- $(PIP) install -i https://download.pytorch.org/whl/cu118 torch
+ $(PIP) install -i https://download.pytorch.org/whl/cu118 torch torchvision
 
 # Dependencies for deployment in an ubuntu/debian linux
 deps-ubuntu:
@@ -178,7 +178,7 @@ build:
 
 # (Re)install the tool
 install: #build
- # not stricttly necessary but a precaution against outdated python build tools, https:/OCR-D/core/pull/1166
+ # not strictly necessary but a precaution against outdated python build tools, https:/OCR-D/core/pull/1166
  $(PIP) install -U pip wheel
  $(PIP_INSTALL) . $(PIP_INSTALL_CONFIG_OPTION)
  @# workaround for shapely#1598

src/ocrd/cli/__init__.py

@@ -83,6 +83,8 @@ def get_help(self, ctx):
 \b
 {config.describe('OCRD_NETWORK_RABBITMQ_CLIENT_CONNECT_ATTEMPTS')}
 \b
+{config.describe('OCRD_NETWORK_RABBITMQ_HEARTBEAT')}
+\b
 {config.describe('OCRD_PROFILE_FILE')}
 \b
 {config.describe('OCRD_PROFILE', wrap_text=False)}

src/ocrd/decorators/__init__.py

@@ -13,7 +13,6 @@
  redirect_stderr_and_stdout_to_file,
 )
 from ocrd_validators import WorkspaceValidator
-from ocrd_network import ProcessingWorker, ProcessorServer, AgentType
 
 from ..resolver import Resolver
 from ..processor.base import ResourceNotFoundError, run_processor
@@ -23,8 +22,6 @@
 from .ocrd_cli_options import ocrd_cli_options
 from .mets_find_options import mets_find_options
 
-SUBCOMMANDS = [AgentType.PROCESSING_WORKER, AgentType.PROCESSOR_SERVER]
-
 
 def ocrd_cli_wrap_processor(
  processorClass,
@@ -88,11 +85,9 @@ def ocrd_cli_wrap_processor(
  if list_resources:
  processor.list_resources()
  sys.exit()
- if subcommand:
+ if subcommand or address or queue or database:
  # Used for checking/starting network agents for the WebAPI architecture
  check_and_run_network_agent(processorClass, subcommand, address, database, queue)
- elif address or queue or database:
- raise ValueError(f"Subcommand options --address --queue and --database are only valid for subcommands: {SUBCOMMANDS}")
 
  # from here: single-run processing context
  initLogging()
@@ -162,6 +157,11 @@ def goexit():
 def check_and_run_network_agent(ProcessorClass, subcommand: str, address: str, database: str, queue: str):
  """
  """
+ from ocrd_network import ProcessingWorker, ProcessorServer, AgentType
+ SUBCOMMANDS = [AgentType.PROCESSING_WORKER, AgentType.PROCESSOR_SERVER]
+
+ if not subcommand:
+ raise ValueError(f"Subcommand options --address --queue and --database are only valid for subcommands: {SUBCOMMANDS}")
  if subcommand not in SUBCOMMANDS:
  raise ValueError(f"SUBCOMMAND can only be one of {SUBCOMMANDS}")