implement --log-filename, wip

ocrd/ocrd/decorators/__init__.py

@@ -39,6 +39,7 @@ def ocrd_cli_wrap_processor(
  subcommand=None,
  address=None,
  queue=None,
+ log_filename=None,
  database=None,
  # ocrd_network params end #
  **kwargs

ocrd/ocrd/decorators/ocrd_cli_options.py

@@ -48,6 +48,7 @@ def cli(mets_url):
  option('-D', '--dump-module-dir', is_flag=True, default=False),
  option('-h', '--help', is_flag=True, default=False),
  option('-V', '--version', is_flag=True, default=False),
+ option('--log-filename', default=None),
  # Subcommand, only used for 'worker'/'server'. Cannot be handled in
  # click because processors use the @command decorator and even if they
  # were using `group`, you cannot combine have a command with

ocrd/ocrd/lib.bash

@@ -145,6 +145,7 @@ ocrd__parse_argv () {
  -I|--input-file-grp) ocrd__argv[input_file_grp]=$2 ; shift ;;
  -w|--working-dir) ocrd__argv[working_dir]=$(realpath "$2") ; shift ;;
  -m|--mets) ocrd__argv[mets_file]=$(realpath "$2") ; shift ;;
+ --log-filename) ocrd__argv[log_filename]="$2" ; shift ;;
  --mets-server-url) ocrd_argv[mets_server_url]="$2" ; shift ;;
  --overwrite) ocrd__argv[overwrite]=true ;;
  --profile) ocrd__argv[profile]=true ;;
@@ -168,7 +169,7 @@ ocrd__parse_argv () {
  if ! [ -v ocrd__worker_queue ]; then
  ocrd__raise "For the Processing Worker --queue is required"
  fi
- ocrd network processing-worker $OCRD_TOOL_NAME --queue "${ocrd__worker_queue}" --database "${ocrd__worker_database}"
+ ocrd network processing-worker $OCRD_TOOL_NAME --queue "${ocrd__worker_queue}" --database "${ocrd__worker_database}" --log-filename "${ocrd__argv[log_filename]}"
  elif [ ${ocrd__subcommand} = "server" ]; then
  if ! [ -v ocrd__worker_address ]; then
  ocrd__raise "For the Processor Server --address is required"

ocrd/ocrd/processor/helpers.py

@@ -251,6 +251,8 @@ def generate_processor_help(ocrd_tool, processor_instance=None, subcommand=None)
  --database The MongoDB server address in format
  "mongodb://{host}:{port}"
  [mongodb://localhost:27018]
+ --log-filename Filename to redirect STDOUT/STDERR to,
+ if specified.
 '''
 
  processing_server_options = '''\

ocrd_network/ocrd_network/process_helpers.py

@@ -1,9 +1,11 @@
 import json
 from typing import List, Optional
 import logging
+from contextlib import nullcontext
 
 from ocrd.processor.helpers import run_cli, run_processor
 from .utils import get_ocrd_workspace_instance
+from ocrd_utils import redirect_stderr_and_stdout_to_file
 
 
 # A wrapper for run_processor() and run_cli()
@@ -15,44 +17,47 @@ def invoke_processor(
  output_file_grps: List[str],
  page_id: str,
  parameters: dict,
- mets_server_url: Optional[str] = None
+ mets_server_url: Optional[str] = None,
+ log_filename : str = None,
 ) -> None:
  if not (processor_class or executable):
- raise ValueError(f'Missing processor class and executable')
+ raise ValueError('Missing processor class and executable')
  input_file_grps_str = ','.join(input_file_grps)
  output_file_grps_str = ','.join(output_file_grps)
 
- workspace = get_ocrd_workspace_instance(
- mets_path=abs_path_to_mets,
- mets_server_url=mets_server_url
- )
+ ctx_mgr = redirect_stderr_and_stdout_to_file(log_filename) if log_filename else nullcontext()
+ with ctx_mgr:
+ workspace = get_ocrd_workspace_instance(
+ mets_path=abs_path_to_mets,
+ mets_server_url=mets_server_url
+ )
 
- if processor_class:
- try:
- run_processor(
- processorClass=processor_class,
+ if processor_class:
+ try:
+ run_processor(
+ processorClass=processor_class,
+ workspace=workspace,
+ input_file_grp=input_file_grps_str,
+ output_file_grp=output_file_grps_str,
+ page_id=page_id,
+ parameter=parameters,
+ instance_caching=True,
+ mets_server_url=mets_server_url,
+ log_level=logging.DEBUG
+ )
+ except Exception as e:
+ raise RuntimeError(f"Python executable '{processor_class.__dict__}' exited with: {e}")
+ else:
+ return_code = run_cli(
+ executable=executable,
  workspace=workspace,
+ mets_url=abs_path_to_mets,
  input_file_grp=input_file_grps_str,
  output_file_grp=output_file_grps_str,
  page_id=page_id,
- parameter=parameters,
- instance_caching=True,
+ parameter=json.dumps(parameters),
  mets_server_url=mets_server_url,
  log_level=logging.DEBUG
  )
- except Exception as e:
- raise RuntimeError(f"Python executable '{processor_class.__dict__}' exited with: {e}")
- else:
- return_code = run_cli(
- executable=executable,
- workspace=workspace,
- mets_url=abs_path_to_mets,
- input_file_grp=input_file_grps_str,
- output_file_grp=output_file_grps_str,
- page_id=page_id,
- parameter=json.dumps(parameters),
- mets_server_url=mets_server_url,
- log_level=logging.DEBUG
- )
- if return_code != 0:
- raise RuntimeError(f"CLI executable '{executable}' exited with: {return_code}")
+ if return_code != 0:
+ raise RuntimeError(f"CLI executable '{executable}' exited with: {return_code}")

ocrd_network/ocrd_network/processing_worker.py

@@ -43,11 +43,13 @@
 
 
 class ProcessingWorker:
- def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, processor_class=None) -> None:
+ def __init__(self, rabbitmq_addr, mongodb_addr, processor_name, ocrd_tool: dict, processor_class=None, log_filename:str=None) -> None:
  initLogging()
- logging_suffix = f'{processor_name}.{getpid()}'
  self.log = getLogger(f'ocrd_network.processing_worker')
- file_handler = logging.FileHandler(f'/tmp/ocrd_worker_{logging_suffix}.log', mode='a')
+ if not log_filename:
+ log_filename = f'/tmp/ocrd_worker_{processor_name}.{getpid()}.log'
+ self.log_filename = log_filename
+ file_handler = logging.FileHandler(log_filename, mode='a')
  file_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
  self.log.addHandler(file_handler)
 
@@ -219,6 +221,7 @@ def process_message(self, processing_message: OcrdProcessingMessage) -> None:
  input_file_grps=input_file_grps,
  output_file_grps=output_file_grps,
  page_id=page_id,
+ log_filename=self.log_filename,
  parameters=processing_message.parameters,
  mets_server_url=mets_server_url
  )