ocrd.processor.base: add property zip_input_files

ocrd/ocrd/processor/base.py

@@ -6,7 +6,7 @@
 
 import os
 import json
-from ocrd_utils import VERSION as OCRD_VERSION, MIMETYPE_PAGE
+from ocrd_utils import VERSION as OCRD_VERSION, MIMETYPE_PAGE, getLogger
 from ocrd_validators import ParameterValidator
 from ocrd_models.ocrd_page import MetadataItemType, LabelType, LabelsType
 
@@ -15,9 +15,11 @@
 
 class Processor():
  """
- A processor runs an algorithm based on the workspace, the mets.xml in the
- workspace (and the input files defined therein) as well as optional
- parameter.
+ A processor is an OCR-D compliant command-line-interface for executing
+ a single workflow step on the workspace (represented by local METS). It
+ reads input files for all or requested physical pages of the input fileGrp(s),
+ and writes output files for them into the output fileGrp(s). It may take 
+ a number of optional or mandatory parameters.
  """
 
  def __init__(
@@ -123,3 +125,47 @@ def input_files(self):
  self.input_file_grp
  ))
  return ret
+
+ def zip_input_files(self, require_first=True, mimetype=MIMETYPE_PAGE):
+ """
+ List tuples of input files (for multiple input file groups).
+
+ Processors that expect/need multiple input file groups,
+ cannot use ``input_files``. They must align (zip) input files
+ across pages. This includes the case where not all pages
+ are equally present in all file groups.
+
+ This function does not make much sense for non-PAGE fileGrps,
+ so it uses a fixed MIME type filter for PAGE-XML.
+
+ Args:
+ require_first (bool): If true, then skip a page entirely
+ whenever it is not available in the first input fileGrp.
+ """
+
+ LOG = getLogger('ocrd.processor.base')
+ ifgs = self.input_file_grp.split(",")
+ # Iterating over all files repeatedly may seem inefficient at first sight,
+ # but the unnecessary OcrdFile instantiations for posterior fileGrp filtering
+ # can actually be much more costly than traversing the ltree.
+ # This might depend on the number of pages vs number of fileGrps.
+
+ pages = dict()
+ for i, ifg in enumerate(ifgs):
+ for file_ in self.workspace.mets.find_all_files(
+ pageId=self.page_id, fileGrp=ifg, mimetype=mimetype):
+ if not file_.pageId:
+ continue
+ LOG.debug("adding page %s to input file group %s", file_.pageId, ifg)
+ ift = pages.setdefault(file_.pageId, [None]*len(ifgs))
+ ift[i] = file_
+ ifts = list()
+ for page, ifiles in pages.items():
+ for i, ifg in enumerate(ifgs):
+ if not ifiles[i]:
+ # other fallback options?
+ LOG.error('found no page %s in file group %s',
+ page, ifg)
+ if ifiles[0] or not require_first:
+ ifts.append(tuple(ifiles))
+ return ifts

tests/processor/test_processor.py

@@ -2,10 +2,10 @@
 
 from tempfile import TemporaryDirectory
 from os.path import join
-from tests.base import TestCase, assets, main # pylint: disable=import-error, no-name-in-module
+from tests.base import CapturingTestCase as TestCase, assets, main # pylint: disable=import-error, no-name-in-module
 from tests.data import DummyProcessor, DummyProcessorWithRequiredParameters, IncompleteProcessor, DUMMY_TOOL
 
-from ocrd_utils import MIMETYPE_PAGE
+from ocrd_utils import MIMETYPE_PAGE, pushd_popd, initLogging
 from ocrd.resolver import Resolver
 from ocrd.processor.base import Processor, run_processor, run_cli
 
@@ -89,5 +89,32 @@ def test_run_cli(self):
  resolver=Resolver(),
  )
 
+ def test_zip_input_files(self):
+ class ZipTestProcessor(Processor): pass
+ with pushd_popd(tempdir=True) as tempdir:
+ ws = self.resolver.workspace_from_nothing(directory=tempdir)
+ ws.add_file('GRP1', mimetype=MIMETYPE_PAGE, ID='foobar1', pageId='phys_0001')
+ ws.add_file('GRP2', mimetype='application/alto+xml', ID='foobar2', pageId='phys_0001')
+ ws.add_file('GRP1', mimetype=MIMETYPE_PAGE, ID='foobar3', pageId='phys_0002')
+ ws.add_file('GRP2', mimetype=MIMETYPE_PAGE, ID='foobar4', pageId='phys_0002')
+ proc = ZipTestProcessor(workspace=ws, input_file_grp='GRP1,GRP2')
+ tuples = [(one.ID, two.ID) for one, two in proc.zip_input_files(mimetype=r'//application/(vnd.prima.page|alto)\+xml')]
+ assert ('foobar1', 'foobar2') in tuples
+ assert ('foobar3', 'foobar4') in tuples
+
+ def test_zip_input_files_require_first(self):
+ initLogging()
+ class ZipTestProcessor(Processor): pass
+ self.capture_out_err()
+ with pushd_popd(tempdir=True) as tempdir:
+ ws = self.resolver.workspace_from_nothing(directory=tempdir)
+ ws.add_file('GRP1', mimetype=MIMETYPE_PAGE, ID='foobar1', pageId=None)
+ ws.add_file('GRP2', mimetype=MIMETYPE_PAGE, ID='foobar2', pageId='phys_0001')
+ proc = ZipTestProcessor(workspace=ws, input_file_grp='GRP1,GRP2')
+ assert [(one, two.ID) for one, two in proc.zip_input_files(require_first=False)] == [(None, 'foobar2')]
+ r = self.capture_out_err()
+ assert 'ERROR ocrd.processor.base - found no page phys_0001 in file group GRP1' in r.err
+
+
 if __name__ == "__main__":
  main(__file__)