Add min length flag

chunk_convert.sh

@@ -37,6 +37,7 @@ for (( i=0; i<$NUM_DEVICES; i++ )); do
     echo "Running convert.py on GPU $DEVICE_NUM"
     cmd="CUDA_VISIBLE_DEVICES=$DEVICE_NUM python convert.py $INPUT_FOLDER $OUTPUT_FOLDER --num_chunks $NUM_DEVICES --chunk_idx $DEVICE_NUM --workers $NUM_WORKERS"
     [[ -n "$METADATA_FILE" ]] && cmd="$cmd --metadata_file $METADATA_FILE"
+    [[ -n "$MIN_LENGTH" ]] && cmd="$cmd --min_length $MIN_LENGTH"
     eval $cmd &
 
     sleep 5

convert.py

@@ -6,7 +6,7 @@ import ray
 from tqdm import tqdm
 import math
 
-from marker.convert import convert_single_pdf
+from marker.convert import convert_single_pdf, get_length_of_text
 from marker.segmentation import load_layout_model
 from marker.cleaners.equations import load_nougat_model
 from marker.settings import settings
@@ -16,14 +16,23 @@ import json
 
 configure_logging()
 
+
 @ray.remote(num_cpus=settings.RAY_CORES_PER_WORKER, num_gpus=.05 if settings.CUDA else 0)
-def process_single_pdf(fname: str, out_folder: str, nougat_model, layout_model, metadata: Dict | None=None):
+def process_single_pdf(fname: str, out_folder: str, nougat_model, layout_model, metadata: Dict | None=None, min_length: int | None = None):
     out_filename = fname.rsplit(".", 1)[0] + ".md"
     out_filename = os.path.join(out_folder, os.path.basename(out_filename))
     out_meta_filename = out_filename.rsplit(".", 1)[0] + "_meta.json"
     if os.path.exists(out_filename):
         return
     try:
+        # Skip trying to convert files that don't have a lot of embedded text
+        # This can indicate that they were scanned, and not OCRed properly
+        # Usually these files are not recent/high-quality
+        if min_length:
+            length = get_length_of_text(fname)
+            if length < min_length:
+                return
+
         full_text, out_metadata = convert_single_pdf(fname, layout_model, nougat_model, metadata=metadata)
         if len(full_text.strip()) > 0:
             with open(out_filename, "w+") as f:
@@ -46,6 +55,7 @@ if __name__ == "__main__":
     parser.add_argument("--max", type=int, default=None, help="Maximum number of pdfs to convert")
     parser.add_argument("--workers", type=int, default=5, help="Number of worker processes to use")
     parser.add_argument("--metadata_file", type=str, default=None, help="Metadata file to use for filtering")
+    parser.add_argument("--min_length", type=int, default=None, help="Minimum length of pdf to convert")
 
     args = parser.parse_args()
 
@@ -95,7 +105,8 @@ if __name__ == "__main__":
             out_folder,
             nougat_ref,
             layoutlm_ref,
-            metadata.get(os.path.basename(filename))
+            metadata=metadata.get(os.path.basename(filename)),
+            min_length=args.min_length
         ) for filename in files_to_convert
     ]
 

marker/convert.py

@@ -39,6 +39,19 @@ def annotate_spans(blocks: List[Page], block_types: List[BlockType]):
         page.add_block_types(page_block_types)
 
 
+def get_length_of_text(fname: str) -> int:
+    filetype = find_filetype(fname)
+    if filetype == "other":
+        return 0
+
+    doc = pymupdf.open(fname, filetype=filetype)
+    full_text = ""
+    for page in doc:
+        full_text += page.get_text("text", sort=True, flags=settings.TEXT_FLAGS)
+
+    return len(full_text)
+
+
 def convert_single_pdf(fname: str, layoutlm_model, nougat_model, max_pages=None, metadata: Dict | None=None) -> Tuple[str, Dict]:
     lang = settings.DEFAULT_LANG
     if metadata:

marker/extract_text.py

@@ -1,7 +1,6 @@
 import os
 from typing import Tuple, List
 
-from marker.ocr.segment import ocr_bbox
 from marker.ocr.page import ocr_entire_page_ocrmp
 from marker.ocr.utils import detect_bad_ocr, font_flags_decomposer
 from marker.settings import settings
@@ -61,6 +60,8 @@ def get_text_blocks(doc, tess_lang: str, spell_lang: str, max_pages: int | None=
     extracted = [False]
     ocr_pages = 0
     min_ocr_page = 2
+    ocr_failed = 0
+    ocr_success = 0
     for pnum, page in enumerate(doc):
         if max_pages and pnum >= max_pages:
             break
@@ -81,9 +82,13 @@ def get_text_blocks(doc, tess_lang: str, spell_lang: str, max_pages: int | None=
             page_obj = Page(blocks=blocks, pnum=pnum)
             extracted.append(False)
             ocr_pages += 1
+            if len(blocks) == 0:
+                ocr_failed += 1
+            else:
+                ocr_success += 1
         else:
             if pnum > min_ocr_page:
                 extracted.append(True)
 
         all_blocks.append(page_obj)
-    return all_blocks, toc, {"ocr_pages": ocr_pages}
+    return all_blocks, toc, {"ocr_pages": ocr_pages, "ocr_failed": ocr_failed, "ocr_success": ocr_success}

marker/ocr/segment.py

@@ -1,27 +0,0 @@
-import fitz as pymupdf
-
-from marker.settings import settings
-
-
-def ocr_bbox(page, old_text, bbox, lang: str):
-    pix = page.get_pixmap(dpi=settings.SEGMENT_DPI, clip=bbox)
-
-    try:
-        ocrpdf = pymupdf.open("pdf", pix.pdfocr_tobytes(language=lang))
-        ocrpage = ocrpdf[0]
-        new_text = ocrpage.get_text()  # extract OCR-ed text
-    except RuntimeError:
-        # If the OCR fails, just return the original text
-        return old_text
-
-    if not new_text.strip():
-        # If the OCR data is blank, return old text
-        return old_text
-
-    # Tesseract ignores leading spaces, hence some corrections
-    lblanks = len(old_text) - len(old_text.lstrip())
-    rblanks = len(old_text) - len(old_text.rstrip())
-
-    # prefix/suffix OCRed text with this many spaces
-    new_text = " " * lblanks + new_text + " " * rblanks
-    return new_text

marker/ocr/utils.py

@@ -36,7 +36,7 @@ def detect_bad_ocr(text, spell_lang: str | None, misspell_threshold=.8, space_th
         if char in settings.INVALID_CHARS:
             invalid_chars += 1
 
-    if invalid_chars > 2:
+    if invalid_chars > max(2.0, len(text) * .02):
         return True
 
     return False

marker/settings.py

@@ -24,7 +24,7 @@ class Settings(BaseSettings):
     TEXT_FLAGS: int = pymupdf.TEXTFLAGS_DICT & ~pymupdf.TEXT_PRESERVE_LIGATURES & ~pymupdf.TEXT_PRESERVE_IMAGES
 
     # OCR
-    INVALID_CHARS: List[str] = [chr(0xfffd), chr(65533)]
+    INVALID_CHARS: List[str] = [chr(0xfffd)]
     DPI: int = 800
     SEGMENT_DPI: int = 1200
     TESSDATA_PREFIX: str = ""