with GUI - only windows, import libs if necessary
Browse files- parser_sevenof9_v1_52_en.py +656 -0
parser_sevenof9_v1_52_en.py
ADDED
@@ -0,0 +1,656 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os # OS module for interacting with the operating system (file management, etc.)
|
2 |
+
import re
|
3 |
+
import sys # Provides access to system-specific parameters and functions
|
4 |
+
import tkinter as tk # GUI module for creating desktop applications
|
5 |
+
from tkinter import filedialog, messagebox # Additional tkinter components for file dialogs and message boxes
|
6 |
+
import subprocess # Module to run system commands
|
7 |
+
import threading # Threading module to run tasks concurrently
|
8 |
+
import json # JSON module for working with JSON data
|
9 |
+
import logging # Logging module for tracking events and errors
|
10 |
+
import pdfplumber # Library for extracting text and tables from PDFs
|
11 |
+
#from pdfplumber.utils import get_bbox_overlap, obj_to_bbox # Helper functions from pdfplumber for working with bounding boxes
|
12 |
+
from pdfplumber.utils.exceptions import PdfminerException # Exception related to PDF processing
|
13 |
+
from joblib import delayed, cpu_count, parallel_backend, Parallel # Joblib for parallel processing and optimization
|
14 |
+
import customtkinter as ctk
|
15 |
+
import tkinter.font as tkfont
|
16 |
+
|
17 |
+
|
18 |
+
|
19 |
+
# ========================
|
20 |
+
# Parser Configuration
|
21 |
+
# ========================
|
22 |
+
|
23 |
+
# Function to suppress PDFMiner logging, reducing verbosity
|
24 |
+
def suppress_pdfminer_logging():
|
25 |
+
for logger_name in [
|
26 |
+
#"pdfminer", # Various pdfminer modules to suppress logging from
|
27 |
+
#"pdfminer.pdfparser",
|
28 |
+
#"pdfminer.pdfdocument",
|
29 |
+
"pdfminer.pdfpage",
|
30 |
+
#"pdfminer.converter",
|
31 |
+
#"pdfminer.layout",
|
32 |
+
#"pdfminer.cmapdb",
|
33 |
+
#"pdfminer.utils"
|
34 |
+
]:
|
35 |
+
logging.getLogger(logger_name).setLevel(logging.ERROR) # Set logging level to ERROR to suppress lower levels
|
36 |
+
|
37 |
+
|
38 |
+
PARALLEL_THRESHOLD = 16 # Number of pages to use for deciding between serial or parallel processing
|
39 |
+
|
40 |
+
|
41 |
+
TEXT_EXTRACTION_SETTINGS = {
|
42 |
+
"x_tolerance": 1.5, # Horizontal tolerance for text extraction
|
43 |
+
"y_tolerance": 2.5, # Vertical tolerance for text extraction
|
44 |
+
"char_dir": "ltr",
|
45 |
+
"keep_blank_chars": False, # Option to retain blank characters in the extracted text
|
46 |
+
"use_text_flow": True, # Option to use text flow for better structure
|
47 |
+
}
|
48 |
+
|
49 |
+
# Regex zur Entfernung unerwünschter Zeichen:
|
50 |
+
EUROPEAN_PRINTABLES_PATTERN = re.compile(r"[^\n\r\t \w\u0000-\uFFFF]")
|
51 |
+
# EUROPEAN_PRINTABLES_PATTERN = re.compile(r"[^ \t\n\r0-9a-zA-ZäöüÄÖÜßÀ-ÿ.,;:!?(){}\[\]„“‚‘\"'´`\-_/+*=%^~|&§#@€$£<>\\]")
|
52 |
+
|
53 |
+
CID_PATTERN = re.compile(r"\(cid:\d+\)")
|
54 |
+
|
55 |
+
# Function to clean up text by removing unwanted hyphenations and newlines
|
56 |
+
def clean_cell_text(text):
|
57 |
+
if not isinstance(text, str):
|
58 |
+
return ""
|
59 |
+
|
60 |
+
# Entferne Silbentrennzeichen am Zeilenende
|
61 |
+
text = text.replace("-\n", "")
|
62 |
+
text = text.replace("\n", " ")
|
63 |
+
text = CID_PATTERN.sub("", text)
|
64 |
+
|
65 |
+
# Entferne alle Zeichen, die nicht zu den definierten druckbaren Zeichen gehören
|
66 |
+
cleaned_text = EUROPEAN_PRINTABLES_PATTERN.sub("", text)
|
67 |
+
|
68 |
+
return cleaned_text
|
69 |
+
|
70 |
+
# Function to safely clean and join row cell data
|
71 |
+
def safe_join(row):
|
72 |
+
return [clean_cell_text(str(cell)) if cell is not None else "" for cell in row] # Clean each cell in the row, or return empty if None
|
73 |
+
|
74 |
+
|
75 |
+
# Function to clamp bounding box coordinates within page boundaries
|
76 |
+
def clamp_bbox(bbox, page_width, page_height, precision=3):
|
77 |
+
x0, top, x1, bottom = bbox
|
78 |
+
x0 = max(0, min(x0, page_width))
|
79 |
+
x1 = max(0, min(x1, page_width))
|
80 |
+
top = max(0, min(top, page_height))
|
81 |
+
bottom = max(0, min(bottom, page_height))
|
82 |
+
|
83 |
+
# Hier runden
|
84 |
+
return (
|
85 |
+
round(x0, precision),
|
86 |
+
round(top, precision),
|
87 |
+
round(x1, precision),
|
88 |
+
round(bottom, precision)
|
89 |
+
)
|
90 |
+
|
91 |
+
# Hauptfunktion zur Verarbeitung einer einzelnen PDF-Seite
|
92 |
+
def process_page(args):
|
93 |
+
# suppress_pdfminer_logging()
|
94 |
+
try:
|
95 |
+
page_number, pdf_path, TEXT_EXTRACTION_SETTINGS = args
|
96 |
+
with pdfplumber.open(pdf_path) as pdf:
|
97 |
+
page = pdf.pages[page_number]
|
98 |
+
output = f"\n\nPage {page_number + 1}\n"
|
99 |
+
width, height = page.width, page.height
|
100 |
+
|
101 |
+
# Seitenränder (5 %) abschneiden
|
102 |
+
margin_x = width * 0.05
|
103 |
+
margin_y = height * 0.05
|
104 |
+
content_bbox = (margin_x, margin_y, width - margin_x, height - margin_y)
|
105 |
+
cropped_page = page.crop(content_bbox)
|
106 |
+
|
107 |
+
|
108 |
+
# 1. Bounding Boxes aus find_tables(für späteres ausschließen der fließtexterkennung)
|
109 |
+
table_bboxes = []
|
110 |
+
for table in cropped_page.find_tables():
|
111 |
+
bbox = clamp_bbox(table.bbox, width, height)
|
112 |
+
# if cropped_page.crop(bbox).chars:
|
113 |
+
# table_bboxes.append(bbox)
|
114 |
+
cropped_chars = cropped_page.crop(bbox).chars
|
115 |
+
valid_chars = [
|
116 |
+
c for c in cropped_chars
|
117 |
+
if not EUROPEAN_PRINTABLES_PATTERN.search(c["text"])
|
118 |
+
]
|
119 |
+
if valid_chars:
|
120 |
+
table_bboxes.append(bbox)
|
121 |
+
|
122 |
+
|
123 |
+
|
124 |
+
# 2. Tabelleninhalt unabhängig extrahieren
|
125 |
+
table_json_outputs = []
|
126 |
+
|
127 |
+
for table_data_raw in cropped_page.extract_tables({"text_x_tolerance": 1.5}):
|
128 |
+
if table_data_raw and len(table_data_raw) >= 1:
|
129 |
+
# Bereinige bereits hier alle Zellen der rohen Tabelle
|
130 |
+
table_data = [[clean_cell_text(cell) for cell in row] for row in table_data_raw]
|
131 |
+
|
132 |
+
headers = table_data[0]
|
133 |
+
rows = table_data[1:]
|
134 |
+
json_table = [dict(zip(headers, row)) for row in rows]
|
135 |
+
table_json_outputs.append(json.dumps(json_table, indent=1, ensure_ascii=False))
|
136 |
+
|
137 |
+
|
138 |
+
|
139 |
+
|
140 |
+
# fließtexterkennunf ohne tabellenbereiche
|
141 |
+
words_outside_tables = [
|
142 |
+
word for word in cropped_page.extract_words(**TEXT_EXTRACTION_SETTINGS)
|
143 |
+
if not any(
|
144 |
+
bbox[0] <= float(word['x0']) <= bbox[2] and
|
145 |
+
bbox[1] <= float(word['top']) <= bbox[3]
|
146 |
+
for bbox in table_bboxes
|
147 |
+
) and not EUROPEAN_PRINTABLES_PATTERN.search(word['text']) # <== Filter auf gültige Zeichen
|
148 |
+
]
|
149 |
+
|
150 |
+
# alle buchstaben je seite
|
151 |
+
characters = [
|
152 |
+
c for c in cropped_page.chars
|
153 |
+
if not any(
|
154 |
+
bbox[0] <= float(c['x0']) <= bbox[2] and
|
155 |
+
bbox[1] <= float(c['top']) <= bbox[3]
|
156 |
+
for bbox in table_bboxes
|
157 |
+
) and not EUROPEAN_PRINTABLES_PATTERN.search(c['text']) # <== Filter auf gültige Zeichen
|
158 |
+
]
|
159 |
+
|
160 |
+
|
161 |
+
# durchschnittliche schriftgröße je seite
|
162 |
+
letter_chars = [c for c in characters if c.get('text', '').isalpha()]
|
163 |
+
average_font_size = (
|
164 |
+
sum(float(c.get('size', 0)) for c in letter_chars) / len(letter_chars)
|
165 |
+
if letter_chars else 0
|
166 |
+
)
|
167 |
+
|
168 |
+
# wann ist ein wort bold oder groß-geschrieben, um das label wichtig oder kapitel zu erhalten
|
169 |
+
def classify_word(word, is_first_word_in_line):
|
170 |
+
"""Klassifiziere ein Wort individuell mit Stil, nur beim ersten Wort der Zeile."""
|
171 |
+
word_top = float(word['top'])
|
172 |
+
word_mid = (float(word['x0']) + float(word['x1'])) / 2
|
173 |
+
|
174 |
+
# Alle Zeichen in Zeilenhöhe
|
175 |
+
line_chars = sorted([
|
176 |
+
c for c in characters
|
177 |
+
if abs(c['top'] - word_top) < 2
|
178 |
+
], key=lambda c: c['x0'])
|
179 |
+
|
180 |
+
# Funktion: Finde mind. 3 aufeinanderfolgende fettgedruckte Zeichen
|
181 |
+
def has_consecutive_bold(chars):
|
182 |
+
count = 0
|
183 |
+
for c in chars:
|
184 |
+
if "bold" in c.get("fontname", "").lower() and float(c.get("size", 0)) >= average_font_size:
|
185 |
+
count += 1
|
186 |
+
if count >= 3:
|
187 |
+
return True
|
188 |
+
else:
|
189 |
+
count = 0
|
190 |
+
return False
|
191 |
+
|
192 |
+
# Funktion: Finde mind. 3 aufeinanderfolgende große Buchstaben
|
193 |
+
def has_consecutive_large_alpha(chars):
|
194 |
+
count = 0
|
195 |
+
for c in chars:
|
196 |
+
if c.get('text', '').isalpha() and float(c.get("size", 0)) >= average_font_size * 1.16:
|
197 |
+
count += 1
|
198 |
+
if count >= 3:
|
199 |
+
return True
|
200 |
+
else:
|
201 |
+
count = 0
|
202 |
+
return False
|
203 |
+
|
204 |
+
prefix = ""
|
205 |
+
if is_first_word_in_line:
|
206 |
+
if has_consecutive_bold(line_chars):
|
207 |
+
prefix += "important: "
|
208 |
+
if has_consecutive_large_alpha(line_chars):
|
209 |
+
prefix += "chapter: "
|
210 |
+
|
211 |
+
return prefix + word['text']
|
212 |
+
|
213 |
+
|
214 |
+
# Gruppierung in Zeilen mit Wortbasierter Analyse
|
215 |
+
current_y = None
|
216 |
+
line = []
|
217 |
+
text_content = ""
|
218 |
+
|
219 |
+
for word in words_outside_tables:
|
220 |
+
word_y = float(word['top'])
|
221 |
+
if current_y is None or abs(word_y - current_y) > 10:
|
222 |
+
if line:
|
223 |
+
text_content += " ".join(line).strip() + "\n"
|
224 |
+
current_y = word_y
|
225 |
+
line = [classify_word(word, is_first_word_in_line=True)]
|
226 |
+
else:
|
227 |
+
line.append(classify_word(word, is_first_word_in_line=False))
|
228 |
+
|
229 |
+
|
230 |
+
if line:
|
231 |
+
text_content += " ".join(line).strip() + "\n"
|
232 |
+
|
233 |
+
output += text_content.strip() + "\n"
|
234 |
+
|
235 |
+
# Add table JSON outputs to the page output
|
236 |
+
for idx, table in enumerate(table_json_outputs, start=1):
|
237 |
+
output += f'"table {idx}":\n{table}\n'
|
238 |
+
|
239 |
+
return page_number, output # Return the processed page number and output content
|
240 |
+
|
241 |
+
# except Exception as e:
|
242 |
+
# return args[0], f"[ERROR] Page {args[0]+1} ({args[1]}): {str(e)}" # Return an error message if an exception occurs
|
243 |
+
|
244 |
+
except Exception as e:
|
245 |
+
error_msg = str(e)
|
246 |
+
if "Cannot set gray non-stroke color because" in error_msg and "invalid float value" in error_msg:
|
247 |
+
friendly_msg = f"[ERROR] Seite {args[0]+1} ({args[1]}): Ungültiger Farbwert in PDF-Inhalt erkannt (möglicherweise beschädigte Farbdefinition). Verarbeitung nicht möglich."
|
248 |
+
return args[0], friendly_msg
|
249 |
+
else:
|
250 |
+
return args[0], f"[ERROR] Seite {args[0]+1} ({args[1]}): {error_msg}"
|
251 |
+
|
252 |
+
|
253 |
+
|
254 |
+
|
255 |
+
# Function to process the entire PDF document
|
256 |
+
def process_pdf(pdf_path):
|
257 |
+
suppress_pdfminer_logging() # Suppress unnecessary logging
|
258 |
+
try:
|
259 |
+
if not os.path.exists(pdf_path): # Check if the file exists
|
260 |
+
return f"[ERROR] File not found: {pdf_path}" # Return error message if file does not exist
|
261 |
+
|
262 |
+
print(f"[INFO] Starting processing: {pdf_path}") # Log the start of processing
|
263 |
+
try:
|
264 |
+
with pdfplumber.open(pdf_path) as pdf: # Open the PDF using pdfplumber
|
265 |
+
num_pages = len(pdf.pages) # Get the number of pages in the PDF
|
266 |
+
except PdfminerException as e:
|
267 |
+
return f"[ERROR] Cannot open PDF: {pdf_path} – {str(e)}" # Return error if the PDF cannot be opened
|
268 |
+
except Exception as e:
|
269 |
+
return f"[ERROR] General error opening PDF: {pdf_path} – {str(e)}" # Return general error if any exception occurs
|
270 |
+
|
271 |
+
pages = [(i, pdf_path, TEXT_EXTRACTION_SETTINGS) for i in range(num_pages)] # Prepare the pages for processing
|
272 |
+
|
273 |
+
try:
|
274 |
+
results = run_serial(pages) if num_pages <= PARALLEL_THRESHOLD else run_parallel(pages) # Run serial or parallel processing
|
275 |
+
except (EOFError, BrokenPipeError, KeyboardInterrupt):
|
276 |
+
return "[INFO] Processing was interrupted." # Handle interruptions during processing
|
277 |
+
|
278 |
+
sorted_results = sorted(results, key=lambda x: x[0]) # Sort results by page number
|
279 |
+
final_output = "\n".join(text for _, text in sorted_results) # Combine all page results into a single string
|
280 |
+
|
281 |
+
base_name = os.path.splitext(os.path.basename(pdf_path))[0] # Get the base name of the PDF file
|
282 |
+
output_dir = os.path.dirname(pdf_path) # Get the directory of the PDF file
|
283 |
+
output_path = os.path.join(output_dir, f"{base_name}.txt") # Generate the output file path
|
284 |
+
|
285 |
+
with open(output_path, "w", encoding="utf-8", errors="ignore") as f: # Open the output file for writing
|
286 |
+
f.write(final_output) # Write the final output to the file
|
287 |
+
|
288 |
+
print(f"[INFO] Processing complete: {output_path}") # Log the successful processing completion
|
289 |
+
return "complete"
|
290 |
+
|
291 |
+
except (EOFError, BrokenPipeError, KeyboardInterrupt):
|
292 |
+
return "[INFO] Processing interrupted by user." # Handle user interruptions
|
293 |
+
except Exception as e:
|
294 |
+
return f"[ERROR] Unexpected error with '{pdf_path}': {str(e)}" # Handle unexpected errors during processing
|
295 |
+
|
296 |
+
# Function to run the PDF processing serially (one page at a time)
|
297 |
+
def run_serial(pages):
|
298 |
+
return [process_page(args) for args in pages] # Process each page in sequence
|
299 |
+
|
300 |
+
# Function to run the PDF processing in parallel (across multiple cores)
|
301 |
+
def run_parallel(pages):
|
302 |
+
available_cores = max(1, cpu_count() - 2)
|
303 |
+
num_cores = min(available_cores, len(pages))
|
304 |
+
print(f"Starting parallel processing with {num_cores} cores...")
|
305 |
+
with parallel_backend('loky'): # 'loky' ist der Standard-Backend bei joblib
|
306 |
+
return Parallel(n_jobs=num_cores)(
|
307 |
+
delayed(process_page)(args) for args in pages
|
308 |
+
)
|
309 |
+
|
310 |
+
# Main function to process a list of PDFs
|
311 |
+
def process_pdfs_main():
|
312 |
+
suppress_pdfminer_logging() # Suppress unnecessary logging
|
313 |
+
pdf_files = sys.argv[1:] # Get PDF file paths from command-line arguments
|
314 |
+
if not pdf_files: # Check if any PDFs are provided
|
315 |
+
print("No PDF files provided.") # Log message if no PDFs are provided
|
316 |
+
return
|
317 |
+
|
318 |
+
small_pdfs = [] # List to store small PDFs (less than the parallel threshold)
|
319 |
+
large_pdfs = [] # List to store large PDFs (greater than the parallel threshold)
|
320 |
+
|
321 |
+
# Categorize PDFs into small and large based on the number of pages
|
322 |
+
for path in pdf_files:
|
323 |
+
if not os.path.exists(path): # Check if the file exists
|
324 |
+
print(f"File not found: {path}") # Log error if file does not exist
|
325 |
+
continue
|
326 |
+
try:
|
327 |
+
with pdfplumber.open(path) as pdf: # Open the PDF
|
328 |
+
if len(pdf.pages) <= PARALLEL_THRESHOLD: # If the PDF has fewer pages than the threshold
|
329 |
+
small_pdfs.append(path) # Add to small PDFs list
|
330 |
+
else:
|
331 |
+
large_pdfs.append(path) # Add to large PDFs list
|
332 |
+
except PdfminerException:
|
333 |
+
print(f"[ERROR] Password-protected PDF skipped: {path}") # Log if the PDF is password-protected
|
334 |
+
except Exception as e:
|
335 |
+
print(f"[ERROR] Error opening {path}: {str(e)}") # Log any other errors when opening the PDF
|
336 |
+
|
337 |
+
# Process small PDFs in parallel (each on one core)
|
338 |
+
if small_pdfs:
|
339 |
+
available_cores = max(1, cpu_count() - 2) # Determine the number of available cores
|
340 |
+
num_cores = min(available_cores, len(small_pdfs)) # Use the lesser of available cores or small PDFs count
|
341 |
+
print(f"\n[Phase 1] Starting parallel processing of small PDFs with {num_cores} cores, 2 leaving for system processes...") # Log processing start
|
342 |
+
results = Parallel(n_jobs=num_cores)( # Run parallel processing for small PDFs
|
343 |
+
delayed(process_pdf)(path) for path in small_pdfs
|
344 |
+
)
|
345 |
+
for r in results:
|
346 |
+
print(r) # Print the results for each small PDF
|
347 |
+
|
348 |
+
# Process large PDFs one by one (in serial on all cores)
|
349 |
+
for path in large_pdfs:
|
350 |
+
print(f"\n[Phase 2] Processing large PDF: {os.path.basename(path)}") # Log processing of large PDF
|
351 |
+
print(process_pdf(path)) # Process the large PDF
|
352 |
+
|
353 |
+
|
354 |
+
# GUI
|
355 |
+
|
356 |
+
|
357 |
+
ctk.set_appearance_mode("System")
|
358 |
+
ctk.set_default_color_theme("dark-blue")
|
359 |
+
|
360 |
+
class FileManager:
|
361 |
+
def __init__(self, master):
|
362 |
+
self.master = master
|
363 |
+
self.master.protocol("WM_DELETE_WINDOW", self.on_close)
|
364 |
+
self.master.title("Parser-Sevenof9")
|
365 |
+
self.master.geometry("1000x800+200+100")
|
366 |
+
self.master.minsize(1000, 800)
|
367 |
+
custom_font = tkfont.Font(family="Courier New", size=14)
|
368 |
+
self.files = []
|
369 |
+
self.last_selected_index = None
|
370 |
+
self.parser_process = None
|
371 |
+
|
372 |
+
self.master.grid_rowconfigure(1, weight=0)
|
373 |
+
self.master.grid_columnconfigure(0, weight=1)
|
374 |
+
|
375 |
+
# Label: "Selected PDF files"
|
376 |
+
self.label = ctk.CTkLabel(master, text="Selected PDF files: (right mouse, you can copy path or open PDF)", height=30)
|
377 |
+
self.label.grid(row=0, column=0, sticky="nw", padx=10, pady=(10, 0))
|
378 |
+
|
379 |
+
# Listbox Frame
|
380 |
+
listbox_frame = ctk.CTkFrame(master, height=200)
|
381 |
+
listbox_frame.grid(row=1, column=0, sticky="nsew", padx=10)
|
382 |
+
listbox_frame.grid_propagate(False)
|
383 |
+
listbox_frame.grid_rowconfigure(0, weight=1)
|
384 |
+
listbox_frame.grid_columnconfigure(0, weight=1)
|
385 |
+
|
386 |
+
self.listbox = tk.Listbox(listbox_frame, selectmode=tk.MULTIPLE, font=custom_font,)
|
387 |
+
scrollbar_listbox = tk.Scrollbar(listbox_frame, command=self.listbox.yview)
|
388 |
+
self.listbox.config(yscrollcommand=scrollbar_listbox.set)
|
389 |
+
|
390 |
+
self.listbox.grid(row=0, column=0, sticky="nsew")
|
391 |
+
scrollbar_listbox.grid(row=0, column=1, sticky="ns")
|
392 |
+
|
393 |
+
# Context Menu
|
394 |
+
self.context_menu = tk.Menu(master, tearoff=0)
|
395 |
+
self.context_menu.add_command(label="Remove selected", command=self.remove_file)
|
396 |
+
self.context_menu.add_separator()
|
397 |
+
self.context_menu.add_command(label="Copy file location", command=self.copy_file_location)
|
398 |
+
self.context_menu.add_command(label="Open in default PDF app", command=self.open_file_in_default_app)
|
399 |
+
self.listbox.bind("<Button-3>", self.show_context_menu)
|
400 |
+
|
401 |
+
self.listbox.bind("<<ListboxSelect>>", self.show_text_file)
|
402 |
+
self.listbox.bind("<Button-1>", self.on_listbox_click)
|
403 |
+
self.listbox.bind("<Shift-Button-1>", self.on_listbox_shift_click)
|
404 |
+
|
405 |
+
# Button Frame
|
406 |
+
button_frame = ctk.CTkFrame(master, height=40)
|
407 |
+
button_frame.grid(row=2, column=0, sticky="nsew", padx=10, pady=5)
|
408 |
+
button_frame.grid_propagate(False)
|
409 |
+
|
410 |
+
button_frame.grid_columnconfigure((0, 1, 2, 3, 4, 5), weight=1)
|
411 |
+
|
412 |
+
ctk.CTkButton(button_frame, text="Add Folder", command=self.add_folder).grid(row=0, column=0, padx=5, pady=5)
|
413 |
+
ctk.CTkButton(button_frame, text="Select Files", command=self.add_file).grid(row=0, column=1, padx=5, pady=5)
|
414 |
+
ctk.CTkButton(button_frame, text="Remove Selected", command=self.remove_file).grid(row=0, column=2, padx=5, pady=5)
|
415 |
+
ctk.CTkButton(button_frame, text="Remove All", command=self.remove_all).grid(row=0, column=3, padx=5, pady=5)
|
416 |
+
ctk.CTkButton(button_frame, text="Stop", command=self.stop_parser, fg_color="darkred", hover_color="red").grid(row=0, column=4, padx=5, pady=5)
|
417 |
+
ctk.CTkButton(button_frame, text="Start Parser", command=self.start_parser, fg_color="darkgreen", hover_color="green").grid(row=0, column=5, padx=5, pady=5)
|
418 |
+
|
419 |
+
# Label: "Text Frame"
|
420 |
+
self.progress_label = ctk.CTkLabel(master, text="Text Frame: (select a PDF, you can copy text parts)", height=30)
|
421 |
+
self.progress_label.grid(row=3, column=0, sticky="nw", padx=10)
|
422 |
+
|
423 |
+
# Text Frame
|
424 |
+
text_frame = ctk.CTkFrame(master, height=250)
|
425 |
+
text_frame.grid(row=4, column=0, sticky="nsew", padx=10, pady=5)
|
426 |
+
text_frame.grid_propagate(False)
|
427 |
+
text_frame.grid_rowconfigure(0, weight=1)
|
428 |
+
text_frame.grid_columnconfigure(0, weight=1)
|
429 |
+
|
430 |
+
self.text_widget = tk.Text(text_frame, wrap=tk.WORD, font=custom_font,)
|
431 |
+
scrollbar_text = tk.Scrollbar(text_frame, command=self.text_widget.yview)
|
432 |
+
self.text_widget.config(yscrollcommand=scrollbar_text.set)
|
433 |
+
|
434 |
+
self.text_widget.grid(row=0, column=0, sticky="nsew")
|
435 |
+
scrollbar_text.grid(row=0, column=1, sticky="ns")
|
436 |
+
|
437 |
+
# Label: "Progress"
|
438 |
+
self.progress_label = ctk.CTkLabel(master, text="Progress: (Error and success messages are not always correct)", height=30)
|
439 |
+
self.progress_label.grid(row=5, column=0, sticky="nw", padx=10)
|
440 |
+
|
441 |
+
# Progress Frame
|
442 |
+
progress_frame = ctk.CTkFrame(master, height=160)
|
443 |
+
progress_frame.grid(row=6, column=0, sticky="nsew", padx=10, pady=(0, 10))
|
444 |
+
progress_frame.grid_propagate(False)
|
445 |
+
progress_frame.grid_rowconfigure(0, weight=1)
|
446 |
+
progress_frame.grid_columnconfigure(0, weight=1)
|
447 |
+
|
448 |
+
self.progress_text = tk.Text(progress_frame, state=tk.DISABLED)
|
449 |
+
scrollbar_progress = tk.Scrollbar(progress_frame, command=self.progress_text.yview)
|
450 |
+
self.progress_text.config(yscrollcommand=scrollbar_progress.set)
|
451 |
+
|
452 |
+
self.progress_text.grid(row=0, column=0, sticky="nsew")
|
453 |
+
scrollbar_progress.grid(row=0, column=1, sticky="ns")
|
454 |
+
|
455 |
+
def on_close(self):
|
456 |
+
if self.parser_process:
|
457 |
+
self.stop_parser()
|
458 |
+
# ggf. geplante after-Aufrufe canceln
|
459 |
+
if hasattr(self, 'after_id'):
|
460 |
+
self.master.after_cancel(self.after_id)
|
461 |
+
self.master.destroy()
|
462 |
+
|
463 |
+
|
464 |
+
def on_listbox_click(self, event):
|
465 |
+
# Handle single left-click selection; clear previous selection
|
466 |
+
index = self.listbox.nearest(event.y)
|
467 |
+
self.listbox.selection_clear(0, tk.END)
|
468 |
+
self.listbox.selection_set(index)
|
469 |
+
self.last_selected_index = index
|
470 |
+
self.show_text_file(None)
|
471 |
+
return "break" # Prevent default event propagation
|
472 |
+
|
473 |
+
def on_listbox_shift_click(self, event):
|
474 |
+
# Handle shift-click for range selection
|
475 |
+
index = self.listbox.nearest(event.y)
|
476 |
+
if self.last_selected_index is None:
|
477 |
+
self.last_selected_index = index
|
478 |
+
start, end = sorted((self.last_selected_index, index))
|
479 |
+
self.listbox.selection_clear(0, tk.END)
|
480 |
+
for i in range(start, end + 1):
|
481 |
+
self.listbox.selection_set(i)
|
482 |
+
return "break"
|
483 |
+
|
484 |
+
def show_context_menu(self, event):
|
485 |
+
# Show right-click context menu if any item is selected
|
486 |
+
if self.listbox.curselection():
|
487 |
+
self.context_menu.tk_popup(event.x_root, event.y_root)
|
488 |
+
|
489 |
+
def add_folder(self):
|
490 |
+
# Add all PDFs from a selected folder
|
491 |
+
folder = filedialog.askdirectory(title="Select Folder")
|
492 |
+
if not folder:
|
493 |
+
return
|
494 |
+
for root, _, files in os.walk(folder):
|
495 |
+
for file in files:
|
496 |
+
if file.lower().endswith(".pdf"):
|
497 |
+
path = os.path.normpath(os.path.join(root, file))
|
498 |
+
if path not in self.files:
|
499 |
+
self.files.append(path)
|
500 |
+
self.listbox.insert(tk.END, path)
|
501 |
+
|
502 |
+
def add_file(self):
|
503 |
+
# Add selected individual PDF files
|
504 |
+
paths = filedialog.askopenfilenames(title="Select PDF Files", filetypes=[("PDF Files", "*.pdf")])
|
505 |
+
for path in paths:
|
506 |
+
path = os.path.normpath(path)
|
507 |
+
if path not in self.files:
|
508 |
+
self.files.append(path)
|
509 |
+
self.listbox.insert(tk.END, path)
|
510 |
+
|
511 |
+
def remove_file(self):
|
512 |
+
# Remove selected files from list and internal storage
|
513 |
+
selection = self.listbox.curselection()
|
514 |
+
if not selection:
|
515 |
+
messagebox.showwarning("Notice", "Please select an entry to remove.")
|
516 |
+
return
|
517 |
+
for index in reversed(selection): # Reverse to avoid index shifting
|
518 |
+
self.listbox.delete(index)
|
519 |
+
del self.files[index]
|
520 |
+
self.text_widget.delete(1.0, tk.END)
|
521 |
+
|
522 |
+
def copy_file_location(self):
|
523 |
+
selection = self.listbox.curselection()
|
524 |
+
if not selection:
|
525 |
+
return
|
526 |
+
index = selection[0]
|
527 |
+
path = self.files[index]
|
528 |
+
self.master.clipboard_clear()
|
529 |
+
self.master.clipboard_append(path)
|
530 |
+
self.master.update() # erforderlich, um Inhalt im Clipboard zu halten
|
531 |
+
|
532 |
+
def open_file_in_default_app(self):
|
533 |
+
selection = self.listbox.curselection()
|
534 |
+
if not selection:
|
535 |
+
return
|
536 |
+
index = selection[0]
|
537 |
+
path = self.files[index]
|
538 |
+
if os.path.exists(path):
|
539 |
+
try:
|
540 |
+
os.startfile(path)
|
541 |
+
except Exception as e:
|
542 |
+
messagebox.showerror("Error", f"Cannot open file:\n{e}")
|
543 |
+
else:
|
544 |
+
messagebox.showwarning("File not found", "The selected file could not be found.")
|
545 |
+
|
546 |
+
def remove_all(self):
|
547 |
+
# Remove all files from the list
|
548 |
+
self.listbox.delete(0, tk.END)
|
549 |
+
self.files.clear()
|
550 |
+
self.text_widget.delete(1.0, tk.END)
|
551 |
+
|
552 |
+
def start_parser(self):
|
553 |
+
# Validate input and launch parser in separate thread
|
554 |
+
if not self.files:
|
555 |
+
messagebox.showinfo("No Files", "Please select at least one file.")
|
556 |
+
return
|
557 |
+
self.progress_text.config(state=tk.NORMAL)
|
558 |
+
self.progress_text.delete(1.0, tk.END)
|
559 |
+
self.progress_text.insert(tk.END, "Starting parser...\n")
|
560 |
+
self.progress_text.config(state=tk.DISABLED)
|
561 |
+
|
562 |
+
# Launch parsing in background to avoid UI freeze
|
563 |
+
thread = threading.Thread(target=self.run_parser)
|
564 |
+
thread.start()
|
565 |
+
|
566 |
+
def stop_parser(self):
|
567 |
+
# Terminate running parser process if active
|
568 |
+
if self.parser_process and self.parser_process.poll() is None:
|
569 |
+
self.parser_process.terminate()
|
570 |
+
self.append_progress_text("Parser process was stopped.\n")
|
571 |
+
else:
|
572 |
+
self.append_progress_text("No active parser process to stop.\n")
|
573 |
+
|
574 |
+
def run_parser(self):
|
575 |
+
# Internal method to run the external parser script
|
576 |
+
try:
|
577 |
+
self.parser_process = subprocess.Popen(
|
578 |
+
[sys.executable, __file__] + self.files,
|
579 |
+
stdout=subprocess.PIPE,
|
580 |
+
stderr=subprocess.STDOUT,
|
581 |
+
text=True,
|
582 |
+
encoding='utf-8',
|
583 |
+
errors='ignore',
|
584 |
+
bufsize=4096
|
585 |
+
)
|
586 |
+
for line in self.parser_process.stdout:
|
587 |
+
self.append_progress_text(line)
|
588 |
+
self.parser_process.stdout.close()
|
589 |
+
self.parser_process.wait()
|
590 |
+
|
591 |
+
if self.parser_process.returncode == 0:
|
592 |
+
self.append_progress_text("\nParser finished successfully.\n")
|
593 |
+
self.show_messagebox_threadsafe("Parser Done", "The parser was executed successfully.")
|
594 |
+
else:
|
595 |
+
self.append_progress_text("\nError while running the parser.\n")
|
596 |
+
self.show_messagebox_threadsafe("Error", "Error while running the parser.")
|
597 |
+
except Exception as e:
|
598 |
+
self.append_progress_text(f"Error: {e}\n")
|
599 |
+
self.show_messagebox_threadsafe("Error", f"Error during execution:\n{e}")
|
600 |
+
finally:
|
601 |
+
self.parser_process = None
|
602 |
+
|
603 |
+
def append_progress_text(self, text):
|
604 |
+
# Thread-safe method to append text to the progress view
|
605 |
+
self.progress_text.after(0, lambda: self._insert_text(text))
|
606 |
+
|
607 |
+
def _insert_text(self, text):
|
608 |
+
# Append text and scroll to bottom
|
609 |
+
self.progress_text.config(state=tk.NORMAL)
|
610 |
+
self.progress_text.insert(tk.END, text)
|
611 |
+
self.progress_text.see(tk.END)
|
612 |
+
self.progress_text.config(state=tk.DISABLED)
|
613 |
+
|
614 |
+
def show_messagebox_threadsafe(self, title, message):
|
615 |
+
# Display a messagebox from a background thread
|
616 |
+
self.master.after(0, lambda: messagebox.showinfo(title, message))
|
617 |
+
|
618 |
+
def show_text_file(self, event):
|
619 |
+
# Load and show the content of the corresponding .txt file (if available)
|
620 |
+
selection = self.listbox.curselection()
|
621 |
+
if not selection:
|
622 |
+
return
|
623 |
+
index = selection[0]
|
624 |
+
path = self.files[index]
|
625 |
+
txt_path = os.path.splitext(path)[0] + ".txt"
|
626 |
+
self.text_widget.delete(1.0, tk.END)
|
627 |
+
if os.path.exists(txt_path):
|
628 |
+
try:
|
629 |
+
with open(txt_path, "r", encoding="utf-8", errors="ignore") as f:
|
630 |
+
self.text_widget.insert(tk.END, f.read())
|
631 |
+
except Exception as e:
|
632 |
+
self.text_widget.insert(tk.END, f"Error loading text file:\n{e}")
|
633 |
+
else:
|
634 |
+
self.text_widget.insert(tk.END, "[No corresponding .txt file found]")
|
635 |
+
|
636 |
+
|
637 |
+
|
638 |
+
|
639 |
+
|
640 |
+
def main():
|
641 |
+
if len(sys.argv) > 1:
|
642 |
+
process_pdfs_main()
|
643 |
+
else:
|
644 |
+
launch_gui()
|
645 |
+
|
646 |
+
def launch_gui():
|
647 |
+
root = ctk.CTk()
|
648 |
+
app = FileManager(root)
|
649 |
+
root.mainloop()
|
650 |
+
|
651 |
+
# MAIN
|
652 |
+
|
653 |
+
if __name__ == "__main__":
|
654 |
+
main()
|
655 |
+
|
656 |
+
|