`Added new parsers and updated existing ones for Noc timesheet and invoice extraction`

app.py

@@ -36,21 +36,63 @@ def Noc_timeSheet_pdf_to_img(pdf_path,dpi: int = 300, quality: int = 95):
 
     # Save the combined image
     combined_image.save(output_path, "JPEG", quality=quality)
-
+ 
 def Clauses_in_invoice(pdf_path: str) -> bool:
     """
     Extract text from the last page of a PDF.
     """
-    pdf_document = pymupdf.open(pdf_path)
-    total_pages = pdf_document.page_count 
-    last_page = pdf_document.load_page(total_pages - 1)  
-    text = last_page.get_text() 
-    pdf_document.close() 
-    if "clauses" in text.lower():
-        return True
-    else:
+    try:
+        
+        pdf_document = pymupdf.open(pdf_path)
+        total_pages = pdf_document.page_count 
+        if total_pages < 2:
+                print("The PDF has fewer than 2 pages.")
+                return False
+        last_page = pdf_document.load_page(total_pages - 1)
+        text = last_page.get_text() 
+        
+        last_page = text.lower()
+        
+        if "clauses" in last_page:
+            return True
+        else:
+            return False
+    except Exception as e: 
+        print(f"error :{e}")
         return False
-
+    
+    finally:
+        # Ensure the PDF document is closed
+        if 'pdf_document' in locals():
+            pdf_document.close()
+            
+def Clauses_in_invoice_2nd_version(pdf_path: str) -> bool:
+    """
+    Extract text from the last page of a PDF.
+    """
+    try:
+        
+        pdf_document = pymupdf.open(pdf_path)
+        total_pages = pdf_document.page_count 
+        if total_pages < 2:
+                print("The PDF has fewer than 2 pages.")
+                return False
+        second_to_last_page = pdf_document.load_page(total_pages - 2)
+        text = second_to_last_page.get_text() 
+        
+        if "clauses" in text.lower():
+            return True
+        else:
+            return False
+    except Exception as e: 
+        print(f"error :{e}")
+        return False
+    
+    finally:
+        # Ensure the PDF document is closed
+        if 'pdf_document' in locals():
+            pdf_document.close()
+ 
 def Noc_invoice_pdf_to_img(pdf_path: str, folder_path: str, dpi: int = 300, quality: int = 95):
     pdf_document = pymupdf.open(pdf_path)
     folder_path = folder_path.rstrip(os.sep)
@@ -107,11 +149,26 @@ def noc_invoice_extraction(pdf_path: str, folder_path):
             new_item = get_image_informations(image_paths[pic + 2], invoice_item_pages_prompt, Noc_PurchaseOrder_items_parser)
             for item in new_item["items"]:
                 data["items"].append(item)
-        result = get_image_informations(image_paths[-2], invoice_total_page_prompt, Noc_PurchaseOrder_total_parser)
+        result = get_image_informations(image_paths[-1], invoice_total_page_prompt, Noc_PurchaseOrder_total_parser)
         data.update(result)
         delete_images(image_paths)
         return data
 
+       
+def pdf_to_img(pdf_path, dpi: float = 300, quality: float = 95):
+    pdf_document = pymupdf.open(pdf_path)
+    page = pdf_document.load_page(0)  # Load the first page
+    output_path = "output.jpg"
+
+    # Convert the page to a pixmap (image)
+    pix = page.get_pixmap(dpi=dpi)
+    
+    # Convert the pixmap to a PIL Image
+    image = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
+
+    image.save(output_path, "JPEG",quality=quality)
+
+
 def process_file(file, option):
     if file is None:
         return "Please upload a PDF or image file."
@@ -125,31 +182,45 @@ def process_file(file, option):
         print(file_extension)
         if file_extension in ['.pdf']:
             # Process PDF files
-            if option == "Noc_timesheet_residential":
+            if option == "Noc_timesheet_residential_old":
                 print(file_path)
                 Noc_timeSheet_pdf_to_img(file_path)
                 print("here 2")
                 result = get_image_informations("output.jpg", Noc_Res_timesheet_prompt, Noc_Res_timeSheet_parser)
                 return result
-            elif option == "Noc_timesheet_rotational":
+            elif option == "Noc_timesheet_rotational_old":
                 Noc_timeSheet_pdf_to_img(file_path)
                 result = get_image_informations("output.jpg", Noc_Rot_timesheet_prompt, Noc_Rot_timeSheet_parser)
                 return result
-            elif option == "Noc_invoice":
+            elif option == "Noc_PO":
                 result = noc_invoice_extraction(file_path, save_dir)
                 return result
+            elif option =="Noc_timesheet_new":
+                pdf_to_img(file_path)
+                result = get_image_informations("output.jpg", Noc_timesheet_prompt, Noc_timesheet_parser_v1)
+                return result
+            elif option == "Noc_invoice":
+                pdf_to_img(file_path)
+                result = get_image_informations("output.jpg", Noc_invoice_prompt, Noc_invoice_parser_v1)
+                return result
         elif file_extension in ['.jpg', '.jpeg', '.png']:
             # Process image files directly
-            if option == "Noc_timesheet_residential":
+            if option == "Noc_timesheet_residential_old":
                 result = get_image_informations(file_path, Noc_Res_timesheet_prompt, Noc_Res_timeSheet_parser)
                 return result
-            elif option == "Noc_timesheet_rotational":
+            elif option == "Noc_timesheet_rotational_old":
                 result = get_image_informations(file_path, Noc_Rot_timesheet_prompt, Noc_Rot_timeSheet_parser)
                 return result
-            elif option == "Noc_invoice":
+            elif option == "Noc_PO":
                 # For invoice images, we assume it's a single page
                 result = get_image_informations(file_path, invoice_first_page_prompt, Noc_PurchaseOrder_information_parser)
                 return result
+            elif option =="Noc_timesheet_new":
+                result = get_image_informations(file_path, Noc_timesheet_prompt, Noc_timesheet_parser_v1)
+                return result
+            elif option == "Noc_invoice":
+                result = get_image_informations(file_path, Noc_invoice_prompt, Noc_invoice_parser_v1)
+                return result
         else:
             return "Unsupported file type. Please upload a PDF or image file."
     except Exception as e:
@@ -160,7 +231,7 @@ demo = gr.Interface(
     fn=process_file,
     inputs=[
         gr.File(label="Upload PDF or Image"),  # File upload input
-        gr.Radio(["Noc_timesheet_residential", "Noc_timesheet_rotational", "Noc_invoice"], label="Choose an option")  # Radio buttons for options
+        gr.Radio(["Noc_timesheet_new","Noc_invoice","Noc_timesheet_residential_old", "Noc_timesheet_rotational_old", "Noc_PO"], label="Choose an option")  # Radio buttons for options
     ],
     outputs="text",  # Text output
     title="PDF/Image Processor",

dataSchema.py

@@ -1,6 +1,6 @@
 from pydantic import BaseModel, Field
 from typing import Optional,List
-from langchain_core.output_parsers import JsonOutputParser
+from langchain_core.output_parsers import JsonOutputParser # type: ignore
 
 class Noc_Residential_TimeSheetInformation(BaseModel):
     """Details of a timesheet entry."""
@@ -53,6 +53,29 @@ class Noc_Rotational_TimeSheetInformation(BaseModel):
     approved_on : str = Field(...,description="DD/MM/YY of the stamp")
     approved_by : str = Field(...,description="Name of the person who approved the document") 
 
+class Noc_TimeSheetInformation(BaseModel):
+    """Details of a timesheet entry."""
+    position_title: str = Field(..., description="Position title of the person.")
+    work_location: str = Field(..., description="Work location ")
+    agency: str = Field(..., description="the agency")
+    noc_id: str = Field(..., description="NOC ID of the person.")
+    full_name: str = Field(..., description="Full name of the person.")
+    approval_status: str = Field("not approved",description="(e.g., 'approved', 'not approved').")
+    approved_on : str = Field(...,description="DD/MM/YY of the stamp")
+    approved_by : str = Field(...,description="Name of the person who approved the document")
+    
+    service_days_onshore: int = Field(0, description="Number of service days onshore.")
+    standby_days_onshore: int = Field(0, description="Number of standby days onshore in Doha.")
+    service_days_offshore: int = Field(0, description="Number of service days offshore.")
+    service_days_weekend_public_holiday: int = Field(0, description="Number of service days during weekends or public holidays.")
+    standby_extended_hitch_days_offshore: int = Field(0, description="Number of standby and extended hitch days offshore.")
+    extended_hitch_days_onshore: int = Field(0, description="Number of extended hitch days onshore for rotational personnel.")
+    overtime_Hourly_Rate_ONSHORE:int = Field(0,description="number of over time hours onshore (Over 8 hours) ")
+    overtime_Hourly_Rate_OFFSHORE:int = Field(0,description="number of over time hours offshore (Over 12 hours) ")
+    per_diem_days: int = Field(0, description="Number of Per Diem days for onshore/offshore rotational personnel.")
+    training_days: int = Field(0, description="Number of training days.")
+    travel_days: int = Field(0, description="Number of travel days.")
+
 class Noc_Item_Information(BaseModel):
     """Details of each item in the document."""
     item_number : int = Field(...,description="the number of the item")
@@ -106,7 +129,103 @@ class Noc_PurchaseOrderInformation(BaseModel):
 class Noc_Clauses(BaseModel):
     Clauses: str = Field(..., description="the contract clauses.")
 
+class service(BaseModel):
+    service: str = Field(..., description="the service name.")
+    from_date: str = Field(None,description="starting date in DD/MM/YYYY format..")
+    to_date: str = Field(None,description="ending date in DD/MM/YYYY format.")
+    currency : str =Field(...,description="currency of the rate.")
+    fx: str = Field(None,description="foreign exchange.")
+    Number_of_days_hours: int = Field(...,description="number of hours or days for the service.")
+    rate: float = Field(..., description="the rate of the service.")
+    total: float = Field(...,description="total which is the rate* No of days/hours .")
+    
+class bank_details:
+    bank_name: str = Field(..., description="Name of the bank.")
+    swift_bic_code: str = Field(None, description=" SWIFT/BIC CODE.")
+    iban_number: str = Field(...,description="IBAN Number.")
+    beneficiary_name:str = Field(...,description="full name")
+    account_currency: str = Field(...,description="Account currency.")
+    expected_amount: str = Field(...,description="the amount.")
+    
+class Noc_Invoice(BaseModel):
+    """Details of an invoice."""
+    invoice_date: str = Field(..., description="Date of the invoice in DD/MM/YYYY format")
+    invoice_number: str = Field(..., description="Unique identifier for the invoice.")
+    full_name: str = Field(..., description="Full name of the person.")
+    invoice_to:str = Field(...,description="email to send the invoice forward to.")    
+    company_name:str = Field(...,description="company name")
+    address: str = Field(..., description="Address of the company.")
+    
+    services : List[service] = Field(None,description="list of services in the table.")
+    sub_total: float = Field(...,description="the sub total.")
+    vat: float = Field(None, description="the vat.")
+    total_due:float= Field(...,description="the total due.")
+    
+    first_bank : List[bank_details]=Field(...,description="first bank informations.")
+    second_details : List[bank_details]=Field(...,description="second bank details") 
+    
+
+
+Noc_timesheet_prompt = """
+Based on the provided timesheet details, extract the following information:
+- Position title of the person.
+- Work location .
+- the agency.
+- NOC ID of the person.
+- Name of the person.
+- approval status
+- date of the approval
+- approved by
+- Number of service days onshore
+- Number of standby days onshore in Doha
+- Number of service days offshore
+- Number of service days during weekends or public holidays
+- Number of standby and extended hitch days offshore
+- Number of extended hitch days onshore for rotational personnel
+- Number of over time hours onshore (Over 8 hours)
+- Number of over time hours offshore (Over 12 hours) 
+- Number of Per Diem days for onshore/offshore rotational personnel
+- Number of training days
+- Number of travel days
+
+"""
 
+Noc_invoice_prompt ="""
+Based on the provided timesheet details, extract the following information:
+- Invoice date
+- Invoice number
+- Full name of the person
+- Email to send the invoice forward to.
+- Company name
+- Address of the company
+- List of services,for each existing service provide:
+    - the service name.
+    - starting date in DD/MM/YYYY format.
+    - ending date in DD/MM/YYYY format.
+    - currency of the rate.
+    - foreign exchange.
+    - number of hours or days for the service.
+    - the rate of the service.
+    - total which is the rate* No of days/hours .
+- Sub total
+- VAT
+- Total due
+- First bank informations, for each existing bank provide, do not mess up the iban:
+    - Name of the bank.
+    - SWIFT/BIC CODE.
+    - IBAN Number.
+    - full name.
+    - Account currency.
+    - the amount.
+- Second bank informations, for each existing bank provide,do not mess up the iban:
+    - Name of the bank.
+    - SWIFT/BIC CODE.
+    - IBAN Number.
+    - full name.
+    - Account currency.
+    - the amount. 
+"""
+    
 Noc_Res_timesheet_prompt = """
    Based on the provided timesheet details, extract the following information:
    - Full name of the person
@@ -221,6 +340,7 @@ extract from the document:
 invoice_clauses_page_prompt = """ 
 extract from the document the clauses  """
 
+
 # CHOOSING PARSER DEPENDING ON THE TYPE OF DOCUMENT
 Noc_Res_timeSheet_parser = JsonOutputParser(pydantic_object=Noc_Residential_TimeSheetInformation)
 Noc_Rot_timeSheet_parser = JsonOutputParser(pydantic_object=Noc_Rotational_TimeSheetInformation)
@@ -228,4 +348,6 @@ Noc_PurchaseOrder_information_parser = JsonOutputParser(pydantic_object=Noc_Purc
 Noc_PurchaseOrder_item1_parser = JsonOutputParser(pydantic_object=Noc_Document_Information)
 Noc_PurchaseOrder_items_parser = JsonOutputParser(pydantic_object=Noc_items)
 Noc_PurchaseOrder_total_parser = JsonOutputParser(pydantic_object=Noc_total)
-Noc_PurchaseOrder_clauses_parser = JsonOutputParser(pydantic_object=Noc_Clauses)
+Noc_PurchaseOrder_clauses_parser = JsonOutputParser(pydantic_object=Noc_Clauses)
+Noc_invoice_parser_v1 = JsonOutputParser(pydantic_object=Noc_Invoice)
+Noc_timesheet_parser_v1 = JsonOutputParser(pydantic_object=Noc_TimeSheetInformation)