Upload 17 files

app.py

@@ -1,6 +1,6 @@
 import streamlit as st
 import tensorflow as tf
-from button_click_alt import find_order_id
+from button_click_alt import find_order_id_2
 from button_click_alt import find_order_id_similarity
 from flann import generate_images, flann_matching, flann_matching_3, flann_matching_alt, flann_matching_4
 import cv2
@@ -8,6 +8,7 @@ import numpy as np
 import pandas as pd
 import plotly.express as px
 import os
+import altair as alt
 
 FONTS_FOLDER = "fonts"
 NUM_IMAGES_PER_FONT = 5
@@ -16,7 +17,7 @@ def main():
     st.set_page_config(page_title='Order ID Finder', layout='wide')
     st.title('OCR + Font type demo')
 
-    tabs = st.tabs(["Intro", "Find Order", "Try FLANN Matching"])
+    tabs = st.tabs(["Intro", "Find Order", "Try FLANN Matching", "Results"])
 
     with tabs[0]:
         col1, col2 = st.columns([1, 2])
@@ -54,14 +55,35 @@ def main():
             It calculates the threshold that minimizes the intra-class variance or maximizes the inter-class variance. 
             By choosing the threshold that maximizes the inter-class variance, Otsu's thresholding effectively separates the two classes, resulting in a binary image.</p> """, unsafe_allow_html=True)
 
-        colab_link = '[<img src="https://colab.research.google.com/assets/colab-badge.svg">](https://colab.research.google.com/drive/1tq35g7ym1c73uDlAcy2KChIXsqlNY-RL?usp=sharing)'
+            st.subheader("FLANN Matching")
+            st.markdown("""<p>FLANN (Fast Library for Approximate Nearest Neighbors) is a popular library for performing fast and efficient nearest neighbor searches in high-dimensional spaces. 
+            It is often used in computer vision tasks such as feature matching, where the goal is to find corresponding features between two images.
+            In feature matching, one commonly used algorithm is SIFT (Scale-Invariant Feature Transform), which extracts keypoint descriptors from an image. 
+            However, because SIFT produces a large number of keypoints, it can be computationally expensive to match them between images.
+            To address this issue, the Lowe ratio test is often used in conjunction with SIFT and FLANN. 
+            The ratio test involves comparing the distance between the two closest matches of a given keypoint descriptor. 
+            If the ratio of these distances is below a certain threshold (typically 0.7), then the match is considered to be valid. 
+            This helps to filter out false matches and improve the accuracy of the feature matching process.</p>
+            <p> The ORB descriptor provides information about the local orientation and intensity of image features, which can be used to identify reliable matching points between two images. 
+            By using RANSAC to estimate the homography between the matched keypoints, we can eliminate outliers and improve the accuracy of the registration process.</P>
+            """, unsafe_allow_html=True)
+
+        colab_link = '[<img src="https://colab.research.google.com/assets/colab-badge.svg">](https://colab.research.google.com/drive/11aO-QNRl2qMK0tgJ03RvcRLUuUSPvWMc?usp=sharing)'
         st.markdown(colab_link, unsafe_allow_html=True)
 
 
 
     with tabs[1]:
         st.write('## Find Order')
-        st.write('This is the find order tab')
+        st.markdown("""<p>The pretrained model has been trained with 21,000 synthetic generated images. The goal is to detect the font type from a given list of 21 font types.
+        The model comprises several layers, starting with a Conv2D layer with a filter size of 32 and activation function of ReLU. 
+        Input shape is specified here. Batch normalization is added to standardize output values. 
+        This layer is followed by another Conv2D layer with the same filter size, activation function, and batch normalization. 
+        MaxPooling2D is used to reduce the dimensionality of the data, with a pool size of (2,2). Dropout is then applied, with a rate of 0.25.
+        This process is repeated with two more sets of Conv2D, BatchNormalization, MaxPooling2D and Dropout layers, increasing the filter size to 64 and 128 for each set respectively.
+        Finally, the model is flattened to transform the multidimensional output into a one-dimensional array, and a Dense layer with 512 nodes is added with an activation function of ReLU. 
+        Batch normalization and dropout are also included in this layer. Finally, a last dense layer with a softmax activation function is added, which outputs the predicted class probabilities.</p>
+        """, unsafe_allow_html=True)
         
         with st.sidebar:
             st.write('## Input')
@@ -80,7 +102,7 @@ def main():
             if st.button('Find Order ID by OCR + font type') and uploaded_file and input_file:
                 st.write('## Output')
                 model = tf.keras.models.load_model('model.h5')
-                result = find_order_id(uploaded_file, input_file, model, ocre)
+                result = find_order_id_2(uploaded_file, input_file, model, ocre)
                 if result['status'] == 'success':
                     st.success(result['message'])
                 elif result['status'] == 'warning':
@@ -96,6 +118,10 @@ def main():
 
     with tabs[2]:
         st.write('## Try FLANN Matching')
+        st.markdown("""<p>Multiple images are generated for a given text, or detected text, with slight variations for each font type. Specifically, five images are created for each font, across all 21 font types. 
+        To detect features, SIFT descriptors are utilized and matched using flann method. 
+        Depending on the selected options, Lowe's ratio test, KNN matching, or ORB descriptor is then employed. 
+        Average matching percentages are calculated for each font type, and the font type with the highest percentage is returned as the most likely one.</p>""", unsafe_allow_html=True)
         text_input = st.text_input("Enter your text:")
         upload_image = st.file_uploader("Choose an image:", type=["jpg", "jpeg", "png"])
         col1, col2, col3 = st.columns(3)
@@ -148,7 +174,62 @@ def main():
                 st.plotly_chart(fig)
             max_match_font = avg_df['Match Percent'].idxmax()
             st.success(f"The most likely font type is: {max_match_font}")
+    
+    with tabs[3]:
+        st.title('Results')
+        df = pd.read_csv('re.csv')
+        st.dataframe(df)
+
+        def calculate_accuracy(df, method):
+            correct = df['correct font']
+            predicted = df[method]
+            accuracy = np.mean(correct == predicted)
+            return round(accuracy, 3)
+
+        col1, col2 = st.columns(2)
+        with col1:
+            data = pd.DataFrame({
+                'Method': ['CNN + OTSU', 'CNN Median blur + Adaptive thresh', 'FLANN + SIFT + LOWE'],
+                'Accuracy': [calculate_accuracy(df, 'CNN + OTSU'),
+                            calculate_accuracy(df, 'CNN Median blur + Adaptive thresh'),
+                            calculate_accuracy(df, 'FLANN + SIFT + LOWE')]
+            })
+
+            bar_chart = alt.Chart(data).mark_bar().encode(
+                x='Method',
+                y='Accuracy',
+                color=alt.condition(
+                    alt.datum.Accuracy >= 0.8,
+                    alt.value('green'),
+                    alt.value('red')
+                )
+            ).properties(title='Accuracy by Method')
+
+            st.altair_chart(bar_chart)
+
+        with col2:
+            font_counts = df.groupby(['correct font']).size().reset_index(name='counts')
+            font_counts_chart = alt.Chart(font_counts).mark_bar().encode(
+                x=alt.X('correct font', sort='-y'),
+                y='counts'
+            ).properties(title='Number of Correct Predictions by Font')
+
+            st.altair_chart(font_counts_chart)       
+
+        # create a stacked bar chart showing the distribution of predicted fonts for each image
+        melted = df.melt(id_vars=['Images', 'correct font'], var_name='Method', value_name='Predicted Font')
+        melted_counts = melted.groupby(['Images', 'correct font', 'Predicted Font']).size().reset_index(name='counts')
+        stacked_bar_chart = alt.Chart(melted_counts).mark_bar().encode(
+            x=alt.X('Images', sort='-y'),
+            y='counts',
+            color='Predicted Font',
+            order=alt.Order(
+                'Predicted Font',
+                sort='ascending'
+            )
+        ).properties(title='Distribution of Predicted Fonts by Image')
 
+        st.altair_chart(stacked_bar_chart)
 
 if __name__ == '__main__':
     main()

button_click_alt.py

@@ -133,4 +133,69 @@ def find_order_id_similarity(uploaded_file, input_file, similarity_method, ocre)
             else:
                 message = f'Detected Text: {text.strip()}\n, Could not find the Order ID.'
                 result = {'status': 'error', 'message': message}
-                return result
+                return result
+
+def find_order_id_2(uploaded_file, input_file, model, ocre):
+    if ocre == 'Hive':
+        uploaded_image = Image.open(uploaded_file)
+        text = infer_text(uploaded_image)
+    else:
+        rotated = preprocess_image(uploaded_file)
+        text = pytesseract.image_to_string(rotated)
+        
+    with input_file as file:
+        file_contents = file.read().decode()
+        lines = file_contents.split('\n')
+        found = False
+        possible_order_ids = []
+        for line in lines:
+            order_id, name, font = line.strip().split(',')
+            if name.strip() in text:
+                image = load_img(uploaded_file, target_size=(64, 64))
+                image = img_to_array(image)
+                image = np.expand_dims(image, axis=0)
+                image = image / 255.0
+                prediction = model.predict(image)
+                
+                class_names = ['Allibretto1.8.otf', 'Bella1.1.otf', 'Buffalo Nickel1.2.otf', 'Cervanttis1.18.otf', 'Claster1.6.otf', 'Fairy4.5.otf', 'Mon-Amour-April1.7.otf', 'Mon-Amour-Aug1.1.otf', 'Mon-Amour-Dec1.2.otf', 'Mon-Amour-Feb1.1.otf', 'Mon-Amour-January1.2.otf', 'Mon-Amour-July1.1.otf', 'Mon-Amour-June1.1.otf', 'Mon-Amour-Mar1.2.otf', 'Mon-Amour-May1.1.otf', 'Mon-Amour-Nov1.1.otf', 'Mon-Amour-Oct1.1.otf', 'Mon-Amour-Sept1.1.otf', 'Mon-Amour2.3.otf', 'Shelby1.3.otf', 'UKIJJ-Quill1.7.otf']
+                predicted_class_index = np.argmax(prediction[0])
+                predicted_class_name = class_names[predicted_class_index]
+                
+                if predicted_class_name.strip() == font.strip():
+                    result = {
+                        'status': 'success',
+                        'message': f'Detected Text: {text.strip()}\n, Order ID: {order_id}, Predicted Font Type: {predicted_class_name.strip()}'
+                    }
+                    found = True
+                    break
+                else:
+                    possible_order_ids.append(order_id)
+        
+        if not found:
+            image = load_img(uploaded_file, target_size=(64, 64))
+            image = img_to_array(image)
+            image = np.expand_dims(image, axis=0)
+            image = image / 255.0
+            prediction = model.predict(image)
+            
+            class_names = ['Allibretto1.8.otf', 'Bella1.1.otf', 'Buffalo Nickel1.2.otf', 'Cervanttis1.18.otf', 'Claster1.6.otf', 'Fairy4.5.otf', 'Mon-Amour-April1.7.otf', 'Mon-Amour-Aug1.1.otf', 'Mon-Amour-Dec1.2.otf', 'Mon-Amour-Feb1.1.otf', 'Mon-Amour-January1.2.otf', 'Mon-Amour-July1.1.otf', 'Mon-Amour-June1.1.otf', 'Mon-Amour-Mar1.2.otf', 'Mon-Amour-May1.1.otf', 'Mon-Amour-Nov1.1.otf', 'Mon-Amour-Oct1.1.otf', 'Mon-Amour-Sept1.1.otf', 'Mon-Amour2.3.otf', 'Shelby1.3.otf', 'UKIJJ-Quill1.7.otf']
+            predicted_class_index = np.argmax(prediction[0])
+            predicted_class_name = class_names[predicted_class_index]
+            
+            for line in lines:
+                order_id, name, font = line.strip().split(',')
+                if font.strip() == predicted_class_name.strip():
+                    possible_order_ids.append(order_id)
+            
+            if len(possible_order_ids) > 0:
+                result = {
+                    'status': 'warning',
+                    'message': f'Detected Text: {text.strip()}\n, Possible Order IDs: {",".join(possible_order_ids)}, Predicted Font Type: {predicted_class_name.strip()}'
+                }
+            else:
+                result = {
+                    'status': 'error',
+                    'message': f'Detected Text: {text.strip()}\n, Could not find the Order ID and possible font matches.'
+                }
+
+    return result

model.h5

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:148ca2bd9a8292808dd8ab5e1cfde8164fc92c3a8658ea1007cd2f0b4caacd14
-size 8253400
+oid sha256:638f6f58906d91e654ba974d39dfefd04d724cc5f544dc594d629d212f88be79
+size 16325952

model_1.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:148ca2bd9a8292808dd8ab5e1cfde8164fc92c3a8658ea1007cd2f0b4caacd14
+size 8253400

process.py

@@ -11,16 +11,11 @@ def preprocess_image(image_file):
     img = cv2.imdecode(np.fromstring(image_file.read(), np.uint8), 1)
     gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
     
-    # Thresholding
     _, thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
-
-    # Image Denoising
     blur = cv2.GaussianBlur(thresh, (3,3), 0)
 
-    # Image Binarization
     thresh = cv2.adaptiveThreshold(blur, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, 11, 4)
 
-    # Skew Correction
     coords = np.column_stack(np.where(thresh > 0))
     angle = cv2.minAreaRect(coords)[-1]
     if angle < -45:

re.csv

@@ -0,0 +1,79 @@
+Images,correct font,CNN + OTSU,CNN Median blur + Adaptive thresh,FLANN + SIFT + LOWE
+image01,Claster1.6.otf,Claster1.6.otf,Claster1.6.otf,Allibretto1.8.otf
+image02,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image03,Buffalo Nickel1.2.otf,Cervanttis1.18.otf,Cervanttis1.18.otf,Buffalo Nickel1.2.otf
+image04,Fairy4.5.otf,Shelby1.3.otf,Mon-Amour2.3.otf,Shelby1.3.otf
+image05,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf,Fairy4.5.otf
+image06,UKIJJ-Quill1.7.otf,Claster1.6.otf,Claster1.6.otf,UKIJJ-Quill1.7.otf
+image07,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image08,Cervanttis1.18.otf,Claster1.6.otf,Claster1.6.otf,Cervanttis1.18.otf
+image09,Buffalo Nickel1.2.otf,Mon-Amour-Dec1.2.otf,Mon-Amour-Dec1.2.otf,Buffalo Nickel1.2.otf
+image10,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf,Fairy4.5.otf
+image11,Mon Amour2.3.otf,Claster1.6.otf,Fairy4.5.otf,Mon-Amour-January1.2.otf
+image12,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image13,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf,Fairy4.5.otf
+image14,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf,Mon-Amour-Sept1.1.otf
+image15,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image16,Claster1.6.otf,Fairy4.5.otf,Claster1.6.otf,Mon-Amour-April1.7.otf
+image17,Fairy4.5.otf,Claster1.6.otf,Mon-Amour2.3.otf,Fairy4.5.otf
+image18,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf
+image19,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf,Mon-Amour-January1.2.otf
+image20,Cervanttis1.18.otf,Claster1.6.otf,Claster1.6.otf,Mon-Amour-Aug1.1.otf
+image21,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf,Fairy4.5.otf
+image22,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image23,Cervanttis1.18.otf,Claster1.6.otf,Fairy4.5.otf,Cervanttis1.18.otf
+image24,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image25,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf,Fairy4.5.otf
+image26,Fairy4.5.otf,Fairy4.5.otf,Mon-Amour2.3.otf,Mon-Amour-May1.1.otf
+image27,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf
+image28,Fairy4.5.otf,Fairy4.5.otf,Mon-Amour2.3.otf,Fairy4.5.otf
+image29,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf,Fairy4.5.otf
+image30,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image31,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image32,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image33,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf,Fairy4.5.otf
+image34,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf,Fairy4.5.otf
+image35,Fairy4.5.otf,Shelby1.3.otf,Claster1.6.otf,Allibretto1.8.otf
+image36,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf,Fairy4.5.otf
+image37,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf,Fairy4.5.otf
+image38,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image39,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image40,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image41,Fairy4.5.otf,Claster1.6.otf,Mon-Amour-Dec1.2.otf,Fairy4.5.otf
+image42,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image43,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image44,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf,Fairy4.5.otf
+image45,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf
+image46,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf
+image47,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image48,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image49,Claster1.6.otf,Fairy4.5.otf,Claster1.6.otf,UKIJJ-Quill1.7.otf
+image50,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf,Fairy4.5.otf
+image51,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf,Fairy4.5.otf
+image52,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Allibretto1.8.otf
+image53,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Shelby1.3.otf
+image54,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf,Fairy4.5.otf
+image55,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf
+image56,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf,Fairy4.5.otf
+image57,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image58,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image59,Cervanttis1.18.otf,Shelby1.3.otf,Fairy4.5.otf,Cervanttis1.18.otf
+image60,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image61,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Bella1.1.otf
+image62,Allibretto1.8.otf,Claster1.6.otf,Claster1.6.otf,Allibretto1.8.otf
+image63,Cervanttis1.18.otf,Fairy4.5.otf,Claster1.6.otf,Mon-Amour-Mar1.2.otf
+image64,Claster1.6.otf,Claster1.6.otf,Claster1.6.otf,Shelby1.3.otf
+image65,Claster1.6.otf,Claster1.6.otf,Mon-Amour-Dec1.2.otf,Claster1.6.otf
+image66,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf,Mon-Amour-Aug1.1.otf
+image67,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Cervanttis1.18.otf
+image68,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Mon-Amour-June1.1.otf
+image69,UKIJJ-Quill1.7.otf,Fairy4.5.otf,Mon-Amour2.3.otf,UKIJJ-Quill1.7.otf
+image70,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Claster1.6.otf
+image71,Cervanttis1.18.otf,Claster1.6.otf,Claster1.6.otf,Cervanttis1.18.otf
+image72,Fairy4.5.otf,Fairy4.5.otf,Claster1.6.otf,Fairy4.5.otf
+image73,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf,Fairy4.5.otf
+image74,Allibretto1.8.otf,Mon-Amour-April1.7.otf,Mon-Amour-Sept1.1.otf,Allibretto1.8.otf
+image75,Allibretto1.8.otf,Mon-Amour-January1.2.otf,Mon-Amour-January1.2.otf,Allibretto1.8.otf
+image76,Allibretto1.8.otf,Mon-Amour-January1.2.otf,Fairy4.5.otf,Allibretto1.8.otf
+image77,Allibretto1.8.otf,Mon-Amour-Sept1.1.otf,Mon-Amour-Sept1.1.otf,Allibretto1.8.otf
+image78,Mon-Amour-April1.7.otf,Claster1.6.otf,Claster1.6.otf,Mon-Amour-April1.7.otf