<file_sep>import numpy as np
from numpy import array
import pandas as pd
import matplotlib.pyplot as plt
import string
import os
from PIL import Image
import glob
from pickle import dump, load
from time import time
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import LSTM, Embedding, TimeDistributed, Dense, RepeatVector,\
                         Activation, Flatten, Reshape, concatenate, Dropout, BatchNormalization
from keras.optimizers import Adam, RMSprop
from keras.layers.wrappers import Bidirectional
from keras.layers.merge import add
from keras.applications.inception_v3 import InceptionV3
from keras.applications.resnet50 import ResNet50
from keras.preprocessing import image
from keras.models import Model
from keras import Input, layers
from keras import optimizers
from keras.applications.inception_v3 import preprocess_input
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
import tensorflow as tf
from tensorflow.keras.preprocessing.image import load_img,img_to_array
from flask import Flask, request, render_template
from werkzeug.utils import secure_filename
from gevent.pywsgi import WSGIServer

# from gtts import gTTS 
# language = 'en'
# from playsound import playsound

model = ResNet50(weights='imagenet')
model_new = Model(model.input, model.layers[-2].output)

train_features = load(open("Caption/encoded_train_images.pkl", "rb"))
train_descriptions = load(open("Caption/encoded_train_desc.pkl","rb"))

all_train_captions = []
for key, val in train_descriptions.items():
    for cap in val:
        all_train_captions.append(cap)
#consider only words which occur at least 10 times in the corpus        
word_count_threshold = 10
word_counts = {}
nsents = 0
for sent in all_train_captions:
    nsents += 1
    for w in sent.split(' '):
        word_counts[w] = word_counts.get(w, 0) + 1

vocab = [w for w in word_counts if word_counts[w] >= word_count_threshold]  

ixtoword = {}
wordtoix = {}

ix = 1
for w in vocab:
    wordtoix[w] = ix
    ixtoword[ix] = w
    ix += 1      

vocab_size = len(ixtoword) + 1 # one for appended 0's
vocab_size

# convert a dictionary of clean descriptions to a list of descriptions
def to_lines(descriptions):
	all_desc = list()
	for key in descriptions.keys():
		[all_desc.append(d) for d in descriptions[key]]
	return all_desc

# calculate the length of the description with the most words
def max_length(descriptions):
	lines = to_lines(descriptions)
	return max(len(d.split()) for d in lines)

# determine the maximum sequence length
max_length = max_length(train_descriptions)
#print('Description Length: %d' % max_length)

from keras.applications.imagenet_utils import preprocess_input, decode_predictions
from keras.models import load_model
from keras.preprocessing import image
# Model saved with Keras model.save()
MODEL_PATH = 'Caption/my_model_30'

# Load your trained model
model = load_model(MODEL_PATH)



def greedySearch(photo):
    in_text = 'startseq'
    for i in range(max_length):
        sequence = [wordtoix[w] for w in in_text.split() if w in wordtoix]
        sequence = pad_sequences([sequence], maxlen=max_length)
        yhat = model.predict([photo,sequence], verbose=0)
        yhat = np.argmax(yhat)
        word = ixtoword[yhat]
        in_text += ' ' + word
        if word == 'endseq':
            break
    final = in_text.split()
    final = final[1:-1]
    final = ' '.join(final)
    return final

def predict(img):
    #img_y = load_img(img_path,target_size=(224,224))
   # plt.imshow(img_y)
    #plt.show()
    
    img_y1=img_to_array(img)
    img_y1=np.expand_dims(img_y1, axis=0)
    img_y1=preprocess_input(img_y1)
    fea_vec_test = model_new.predict(img_y1) # Get the encoding vector for the image
    fea_vec_test = np.reshape(fea_vec_test, fea_vec_test.shape[1])
    test_reshape = fea_vec_test.reshape(1,2048)
    text = greedySearch(test_reshape)
    #speech = gTTS(text = text, lang = language, slow = False)
    #speech.save('text.mp3')
    
    return text


# with open("Caption/encoded_test_images.pkl", "rb") as encoded_pickle:
#     encoding_test = load(encoded_pickle)

#print("prediction is ",predict('sample_1.jpeg'))
#print("prediction is ",predict('horse.jpg'))
#print("prediction is ",predict('worker.jpg'))
#print("prediction is ",predict('download.png'))
#print("prediction is ",predict('soccer.jpg'))
#print("prediction is ",predict('cat.jpg'))
#print("prediction is ",predict('Ducati.jpg'))
#image = Image.open('photo-1604741637599-b7e53240e1c4.jpg')
#image =image.resize((224,224))
#print("prediction is ",predict(image))





#images = 'Flicker8k_Dataset/'

#z=101
#pic = list(encoding_test.keys())[z]
#image = encoding_test[pic].reshape((1,2048))
#x=plt.imread(images+pic)
#plt.imshow(x)
#plt.show()
#print("Greedy:",greedySearch(image))


<file_sep># -*- coding: utf-8 -*-
"""
Created on Fri Nov 13 11:21:07 2020

@author: Vandhana
"""

import streamlit as st
import os
from PIL import Image
from app import predict
import tensorflow as tf
from PIL import Image
import requests
import base64
from io import BytesIO

main_bg = "background.jpg"
main_bg_ext = "jpg"


st.markdown(
    f"""
    <style>
    .reportview-container {{
        background: url(data:image/{main_bg_ext};base64,{base64.b64encode(open(main_bg, "rb").read()).decode()})
    }}
    </style>
    """,
    unsafe_allow_html=True
    )

st.sidebar.info("This is an Image Captioning web deployment Model.The application identifies the objects in \
                the picture and generates Caption. It was built using a Convolution Neural Network (CNN) for object identification and RNN to generate captions using sequence to sequence model (LSTM)")

st.set_option('deprecation.showfileUploaderEncoding', False)

#st.title("Image Captioning")
st.markdown("<h1 style='text-align: center; color: green;'>Image Captioning</h1>", unsafe_allow_html=True)
st.write("")

#def file_selector(folder_path='.'):
#    filenames = os.listdir(folder_path)
#    selected_filename = st.selectbox('Select a file', filenames)
#    return os.path.join(folder_path, selected_filename)
#
#
#if __name__ == '__main__':
#    # Select a file
#    if st.checkbox('Select a file in current directory'):
#        folder_path = '.'
#        if st.checkbox('Change directory'):
#            folder_path = st.text_input('Enter folder path', '.')
#        filename = file_selector(folder_path=folder_path)
#        st.write('You selected `%s`' % filename)
#        image = Image.open(filename)
#        st.image(image, caption='Selected Image.', use_column_width=True)
#        st.write("")
#        st.write("Just a second...")
#        label = predict('bike.jpg')
#        st.write("Caption for above image is : ",label)
        
#st.write()

def load_img(path_to_img):
  img = tf.io.read_file(path_to_img)
  #img = tf.image.decode_image(img, channels=3)
  #img = tf.image.resize(img, (224,224))
  return img
st.write('<style>div.Widget.row-widget.stRadio > div{flex-direction:row;}</style>', unsafe_allow_html=True)
status = st.radio("Hello, Do you want to Upload an Image or Insert an Image URL?",("Upload Image","Insert URL"))
if status == 'Upload Image':
    st.success("Please Upload an Image")
    file_up = st.file_uploader("Upload an image", type="jpg")
    if file_up is not None:
            image = Image.open(file_up)
            st.image(image, caption='Uploaded Image.', use_column_width=True)
            st.write("")
            image = image.resize((224,224))
            st.write("Just a second...")
            label = predict(image)
            st.write("Caption for above image is : ",label)
#             audio_file = open("text.mp3","rb").read()
#             st.audio(audio_file,format='audio/mp3')
  
else:
    st.success("Please Insert Web URL")
    url = st.text_input("Insert URL below")
    if url:
        response = requests.get(url)
        image = Image.open(BytesIO(response.content))
        st.image(image, caption='Uploaded Image.', use_column_width=True)
        st.write("")
        image = image.resize((224,224))
        st.write("Just a second...")
        label = predict(image)
        st.write("Caption for above image is : ",label)
#         audio_file = open("text.mp3","rb").read()
#         st.audio(audio_file,format='audio/mp3')
        
<file_sep># Image_Captioning
Image_Captioning
<file_sep>requests==2.22.0
tensorflow-cpu==2.3.0
Werkzeug==0.16.0
gTTS==2.2.1
streamlit==0.70.0
playsound==1.2.2
Keras==2.4.3
gevent==1.4.0
matplotlib==3.1.1
pandas==0.25.1
numpy==1.16.5
Pillow==8.1.2
Flask==1.1.1
