readme: 
  # Visual semantic with BERT-CNN
  
  This model can be used to assign an object-to-caption semantic relatedness score, which is valuable for
  (1) caption diverse re-ranking, and (2) generate soft labels for caption filtering when scraping text-to-captions from the internet.
  
  To take advantage of the overlapping between the visual context and the caption, and to extract global information from each visual (i.e., object, scene, etc) we use BERT  as an embedding layer followed by a shallow CNN (tri-gram kernel) (Kim, 2014).
   
  Please refer to [Github](https://github.com/ahmedssabir/Visual-Semantic-Relatedness-Dataset-for-Image-Captioning) for more information. 
  
  [![arXiv](https://img.shields.io/badge/arXiv-2301.08784-b31b1b.svg)](https://arxiv.org/abs/2301.08784)  [![Website shields.io](https://img.shields.io/website-up-down-green-red/http/shields.io.svg)](https://ahmed.jp/project_page/Dataset_2022/index.html)
  
  
   For datasets that are less than 100K please have look at our [shallow model](https://github.com/ahmedssabir/Semantic-Relatedness-Based-Reranker-for-Text-Spotting) 
  
  
  The model is trained with a strict filter of 0.4 similarity distance thresholds between the object and its related caption.   
  
   For a quick start please have a look at this [demo](https://github.com/ahmedssabir/Textual-Visual-Semantic-Dataset/blob/main/BERT_CNN_Visual_re_ranker_demo.ipynb)
  
   
   For the [dataset](https://huggingface.co/datasets/AhmedSSabir/Textual-Image-Caption-Dataset)
   
  
  ```
  conda create -n BERT_visual python=3.6 anaconda
  conda activate BERT_visual
  pip install tensorflow==1.15.0
  pip install --upgrade tensorflow_hub==0.7.0
  ```
  
  ```
  git clone https://github.com/gaphex/bert_experimental/
  ```
  
  ```python
  import tensorflow as tf
  import numpy as np
  import pandas as pd
  import sys
  from sklearn.model_selection import train_test_split
  
  sys.path.insert(0, "bert_experimental")
  
  from bert_experimental.finetuning.text_preprocessing import build_preprocessor
  from bert_experimental.finetuning.graph_ops import load_graph
  
  df = pd.read_csv("test.tsv", sep='\t')
  
  texts = []
  delimiter = " ||| "
  
  for vis, cap  in zip(df.visual.tolist(), df.caption.tolist()):
    texts.append(delimiter.join((str(vis), str(cap))))
  
  texts = np.array(texts)
  
  trX, tsX = train_test_split(texts, shuffle=False, test_size=0.01)
  
  restored_graph = load_graph("frozen_graph.pb")
  
  graph_ops = restored_graph.get_operations()
  input_op, output_op = graph_ops[0].name, graph_ops[-1].name
  print(input_op, output_op)
  
  x = restored_graph.get_tensor_by_name(input_op + ':0')
  y = restored_graph.get_tensor_by_name(output_op + ':0')
  
  preprocessor = build_preprocessor("vocab.txt", 64)
  py_func = tf.numpy_function(preprocessor, [x], [tf.int32, tf.int32, tf.int32], name='preprocessor')
  
  ##predictions
  sess = tf.Session(graph=restored_graph)
  
  print(trX[:4])
  
  y = tf.print(y, summarize=-1)
  y_out = sess.run(y, feed_dict={
          x: trX[:4].reshape((-1,1))
   	
      })
  
  
  print(y_out)
  ````
  
  For training and inference 
  
  ```
  python BERT_CNN.py --train train_0.4.tsv --epochs 5
  ```
  
  
  ```python 
  # -*- coding: utf-8 -*-
  #!/bin/env python
  import sys
  import argparse
  import re
  import os
  import sys
  import json
  
  import logging
  import numpy as np
  import pandas as pd
  import tensorflow as tf
  import tensorflow_hub as hub
  from BertLayer import BertLayer
  from BertLayer import build_preprocessor
  from freeze_keras_model import freeze_keras_model
  
  from data_pre import * 
  from tensorflow import keras
  from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
  from sklearn.model_selection import train_test_split
  
  
  if not 'bert_repo' in sys.path:
      sys.path.insert(0, 'bert_repo')
  
  from modeling import BertModel, BertConfig
  from tokenization import FullTokenizer, convert_to_unicode
  from extract_features import InputExample, convert_examples_to_features
  
  
  # get TF logger 
  log = logging.getLogger('tensorflow')
  log.handlers = []
  
  
  parser=argparse.ArgumentParser()
  parser.add_argument('--train',  default='train.tsv', help='beam serach', type=str,required=False)  
  parser.add_argument('--num_bert_layer', default='12', help='truned layers', type=int,required=False)  
  parser.add_argument('--batch_size', default='128', help='truned layers', type=int,required=False) 
  parser.add_argument('--epochs', default='5', help='', type=int,required=False) 
  parser.add_argument('--seq_len', default='64', help='', type=int,required=False) 
  parser.add_argument('--CNN_kernel_size', default='3', help='', type=int,required=False) 
  parser.add_argument('--CNN_filters', default='32', help='', type=int,required=False)
  args = parser.parse_args()
  
  
  # Downlaod the pre-trained model
  
  #!wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip
  #!unzip uncased_L-12_H-768_A-12.zip
  
  
  # tf.Module
  def build_module_fn(config_path, vocab_path, do_lower_case=True):
  
      def bert_module_fn(is_training):
          """Spec function for a token embedding module."""
  
          input_ids = tf.placeholder(shape=[None, None], dtype=tf.int32, 