title: Extract words in rectangles from text
body: <p>I am struggling to extract fast and efficiently words that are in rectangles from a BufferedImage.<br>
  For example I have the following page : ( edit! ) the image is scanned, so it can contain noise, skewing and distortion.  <br>
  <img src="https://i.stack.imgur.com/gcnc2.jpg" alt="enter image description here"></p>
  
  <p><br> 
  How can I extract the following images without the rectangle :
  ( edit! ) I can use OpenCv or any other library, but I'm absolutely new to advanced image processing techniques. 
  <img src="https://i.stack.imgur.com/s3Ihr.jpg" alt="enter image description here"></p>
  
  <p><strong>EDIT</strong></p>
  
  <p>I've used the method suggested by <code>karlphillip</code> <a href="https://stackoverflow.com/questions/20427759/image-processing-and-extraction-of-characters/20624819#20624819">here</a> and it works decent.<br>
  Here is the code :</p>
  
  <pre><code>    package ro.ubbcluj.detection;
  
  import java.awt.FlowLayout;
  import java.awt.image.BufferedImage;
  import java.io.ByteArrayInputStream;
  import java.io.IOException;
  import java.io.InputStream;
  import java.util.ArrayList;
  import java.util.List;
  
  import javax.imageio.ImageIO;
  import javax.swing.ImageIcon;
  import javax.swing.JFrame;
  import javax.swing.JLabel;
  import javax.swing.WindowConstants;
  
  import org.opencv.core.Core;
  import org.opencv.core.Mat;
  import org.opencv.core.MatOfByte;
  import org.opencv.core.MatOfPoint;
  import org.opencv.core.Point;
  import org.opencv.core.Scalar;
  import org.opencv.core.Size;
  import org.opencv.highgui.Highgui;
  import org.opencv.imgproc.Imgproc;
  
  public class RectangleDetection {
  
  public static void main(String[] args) throws IOException {
      System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
      Mat image = loadImage();
      Mat grayscale = convertToGrayscale(image);
  
      Mat treshold = tresholdImage(grayscale);
      List&lt;MatOfPoint&gt; contours = findContours(treshold);
      Mat contoursImage = fillCountours(contours, grayscale);
      Mat grayscaleWithContours = convertToGrayscale(contoursImage);
      Mat tresholdGrayscaleWithContours = tresholdImage(grayscaleWithContours);
      Mat eroded = erodeAndDilate(tresholdGrayscaleWithContours);
      List&lt;MatOfPoint&gt; squaresFound = findSquares(eroded);
      Mat squaresDrawn = Rectangle.drawSquares(grayscale, squaresFound);
      BufferedImage convertedImage = convertMatToBufferedImage(squaresDrawn);
      displayImage(convertedImage);
  }
  
  private static List&lt;MatOfPoint&gt; findSquares(Mat eroded) {
      return Rectangle.findSquares(eroded);
  }
  
  private static Mat erodeAndDilate(Mat input) {
      int erosion_type = Imgproc.MORPH_RECT;
      int erosion_size = 5;
      Mat result = new Mat();
      Mat element = Imgproc.getStructuringElement(erosion_type, new Size(2 * erosion_size + 1, 2 * erosion_size + 1));
      Imgproc.erode(input, result, element);
      Imgproc.dilate(result, result, element);
      return result;
  }
  
  private static Mat convertToGrayscale(Mat input) {
      Mat grayscale = new Mat();
      Imgproc.cvtColor(input, grayscale, Imgproc.COLOR_BGR2GRAY);
      return grayscale;
  }
  
  private static Mat fillCountours(List&lt;MatOfPoint&gt; contours, Mat image) {
      Mat result = image.clone();
      Imgproc.cvtColor(result, result, Imgproc.COLOR_GRAY2RGB);
      for (int i = 0; i &lt; contours.size(); i++) {
          Imgproc.drawContours(result, contours, i, new Scalar(255, 0, 0), -1, 8, new Mat(), 0, new Point());
      }
      return result;
  }
  
  private static List&lt;MatOfPoint&gt; findContours(Mat image) {
      List&lt;MatOfPoint&gt; contours = new ArrayList&lt;&gt;();
      Mat hierarchy = new Mat();
      Imgproc.findContours(image, contours, hierarchy, Imgproc.RETR_TREE, Imgproc.CHAIN_APPROX_NONE);
      return contours;
  }
  
  private static Mat detectLinesHough(Mat img) {
      Mat lines = new Mat();
      int threshold = 80;
      int minLineLength = 10;
      int maxLineGap = 5;
      double rho = 0.4;
      Imgproc.HoughLinesP(img, lines, rho, Math.PI / 180, threshold, minLineLength, maxLineGap);
      Imgproc.cvtColor(img, img, Imgproc.COLOR_GRAY2RGB);
      System.out.println(lines.cols());
      for (int x = 0; x &lt; lines.cols(); x++) {
          double[] vec = lines.get(0, x);
          double x1 = vec[0], y1 = vec[1], x2 = vec[2], y2 = vec[3];
          Point start = new Point(x1, y1);
          Point end = new Point(x2, y2);
          Core.line(lines, start, end, new Scalar(0, 255, 0), 3);
      }
      return img;
  }
  
  static BufferedImage convertMatToBufferedImage(Mat mat) throws IOException {
      MatOfByte matOfByte = new MatOfByte();
      Highgui.imencode(".jpg", mat, matOfByte);
      byte[] byteArray = matOfByte.toArray();
      InputStream in = new ByteArrayInputStream(byteArray);
      return ImageIO.read(in);
  
  }
  
  static void displayImage(BufferedImage image) {
      JFrame frame = new JFrame();
      frame.getContentPane().setLayout(new FlowLayout());
      frame.getContentPane().add(new JLabel(new ImageIcon(image)));
      frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);
      frame.pack();
      frame.setVisible(true);
  
  }
  
  private static Mat tresholdImage(Mat img) {
      Mat treshold = new Mat();
      Imgproc.threshold(img, treshold, 225, 255, Imgproc.THRESH_BINARY_INV);
      return treshold;
  }
  
  private static Mat tresholdImage2(Mat img) {
      Mat treshold = new Mat();
      Imgproc.threshold(img, treshold, -1, 255, Imgproc.THRESH_BINARY_INV + Imgproc.THRESH_OTSU);
      return treshold;
  }
  
  private static Mat loadImage() {
      return Highgui
              .imread("E:\\Programs\\Eclipse Workspace\\LicentaWorkspace\\OpenCvRectangleDetection\\src\\img\\form3.jpg");
  }
  </code></pre>
  
  <p>}</p>
  
  <p><br> and the Rectangle class </p>
  
  <pre><code>    package ro.ubbcluj.detection;
  
  import java.awt.image.BufferedImage;
  import java.io.IOException;
  import java.util.ArrayList;
  import java.util.List;
  
  import org.opencv.core.Core;
  import org.opencv.core.Mat;
  import org.opencv.core.MatOfPoint;
  import org.opencv.core.MatOfPoint2f;
  import org.opencv.core.Point;
  import org.opencv.core.Scalar;
  import org.opencv.core.Size;
  import org.opencv.imgproc.Imgproc;
  
  public class Rectangle {
  static List&lt;MatOfPoint&gt; findSquares(Mat input) {
      Mat pyr = new Mat();
      Mat timg = new Mat();
  
      // Down-scale and up-scale the image to filter out small noises
      Imgproc.pyrDown(input, pyr, new Size(input.cols() / 2, input.rows() / 2));
      Imgproc.pyrUp(pyr, timg, input.size());
      // Apply Canny with a threshold of 50
      Imgproc.Canny(timg, timg, 0, 50, 5, true);
  
      // Dilate canny output to remove potential holes between edge segments
      Imgproc.dilate(timg, timg, new Mat(), new Point(-1, -1), 1);
  
      // find contours and store them all as a list
      Mat hierarchy = new Mat();
      List&lt;MatOfPoint&gt; contours = new ArrayList&lt;&gt;();
      Imgproc.findContours(timg, contours, hierarchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
      List&lt;MatOfPoint&gt; squaresResult = new ArrayList&lt;MatOfPoint&gt;();
      for (int i = 0; i &lt; contours.size(); i++) {
  
          // Approximate contour with accuracy proportional to the contour
          // perimeter
          MatOfPoint2f contour = new MatOfPoint2f(contours.get(i).toArray());
          MatOfPoint2f approx = new MatOfPoint2f();
          double epsilon = Imgproc.arcLength(contour, true) * 0.02;
          boolean closed = true;
          Imgproc.approxPolyDP(contour, approx, epsilon, closed);
          List&lt;Point&gt; approxCurveList = approx.toList();
  
          // Square contours should have 4 vertices after approximation
          // relatively large area (to filter out noisy contours)
          // and be convex.
          // Note: absolute value of an area is used because
          // area may be positive or negative - in accordance with the
          // contour orientation
          boolean aproxSize = approx.rows() == 4;
          boolean largeArea = Math.abs(Imgproc.contourArea(approx)) &gt; 200;
          boolean isConvex = Imgproc.isContourConvex(new MatOfPoint(approx.toArray()));
          if (aproxSize &amp;&amp; largeArea &amp;&amp; isConvex) {
              double maxCosine = 0;
              for (int j = 2; j &lt; 5; j++) {
                  // Find the maximum cosine of the angle between joint edges
                  double cosine = Math.abs(getAngle(approxCurveList.get(j % 4), approxCurveList.get(j - 2),
                          approxCurveList.get(j - 1)));
                  maxCosine = Math.max(maxCosine, cosine);
              }
              // If cosines of all angles are small
              // (all angles are ~90 degree) then write quandrange
              // vertices to resultant sequence
              if (maxCosine &lt; 0.3) {
                  Point[] points = approx.toArray();
                  squaresResult.add(new MatOfPoint(points));
              }
          }
      }
      return squaresResult;
  }
  
  // angle: helper function.
  // Finds a cosine of angle between vectors from pt0-&gt;pt1 and from pt0-&gt;pt2.
  private static double getAngle(Point point1, Point point2, Point point0) {
      double dx1 = point1.x - point0.x;
      double dy1 = point1.y - point0.y;
      double dx2 = point2.x - point0.x;
      double dy2 = point2.y - point0.y;
      return (dx1 * dx2 + dy1 * dy2) / Math.sqrt((dx1 * dx1 + dy1 * dy1) * (dx2 * dx2 + dy2 * dy2) + 1e-10);
  }
  
  public static Mat drawSquares(Mat image, List&lt;MatOfPoint&gt; squares) {
      Mat result = new Mat();
      Imgproc.cvtColor(image, result, Imgproc.COLOR_GRAY2RGB);
      int thickness = 2;
      Core.polylines(result, squares, false, new Scalar(0, 255, 0), thickness);
      return result;
  }
  }
  </code></pre>
  
  <p>Example of result : </p>
  
  <p><img src="https://i.stack.imgur.com/oL6Xg.jpg" alt="enter image description here">
  <img src="https://i.stack.imgur.com/ANHLX.jpg" alt="enter image description here">
  <br>
  <strong><br>... though, it doesn't work so great for smaller images :</strong>
  <img src="https://i.stack.imgur.com/Ecmb8.jpg" alt="enter image description here"><img src="https://i.stack.imgur.com/MBbiz.jpg" alt="enter image description here"></p>
  
  <p><strong>Maybe some enhancements can be suggested? Or how to make the algorithm faster in case I have a batch of images to process?</strong></p>