title: Mouse picking with ray casting in OpenGL?
body: <p>I am trying to follow <a href="http://antongerdelan.net/opengl/raycasting.html" rel="nofollow noreferrer">this tutorial</a> regarding mouse picking in OpenGL but I am failing to get it to work properly. </p>
  
  <p>In my basic understanding to get world space coordinates from mouse coordinates I would need to multiply a bunch of inverse matrices. Then to get world space coordinates from local space I would multiply the local space coordinates with the model matrix. But the coordinates that I got don't line up with each other.</p>
  
  <p>I tried to normalize the world coordinates for the model because the mouse coordinates is also normalized but I'm not sure if that is the case.</p>
  
  <pre><code>if (instance &amp;&amp; button == GLFW_MOUSE_BUTTON_LEFT
      &amp;&amp; action == GLFW_PRESS) {
      glfwGetCursorPos(window, &amp;x, &amp;y);
  
      pMat = instance-&gt;graphics-&gt;getPMat();
  
      cameraPos = instance-&gt;graphics-&gt;getCameraPos();
  
      //View matrix
      vMat = glm::translate(glm::mat4(1.0f), glm::vec3(cameraPos.x, cameraPos.y, -cameraPos.z));
      //vMat = glm::rotate(vMat, 0.6f, glm::vec3(1.0f, 0.0f, 0.0f));
  
      //normalized mouse coordinates
      x = (2.0f * x) / width - 1.0f;
      y = 1.0 - (2.0f * y) / height;
  
      glm::vec3 ray_nds = glm::vec3(x, y, 1.0f);
  
      glm::vec4 ray_clip = glm::vec4(ray_nds.x, ray_nds.y, -1.0, 1.0);
  
      glm::vec4 ray_eye = glm::inverse(pMat) * ray_clip;
      ray_eye = glm::vec4(ray_eye.x, ray_eye.y, -1.0, 0.0f);
  
      glm::vec3 ray_wor = glm::vec4(glm::inverse(vMat) * ray_eye);
      ray_wor = glm::normalize(ray_wor);
  
      Entity* e = instance-&gt;getCurrentScene().getEntities()[0];
  
      sMat = glm::scale(glm::mat4(1.0f), glm::vec3(e-&gt;getScaleX(), e-&gt;getScaleY(), e-&gt;getScaleZ()));
      tMat = glm::translate(glm::mat4(1.0f), glm::vec3(e-&gt;getX(), e-&gt;getY(), e-&gt;getZ()));
  
      rMat = glm::rotate(glm::mat4(1.0f), e-&gt;getRotY(), glm::vec3(0.0f, 1.0f, 0.0f));
      rMat = glm::rotate(rMat, e-&gt;getRotX(), glm::vec3(1.0f, 0.0f, 0.0f));
      rMat = glm::rotate(rMat, e-&gt;getRotZ(), glm::vec3(0.0f, 0.0f, 1.0f));
  
      mMat = tMat * rMat * sMat;
      mvMat = vMat * mMat;
  
      startCoords = mMat * glm::vec4(0.0f, 0.0f, 0.0f, 1.0f);
      startCoords = glm::normalize(startCoords);
      endCoords = mMat * glm::vec4(1.0f, -1.0f, 1.0f, 1.0f);
      endCoords = glm::normalize(endCoords);
  
      if (ray_wor.x &gt;= startCoords.x
          &amp;&amp; ray_wor.y &lt;= startCoords.y
          &amp;&amp; ray_wor.x &lt;= endCoords.x
          &amp;&amp; ray_wor.y &gt;= endCoords.y) {
          std::cout &lt;&lt; "\nFound!\n";
      }
  
      std::cout &lt;&lt; "x: " &lt;&lt; ray_wor.x &lt;&lt; "\n"
          &lt;&lt; "y: " &lt;&lt; ray_wor.y &lt;&lt; "\n"
          &lt;&lt; "z: " &lt;&lt; ray_wor.z &lt;&lt; "\n\n";
  
      std::cout &lt;&lt; "x: " &lt;&lt; endCoords.x &lt;&lt; "\n"
          &lt;&lt; "y: " &lt;&lt; endCoords.y &lt;&lt; "\n"
          &lt;&lt; "z: " &lt;&lt; endCoords.z &lt;&lt; "\n\n";
  
      std::cout &lt;&lt; "x: " &lt;&lt; startCoords.x &lt;&lt; "\n"
          &lt;&lt; "y: " &lt;&lt; startCoords.y &lt;&lt; "\n"
          &lt;&lt; "z: " &lt;&lt; startCoords.z &lt;&lt; "\n\n";
  }
  </code></pre>