About 3D Picking in Android (with OpenGL ES 2)
안드로이드 OpenGL ES 2에서 화면 터치 이벤트로 오브젝트의 선택여부 확인하는 예
I need some help on 3D picking.
I am using the way it works here. In short, what I have is:
normalizedPoint[0] = (x * 2 / screenW) -1;
normalizedPoint[1] = 1 - (y * 2 / screenH);
normalizedPoint[2] = ?
normalizedPoint[3] = ?
for 2 and 3, I have no idea what it should be (I put 1, -1 just like the reference, and it doesn't work)
Then, for my root object (following just psuedo code):
matrix = perspective_matrix x model_matrix
inv_matrix = inverse(matrix)
outpoint = inv_matrix x normalizedPoint
That's what I have, but it doesn't work, the outPoint I receive is not even close to the point I am suppose clicking. I've been searching in web for more than a week. but no idea how to solve it. HELP!
Answers
Oh. I actually solved the problem, sort of.
I first, modify from the source code of glUnproject to have the following:
public static Vec3 unProject(
float winx, float winy, float winz,
Matrix44 resultantMatrix,
int width, int height){
float[] m = new float[16],
in = new float[4],
out = new float[4];
m = Matrix44.invert(resultantMatrix.get());
in[0] = (winx / (float)width) * 2 - 1;
in[1] = (winy / (float)height) * 2 - 1;
in[2] = 2 * winz - 1;
in[3] = 1;
Matrix.multiplyMV(out, 0, m, 0, in, 0);
if (out[3]==0)
return null;
out[3] = 1/out[3];
return new Vec3(out[0] * out[3], out[1] * out[3], out[2] * out[3]);
}
Input to the above would be the point in the Projected View Frustum Coordinates (i.e., screen input). For example:
unProject(30, 50, 0, mvpMatrix, 800, 480)
will translate the screen input (click) at (30,50) to the world coordinate where the object is sitting at. The third parameter, winz is actually on which projected plane the click is occured, here, 0 means the nearZ of the projection plane.
The way I make picking functions, is by unprojecting two points, using the above function, on the far and near clipping plane, so:
Vec3 near = unProject(30, 50, 0, mvpMatrix, 800, 480);
Vec3 far = unProject(30, 50, 1, mvpMatrix, 800, 480); // 1 for winz means projected on the far plane
Vec3 pickingRay = Subtract(far, near); // Vector subtraction
Once we have the picking ray, what I am doing is simply testing the distance between the picking ray and the "center" of those "pickable" objects. (of course, you can have some more sophisticated testing algorithm).
This works great xandy! Two comments on this solution. For Android, you'll want to invert winy (winy = screenheight-winy) for compatibility between android screen origin (top left of screen) and Opengl's origin (bottom left). Also, if you have the position of the camera, you can save one call to unProject, and build your pickingRay using the camera's eye position: Vec3 pickingRay = Subtract(near, eye)
위의 방법을 실제로 적용한 예
아래의 예제를 예제를 안드로이드 기기에서 실행하고 화면을 터치하면 카메라로부터 시야 절두체(View Frustum) 상의 원거리 평면에 이르는 벡터를 구할 수 있다.
아래의 코드에서 사용된 unProject() 는 화면상에서 터치된 좌표를 OpenGL ES 2의 월드좌표 상의 좌표로 변환해준다.
ObjectPickingActivity class
package gl.test1;
import android.app.Activity;
import android.app.ActivityManager;
import android.content.Context;
import android.content.pm.ConfigurationInfo;
import android.opengl.GLSurfaceView;
import android.opengl.Matrix;
import android.os.Bundle;
import android.util.Log;
import android.view.MotionEvent;
import android.view.Window;
import android.view.WindowManager;
public class ObjectPickingActivity extends Activity {
private GLSurfaceView mGLSurfaceView;
private SquareRenderer renderer;
@Override
public void onCreate(Bundle savedInstanceState)
{
super.onCreate(savedInstanceState);
this.requestWindowFeature(Window.FEATURE_NO_TITLE);
getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN,
WindowManager.LayoutParams.FLAG_FULLSCREEN);
mGLSurfaceView = new GLSurfaceView(this);
// Check if the system supports OpenGL ES 2.0.
final ActivityManager activityManager = (ActivityManager) getSystemService(Context.ACTIVITY_SERVICE);
final ConfigurationInfo configurationInfo = activityManager.getDeviceConfigurationInfo();
final boolean supportsEs2 = configurationInfo.reqGlEsVersion >= 0x20000;
if (supportsEs2)
{
// Request an OpenGL ES 2.0 compatible context.
mGLSurfaceView.setEGLContextClientVersion(2);
renderer = new SquareRenderer();
mGLSurfaceView.setRenderer(renderer);
}
else
{
return;
}
setContentView(mGLSurfaceView);
}
@Override
protected void onResume()
{
// The activity must call the GL surface view's onResume() on activity onResume().
super.onResume();
mGLSurfaceView.onResume();
}
@Override
protected void onPause()
{
// The activity must call the GL surface view's onPause() on activity onPause().
super.onPause();
mGLSurfaceView.onPause();
}
@Override
public boolean onTouchEvent(MotionEvent event) {
int action = event.getAction();
if(action!=MotionEvent.ACTION_DOWN) return false;
float rx = event.getX();
float ry = event.getY();
float rz = 0f;
// 테스트 편의상 화면의 중앙을 클릭한 것으로 간주한다
rx = renderer.viewWidth/2;
ry = renderer.viewHeight/2;
float [] near_xyz = unProject(rx, ry, 0, renderer.mMVPMatrix, renderer.viewWidth, renderer.viewHeight);
float [] far_xyz = unProject(rx, ry, 1, renderer.mMVPMatrix, renderer.viewWidth, renderer.viewHeight);
//Log.w("가까운 평면상의 좌표", "x:"+near_xyz[0]+", y:"+near_xyz[1]+", z:"+near_xyz[2]);
//Log.w("먼 평면상의 좌표", "x:"+far_xyz[0]+", y:"+far_xyz[1]+", z:"+far_xyz[2]);
// 벡터 뺄셈으로
float[] vec3 = { far_xyz[0]-near_xyz[0], far_xyz[1]-near_xyz[1], far_xyz[2]-near_xyz[2] };
Log.w("Picking Ray", "x:"+vec3[0]+", y:"+vec3[1]+", z:"+vec3[2]);
// 위와 같이 산출된 벡터가 월드좌표상의 오브젝트와 교차하는지 판단하면
// 오브젝트의 선택여부를 확인할 수 있다.
return false;
}
private float[] unProject( float winx, float winy, float winz,
float[] mvpMatrix,
int width, int height) {
float[] m = new float[16];
float[] in = new float[4];
float[] out = new float[4];
Matrix.invertM(m, 0, mvpMatrix, 0);
in[0] = (winx / (float)width) * 2 - 1;
in[1] = (winy / (float)height) * 2 - 1;
in[2] = 2 * winz - 1;
in[3] = 1;
Matrix.multiplyMV(out, 0, m, 0, in, 0);
if (out[3]==0) return null;
out[3] = 1/out[3];
return new float[] {out[0] * out[3], out[1] * out[3], out[2] * out[3]};
}
}
SquareRenderer
package gl.test1;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.opengl.GLSurfaceView.Renderer;
public class SquareRenderer implements Renderer {
public float mAngle;
int viewWidth, viewHeight;
private int mProgram;
private int maPositionHandle;
private int muMVPMatrixHandle;
private FloatBuffer vbuf;
float[] mMVPMatrix = new float[16];
private float[] mMMatrix = new float[16];
private float[] mVMatrix = new float[16];
private float[] mMVMatrix = new float[16];
private float[] mProjMatrix = new float[16];
private final String vertexShaderCode =
"uniform mat4 uMVPMatrix; \n" +
"attribute vec4 aPosition; \n" +
"void main(){ \n" +
" gl_Position = uMVPMatrix * aPosition; \n" +
"} \n";
private final String fragmentShaderCode =
"precision mediump float; \n" +
"void main(){ \n" +
" gl_FragColor = vec4 (0, 0.5, 0, 1.0); \n" +
"} \n";
@Override
public void onDrawFrame(GL10 unused) {
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT );
//glUseProgram
// Add program to OpenGL environment
GLES20.glUseProgram(mProgram);
// Prepare the square data
GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false, 12, vbuf);
GLES20.glEnableVertexAttribArray(maPositionHandle);
Matrix.setIdentityM(mMMatrix, 0);
Matrix.translateM(mMMatrix, 0, 0, 0, -5.0f);
//Matrix.setRotateM(mMMatrix, 0, mAngle++, 0, 0, 1.0f);
Matrix.multiplyMM(mMVMatrix, 0, mVMatrix, 0, mMMatrix, 0);
Matrix.multiplyMM(mMVPMatrix, 0, mProjMatrix, 0, mMVMatrix, 0);
// Apply a ModelView Projection transformation
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
// Draw the square(2개의 삼각형이므로 정점은 6개)
GLES20.glDrawArrays(GLES20.GL_TRIANGLES, 0, 6);
}
@Override
public void onSurfaceChanged(GL10 unused, int width, int height) {
viewWidth = width;
viewHeight = height;
GLES20.glViewport(0,0,width,height);
float ratio = (float) width/height;
//this projection matrix is applied to object coodinates
//in the onDrawFrame() method
Matrix.frustumM(mProjMatrix, 0, -ratio, ratio, -1, 1, 3, 10);
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
Matrix.setLookAtM(mVMatrix, 0, 0, 0, 3, 0f, 0f, -1.0f, 0f, 1.0f, 0.0f);
}
@Override
public void onSurfaceCreated(GL10 unused, EGLConfig config) {
GLES20.glClearColor(0.0f,0.0f,0.0f,1.0f);
initShapes();
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexShaderCode);
int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderCode);
mProgram = GLES20.glCreateProgram(); // create empty OpenGL Program
GLES20.glAttachShader(mProgram, vertexShader); // add the vertex shader to program
GLES20.glAttachShader(mProgram, fragmentShader); // add the fragment shader to program
GLES20.glLinkProgram(mProgram); // creates OpenGL program executables
// get handle to the vertex shader's vPosition member
maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
}
private int loadShader(int type, String shaderCode){
// create a vertex shader type (GLES20.GL_VERTEX_SHADER)
// or a fragment shader type (GLES20.GL_FRAGMENT_SHADER)
int shader = GLES20.glCreateShader(type);
// add the source code to the shader and compile it
GLES20.glShaderSource(shader, shaderCode);
GLES20.glCompileShader(shader);
return shader;
}
private void initShapes(){
float squareCoords[] = {
// X, Y, Z
-0.5f, 0.5f, 0f,
-0.5f, -0.5f, 0f,
0.5f, -0.5f, 0f,
-0.5f, 0.5f, 0f,
0.5f, -0.5f, 0f,
0.5f, 0.5f, 0f
};
// initialize vertex Buffer for triangle
ByteBuffer vbb = ByteBuffer.allocateDirect(squareCoords.length * 4);
vbb.order(ByteOrder.nativeOrder()); // use the device hardware's native byte order
vbuf = vbb.asFloatBuffer(); // create a floating point buffer from the ByteBuffer
vbuf.put(squareCoords); // add the coordinates to the FloatBuffer
vbuf.position(0); // set the buffer to read the first coordinate
}
}
object_picking.zip