Problem With Data Execution Prevention (DEB)
CodeGuru Home VC++ / MFC / C++ .NET / C# Visual Basic VB Forums Developer.com
Results 1 to 4 of 4

Thread: Problem With Data Execution Prevention (DEB)

  1. #1
    Join Date
    Jan 2009
    Posts
    1

    Problem With Data Execution Prevention (DEB)

    I worte a program using opengl that draws a circle. When I run it on my computer it works fine, but when I try and run it on my friend's computer (vista), the program crashes giving a message relating to DEP.

    My friend tried disabling DEP on the program, but vista didn't like that and wouldn't allow it.

    I have the following files added to additional dependenices of the input page of the linker:

    opengl32.lib glu32.lib


    Heres my code:
    Code:
    //
    // Based from:
    //
    // GLSAMPLE.CPP
    //  by Blaine Hodge
    // http://www.nullterminator.net/opengl32.html
    
    // Includes
    
    #include <windows.h>
    #include <gl/gl.h>
    #include <math.h>
    
    // Function Declarations
    
    LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam);
    void EnableOpenGL(HWND hWnd, HDC * hDC, HGLRC * hRC);
    void DisableOpenGL(HWND hWnd, HDC hDC, HGLRC hRC);
    void DrawCircle(float, float, float);
    void DrawFillCircle(float, float, float);
    
    // WinMain
    
    int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, 
    				   LPSTR lpCmdLine, int iCmdShow)
    {
    	WNDCLASS wc;
    	HWND hWnd;
    	HDC hDC;
    	HGLRC hRC;
    	MSG msg;
    	BOOL quit = FALSE;
    	
    	// register window class
    	wc.style = CS_OWNDC;
    	wc.lpfnWndProc = WndProc;
    	wc.cbClsExtra = 0;
    	wc.cbWndExtra = 0;
    	wc.hInstance = hInstance;
    	wc.hIcon = LoadIcon( NULL, IDI_APPLICATION );
    	wc.hCursor = LoadCursor( NULL, IDC_ARROW );
    	wc.hbrBackground = (HBRUSH)GetStockObject( BLACK_BRUSH );
    	wc.lpszMenuName = NULL;
    	wc.lpszClassName = "GLBase";
    	RegisterClass( &wc );
    	
    	// create main window
    	hWnd = CreateWindow( 
    		"GLBase", "OpenGL Base", 
    		WS_CAPTION | WS_POPUPWINDOW | WS_VISIBLE,
    		0, 0, 512, 512,
    		NULL, NULL, hInstance, NULL );
    	
    	// enable OpenGL for the window
    	EnableOpenGL( hWnd, &hDC, &hRC );
    	
    	// program main loop
    	while ( !quit )
    	{
    		
    		// check for messages
    		if ( PeekMessage( &msg, NULL, 0, 0, PM_REMOVE )  )
    		{
    			
    			// handle or dispatch messages
    			if ( msg.message == WM_QUIT ) 
    			{
    				quit = TRUE;
    			} 
    			else 
    			{
    				TranslateMessage( &msg );
    				DispatchMessage( &msg );
    			}
    			
    		} 
    		else 
    		{
    			glClearColor( 0.0f, 0.0f, 0.0f, 0.0f );
    			glClear( GL_COLOR_BUFFER_BIT );
    			
    			glPushMatrix();
    
    			//Background - White
    			glColor3f(1.0f, 1.0f, 1.0f);
    			glBegin(GL_QUADS);
    				glVertex2f(-1.0f, 1.0f);
    				glVertex2f(1.0f, 1.0f);
    				glVertex2f(1.0f, -1.0f);
    				glVertex2f(-1.0f, -1.0f);
    			glEnd();
    			
    			//Circle
    			glColor3f(0.0f, 0.0f, 1.0f);
    			DrawFillCircle(0,0,.5);
    
    			glColor3f(0.0f, 0.0f, 0.0f);
    			DrawCircle(0,0,.5);
    
    			glPopMatrix();
    			
    			SwapBuffers( hDC );
    		}
    		
    	}
    	
    	// shutdown OpenGL
    	DisableOpenGL( hWnd, hDC, hRC );
    	
    	// destroy the window explicitly
    	DestroyWindow( hWnd );
    	
    	return msg.wParam;
    	
    }
    
    // Window Procedure
    
    LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
    {
    	
    	switch (message)
    	{
    		
    	case WM_CREATE:
    		return 0;
    		
    	case WM_CLOSE:
    		PostQuitMessage( 0 );
    		return 0;
    		
    	case WM_DESTROY:
    		return 0;
    		
    	case WM_KEYDOWN:
    		switch ( wParam )
    		{
    			
    		case VK_ESCAPE:
    			PostQuitMessage(0);
    			return 0;
    			
    		}
    		return 0;
    	
    	default:
    		return DefWindowProc( hWnd, message, wParam, lParam );
    			
    	}
    	
    }
    
    // Enable OpenGL
    
    void EnableOpenGL(HWND hWnd, HDC * hDC, HGLRC * hRC)
    {
    	PIXELFORMATDESCRIPTOR pfd;
    	int format;
    	
    	// get the device context (DC)
    	*hDC = GetDC( hWnd );
    	
    	// set the pixel format for the DC
    	ZeroMemory( &pfd, sizeof( pfd ) );
    	pfd.nSize = sizeof( pfd );
    	pfd.nVersion = 1;
    	pfd.dwFlags = PFD_DRAW_TO_WINDOW | PFD_SUPPORT_OPENGL | PFD_DOUBLEBUFFER;
    	pfd.iPixelType = PFD_TYPE_RGBA;
    	pfd.cColorBits = 32;
    	pfd.cDepthBits = 32;
    	pfd.iLayerType = PFD_MAIN_PLANE;
    	format = ChoosePixelFormat( *hDC, &pfd );
    	SetPixelFormat( *hDC, format, &pfd );
    	
    	// create and enable the render context (RC)
    	*hRC = wglCreateContext( *hDC );
    	wglMakeCurrent( *hDC, *hRC );
    	
    }
    
    // Disable OpenGL
    
    void DisableOpenGL(HWND hWnd, HDC hDC, HGLRC hRC)
    {
    	wglMakeCurrent( NULL, NULL );
    	wglDeleteContext( hRC );
    	ReleaseDC( hWnd, hDC );
    }
    
    
    
    ////////////////////
    // Draw Functions //
    ////////////////////
    
    //Circle - http://www.swiftless.com/tutorials/opengl/circle.html
    
    void DrawCircle(float x, float y, float r)
    {
    	glBegin(GL_LINES);
    
    	for (float i = 0.0F; i < 6.28F; i = i + .1)
    	{
    		float X = r * cos(i) + x;
    		float Y = r * sin(i) + y;
    		glVertex2f(X, Y);
    
    		X = r * cos(i + 0.1F) + x;
    		Y = r * sin(i + 0.1F) + y;
    		glVertex2f(X, Y);
    	}
    
    	glEnd();
    }
    
    //Filled Circle
    
    void DrawFillCircle(float x, float y, float r)
    {
    	glBegin(GL_LINES);
    
    	for (float i = 0.0F; i < 3.14F; i = i + .005)
    	{
    		float X = r * cos(i) + x;
    		float Y = r * sin(i) + y;
    		glVertex2f(X, Y);
    
    		X = r * cos(-i) + x;
    		Y = r * sin(-i) + y;
    		glVertex2f(X, Y);
    	}
    
    	glEnd();
    }
    Last edited by ring2ding; January 11th, 2009 at 07:11 PM.

  2. #2
    Join Date
    Jun 2002
    Location
    Stockholm, Sweden
    Posts
    1,621

    Re: Problem With Data Execution Prevention (DEB)

    Nobody cares how it works as long as it works

  3. #3
    Join Date
    May 2002
    Posts
    1,727

    Re: Problem With Data Execution Prevention (DEB)

    I have used an OpenGL SDI application for years. It runs fine on VISTA using VS 2005. When I run the same program on VISTA VS 2008, it automatically updates and runs fine. But when I attempt to build the very same code from scratch on VISTA / 2008, it won't run and gives the DEP error. This makes absolutely no sense to me from a security standpoint. After reading the above noted blog, I really don't know what to think. Is M$ losing it?
    mpliam

  4. #4
    Join Date
    May 2002
    Posts
    1,727

    Re: Problem With Data Execution Prevention (DEB)

    Some further thoughts on this topic:

    Using the Post-Build command as suggested in the blog:

    Code:
    call $(DevEnvDir)..\tools\vsvars32.bat
    editbin.exe /NXCOMPAT:NO $(TargetPath)
    results in:

    1>Microsoft (R) Windows (R) Resource Compiler Version 6.0.5724.0
    1>Copyright (C) Microsoft Corporation. All rights reserved.
    1>Linking...
    1>LINK : fatal error LNK1000: Internal error during IncrBuildImage
    1> Version 9.00.21022.08
    1> ExceptionCode = C0000005
    1> ExceptionFlags = 00000000
    1> ExceptionAddress = 002DFCF7 (00260000) "c:\Program Files\Microsoft Visual Studio 9.0\VC\bin\link.exe"
    1> NumberParameters = 00000002
    1> ExceptionInformation[ 0] = 00000000
    1> ExceptionInformation[ 1] = 012BD670
    1>CONTEXT:
    1> Eax = 401E58D4 Esp = 001EF13C
    1> Ebx = 4000814C Ebp = 001EF164
    1> Ecx = 012BD670 Esi = 401E5764
    1> Edx = 001EF154 Edi = 0026D6C0
    1> Eip = 002DFCF7 EFlags = 00010246
    1> SegCs = 0000001B SegDs = 00000023
    1> SegSs = 00000023 SegEs = 00000023
    1> SegFs = 0000003B SegGs = 00000000
    1> Dr0 = 00000000 Dr3 = 00000000
    1> Dr1 = 00000000 Dr6 = 00000000
    1> Dr2 = 00000000 Dr7 = 00000000
    1>Build log was saved at "file://c:\Users\mbpliam\Documents\Visual Studio 2008\Projects\CPP\GRAPHICS\mog03032010\mog\Debug\BuildLog.htm"
    1>mog - 1 error(s), 0 warning(s)

    (whatever that means ???)

    Apparently DEP can be completely disabled in VISTA using the following method:
    1. Click Start
    2. Click All Programs
    3. Click Accessories
    4. Right Click on Command Prompt
    5. Left click on Run as Administrator
    6. Click the Allow button if it asks you for permission
    7. Type or copy and paste this into the command prompt window:

    bcdedit.exe /set {current} nx AlwaysOff

    8. Hit enter and you should see the confirmation: Operation Completed Successfully
    9. Reboot and DEP should be off.

    but I am reluctant to turn off what is touted to be a useful security fix. But when I try to turn off DEP for only the application in question, a message appears indicating that DEP cannot be turned off for this application.
    mpliam

Posting Permissions

  • You may not post new threads
  • You may not post replies
  • You may not post attachments
  • You may not edit your posts
  •  


Windows Mobile Development Center


Click Here to Expand Forum to Full Width

This is a CodeGuru survey question.


Featured


HTML5 Development Center