-
January 11th, 2009, 03:15 PM
#1
Problem With Data Execution Prevention (DEB)
I worte a program using opengl that draws a circle. When I run it on my computer it works fine, but when I try and run it on my friend's computer (vista), the program crashes giving a message relating to DEP.
My friend tried disabling DEP on the program, but vista didn't like that and wouldn't allow it.
I have the following files added to additional dependenices of the input page of the linker:
opengl32.lib glu32.lib
Heres my code:
Code:
//
// Based from:
//
// GLSAMPLE.CPP
// by Blaine Hodge
// http://www.nullterminator.net/opengl32.html
// Includes
#include <windows.h>
#include <gl/gl.h>
#include <math.h>
// Function Declarations
LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam);
void EnableOpenGL(HWND hWnd, HDC * hDC, HGLRC * hRC);
void DisableOpenGL(HWND hWnd, HDC hDC, HGLRC hRC);
void DrawCircle(float, float, float);
void DrawFillCircle(float, float, float);
// WinMain
int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
LPSTR lpCmdLine, int iCmdShow)
{
WNDCLASS wc;
HWND hWnd;
HDC hDC;
HGLRC hRC;
MSG msg;
BOOL quit = FALSE;
// register window class
wc.style = CS_OWNDC;
wc.lpfnWndProc = WndProc;
wc.cbClsExtra = 0;
wc.cbWndExtra = 0;
wc.hInstance = hInstance;
wc.hIcon = LoadIcon( NULL, IDI_APPLICATION );
wc.hCursor = LoadCursor( NULL, IDC_ARROW );
wc.hbrBackground = (HBRUSH)GetStockObject( BLACK_BRUSH );
wc.lpszMenuName = NULL;
wc.lpszClassName = "GLBase";
RegisterClass( &wc );
// create main window
hWnd = CreateWindow(
"GLBase", "OpenGL Base",
WS_CAPTION | WS_POPUPWINDOW | WS_VISIBLE,
0, 0, 512, 512,
NULL, NULL, hInstance, NULL );
// enable OpenGL for the window
EnableOpenGL( hWnd, &hDC, &hRC );
// program main loop
while ( !quit )
{
// check for messages
if ( PeekMessage( &msg, NULL, 0, 0, PM_REMOVE ) )
{
// handle or dispatch messages
if ( msg.message == WM_QUIT )
{
quit = TRUE;
}
else
{
TranslateMessage( &msg );
DispatchMessage( &msg );
}
}
else
{
glClearColor( 0.0f, 0.0f, 0.0f, 0.0f );
glClear( GL_COLOR_BUFFER_BIT );
glPushMatrix();
//Background - White
glColor3f(1.0f, 1.0f, 1.0f);
glBegin(GL_QUADS);
glVertex2f(-1.0f, 1.0f);
glVertex2f(1.0f, 1.0f);
glVertex2f(1.0f, -1.0f);
glVertex2f(-1.0f, -1.0f);
glEnd();
//Circle
glColor3f(0.0f, 0.0f, 1.0f);
DrawFillCircle(0,0,.5);
glColor3f(0.0f, 0.0f, 0.0f);
DrawCircle(0,0,.5);
glPopMatrix();
SwapBuffers( hDC );
}
}
// shutdown OpenGL
DisableOpenGL( hWnd, hDC, hRC );
// destroy the window explicitly
DestroyWindow( hWnd );
return msg.wParam;
}
// Window Procedure
LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
{
switch (message)
{
case WM_CREATE:
return 0;
case WM_CLOSE:
PostQuitMessage( 0 );
return 0;
case WM_DESTROY:
return 0;
case WM_KEYDOWN:
switch ( wParam )
{
case VK_ESCAPE:
PostQuitMessage(0);
return 0;
}
return 0;
default:
return DefWindowProc( hWnd, message, wParam, lParam );
}
}
// Enable OpenGL
void EnableOpenGL(HWND hWnd, HDC * hDC, HGLRC * hRC)
{
PIXELFORMATDESCRIPTOR pfd;
int format;
// get the device context (DC)
*hDC = GetDC( hWnd );
// set the pixel format for the DC
ZeroMemory( &pfd, sizeof( pfd ) );
pfd.nSize = sizeof( pfd );
pfd.nVersion = 1;
pfd.dwFlags = PFD_DRAW_TO_WINDOW | PFD_SUPPORT_OPENGL | PFD_DOUBLEBUFFER;
pfd.iPixelType = PFD_TYPE_RGBA;
pfd.cColorBits = 32;
pfd.cDepthBits = 32;
pfd.iLayerType = PFD_MAIN_PLANE;
format = ChoosePixelFormat( *hDC, &pfd );
SetPixelFormat( *hDC, format, &pfd );
// create and enable the render context (RC)
*hRC = wglCreateContext( *hDC );
wglMakeCurrent( *hDC, *hRC );
}
// Disable OpenGL
void DisableOpenGL(HWND hWnd, HDC hDC, HGLRC hRC)
{
wglMakeCurrent( NULL, NULL );
wglDeleteContext( hRC );
ReleaseDC( hWnd, hDC );
}
////////////////////
// Draw Functions //
////////////////////
//Circle - http://www.swiftless.com/tutorials/opengl/circle.html
void DrawCircle(float x, float y, float r)
{
glBegin(GL_LINES);
for (float i = 0.0F; i < 6.28F; i = i + .1)
{
float X = r * cos(i) + x;
float Y = r * sin(i) + y;
glVertex2f(X, Y);
X = r * cos(i + 0.1F) + x;
Y = r * sin(i + 0.1F) + y;
glVertex2f(X, Y);
}
glEnd();
}
//Filled Circle
void DrawFillCircle(float x, float y, float r)
{
glBegin(GL_LINES);
for (float i = 0.0F; i < 3.14F; i = i + .005)
{
float X = r * cos(i) + x;
float Y = r * sin(i) + y;
glVertex2f(X, Y);
X = r * cos(-i) + x;
Y = r * sin(-i) + y;
glVertex2f(X, Y);
}
glEnd();
}
Last edited by ring2ding; January 11th, 2009 at 07:11 PM.
-
January 12th, 2009, 06:37 AM
#2
Re: Problem With Data Execution Prevention (DEB)
Nobody cares how it works as long as it works
-
March 18th, 2010, 11:27 AM
#3
Re: Problem With Data Execution Prevention (DEB)
I have used an OpenGL SDI application for years. It runs fine on VISTA using VS 2005. When I run the same program on VISTA VS 2008, it automatically updates and runs fine. But when I attempt to build the very same code from scratch on VISTA / 2008, it won't run and gives the DEP error. This makes absolutely no sense to me from a security standpoint. After reading the above noted blog, I really don't know what to think. Is M$ losing it?
mpliam
-
March 18th, 2010, 11:50 AM
#4
Re: Problem With Data Execution Prevention (DEB)
Some further thoughts on this topic:
Using the Post-Build command as suggested in the blog:
Code:
call $(DevEnvDir)..\tools\vsvars32.bat
editbin.exe /NXCOMPAT:NO $(TargetPath)
results in:
1>Microsoft (R) Windows (R) Resource Compiler Version 6.0.5724.0
1>Copyright (C) Microsoft Corporation. All rights reserved.
1>Linking...
1>LINK : fatal error LNK1000: Internal error during IncrBuildImage
1> Version 9.00.21022.08
1> ExceptionCode = C0000005
1> ExceptionFlags = 00000000
1> ExceptionAddress = 002DFCF7 (00260000) "c:\Program Files\Microsoft Visual Studio 9.0\VC\bin\link.exe"
1> NumberParameters = 00000002
1> ExceptionInformation[ 0] = 00000000
1> ExceptionInformation[ 1] = 012BD670
1>CONTEXT:
1> Eax = 401E58D4 Esp = 001EF13C
1> Ebx = 4000814C Ebp = 001EF164
1> Ecx = 012BD670 Esi = 401E5764
1> Edx = 001EF154 Edi = 0026D6C0
1> Eip = 002DFCF7 EFlags = 00010246
1> SegCs = 0000001B SegDs = 00000023
1> SegSs = 00000023 SegEs = 00000023
1> SegFs = 0000003B SegGs = 00000000
1> Dr0 = 00000000 Dr3 = 00000000
1> Dr1 = 00000000 Dr6 = 00000000
1> Dr2 = 00000000 Dr7 = 00000000
1>Build log was saved at "file://c:\Users\mbpliam\Documents\Visual Studio 2008\Projects\CPP\GRAPHICS\mog03032010\mog\Debug\BuildLog.htm"
1>mog - 1 error(s), 0 warning(s)
(whatever that means ???)
Apparently DEP can be completely disabled in VISTA using the following method:
1. Click Start
2. Click All Programs
3. Click Accessories
4. Right Click on Command Prompt
5. Left click on Run as Administrator
6. Click the Allow button if it asks you for permission
7. Type or copy and paste this into the command prompt window:
bcdedit.exe /set {current} nx AlwaysOff
8. Hit enter and you should see the confirmation: Operation Completed Successfully
9. Reboot and DEP should be off.
but I am reluctant to turn off what is touted to be a useful security fix. But when I try to turn off DEP for only the application in question, a message appears indicating that DEP cannot be turned off for this application.
mpliam
Posting Permissions
- You may not post new threads
- You may not post replies
- You may not post attachments
- You may not edit your posts
-
Forum Rules
|
Click Here to Expand Forum to Full Width
|