Problem With Data Execution Prevention (DEB)

I worte a program using opengl that draws a circle. When I run it on my computer it works fine, but when I try and run it on my friend's computer (vista), the program crashes giving a message relating to DEP.

My friend tried disabling DEP on the program, but vista didn't like that and wouldn't allow it.

I have the following files added to additional dependenices of the input page of the linker:

opengl32.lib glu32.lib


Heres my code:

Code:

---

//
// Based from:
//
// GLSAMPLE.CPP
//  by Blaine Hodge
// http://www.nullterminator.net/opengl32.html

// Includes

#include <windows.h>
#include <gl/gl.h>
#include <math.h>

// Function Declarations

LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam);
void EnableOpenGL(HWND hWnd, HDC * hDC, HGLRC * hRC);
void DisableOpenGL(HWND hWnd, HDC hDC, HGLRC hRC);
void DrawCircle(float, float, float);
void DrawFillCircle(float, float, float);

// WinMain

int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
                                  LPSTR lpCmdLine, int iCmdShow)
{
        WNDCLASS wc;
        HWND hWnd;
        HDC hDC;
        HGLRC hRC;
        MSG msg;
        BOOL quit = FALSE;
       
        // register window class
        wc.style = CS_OWNDC;
        wc.lpfnWndProc = WndProc;
        wc.cbClsExtra = 0;
        wc.cbWndExtra = 0;
        wc.hInstance = hInstance;
        wc.hIcon = LoadIcon( NULL, IDI_APPLICATION );
        wc.hCursor = LoadCursor( NULL, IDC_ARROW );
        wc.hbrBackground = (HBRUSH)GetStockObject( BLACK_BRUSH );
        wc.lpszMenuName = NULL;
        wc.lpszClassName = "GLBase";
        RegisterClass( &wc );
       
        // create main window
        hWnd = CreateWindow(
                "GLBase", "OpenGL Base",
                WS_CAPTION | WS_POPUPWINDOW | WS_VISIBLE,
                0, 0, 512, 512,
                NULL, NULL, hInstance, NULL );
       
        // enable OpenGL for the window
        EnableOpenGL( hWnd, &hDC, &hRC );
       
        // program main loop
        while ( !quit )
        {
               
                // check for messages
                if ( PeekMessage( &msg, NULL, 0, 0, PM_REMOVE )  )
                {
                       
                        // handle or dispatch messages
                        if ( msg.message == WM_QUIT )
                        {
                                quit = TRUE;
                        }
                        else
                        {
                                TranslateMessage( &msg );
                                DispatchMessage( &msg );
                        }
                       
                }
                else
                {
                        glClearColor( 0.0f, 0.0f, 0.0f, 0.0f );
                        glClear( GL_COLOR_BUFFER_BIT );
                       
                        glPushMatrix();

                        //Background - White
                        glColor3f(1.0f, 1.0f, 1.0f);
                        glBegin(GL_QUADS);
                                glVertex2f(-1.0f, 1.0f);
                                glVertex2f(1.0f, 1.0f);
                                glVertex2f(1.0f, -1.0f);
                                glVertex2f(-1.0f, -1.0f);
                        glEnd();
                       
                        //Circle
                        glColor3f(0.0f, 0.0f, 1.0f);
                        DrawFillCircle(0,0,.5);

                        glColor3f(0.0f, 0.0f, 0.0f);
                        DrawCircle(0,0,.5);

                        glPopMatrix();
                       
                        SwapBuffers( hDC );
                }
               
        }
       
        // shutdown OpenGL
        DisableOpenGL( hWnd, hDC, hRC );
       
        // destroy the window explicitly
        DestroyWindow( hWnd );
       
        return msg.wParam;
       
}

// Window Procedure

LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
{
       
        switch (message)
        {
               
        case WM_CREATE:
                return 0;
               
        case WM_CLOSE:
                PostQuitMessage( 0 );
                return 0;
               
        case WM_DESTROY:
                return 0;
               
        case WM_KEYDOWN:
                switch ( wParam )
                {
                       
                case VK_ESCAPE:
                        PostQuitMessage(0);
                        return 0;
                       
                }
                return 0;
       
        default:
                return DefWindowProc( hWnd, message, wParam, lParam );
                       
        }
       
}

// Enable OpenGL

void EnableOpenGL(HWND hWnd, HDC * hDC, HGLRC * hRC)
{
        PIXELFORMATDESCRIPTOR pfd;
        int format;
       
        // get the device context (DC)
        *hDC = GetDC( hWnd );
       
        // set the pixel format for the DC
        ZeroMemory( &pfd, sizeof( pfd ) );
        pfd.nSize = sizeof( pfd );
        pfd.nVersion = 1;
        pfd.dwFlags = PFD_DRAW_TO_WINDOW | PFD_SUPPORT_OPENGL | PFD_DOUBLEBUFFER;
        pfd.iPixelType = PFD_TYPE_RGBA;
        pfd.cColorBits = 32;
        pfd.cDepthBits = 32;
        pfd.iLayerType = PFD_MAIN_PLANE;
        format = ChoosePixelFormat( *hDC, &pfd );
        SetPixelFormat( *hDC, format, &pfd );
       
        // create and enable the render context (RC)
        *hRC = wglCreateContext( *hDC );
        wglMakeCurrent( *hDC, *hRC );
       
}

// Disable OpenGL

void DisableOpenGL(HWND hWnd, HDC hDC, HGLRC hRC)
{
        wglMakeCurrent( NULL, NULL );
        wglDeleteContext( hRC );
        ReleaseDC( hWnd, hDC );
}



////////////////////
// Draw Functions //
////////////////////

//Circle - http://www.swiftless.com/tutorials/opengl/circle.html

void DrawCircle(float x, float y, float r)
{
        glBegin(GL_LINES);

        for (float i = 0.0F; i < 6.28F; i = i + .1)
        {
                float X = r * cos(i) + x;
                float Y = r * sin(i) + y;
                glVertex2f(X, Y);

                X = r * cos(i + 0.1F) + x;
                Y = r * sin(i + 0.1F) + y;
                glVertex2f(X, Y);
        }

        glEnd();
}

//Filled Circle

void DrawFillCircle(float x, float y, float r)
{
        glBegin(GL_LINES);

        for (float i = 0.0F; i < 3.14F; i = i + .005)
        {
                float X = r * cos(i) + x;
                float Y = r * sin(i) + y;
                glVertex2f(X, Y);

                X = r * cos(-i) + x;
                Y = r * sin(-i) + y;
                glVertex2f(X, Y);
        }

        glEnd();
}

---

This may help:

http://blogs.msdn.com/ed_maurer/arch...-compiler.aspx

I have used an OpenGL SDI application for years. It runs fine on VISTA using VS 2005. When I run the same program on VISTA VS 2008, it automatically updates and runs fine. But when I attempt to build the very same code from scratch on VISTA / 2008, it won't run and gives the DEP error. This makes absolutely no sense to me from a security standpoint. After reading the above noted blog, I really don't know what to think. Is M$ losing it? :confused:

Some further thoughts on this topic:

Using the Post-Build command as suggested in the blog:

Code:

---

call $(DevEnvDir)..\tools\vsvars32.bat
editbin.exe /NXCOMPAT:NO $(TargetPath)

---

results in:

1>Microsoft (R) Windows (R) Resource Compiler Version 6.0.5724.0
1>Copyright (C) Microsoft Corporation. All rights reserved.
1>Linking...
1>LINK : fatal error LNK1000: Internal error during IncrBuildImage
1> Version 9.00.21022.08
1> ExceptionCode = C0000005
1> ExceptionFlags = 00000000
1> ExceptionAddress = 002DFCF7 (00260000) "c:\Program Files\Microsoft Visual Studio 9.0\VC\bin\link.exe"
1> NumberParameters = 00000002
1> ExceptionInformation[ 0] = 00000000
1> ExceptionInformation[ 1] = 012BD670
1>CONTEXT:
1> Eax = 401E58D4 Esp = 001EF13C
1> Ebx = 4000814C Ebp = 001EF164
1> Ecx = 012BD670 Esi = 401E5764
1> Edx = 001EF154 Edi = 0026D6C0
1> Eip = 002DFCF7 EFlags = 00010246
1> SegCs = 0000001B SegDs = 00000023
1> SegSs = 00000023 SegEs = 00000023
1> SegFs = 0000003B SegGs = 00000000
1> Dr0 = 00000000 Dr3 = 00000000
1> Dr1 = 00000000 Dr6 = 00000000
1> Dr2 = 00000000 Dr7 = 00000000
1>Build log was saved at "file://c:\Users\mbpliam\Documents\Visual Studio 2008\Projects\CPP\GRAPHICS\mog03032010\mog\Debug\BuildLog.htm"
1>mog - 1 error(s), 0 warning(s)

(whatever that means ???)

Apparently DEP can be completely disabled in VISTA using the following method:
1. Click Start
2. Click All Programs
3. Click Accessories
4. Right Click on Command Prompt
5. Left click on Run as Administrator
6. Click the Allow button if it asks you for permission
7. Type or copy and paste this into the command prompt window:

bcdedit.exe /set {current} nx AlwaysOff

8. Hit enter and you should see the confirmation: Operation Completed Successfully
9. Reboot and DEP should be off.

but I am reluctant to turn off what is touted to be a useful security fix. But when I try to turn off DEP for only the application in question, a message appears indicating that DEP cannot be turned off for this application. :mad: