c - SetWindowLongPtr returning ERROR_ACCESS_DENIED -

i still struggling hooks.

my goal is:

• set hook in notepad.exe
• subclass (my final goal subclass edit class , show content in own window)

disclaimer: know there easier ways text/content notepad, way me learn c, winapi, subclassing , hooks.

my problem setwindowlongptr return error_access_denied error (code 5).

22th of may 2013: has been fixed! problem setwindowlongptr in wrong place. must inside getmsgproc function.

the question became bit long , messy, re-wrote question (with updated code)

the problem getmsgproc not called when target notepad.exe. if change target simple.exe, getmsgproc gets called , works!

(simple.exe simple gui):

the code looks this:

exe.cpp

#include <windows.h> #include "resource.h" #include <stdlib.h> #include "stdafx.h" #include <strsafe.h>  #include "c:\users\kristensen\documents\visual studio 2012\projects\win32d\dll\dllheader.h"  //--------------------------------------------------------------------------- hwnd hwnd;  lresult callback dlgproc(hwnd hwnd, uint msg, wparam wparam, lparam lparam); //--------------------------------------------------------------------------- int winapi winmain(hinstance hinstance, hinstance hprevinstance,                    lpstr lpcmdline, int ncmdshow) {     dialogbox(hinstance, makeintresource(idd_dlgfirst),         hwnd, reinterpret_cast<dlgproc>(dlgproc));      return false; } //--------------------------------------------------------------------------- lresult callback dlgproc(hwnd hwnddlg, uint msg, wparam wparam, lparam lparam) {     switch(msg)     {     case wm_initdialog:         return true;      case wm_command:         switch(wparam)         {         case idok:             hooknotepad();             return true;         case idcancel:             removehook();             enddialog(hwnddlg, 0);         }         break;     }      return false; } //--------------------------------------------------------------------------- 

dllheader.h

    #ifdef dllapi     #else     #define dllapi extern "c" __declspec(dllimport)     #endif     dllapi bool hooknotepad();     dllapi bool removehook(); 

dll.cpp:

#include "stdafx.h" #include <windows.h> #define dllapi extern "c" __declspec(dllexport) #include "dllheader.h"  // shared variables #pragma data_seg("shared") hhook g_hhook = null; // hook notepad  hwnd nphwnd = null; // notepad handle #pragma data_seg() #pragma comment(linker, "/section:shared,rws")  // forward references lresult callback getmsgproc(int ncode, wparam wparam, lparam lparam) ; lresult callback newwndproc(hwnd hwnd, uint message, wparam wparam, lparam lparam); //lresult callback cbtproc(int ncode, wparam wparam, lparam lparam) ;   long oldwndproc;  dword pid; hinstance g_hinstdll = null; // dllmain entry (dll_process_attach) dword npthreadid = null; // notepad thread id  lresult callback getmsgproc(int ncode, wparam wparam, lparam lparam) //testing cbtproc - same issues getmsgproc. {     //if hook notepad.exe, never called. (silence)      //if hook simple.exe, called (beep beep!)      // make noise     static dword dwtickkeep = 0;     if ((gettickcount()-dwtickkeep)>300)     {   dwtickkeep = gettickcount();     beep(2000, 100);     }      //subclassing......     //for simple.exe: (working)     //hwnd hwndedit = ::findwindowex(nphwnd,null,text("windowsforms10.richedit20w.app.0.2bf8098_r14_ad1"), null);     //for notepad.exe: (not working)     hwnd hwndedit = ::findwindowex(nphwnd,null,text("edit"), null);       if (hwndedit)     {         //subclass         oldwndproc = getwindowlongptr(hwndedit, gwlp_wndproc);         setwindowlongptr(hwndedit, gwl_wndproc, (long_ptr)newwndproc);     }     return(callnexthookex(g_hhook, ncode, wparam, lparam)); }  bool apientry dllmain( hmodule hmodule,  dword  ul_reason_for_call,   lpvoid lpreserved  ) {     switch (ul_reason_for_call)     {     case dll_process_attach:         g_hinstdll = hmodule;         break;      case dll_thread_attach:     case dll_thread_detach:     case dll_process_detach:         break;     }     return true; }  bool hooknotepad () {     // if target running     // if (nphwnd = findwindow(null, text("simplegui")))     if (nphwnd = findwindow(text("notepad"), null))     {         // finds threadid target. use in setwindowshookex            npthreadid = getwindowthreadprocessid(nphwnd, &pid);           // sets hook in target         g_hhook = setwindowshookex(wh_getmessage, getmsgproc, g_hinstdll, npthreadid);          //g_hhook = setwindowshookex(wh_cbt, cbtproc, g_hinstdll, npthreadid);           // if hook succesed         if (g_hhook)          {              ////add menu in notepad.exe, not relevant subclassing notepads edit class...             //hmenu hcurrent = getmenu(nphwnd); //get current menu of window.             //hmenu hnew = createmenu(); //create new one.              //appendmenu(hcurrent, mf_string | mf_popup, (unsigned int)hnew, text("mymenu"));              //appendmenu(hnew, mf_string, 2000, l"mybutton"); //2000 id of new button.              //drawmenubar(nphwnd); //redraw menu.              //force msg messagequeue, hook function(getmsgproc) gets called             postthreadmessage(npthreadid, wm_null, 0, 0);             return 1;         }         return 0;     }     else         //notepad not running         return 0; }  bool removehook() {     // removes hook     if (g_hhook != null)     {         unhookwindowshookex(g_hhook);         g_hhook = null;     }     return 0; }    lresult callback newwndproc(hwnd hwnd, uint message, wparam wparam, lparam lparam) {      //we should come here , should able read text edit class...     return callwindowproc((wndproc)oldwndproc, hwnd, message, wparam, lparam);  } 

any hints, comments or tips highly appreciate...

there 3 problems in code:

(1) variable hwnd nphwnd meant shared between host-exe , notepad-exe must placed inside share data segment block. value presently evaluated inside 'hooknotepad' call , exist in host-exe only. problem resulted in nphwnd handle being null in notepad-exe , setwindowlongptr call failed.

(2) there 2 setwindowlongptr calls, 1 of them wrong. 1 inside getmsgproc correct because executed inside notepad-exe context when hook installed. remove other wrong 1 inside hooknotepad.

(3) if (1) , (2) resolved, final behaviour of setwindowlongptr may not expected because main ui interactive element of notepad-exe embedded edit control , not main frame window. should enumerate child windows of notepad-frame , sub-class 1 child window edit class.

edit #1 - add sound indicator code check activity ------------------------------------

add block of code inside getmsgproc

// make noise static dword dwtickkeep = 0; if ((gettickcount()-dwtickkeep)>300) {   dwtickkeep = gettickcount();     beep(2000, 100); }