You can't reuse the class for the main window to create a popup window. You have to register another class and provide to it its own message pump and callback function. Here is an example:
#COMPILE EXE
#DIM ALL
#INCLUDE "win32api.inc"
' ========================================================================================
' Main
' ========================================================================================
FUNCTION WinMain (BYVAL hInstance AS DWORD, BYVAL hPrevInstance AS DWORD, BYVAL lpszCmdLine AS ASCIIZ PTR, BYVAL nCmdShow AS LONG) AS LONG
LOCAL hWndMain AS DWORD
LOCAL hCtl AS DWORD
LOCAL hFont AS DWORD
LOCAL wcex AS WNDCLASSEX
LOCAL szClassName AS ASCIIZ * 80
LOCAL rc AS RECT
LOCAL szCaption AS ASCIIZ * 255
LOCAL nLeft AS LONG
LOCAL nTop AS LONG
LOCAL nWidth AS LONG
LOCAL nHeight AS LONG
hFont = GetStockObject(%ANSI_VAR_FONT)
' Register the window class
szClassName = "MyClassName"
wcex.cbSize = SIZEOF(wcex)
wcex.style = %CS_HREDRAW OR %CS_VREDRAW
wcex.lpfnWndProc = CODEPTR(WndProc)
wcex.cbClsExtra = 0
wcex.cbWndExtra = 0
wcex.hInstance = hInstance
wcex.hCursor = LoadCursor (%NULL, BYVAL %IDC_ARROW)
wcex.hbrBackground = %COLOR_3DFACE + 1
wcex.lpszMenuName = %NULL
wcex.lpszClassName = VARPTR(szClassName)
wcex.hIcon = LoadIcon (%NULL, BYVAL %IDI_APPLICATION) ' Sample, if resource icon: LoadIcon(hInst, "APPICON")
wcex.hIconSm = LoadIcon (%NULL, BYVAL %IDI_APPLICATION) ' Remember to set small icon too..
RegisterClassEx wcex
' Window caption
szCaption = "SDK Main Window"
' Retrieve the size of the working area
SystemParametersInfo %SPI_GETWORKAREA, 0, BYVAL VARPTR(rc), 0
' Calculate the position and size of the window
nWidth = (((rc.nRight - rc.nLeft)) + 2) * 0.75 ' 75% of the client screen width
nHeight = (((rc.nBottom - rc.nTop)) + 2) * 0.70 ' 70% of the client screen height
nLeft = ((rc.nRight - rc.nLeft) \ 2) - nWidth \ 2
nTop = ((rc.nBottom - rc.nTop) \ 2) - (nHeight \ 2)
' Create a window using the registered class
hWndMain = CreateWindowEx(%WS_EX_CONTROLPARENT, _ ' extended style
szClassName, _ ' window class name
szCaption, _ ' window caption
%WS_OVERLAPPEDWINDOW OR _
%WS_CLIPCHILDREN, _ ' window styles
nLeft, _ ' initial x position
nTop, _ ' initial y position
nWidth, _ ' initial x size
nHeight, _ ' initial y size
%NULL, _ ' parent window handle
0, _ ' window menu handle
hInstance, _ ' program instance handle
BYVAL %NULL) ' creation parameters
hCtl = CreateWindowEx(0, "BUTTON", "&Popup", %WS_CHILD OR %WS_VISIBLE OR %WS_TABSTOP OR %BS_FLAT, _
0, 0, 0, 0, hWndMain, %IDOK, hInstance, BYVAL %NULL)
IF hFont THEN SendMessage hCtl, %WM_SETFONT, hFont, 0
hCtl = CreateWindowEx(0, "BUTTON", "&Close", %WS_CHILD OR %WS_VISIBLE OR %WS_TABSTOP OR %BS_FLAT, _
0, 0, 0, 0, hWndMain, %IDCANCEL, hInstance, BYVAL %NULL)
IF hFont THEN SendMessage hCtl, %WM_SETFONT, hFont, 0
' Show the window
ShowWindow hWndMain, nCmdShow
UpdateWindow hWndMain
' Message handler loop
LOCAL uMsg AS tagMsg
WHILE GetMessage(uMsg, %NULL, 0, 0)
IF ISFALSE IsDialogMessage(hWndMain, uMsg) THEN
TranslateMessage uMsg
DispatchMessage uMsg
END IF
WEND
FUNCTION = uMsg.wParam
END FUNCTION
' ========================================================================================
' ========================================================================================
' Main Window procedure
' ========================================================================================
FUNCTION WndProc (BYVAL hWnd AS DWORD, BYVAL wMsg AS DWORD, BYVAL wParam AS DWORD, BYVAL lParam AS LONG) AS LONG
LOCAL rc AS RECT
SELECT CASE wMsg
CASE %WM_CREATE
' -------------------------------------------------------
' A good place to initiate things, declare variables,
' create controls and read/set settings from a file, etc.
' -------------------------------------------------------
CASE %WM_SIZE
' ----------------------------------------------------------------------
' If you have a Toolbar and/or a Statusbar, send the following messages:
' SendMessage hStatusbar, wMsg, wParam, lParam
' SendMessage hToolbar, wMsg, wParam, lParam
' ----------------------------------------------------------------------
' Resize the two sample buttons of the dialog
IF wParam <> %SIZE_MINIMIZED THEN
GetClientRect hWnd, rc
MoveWindow GetDlgItem(hWnd, %IDOK), (rc.nRight - rc.nLeft) - 185, (rc.nBottom - rc.nTop) - 35, 75, 23, %TRUE
MoveWindow GetDlgItem(hWnd, %IDCANCEL), (rc.nRight - rc.nLeft) - 95, (rc.nBottom - rc.nTop) - 35, 75, 23, %TRUE
END IF
CASE %WM_COMMAND
' -------------------------------------------------------
' Messages from controls and menu items are handled here.
' -------------------------------------------------------
SELECT CASE LO(WORD, wParam)
CASE %IDOK
IF HI(WORD, wParam) = %BN_CLICKED THEN
ShowPopupDialog hwnd
END IF
CASE %IDCANCEL
IF HI(WORD, wParam) = %BN_CLICKED THEN
SendMessage hWnd, %WM_CLOSE, 0, 0
EXIT FUNCTION
END IF
END SELECT
CASE %WM_CTLCOLORBTN, %WM_CTLCOLOREDIT, %WM_CTLCOLORLISTBOX, %WM_CTLCOLORSTATIC
' --------------------------------------------------------
' wParam is the handle of the control's display context (hDC)
' lParam is the handle of the control
' Example of how to set the colors of an specific control:
' --------------------------------------------------------
' IF lParam = GetDlgItem(hWnd, CtlId) THEN
' SetBkColor wParam, GetSysColor(%COLOR_INFOBK)
' SetTextColor wParam, GetSysColor(%COLOR_INFOTEXT)
' FUNCTION = GetSysColorBrush(%COLOR_INFOBK)
' EXIT FUNCTION
' END IF
' --------------------------------------------------------
CASE %WM_SYSCOMMAND
' Capture this message and send a WM_CLOSE message
IF (wParam AND &HFFF0) = %SC_CLOSE THEN
SendMessage hWnd, %WM_CLOSE, 0, 0
EXIT FUNCTION
END IF
CASE %WM_CLOSE
' --------------------------------------------------------
' The WM_CLOSE message is processed BEFORE the WM_DESTROY
' message and it can be used to confirm program exit or
' tasks like deallocating memory or similar tasks.
' --------------------------------------------------------
CASE %WM_DESTROY
' ---------------------------------------------------------------------------
' Is sent when program ends - a good place to delete any created objects and
' store settings in file for next run, etc. Must send PostQuitMessage to end
' properly in SDK-style dialogs. The PostQuitMessage function sends a WM_QUIT
' message to the program's (thread's) message queue, and then WM_QUIT causes
' the GetMessage function to return zero in WINMAIN's message loop.
' ---------------------------------------------------------------------------
PostQuitMessage 0 ' This function closes the main window
EXIT FUNCTION
END SELECT
FUNCTION = DefWindowProc(hWnd, wMsg, wParam, lParam)
END FUNCTION
' ========================================================================================
' ========================================================================================
' Popup dialog - Calling example: ShowPopupDialog hWnd
' ========================================================================================
FUNCTION ShowPopupDialog (BYVAL hParent AS LONG) AS LONG
LOCAL hWndPopup AS LONG
LOCAL hCtl AS LONG
LOCAL hFont AS LONG
LOCAL rc AS RECT
LOCAL wcex AS WNDCLASSEX
LOCAL szClassName AS ASCIIZ * 80
LOCAL szCaption AS ASCIIZ * 255
hFont = GetStockObject(%ANSI_VAR_FONT)
szClassName = "MyPopupClassName"
wcex.cbSize = SIZEOF(wcex)
wcex.style = %CS_HREDRAW OR %CS_VREDRAW
wcex.lpfnWndProc = CODEPTR(PopupDlgProc)
wcex.cbClsExtra = 0
wcex.cbWndExtra = 0
wcex.hInstance = GetModuleHandle("")
wcex.hCursor = LoadCursor(%NULL, BYVAL %IDC_ARROW)
wcex.hbrBackground = %COLOR_3DFACE + 1
wcex.lpszMenuName = %NULL
wcex.lpszClassName = VARPTR(szClassName)
wcex.hIcon = 0
wcex.hIconSm = 0
RegisterClassEx wcex
GetWindowRect hParent, rc ' For centering child in parent
rc.nRight = rc.nRight - rc.nLeft ' Parent's width
rc.nBottom = rc.nBottom - rc.nTop ' Parent's height
szCaption = "Popup dialog"
hWndPopup = CreateWindowEx(%WS_EX_DLGMODALFRAME OR %WS_EX_CONTROLPARENT, _
szClassName, szCaption, %WS_CAPTION OR %WS_POPUPWINDOW OR %WS_VISIBLE, _
rc.nLeft + (rc.nRight - 290) / 2, _
rc.nTop + (rc.nBottom - 180) / 2, _
290, 180, hParent, 0, GetModuleHandle(""), BYVAL %NULL)
hCtl = CreateWindowEx(0, "BUTTON", "&Close", %WS_CHILD OR %WS_VISIBLE OR %WS_TABSTOP OR %BS_FLAT, _
200, 118, 75, 23, hWndPopup, %IDCANCEL, GetModuleHandle(""), BYVAL %NULL)
IF hFont THEN SendMessage hCtl, %WM_SETFONT, hFont, 0
ShowWindow hWndPopup, %SW_SHOW
UpdateWindow hWndPopup
' Message handler loop
LOCAL uMsg AS tagMsg
WHILE GetMessage(uMsg, %NULL, 0, 0)
IF ISFALSE IsDialogMessage(hWndPopup, uMsg) THEN
TranslateMessage uMsg
DispatchMessage uMsg
END IF
WEND
FUNCTION = uMsg.wParam
END FUNCTION
' ========================================================================================
' ========================================================================================
' Popup dialog procedure
' ========================================================================================
FUNCTION PopupDlgProc (BYVAL hWnd AS DWORD, BYVAL wMsg AS DWORD, BYVAL wParam AS DWORD, BYVAL lParam AS LONG) AS LONG
SELECT CASE wMsg
CASE %WM_CREATE
EnableWindow GetWindow(hWnd, %GW_OWNER), %FALSE ' To make the popup dialog modal
CASE %WM_COMMAND
SELECT CASE LO(WORD, wParam)
CASE %IDCANCEL
IF HI(WORD, wParam) = %BN_CLICKED THEN
SendMessage hWnd, %WM_CLOSE, 0, 0
EXIT FUNCTION
END IF
END SELECT
CASE %WM_CLOSE
EnableWindow GetWindow(hWnd, %GW_OWNER), %TRUE ' Maintains parent's zorder
CASE %WM_DESTROY
PostQuitMessage 0 ' This function closes the main window
EXIT FUNCTION
END SELECT
FUNCTION = DefWindowProc(hWnd, wMsg, wParam, lParam)
END FUNCTION
' ========================================================================================