You need to handle the WM_SIZING message. Here's some sample code in MFC style :

void CMyDlg::OnSizing(UINT fwSide, LPRECT pRect)
{
   CDialog::OnSizing(fwSide, pRect);

   int iWidth  = (pRect->right)-(pRect->left);
   int iHeight = (pRect->bottom)-(pRect->top);

   if (iWidth > iHeight)
   {
      pRect->bottom = pRect->top + iWidth;
   }
   else
   {
      if (iHeight > iWidth)
      {
         pRect->right = pRect->left + iHeight;
      }
   }
}

Download