Add public ActiveX backend mode (#176)

## Summary
- add `MSRDPEX_AX_BACKEND=public` to disable private ActiveX layout
assumptions for replacement controls
- automatically select public mode for `MSRDPEX_MSTSCAX_DLL` paths named
`ironrdpax.dll`
- retain Microsoft private-layout behavior by default and document the
unavailable private integrations in public mode

## Validation
- `dotnet test .\dotnet\MsRdpEx_Test\MsRdpEx_Test.csproj -c Release
--no-restore` (701 passed)

Native compilation was not run because Visual Studio/MSVC is unavailable
in the environment.

---------

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Author: mamoreau-devolutions

README.md

@@ -135,6 +135,12 @@ $Env:MSRDPEX_LOG_FILE_PATH="C:\Windows\Temp\MsRdpEx.log"
 
 The trace log level is extremely verbose, so it should only be used when necessary. The MsRdpEx logging is very helpful in understanding the Microsoft RDP client internals.
 
+## Public ActiveX backends
+
+Set `MSRDPEX_AX_BACKEND=public` for a non-Microsoft `mstscax`-compatible replacement selected by `MSRDPEX_MSTSCAX_DLL`. MsRdpEx also automatically uses public mode when that replacement DLL is named `ironrdpax.dll`, unless `MSRDPEX_AX_BACKEND` is explicitly set. MsRdpEx continues to load the replacement DLL and forward its public COM class factory, but does not inspect the replacement client's undocumented Microsoft object layout.
+
+Public mode intentionally disables private extended-settings/property hooks and extended `.RDP` option processing, input/output window association (including output mirroring and recording), SSPI session correlation, and other features that depend on Microsoft private ActiveX internals. The default `private` mode retains standard `mstscax.dll` behavior. Any explicit backend value other than `public` retains private mode; automatic `ironrdpax.dll` detection applies only when the backend option is unset.
+
 ## Building from source
 
 Generate the Visual Studio project files for your target platform:

dll/ApiHooks.cpp

@@ -674,7 +674,7 @@ LRESULT CALLBACK Hook_OPWndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lPar
         result = Real_OPWndProc_mstscax(hWnd, uMsg, wParam, lParam);
     }
 
-	if (uMsg == WM_NCCREATE)
+	if ((uMsg == WM_NCCREATE) && MsRdpEx_UsePrivateAxLayout())
 	{
 		void* pUserData = (void*) GetWindowLongPtrW(hWnd, GWLP_USERDATA);
 

dll/AxDll.cpp

@@ -55,7 +55,19 @@ bool CDECL MsRdpEx_mstscax_Load(MsRdpEx_mstscax* dll, const char* filename)
     dll->DllCancelAuthentication = (fnDllCancelAuthentication)GetProcAddress(dll->hModule, "DllCancelAuthentication");
     dll->DllDeleteSavedCreds = (fnDllDeleteSavedCreds)GetProcAddress(dll->hModule, "DllDeleteSavedCreds");
 
-    dll->tscCtlVer = dll->DllGetTscCtlVer();
+    if (dll->DllGetTscCtlVer)
+    {
+        dll->tscCtlVer = dll->DllGetTscCtlVer();
+    }
+    else if (MsRdpEx_UsePrivateAxLayout())
+    {
+        MsRdpEx_LogPrint(ERROR, "mstscax_load(%s): missing DllGetTscCtlVer", filename);
+        goto exit;
+    }
+    else
+    {
+        MsRdpEx_LogPrint(DEBUG, "mstscax_load(%s): DllGetTscCtlVer is unavailable in public mode", filename);
+    }
 
     success = true;
 exit:

dll/MsRdpClient.cpp

@@ -497,13 +497,16 @@ class CMsRdpClient : public IMsRdpClient10
         HRESULT hr;
         MsRdpEx_LogPrint(DEBUG, "CMsRdpClient::Connect");
 
-        m_pMsRdpExtendedSettings->LoadRdpFile(NULL);
-        m_pMsRdpExtendedSettings->LoadRdpFileFromNamedPipe(NULL);
-        m_pMsRdpExtendedSettings->PrepareSspiSessionIdHack();
-        m_pMsRdpExtendedSettings->DiscardCapturedPinIfNotCertLogon();
-        m_pMsRdpExtendedSettings->PrepareMouseJiggler();
-        m_pMsRdpExtendedSettings->PrepareVideoRecorder();
-        m_pMsRdpExtendedSettings->PrepareExtraSystemMenu();
+        if (MsRdpEx_UsePrivateAxLayout())
+        {
+            m_pMsRdpExtendedSettings->LoadRdpFile(NULL);
+            m_pMsRdpExtendedSettings->LoadRdpFileFromNamedPipe(NULL);
+            m_pMsRdpExtendedSettings->PrepareSspiSessionIdHack();
+            m_pMsRdpExtendedSettings->DiscardCapturedPinIfNotCertLogon();
+            m_pMsRdpExtendedSettings->PrepareMouseJiggler();
+            m_pMsRdpExtendedSettings->PrepareVideoRecorder();
+            m_pMsRdpExtendedSettings->PrepareExtraSystemMenu();
+        }
 
         BeginSspiSessionScope("connect");
         hr = m_pMsTscAx->raw_Connect();
@@ -525,6 +528,9 @@ class CMsRdpClient : public IMsRdpClient10
     }
 
     void BeginSspiSessionScope(const char* reason) {
+        if (!MsRdpEx_UsePrivateAxLayout())
+            return;
+
         if (m_sspiSessionActive)
             return;
 

dll/MsRdpEx.cpp

@@ -108,10 +108,12 @@ uint64_t DllGetTscCtlVer()
     uint64_t version = 0;
 
     if (g_IsOOBClient) {
-        version = g_rdclientax.DllGetTscCtlVer();
+        if (g_rdclientax.DllGetTscCtlVer)
+            version = g_rdclientax.DllGetTscCtlVer();
     }
     else {
-        version = g_mstscax.DllGetTscCtlVer();
+        if (g_mstscax.DllGetTscCtlVer)
+            version = g_mstscax.DllGetTscCtlVer();
     }
 
     MsRdpEx_LogPrint(DEBUG, "DllGetTscCtlVer: 0x%04X", (unsigned int)version);
@@ -725,6 +727,33 @@ HRESULT MsRdpEx_PreCleanUp()
     return DllPreCleanUp();
 }
 
+bool CDECL MsRdpEx_UsePrivateAxLayout()
+{
+    bool usePrivateAxLayout = true;
+    char* axBackend = MsRdpEx_GetEnv("MSRDPEX_AX_BACKEND");
+    char* axDll = NULL;
+
+    if (axBackend)
+    {
+        if (MsRdpEx_StringIEquals(axBackend, "public"))
+            usePrivateAxLayout = false;
+    }
+    else
+    {
+        axDll = MsRdpEx_GetEnv("MSRDPEX_MSTSCAX_DLL");
+
+        if (axDll &&
+            MsRdpEx_StringIEquals(MsRdpEx_FileBase(axDll), "ironrdpax.dll"))
+        {
+            usePrivateAxLayout = false;
+        }
+    }
+
+    free(axDll);
+    free(axBackend);
+    return usePrivateAxLayout;
+}
+
 bool MsRdpEx_DetectClientProcess(bool* pIsOOBClient)
 {
     char moduleFilePath[MSRDPEX_MAX_PATH] = { 0 };

dll/MsRdpEx.h

@@ -189,6 +189,8 @@ typedef struct _MsRdpEx_rdclientax MsRdpEx_rdclientax;
 
 void MsRdpEx_SetAxHookEnabled(bool axHookEnabled);
 
+bool CDECL MsRdpEx_UsePrivateAxLayout();
+
 HRESULT MsRdpEx_AxDll_DllGetClassObject(fnDllGetClassObject pfnDllGetClassObject, REFCLSID rclsid, REFIID riid, LPVOID* ppv);
 
 bool CDECL MsRdpEx_mstscax_Load(MsRdpEx_mstscax* dll, const char* filename);

dll/RdpInstance.cpp

@@ -509,6 +509,9 @@ IMsRdpExInstance* MsRdpEx_InstanceManager_AcquireByOutputPresenterHwnd(HWND hWnd
 
 CMsRdpExInstance* MsRdpEx_InstanceManager_AttachOutputWindow(HWND hOutputWnd, void* pUserData)
 {
+    if (!MsRdpEx_UsePrivateAxLayout())
+        return NULL;
+
     MsRdpEx_InstanceManager* ctx = g_InstanceManager;
 
     if (!ctx)
@@ -631,6 +634,9 @@ IMsRdpExInstance* MsRdpEx_InstanceManager_AcquireByInputCaptureHwnd(HWND hWnd)
 
 CMsRdpExInstance* MsRdpEx_InstanceManager_AttachInputWindow(HWND hInputWnd, void* pUserData)
 {
+    if (!MsRdpEx_UsePrivateAxLayout())
+        return NULL;
+
     MsRdpEx_InstanceManager* ctx = g_InstanceManager;
 
     if (!ctx)

dll/RdpSettings.cpp

@@ -1201,6 +1201,9 @@ HRESULT CMsRdpExtendedSettings::AttachRdpClient(IMsTscAx* pMsTscAx)
 {
     m_pMsTscAx = pMsTscAx;
 
+    if (!MsRdpEx_UsePrivateAxLayout())
+        return S_OK;
+
     ITSObjectBase* pTSWin32CoreApi = NULL;
     ITSPropertySet* pTSCoreProps = NULL;
     ITSPropertySet* pTSBaseProps = NULL;
@@ -1595,6 +1598,9 @@ HRESULT CMsRdpExtendedSettings::PrepareSspiSessionIdHack()
     char fakeKdcProxyName[256];
     char sessionId[MSRDPEX_GUID_STRING_SIZE];
 
+    if (!MsRdpEx_UsePrivateAxLayout())
+        return S_OK;
+
     if (!m_CoreProps) {
         MsRdpEx_LogPrint(ERROR, "PrepareSspiSessionIdHack - m_CoreProps is NULL!");
         return E_UNEXPECTED;