Add support for loading Windows dump file

core/CMakeLists.txt

@@ -33,6 +33,8 @@ if(WIN32)
 			adapters/windowskerneladapter.h
 			adapters/localwindowskerneladapter.cpp
 			adapters/localwindowskerneladapter.h
+			adapters/windowsdumpfile.cpp
+			adapters/windowsdumpfile.h
 			)
 else()
 	set(SOURCES ${COMMON_SOURCES} ${ADAPTER_SOURCES})

core/adapters/windowsdumpfile.cpp

@@ -0,0 +1,222 @@
+#include "windowsdumpfile.h"
+#include <filesystem>
+
+using namespace BinaryNinjaDebugger;
+using namespace std;
+
+
+WindowsDumpFileAdapter::WindowsDumpFileAdapter(BinaryView* data) : DbgEngAdapter(data)
+{
+    m_usePDBFileName = false;
+}
+
+
+bool WindowsDumpFileAdapter::ExecuteWithArgsInternal(const std::string& path, const std::string& args,
+                                            const std::string& workingDir, const LaunchConfigurations& configs) {
+    m_aboutToBeKilled = false;
+
+    if (this->m_debugActive) {
+        this->Reset();
+    }
+
+    if (!Start()) {
+        this->Reset();
+        DebuggerEvent event;
+        event.type = LaunchFailureEventType;
+        event.data.errorData.error = fmt::format("Failed to initialize DbgEng");
+        event.data.errorData.shortError = fmt::format("Failed to initialize DbgEng");
+        PostDebuggerEvent(event);
+        return false;
+    }
+
+    if (const auto result = this->m_debugControl->SetEngineOptions(DEBUG_ENGOPT_INITIAL_BREAK); result != S_OK) {
+        this->Reset();
+        DebuggerEvent event;
+        event.type = LaunchFailureEventType;
+        event.data.errorData.error = fmt::format("Failed to engine option DEBUG_ENGOPT_INITIAL_BREAK");
+        event.data.errorData.shortError = fmt::format("Failed to engine option");
+        PostDebuggerEvent(event);
+        return false;
+    }
+
+    if (const auto result = this->m_debugClient->OpenDumpFile(const_cast<char *>(path.c_str()));
+            result != S_OK) {
+        this->Reset();
+        DebuggerEvent event;
+        event.type = LaunchFailureEventType;
+        event.data.errorData.error = fmt::format("OpenDumpFile failed: 0x{:x}", result);
+        event.data.errorData.shortError = fmt::format("OpenDumpFile failed: 0x{:x}", result);
+        PostDebuggerEvent(event);
+        return false;
+    }
+
+    // The WaitForEvent() must be called once before the engine fully attaches to the target.
+    if (!Wait()) {
+        DebuggerEvent event;
+        event.type = LaunchFailureEventType;
+        event.data.errorData.error = fmt::format("WaitForEvent failed");
+        event.data.errorData.shortError = fmt::format("WaitForEvent failed");
+        PostDebuggerEvent(event);
+    }
+
+    return true;
+}
+
+
+bool WindowsDumpFileAdapter::Start()
+{
+	if (this->m_debugActive)
+		this->Reset();
+
+	auto handle = GetModuleHandleA("dbgeng.dll");
+	if (handle == nullptr)
+		false;
+
+	//    HRESULT DebugCreate(
+	//    [in]  REFIID InterfaceId,
+	//    [out] PVOID  *Interface
+	//    );
+	typedef HRESULT(__stdcall * pfunDebugCreate)(REFIID, PVOID*);
+	auto DebugCreate = (pfunDebugCreate)GetProcAddress(handle, "DebugCreate");
+	if (DebugCreate == nullptr)
+		return false;
+
+	if (const auto result = DebugCreate(__uuidof(IDebugClient7), reinterpret_cast<void**>(&this->m_debugClient));
+		result != S_OK)
+		throw std::runtime_error("Failed to create IDebugClient7");
+
+	QUERY_DEBUG_INTERFACE(IDebugControl7, &this->m_debugControl);
+	QUERY_DEBUG_INTERFACE(IDebugDataSpaces, &this->m_debugDataSpaces);
+	QUERY_DEBUG_INTERFACE(IDebugRegisters, &this->m_debugRegisters);
+	QUERY_DEBUG_INTERFACE(IDebugSymbols3, &this->m_debugSymbols);
+	QUERY_DEBUG_INTERFACE(IDebugSystemObjects, &this->m_debugSystemObjects);
+
+	m_debugEventCallbacks.SetAdapter(this);
+	if (const auto result = this->m_debugClient->SetEventCallbacks(&this->m_debugEventCallbacks); result != S_OK)
+	{
+		LogWarn("Failed to set event callbacks");
+		return false;
+	}
+
+	m_outputCallbacks.SetAdapter(this);
+	if (const auto result = this->m_debugClient->SetOutputCallbacks(&this->m_outputCallbacks); result != S_OK)
+	{
+		LogWarn("Failed to set output callbacks");
+		return false;
+	}
+
+	m_inputCallbacks.SetDbgControl(m_debugControl);
+	if (const auto result = this->m_debugClient->SetInputCallbacks(&this->m_inputCallbacks); result != S_OK)
+	{
+		LogWarn("Failed to set input callbacks");
+		return false;
+	}
+
+	this->m_debugActive = true;
+	return true;
+}
+
+
+void WindowsDumpFileAdapter::Reset()
+{
+	m_aboutToBeKilled = false;
+
+	if (!this->m_debugActive)
+		return;
+
+	// Free up the resources if the dbgsrv is launched by the adapter. Otherwise, the dbgsrv is launched outside BN,
+	// we should keep everything active.
+	SAFE_RELEASE(this->m_debugControl);
+	SAFE_RELEASE(this->m_debugDataSpaces);
+	SAFE_RELEASE(this->m_debugRegisters);
+	SAFE_RELEASE(this->m_debugSymbols);
+	SAFE_RELEASE(this->m_debugSystemObjects);
+
+	if (this->m_debugClient)
+	{
+		this->m_debugClient->EndSession(DEBUG_END_PASSIVE);
+		m_server = 0;
+	}
+
+	// There seems to be an internal ref-counting issue in the DbgEng TTD engine, that the reference for the debug
+	// client is not properly freed after the target has exited. To properly free the debug client instance, here we
+	// are calling Release() a few more times to ensure the ref count goes down to 0. Luckily this would not cause
+	// a UAF or crash.
+	// This might be related to the weird behavior of not terminating the target when we call TerminateProcesses(),
+	// (see comment in `DbgEngTTDAdapter::Quit()`).
+	// The same issue is not observed when we do forward debugging using the regular DbgEng. Also, I cannot reproduce
+	// the issue using my script https://github.com/xusheng6/dbgeng_test.
+	for (size_t i = 0; i < 100; i++)
+		m_debugClient->Release();
+
+	SAFE_RELEASE(this->m_debugClient);
+
+	this->m_debugActive = false;
+}
+
+
+bool WindowsDumpFileAdapter::Detach()
+{
+	m_aboutToBeKilled = true;
+	m_lastOperationIsStepInto = false;
+	if (!this->m_debugClient)
+		return false;
+
+	if (this->m_debugClient->EndSession(DEBUG_END_PASSIVE) != S_OK)
+		return false;
+
+	m_debugClient->ExitDispatch(reinterpret_cast<PDEBUG_CLIENT>(m_debugClient));
+	return true;
+}
+
+
+bool WindowsDumpFileAdapter::Quit()
+{
+	m_aboutToBeKilled = true;
+	m_lastOperationIsStepInto = false;
+	if (!this->m_debugClient)
+		return false;
+
+	if (this->m_debugClient->EndSession(DEBUG_END_PASSIVE) != S_OK)
+		return false;
+
+	m_debugClient->ExitDispatch(reinterpret_cast<PDEBUG_CLIENT>(m_debugClient));
+	return true;
+}
+
+
+WindowsDumpFileAdapterType::WindowsDumpFileAdapterType() : DebugAdapterType("WINDOWS_DUMP_FILE") {}
+
+
+DebugAdapter* WindowsDumpFileAdapterType::Create(BinaryNinja::BinaryView* data)
+{
+    // TODO: someone should free this.
+    return new WindowsDumpFileAdapter(data);
+}
+
+
+bool WindowsDumpFileAdapterType::IsValidForData(BinaryNinja::BinaryView* data)
+{
+	return data->GetTypeName() == "PE" || data->GetTypeName() == "Raw";
+}
+
+
+bool WindowsDumpFileAdapterType::CanConnect(BinaryNinja::BinaryView* data)
+{
+    return true;
+}
+
+
+bool WindowsDumpFileAdapterType::CanExecute(BinaryNinja::BinaryView* data)
+{
+#ifdef WIN32
+    return true;
+#endif
+    return false;
+}
+
+void BinaryNinjaDebugger::InitWindowsDumpFileAdapterType()
+{
+    static WindowsDumpFileAdapterType localType;
+    DebugAdapterType::Register(&localType);
+}

core/adapters/windowsdumpfile.h

@@ -0,0 +1,47 @@
+/*
+Copyright 2020-2024 Vector 35 Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#pragma once
+#include "dbgengadapter.h"
+
+namespace BinaryNinjaDebugger {
+    class WindowsDumpFileAdapter: public DbgEngAdapter
+    {
+    public:
+		WindowsDumpFileAdapter(BinaryView* data);
+
+        [[nodiscard]] bool ExecuteWithArgsInternal(const std::string& path, const std::string& args,
+           const std::string& workingDir, const LaunchConfigurations& configs = {}) override;
+
+		bool Start() override;
+		void Reset() override;
+
+		bool Detach() override;
+		bool Quit() override;
+    };
+
+    class WindowsDumpFileAdapterType : public DebugAdapterType
+    {
+    public:
+		WindowsDumpFileAdapterType();
+        virtual DebugAdapter* Create(BinaryNinja::BinaryView* data);
+        virtual bool IsValidForData(BinaryNinja::BinaryView* data);
+        virtual bool CanExecute(BinaryNinja::BinaryView* data);
+        virtual bool CanConnect(BinaryNinja::BinaryView* data);
+    };
+
+    void InitWindowsDumpFileAdapterType();
+};

core/debugger.cpp

@@ -22,6 +22,7 @@ limitations under the License.
 	#include "adapters/dbgengttdadapter.h"
 	#include "adapters/windowskerneladapter.h"
 	#include "adapters/localwindowskerneladapter.h"
+	#include "adapters/windowsdumpfile.h"
 #endif
 
 using namespace BinaryNinja;
@@ -38,6 +39,7 @@ void InitDebugAdapterTypes()
 	InitDbgEngTTDAdapterType();
 	InitWindowsKernelAdapterType();
 	InitLocalWindowsKernelAdapterType();
+	InitWindowsDumpFileAdapterType();
 #endif
 
 	// Disable these adapters because they are not tested, and will get replaced later