Skip to content

Commit

Permalink
Use reccmp as a python requirement
Browse files Browse the repository at this point in the history
  • Loading branch information
madebr committed Oct 24, 2024
1 parent b5fee6b commit 64f8930
Show file tree
Hide file tree
Showing 100 changed files with 134 additions and 14,791 deletions.
8 changes: 6 additions & 2 deletions .github/workflows/analyze.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,14 @@ jobs:
steps:
- uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: '3.12'

- name: Install python libraries
run: |
python -m pip install -r tools/requirements.txt
pip install -r tools/requirements.txt
- name: Run decomplint.py
run: |
tools/decomplint/decomplint.py ${{ matrix.who }} --module ${{ matrix.who }} --warnfail
reccmp-decomplint ${{ matrix.who }} --module ${{ matrix.who }} --warnfail
29 changes: 19 additions & 10 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,10 @@ jobs:
steps:
- uses: actions/checkout@master

- uses: actions/setup-python@v5
with:
python-version: '3.12'

- uses: actions/download-artifact@master
with:
name: Win32
Expand All @@ -125,12 +129,17 @@ jobs:
run: |
pip install -r tools/requirements.txt
- name: Detect binaries
run: |
reccmp-project detect --what original --search-path legobin
reccmp-project detect --what recompiled --search-path build
- name: Summarize Accuracy
shell: bash
run: |
python3 tools/reccmp/reccmp.py -S CONFIGPROGRESS.SVG --svg-icon tools/reccmp/config.png -H CONFIGPROGRESS.HTML legobin/CONFIG.EXE build/CONFIG.EXE build/CONFIG.PDB . | tee CONFIGPROGRESS.TXT
python3 tools/reccmp/reccmp.py -S ISLEPROGRESS.SVG --svg-icon tools/reccmp/isle.png -H ISLEPROGRESS.HTML legobin/ISLE.EXE build/ISLE.EXE build/ISLE.PDB . | tee ISLEPROGRESS.TXT
python3 tools/reccmp/reccmp.py -S LEGO1PROGRESS.SVG -T 4252 --svg-icon tools/reccmp/lego1.png -H LEGO1PROGRESS.HTML legobin/LEGO1.DLL build/LEGO1.DLL build/LEGO1.PDB . | tee LEGO1PROGRESS.TXT
reccmp-reccmp -S CONFIGPROGRESS.SVG --svg-icon assets/config.png --target CONFIG | tee CONFIGPROGRESS.TXT
reccmp-reccmp -S ISLEPROGRESS.SVG --svg-icon assets/isle.png --target ISLE | tee ISLEPROGRESS.TXT
reccmp-reccmp -S LEGO1PROGRESS.SVG -T 4252 --svg-icon assets/lego1.png --target LEGO1 | tee LEGO1PROGRESS.TXT
- name: Compare Accuracy With Current Master
shell: bash
Expand All @@ -147,21 +156,21 @@ jobs:
- name: Test Exports
shell: bash
run: |
tools/verexp/verexp.py legobin/LEGO1.DLL build/LEGO1.DLL
reccmp-verexp --target LEGO1
- name: Check Vtables
shell: bash
run: |
python3 tools/vtable/vtable.py legobin/CONFIG.EXE build/CONFIG.EXE build/CONFIG.PDB .
python3 tools/vtable/vtable.py legobin/ISLE.EXE build/ISLE.EXE build/ISLE.PDB .
python3 tools/vtable/vtable.py legobin/LEGO1.DLL build/LEGO1.DLL build/LEGO1.PDB .
reccmp-vtable --target CONFIG
reccmp-vtable --target ISLE
reccmp-vtable --target LEGO1
- name: Check Variables
shell: bash
run: |
python3 tools/datacmp.py legobin/CONFIG.EXE build/CONFIG.EXE build/CONFIG.PDB .
python3 tools/datacmp.py legobin/ISLE.EXE build/ISLE.EXE build/ISLE.PDB .
python3 tools/datacmp.py legobin/LEGO1.DLL build/LEGO1.DLL build/LEGO1.PDB .
reccmp-datacmp --target CONFIG
reccmp-datacmp --target ISLE
reccmp-datacmp --target LEGO1
- name: Upload Artifact
uses: actions/upload-artifact@master
Expand Down
37 changes: 0 additions & 37 deletions .github/workflows/format.yml

This file was deleted.

60 changes: 0 additions & 60 deletions .github/workflows/unittest.yml

This file was deleted.

2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
reccmp-user.yml
reccmp-build.yml
Debug/
Release/
*.ncb
Expand Down
6 changes: 6 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ project(isle CXX)
include(CheckCXXSourceCompiles)
include(CMakeDependentOption)
include(CMakePushCheckState)
include("${CMAKE_CURRENT_LIST_DIR}/cmake/reccmp.cmake")

set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
option(ENABLE_CLANG_TIDY "Enable clang-tidy")
Expand Down Expand Up @@ -405,6 +406,7 @@ add_library(lego1 SHARED
LEGO1/main.cpp
LEGO1/modeldb/modeldb.cpp
)
reccmp_add_target(lego1 ID LEGO1)
register_lego1_target(lego1)

if (MINGW)
Expand Down Expand Up @@ -447,6 +449,7 @@ if (ISLE_BUILD_APP)
ISLE/res/isle.rc
ISLE/isleapp.cpp
)
reccmp_add_target(isle ID ISLE)

target_compile_definitions(isle PRIVATE ISLE_APP)

Expand Down Expand Up @@ -477,6 +480,7 @@ if (ISLE_BUILD_CONFIG)
CONFIG/StdAfx.cpp
CONFIG/res/config.rc
)
reccmp_add_target(config ID CONFIG)
target_compile_definitions(config PRIVATE _AFXDLL MXDIRECTX_FOR_CONFIG)
target_include_directories(config PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/util" "${CMAKE_CURRENT_SOURCE_DIR}/LEGO1")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14)
Expand Down Expand Up @@ -603,3 +607,5 @@ if(EXISTS "${CLANGFORMAT_BIN}")
endif()
endif()
endif()

reccmp_configure()
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,4 @@ Right click on `LEGO1.DLL`, select `Properties`, and switch to the `Details` tab

* ISLE.EXE `md5: f6da12249e03eed1c74810cd23beb9f5`
* LEGO1.DLL `md5: 4e2f6d969ea2ef8655ba3fc221a0c8fe`
* CONFIG.DLL `md5: 92d958a64a273662c591c88b09100f4a`
File renamed without changes
File renamed without changes
File renamed without changes
58 changes: 58 additions & 0 deletions cmake/reccmp.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
function(reccmp_find_project RESULT)
set(curdir "${CMAKE_CURRENT_SOURCE_DIR}")
while(1)
if(EXISTS "${curdir}/reccmp-project.yml")
break()
endif()
get_filename_component(nextdir "${curdir}" DIRECTORY)
if(nextdir STREQUAL curdir)
set(curdir "${RESULT}-NOTFOUND")
break()
endif()
set(curdir "${nextdir}")
endwhile()
set("${RESULT}" "${curdir}" PARENT_SCOPE)
endfunction()

function(reccmp_add_target TARGET)
cmake_parse_arguments(ARGS "" "ID" "" ${ARGN})
if(NOT ARGS_ID)
message(FATAL_ERROR "Missing ID argument")
endif()
set_property(TARGET ${TARGET} PROPERTY INTERFACE_RECCMP_ID "${ARGS_ID}")
set_property(GLOBAL APPEND PROPERTY RECCMP_TARGETS ${TARGET})
endfunction()

function(reccmp_configure)
cmake_parse_arguments(ARGS "COPY_TO_SOURCE_FOLDER" "DIR" "" ${ARGN})
set(binary_dir "${CMAKE_BINARY_DIR}")
if(ARGS_DIR)
set(binary_dir "${ARGS_DIR}")
endif()

reccmp_find_project(reccmp_project_dir)
if(NOT reccmp_project_dir)
message(FATAL_ERROR "Cannot find reccmp-project.yml")
endif()

if(CMAKE_CONFIGURATION_TYPES)
set(outputdir "${binary_dir}/$<CONFIG>")
else()
set(outputdir "${binary_dir}")
endif()
set(build_yml_txt "project: '${reccmp_project_dir}'\ntargets:\n")
get_property(RECCMP_TARGETS GLOBAL PROPERTY RECCMP_TARGETS)
foreach(target ${RECCMP_TARGETS})
get_property(id TARGET "${target}" PROPERTY INTERFACE_RECCMP_ID)
string(APPEND build_yml_txt " ${id}:\n")
string(APPEND build_yml_txt " path: '$<TARGET_FILE:${target}>'\n")
if(WIN32 AND MSVC)
string(APPEND build_yml_txt " pdb: '$<TARGET_PDB_FILE:${target}>'\n")
endif()
endforeach()
file(GENERATE OUTPUT "${outputdir}/reccmp-build.yml" CONTENT "${build_yml_txt}")

if(ARGS_COPY_TO_SOURCE_FOLDER)
file(GENERATE OUTPUT "${CMAKE_SOURCE_DIR}/reccmp-build.yml" CONTENT "${build_yml_txt}" CONDITION $<CONFIG:Release>)
endif()
endfunction()
16 changes: 16 additions & 0 deletions reccmp-project.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
targets:
ISLE:
filename: ISLE.EXE
source-root: ISLE
hash:
sha256: 5cf57c284973fce9d14f5677a2e4435fd989c5e938970764d00c8932ed5128ca
LEGO1:
filename: LEGO1.DLL
source-root: LEGO1
hash:
sha256: 14645225bbe81212e9bc1919cd8a692b81b8622abb6561280d99b0fc4151ce17
CONFIG:
filename: CONFIG.EXE
source-root: CONFIG
hash:
sha256: 864766d024d78330fed5e1f6efb2faf815f1b1c3405713a9718059dc9a54e52c
66 changes: 25 additions & 41 deletions tools/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,58 +160,42 @@ inline virtual const char* ClassName() const override // vtable+0x0c

Use `pip` to install the required packages to be able to use the Python tools found in this folder:

```
```sh
pip install -r tools/requirements.txt
```

Run the following command to allow reccmp to detect the original LEGO binaries:

```sh
reccmp-project detect --what original --search-path <paths-to-directories0containing-lego-binaries>
```

After building recompiled binaries, run the following command in this repository's root:

```sh
reccmp-project detect --what recompiled --search-path <paths-to-build-directories>
```

The example usages below assume that the current working directory is this repository's root and that the retail binaries have been copied to `./legobin`.

* [`decomplint`](/tools/decomplint): Checks the decompilation annotations (see above)
* e.g. `py -m tools.decomplint.decomplint --module LEGO1 LEGO1`
* [`isledecomp`](/tools/isledecomp): A library that implements a parser to identify the decompilation annotations (see above)
* `reccmp-decomplint`: Checks the decompilation annotations (see above)
* e.g. `reccmp-decomplint --module LEGO1 LEGO1`
* [`ncc`](/tools/ncc): Checks naming conventions based on a set of rules
* [`reccmp`](/tools/reccmp): Compares an original binary with a recompiled binary, provided a PDB file. For example:
* `reccmp-reccmp`: Compares an original binary with a recompiled binary, provided a PDB file. For example:
* Display the diff for a single function: `py -m tools.reccmp.reccmp --verbose 0x100ae1a0 legobin/LEGO1.DLL build/LEGO1.DLL build/LEGO1.PDB .`
* Generate an HTML report: `py -m tools.reccmp.reccmp --html output.html legobin/LEGO1.DLL build/LEGO1.DLL build/LEGO1.PDB .`
* Create a base file for diffs: `py -m tools.reccmp.reccmp --json base.json --silent legobin/LEGO1.DLL build/LEGO1.DLL build/LEGO1.PDB .`
* Diff against a base file: `py -m tools.reccmp.reccmp --diff base.json legobin/LEGO1.DLL build/LEGO1.DLL build/LEGO1.PDB .`
* [`stackcmp`](/tools/stackcmp): Compares the stack layout for a given function that almost matches.
* e.g. `py -m tools.stackcmp.stackcmp legobin/BETA10.DLL build_debug/LEGO1.DLL build_debug/LEGO1.pdb . 0x1007165d`
* [`roadmap`](/tools/roadmap): Compares symbol locations in an original binary with the same symbol locations of a recompiled binary
* [`verexp`](/tools/verexp): Verifies exports by comparing the exports of the original DLL and the recompiled DLL
* [`vtable`](/tools/vtable): Asserts virtual table correctness by comparing a recompiled binary with the original
* e.g. `py -m tools.vtable.vtable legobin/LEGO1.DLL build/LEGO1.DLL build/LEGO1.PDB .`
* [`datacmp.py`](/tools/datacmp.py): Compares global data found in the original with the recompiled version
* e.g. `py -m tools.datacmp legobin/LEGO1.DLL build/LEGO1.DLL build/LEGO1.PDB .`
* `reccmp-stackcmp`: Compares the stack layout for a given function that almost matches.
* e.g. `reccmp-stackcmp legobin/BETA10.DLL build_debug/LEGO1.DLL build_debug/LEGO1.pdb . 0x1007165d`
* `reccmp-roadmap`: Compares symbol locations in an original binary with the same symbol locations of a recompiled binary
* `reccmp-verexp`: Verifies exports by comparing the exports of the original DLL and the recompiled DLL
* `reccmp-vtable`: Asserts virtual table correctness by comparing a recompiled binary with the original
* e.g. `reccmp-vtable legobin/LEGO1.DLL build/LEGO1.DLL build/LEGO1.PDB .`
* `reccmp-datacmp`: Compares global data found in the original with the recompiled version
* e.g. `reccmp-datacmp legobin/LEGO1.DLL build/LEGO1.DLL build/LEGO1.PDB .`
* [`patch_c2.py`](/tools/patch_c2.py): Patches `C2.EXE` (part of MSVC 4.20) to get rid of a bugged warning

## Testing

`isledecomp` comes with a suite of tests. Install `pytest` and run it, passing in the directory:

```
pip install pytest
pytest tools/isledecomp/tests/
```

## Tool Development

In order to keep the Python code clean and consistent, we use `pylint` and `black`:

`pip install black pylint`

### Run pylint (ignores build and virtualenv)

`pylint tools/ --ignore=build,ncc`

### Check Python code formatting without rewriting files

`black --check tools/`

### Apply Python code formatting

`black tools/`

# Modules
The following is a list of all the modules found in the annotations (e.g. `// FUNCTION: [module] [address]`) and which binaries they refer to. See [this list of all known versions of the game](https://www.legoisland.org/wiki/LEGO_Island#Download).

Expand Down Expand Up @@ -243,7 +227,7 @@ cmake <path-to-source> -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=RelWithDebInfo -D
```
**TODO**: If you can figure out how to make a debug build with SmartHeap enabled, please add it here.

If you want to run scripts to compare your debug build to `BETA10` (e.g. `reccmp`), it is advisable to add a copy of `LEGO1D.DLL` to `/legobin` and rename it to `BETA10.DLL`.
If you want to run scripts to compare your debug build to `BETA10` (e.g. `reccmp-reccmp`), it is advisable to add a copy of `LEGO1D.DLL` to `/legobin` and rename it to `BETA10.DLL`.

### Finding matching functions

Expand Down
Loading

0 comments on commit 64f8930

Please sign in to comment.