Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added iOS testing infra & GHA job #964

Draft
wants to merge 12 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions .github/workflows/ios-cpu-arm64-build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: "iOS CPU ARM64 Build"
on:
workflow_dispatch:
push:
branches:
- main
- rel-*
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
ORT_NIGHTLY_REST_API: "https://feeds.dev.azure.com/aiinfra/PublicPackages/_apis/packaging/Feeds/ORT-Nightly/packages?packageNameQuery=Microsoft.ML.OnnxRuntime&api-version=6.0-preview.1"
ORT_PACKAGE_NAME: "Microsoft.ML.OnnxRuntime"
jobs:
ios-cpu-arm64-build:
runs-on: macos-latest # arm64
env:
xcode_version: 14
steps:
- name: Checkout OnnxRuntime GenAI repo
uses: actions/checkout@v4
with:
submodules: true

- name: Get the Latest OnnxRuntime Nightly Version
run: |
ORT_NIGHTLY_VERSION=$(curl -s "${{ env.ORT_NIGHTLY_REST_API }}" | jq -r '.value[0].versions[0].normalizedVersion')
echo "$ORT_NIGHTLY_VERSION"
echo "ORT_NIGHTLY_VERSION=$ORT_NIGHTLY_VERSION" >> $GITHUB_ENV
- name: Download OnnxRuntime Nightly
run: |
nuget install ${{ env.ORT_PACKAGE_NAME }} -version ${{ env.ORT_NIGHTLY_VERSION }} -x

- name: Extract OnnxRuntime library and header files
run: |
mkdir -p ort/lib
mv ${{ env.ORT_PACKAGE_NAME }}/build/native/include ort/
mv ${{ env.ORT_PACKAGE_NAME }}/runtimes/ios/native/* ort/lib/

- name: Build and Run tests
run: |
python3 -m venv genai-ios-venv
source genai-ios-venv/bin/activate
python3 -m pip install requests
python3 build.py --ios --parallel --apple_sysroot iphonesimulator --osx_arch arm64 --apple_deploy_target 15.4 --cmake_generator 'Xcode'
4 changes: 4 additions & 0 deletions benchmark/c/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,8 @@ target_link_libraries(model_benchmark PRIVATE onnxruntime-genai-static ${ONNXRUN

target_link_directories(model_benchmark PRIVATE ${ORT_LIB_DIR})

if(APPLE)
target_link_libraries(model_benchmark PRIVATE "-framework Foundation" "-framework CoreML")
endif()

source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${model_benchmark_srcs})
43 changes: 40 additions & 3 deletions build.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import shlex
import shutil
import sys
import json
import subprocess
import textwrap

from pathlib import Path
Expand Down Expand Up @@ -406,7 +408,6 @@ def _run_android_tests(args: argparse.Namespace):
# the test app loads and runs a test model using the GenAI Java bindings
gradle_executable = str(REPO_ROOT / "src" / "java" / ("gradlew.bat" if util.is_windows() else "gradlew"))
android_test_path = args.build_dir / "src" / "java" / "androidtest"
import subprocess
exception = None
try:
util.run([gradle_executable, "--no-daemon",
Expand All @@ -431,6 +432,39 @@ def _run_android_tests(args: argparse.Namespace):
raise exception


def _run_ios_tests(args: argparse.Namespace):
simulator_device_info = subprocess.check_output(
[
sys.executable,
os.path.join(REPO_ROOT, "tools", "ci_build", "github", "apple", "get_simulator_device_info.py"),
],
text=True,
).strip()
log.debug(f"Simulator device info:\n{simulator_device_info}")

simulator_device_info = json.loads(simulator_device_info)

xc_test_schemes = [
"unit_tests_xc",
]

for xc_test_scheme in xc_test_schemes:
util.run(
[
"xcodebuild",
"test-without-building",
"-project",
"./Generators.xcodeproj",
"-configuration",
args.config,
"-scheme",
xc_test_scheme,
"-destination",
f"platform=iOS Simulator,id={simulator_device_info['device_udid']}",
],
cwd=args.build_dir,
)

def update(args: argparse.Namespace, env: dict[str, str]):
"""
Update the cmake build files.
Expand Down Expand Up @@ -498,7 +532,6 @@ def update(args: argparse.Namespace, env: dict[str, str]):
platform_name = "macabi" if args.macos == "Catalyst" else args.apple_sysroot
command += [
"-DENABLE_PYTHON=OFF",
"-DENABLE_TESTS=OFF",
"-DENABLE_MODEL_BENCHMARK=OFF",
f"-DBUILD_APPLE_FRAMEWORK={'ON' if args.build_apple_framework else 'OFF'}",
"-DPLATFORM_NAME=" + platform_name,
Expand Down Expand Up @@ -601,6 +634,11 @@ def test(args: argparse.Namespace, env: dict[str, str]):
"""
Run the tests.
"""
if args.ios:
_run_ios_tests(args)
# CTest won't work on iOS
return

lib_dir = args.build_dir / "test"
if not args.ort_home:
_ = util.download_dependencies(args.use_cuda, args.use_rocm, args.use_dml, lib_dir)
Expand All @@ -619,7 +657,6 @@ def test(args: argparse.Namespace, env: dict[str, str]):
if args.android:
_run_android_tests(args)


def clean(args: argparse.Namespace, env: dict[str, str]):
"""
Clean the build output.
Expand Down
88 changes: 81 additions & 7 deletions test/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

if (IOS)
find_package(XCTest REQUIRED)
endif()

set(TEST_SRC_DIR ${CMAKE_SOURCE_DIR}/test)
set(TEST_INC_DIR ${ORT_HEADER_DIR} ${CMAKE_SOURCE_DIR}/src)

include(${CMAKE_SOURCE_DIR}/cmake/cxx_standard.cmake)

set(TESTS_ROOT ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
Expand All @@ -6,19 +16,22 @@ file(GLOB test_srcs CONFIGURE_DEPENDS
"${CMAKE_CURRENT_SOURCE_DIR}/*.cpp"
)

# google unit test
add_executable(unit_tests
set(_UT_SOURCES
main.cpp
c_api_tests.cpp
model_tests.cpp
sampling_tests.cpp
sampling_benchmark.cpp
)

target_include_directories(unit_tests PRIVATE
${ORT_HEADER_DIR}
${CMAKE_SOURCE_DIR}/src
)
if (IOS)
add_executable(unit_tests ${_UT_SOURCES} ${TEST_SRC_DIR}/xctest/ortgenaitestmain.mm)
else()
# google unit test
add_executable(unit_tests ${_UT_SOURCES})
endif()

target_include_directories(unit_tests PRIVATE ${TEST_INC_DIR})

target_link_directories(unit_tests PRIVATE ${ORT_LIB_DIR})
target_link_libraries(unit_tests PRIVATE
Expand All @@ -27,7 +40,14 @@ target_link_libraries(unit_tests PRIVATE
)

if(NOT (CMAKE_SYSTEM_NAME STREQUAL "Android" OR CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Darwin"))
target_link_libraries(unit_tests PRIVATE ${ONNXRUNTIME_LIB})
target_link_libraries(unit_tests PRIVATE ${ONNXRUNTIME_LIB})
endif()

if(APPLE)
target_link_libraries(unit_tests PRIVATE "-framework Foundation" "-framework CoreML")
if(IOS)
target_link_libraries(unit_tests PRIVATE "-framework UIKit")
endif()
endif()

if(USE_CUDA AND CMAKE_CUDA_COMPILER)
Expand All @@ -51,4 +71,58 @@ set_target_properties(unit_tests PROPERTIES FOLDER "Tests")
source_group(TREE ${PROJECT_SOURCE_DIR} FILES ${test_srcs})
set_property(DIRECTORY ${CMAKE_SOURCE_DIR} PROPERTY VS_STARTUP_PROJECT unit_tests)

if (IOS)
set_target_properties(unit_tests PROPERTIES FOLDER "ONNXRuntimeGenAITest"
MACOSX_BUNDLE_BUNDLE_NAME unit_tests
MACOSX_BUNDLE_GUI_IDENTIFIER com.onnxruntimegenai.utest.unit_tests
MACOSX_BUNDLE_LONG_VERSION_STRING ${VERSION_STR}
MACOSX_BUNDLE_BUNDLE_VERSION ${VERSION_STR}
MACOSX_BUNDLE_SHORT_VERSION_STRING ${VERSION_STR}
XCODE_ATTRIBUTE_CLANG_ENABLE_MODULES "YES"
XCODE_ATTRIBUTE_ENABLE_BITCODE "NO"
XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO")

xctest_add_bundle(unit_tests_xc unit_tests
${TEST_SRC_DIR}/xctest/ortgenaixctest.m
${TEST_SRC_DIR}/xctest/xcgtest.mm
${_UT_SOURCES})

get_target_property(srcs unit_tests_xc SOURCES)
set(objective_c_cc_srcs ${srcs})
list(FILTER objective_c_cc_srcs INCLUDE REGEX "\\.mm?$")
set_property(SOURCE ${objective_c_cc_srcs} APPEND PROPERTY COMPILE_OPTIONS "-fobjc-arc")

target_include_directories(unit_tests_xc PRIVATE ${TEST_INC_DIR})
target_link_directories(unit_tests_xc PRIVATE ${ORT_LIB_DIR})
target_link_libraries(unit_tests_xc PRIVATE onnxruntime-genai-static GTest::gtest)
target_link_libraries(unit_tests_xc PRIVATE ${ONNXRUNTIME_LIB})
set_target_properties(unit_tests_xc PROPERTIES FOLDER "ONNXRuntimeGenAIXCTest"
MACOSX_BUNDLE_BUNDLE_NAME unit_tests_xc
MACOSX_BUNDLE_GUI_IDENTIFIER com.onnxruntimegenai.utest.unit_tests
MACOSX_BUNDLE_LONG_VERSION_STRING ${VERSION_STR}
MACOSX_BUNDLE_BUNDLE_VERSION ${VERSION_STR}
MACOSX_BUNDLE_SHORT_VERSION_STRING ${VERSION_STR}
XCODE_ATTRIBUTE_ENABLE_BITCODE "NO")

# This is a workaround for an Xcode 16 / CMake issue:
# error: Multiple commands produce '<build>/Debug/Debug-iphonesimulator/onnxruntime_test_all.app/PlugIns'
# note: CreateBuildDirectory <build>/Debug/Debug-iphonesimulator/onnxruntime_test_all.app/PlugIns
# note: Target 'onnxruntime_test_all' (project 'onnxruntime') has create directory command with output
# '<build>/Debug/Debug-iphonesimulator/onnxruntime_test_all.app/PlugIns'
#
# It seems related to the test target (e.g., unit_tests_xc) LIBRARY_OUTPUT_DIRECTORY property getting set
# to "$<TARGET_BUNDLE_CONTENT_DIR:${testee}>/PlugIns" in xctest_add_bundle():
# https://github.com/Kitware/CMake/blob/9c4a0a9ff09735b847bbbc38caf6da7f6c7238f2/Modules/FindXCTest.cmake#L159-L168
#
# This is the related CMake issue: https://gitlab.kitware.com/cmake/cmake/-/issues/26301
#
# Unsetting LIBRARY_OUTPUT_DIRECTORY avoids the build error.
set_property(TARGET unit_tests_xc PROPERTY LIBRARY_OUTPUT_DIRECTORY)

# Don't bother calling xctest_add_test() because we don't use CTest to run tests on iOS.
# Instead, we can call 'xcodebuild test-without-building' and specify a '-destination' referring to an iOS
# simulator or device.
# xctest_add_test(xctest.unit_tests unit_tests_xc)
endif()

include(GoogleTest)
12 changes: 11 additions & 1 deletion test/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,17 @@
#include <generators.h>
#include <iostream>

int main(int argc, char** argv) {
#define TEST_MAIN main

#if defined(__APPLE__)
#include <TargetConditionals.h>
#if TARGET_OS_SIMULATOR || TARGET_OS_IOS
#undef TEST_MAIN
#define TEST_MAIN main_no_link_ // there is a UI test app for iOS.
#endif
#endif

int TEST_MAIN(int argc, char** argv) {
std::cout << "Generators Utility Library" << std::endl;
std::cout << "Initializing OnnxRuntime... ";
std::cout.flush();
Expand Down
90 changes: 90 additions & 0 deletions test/xctest/ortgenaitestmain.mm
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#import <UIKit/UIKit.h>

static const size_t kMaxStrLen = 2048;

static void set_test_rootdir(const char* image_path) {
size_t n = strnlen(image_path, kMaxStrLen);
for (; n >= 0; n--) {
if (image_path[n] == '/') {
break;
}
}

char* bundle_dir = (char*)malloc(n + 1);
if (bundle_dir != NULL) {
strncpy(bundle_dir, image_path, n);
bundle_dir[n] = 0;
chdir(bundle_dir);
free(bundle_dir);
}
}

@interface ViewController : UIViewController

@end

@implementation ViewController

- (void)viewDidLoad {
[super viewDidLoad];

self.view.backgroundColor = [UIColor whiteColor];
}

- (void)didReceiveMemoryWarning {
[super didReceiveMemoryWarning];
}

@end

@interface AppDelegate : UIResponder <UIApplicationDelegate>

@property(strong, nonatomic) UIWindow* window;

@property(nonatomic, strong) UIViewController* rootViewController;

@end

@implementation AppDelegate

- (BOOL)application:(UIApplication*)application didFinishLaunchingWithOptions:(NSDictionary*)launchOptions {
self.window = [[UIWindow alloc] initWithFrame:[UIScreen mainScreen].bounds];
self.window.tintAdjustmentMode = UIViewTintAdjustmentModeNormal;
self.window.rootViewController = [[ViewController alloc] init];

self.window.backgroundColor = [UIColor whiteColor];
self.window.clipsToBounds = NO;
[self.window makeKeyAndVisible];

return YES;
}

- (void)applicationWillResignActive:(UIApplication*)application {
}

- (void)applicationDidEnterBackground:(UIApplication*)application {
}

- (void)applicationWillEnterForeground:(UIApplication*)application {
}

- (void)applicationDidBecomeActive:(UIApplication*)application {
}

- (void)applicationWillTerminate:(UIApplication*)application {
}

@end

int main(int argc, char* argv[]) {
set_test_rootdir(argv[0]);
int ret = 0;
@autoreleasepool {
ret = UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class]));
}

return ret;
}
22 changes: 22 additions & 0 deletions test/xctest/ortgenaixctest.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License

#import <Foundation/Foundation.h>
#import <XCTest/XCTest.h>

// This is the stub test case which will let the xcode command line tool start testing on Simulator
@interface ONNXRuntimeGenAITestXCWrapper : XCTestCase

@end

@implementation ONNXRuntimeGenAITestXCWrapper

- (void)setUp {
// Put setup code here. This method is called before the invocation of each test method in the class.
}

- (void)tearDown {
// Put teardown code here. This method is called after the invocation of each test method in the class.
}

@end
Loading
Loading