Skip to content

Commit 2fbab0a

Browse files
hipuddingwangshuai09
hipudding
and
wangshuai09
committed
[CANN] Add Ascend NPU backend
Ascend is a full-stack AI computing infrastructure for industry applications and services based on Huawei Ascend processors and software. CANN (Compute Architecture of Neural Networks), developped by Huawei, is a heterogeneous computing architecture for AI. Co-authored-by: wangshuai09 <[email protected]>
1 parent 97bdd26 commit 2fbab0a

26 files changed

+10702
-8
lines changed

‎examples/llama-bench/llama-bench.cpp

+15
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@
2323
#include "ggml-cuda.h"
2424
#include "ggml-sycl.h"
2525

26+
#ifdef GGML_USE_CANN
27+
#include "ggml-cann.h"
28+
#endif
29+
2630
// utils
2731
static uint64_t get_time_ns() {
2832
using clock = std::chrono::high_resolution_clock;
@@ -120,6 +124,17 @@ static std::string get_gpu_info() {
120124
id += "/";
121125
}
122126
}
127+
#endif
128+
#ifdef GGML_USE_CANN
129+
uint32_t count = ggml_backend_cann_get_device_count();
130+
for (uint32_t i = 0; i < count; i++) {
131+
char buf[128];
132+
ggml_backend_cann_get_device_description(i, buf, sizeof(buf));
133+
id += buf;
134+
if (i < count - 1) {
135+
id += "/";
136+
}
137+
}
123138
#endif
124139
// TODO: other backends
125140
return id;

‎examples/llava/clip.cpp

+9
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@
1616
#include "ggml-metal.h"
1717
#endif
1818

19+
#ifdef GGML_USE_CANN
20+
#include "ggml-cann.h"
21+
#endif
22+
1923
#define STB_IMAGE_IMPLEMENTATION
2024
#include "stb_image.h"
2125

@@ -1001,6 +1005,11 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
10011005
LOG_TEE("%s: CLIP using Metal backend\n", __func__);
10021006
#endif
10031007

1008+
#ifdef GGML_USE_CANN
1009+
new_clip->backend = ggml_backend_cann_init(0);
1010+
printf("%s: CLIP using CANN backend\n", __func__);
1011+
#endif
1012+
10041013

10051014
if (!new_clip->backend) {
10061015
new_clip->backend = ggml_backend_cpu_init();

‎ggml/include/ggml-cann.h

+148
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/*
2+
* Copyright (c) 2023-2024 The ggml authors
3+
*
4+
* Permission is hereby granted, free of charge, to any person obtaining a copy
5+
* of this software and associated documentation files (the "Software"), to
6+
* deal in the Software without restriction, including without limitation the
7+
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8+
* sell copies of the Software, and to permit persons to whom the Software is
9+
* furnished to do so, subject to the following conditions:
10+
*
11+
* The above copyright notice and this permission notice shall be included in
12+
* all copies or substantial portions of the Software.
13+
*
14+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19+
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
20+
* IN THE SOFTWARE.
21+
*/
22+
23+
#pragma once
24+
25+
#include "ggml-backend.h"
26+
#include "ggml.h"
27+
28+
#ifdef __cplusplus
29+
extern "C" {
30+
#endif
31+
32+
/**
33+
* @def GGML_CANN_NAME
34+
* @brief Define for the name of the CANN backend.
35+
*/
36+
#define GGML_CANN_NAME "CANN"
37+
38+
/**
39+
* @brief Maximum number of CANN devices supported.
40+
*/
41+
#define GGML_CANN_MAX_DEVICES 16
42+
43+
/**
44+
* @brief Structure for QK4_0 data format.
45+
*/
46+
#define QK4_0 32
47+
typedef struct {
48+
uint16_t d; /**< Delta */
49+
uint8_t qs[QK4_0 / 2]; /**< Nibbles / quants */
50+
} block_q4_0;
51+
52+
/**
53+
* @brief Structure for QK8_0 data format.
54+
*/
55+
#define QK8_0 32
56+
typedef struct {
57+
uint16_t d; /**< Delta */
58+
int8_t qs[QK8_0]; /**< Quants */
59+
} block_q8_0;
60+
61+
/**
62+
* @brief Initializes the CANN backend for a specified device.
63+
*
64+
* This function initializes the CANN backend for the given device.
65+
* It verifies the device index, allocates a context, and creates a backend
66+
* instance.
67+
*
68+
* @param device The index of the device to initialize.
69+
* @return A pointer to the initialized backend instance, or nullptr on failure.
70+
*/
71+
GGML_API GGML_CALL ggml_backend_t ggml_backend_cann_init(int32_t device);
72+
73+
/**
74+
* @brief Checks if a given backend is a CANN backend.
75+
*
76+
* This function verifies if the provided backend is a CANN backend by comparing
77+
* its GUID with the CANN backend's GUID.
78+
*
79+
* @param backend The backend instance to check.
80+
* @return True if the backend is a CANN backend, false otherwise.
81+
*/
82+
GGML_API GGML_CALL bool ggml_backend_is_cann(ggml_backend_t backend);
83+
84+
/**
85+
* @brief Retrieves the CANN buffer type for a specified device.
86+
*
87+
* This function initializes and returns the buffer type interface associated
88+
* with the given device. It ensures thread-safe access using a mutex.
89+
*
90+
* @param device The device index for which to retrieve the buffer type.
91+
* @return A pointer to the buffer type interface for the specified device, or
92+
* nullptr if the device index is out of range.
93+
*/
94+
GGML_API GGML_CALL ggml_backend_buffer_type_t
95+
ggml_backend_cann_buffer_type(int32_t device);
96+
97+
/**
98+
* @brief Retrieves the number of CANN devices available.
99+
*
100+
* This function returns the number of CANN devices available based on
101+
* information obtained from `ggml_cann_info()`.
102+
*
103+
* @return The number of CANN devices available.
104+
*/
105+
GGML_API GGML_CALL int32_t ggml_backend_cann_get_device_count(void);
106+
107+
/**
108+
* @brief Retrieves the description of a specific CANN device.
109+
*
110+
* This function sets the specified device, retrieves the SoC name,
111+
* and writes it into the provided description buffer.
112+
*
113+
* @param device The device index to retrieve the description for.
114+
* @param description Pointer to a buffer where the description will be written.
115+
* @param description_size Size of the description buffer.
116+
*/
117+
GGML_API GGML_CALL void ggml_backend_cann_get_device_description(
118+
int32_t device, char* description, size_t description_size);
119+
120+
/**
121+
* @brief Retrieves the memory information of a specific CANN device.
122+
*
123+
* This function sets the specified device, retrieves the free and total
124+
* memory information of the specified type (ACL_HBM_MEM), and stores them
125+
* in the provided pointers.
126+
*
127+
* @param device The device index to retrieve memory information for.
128+
* @param free Pointer to a variable where the free memory size will be stored.
129+
* @param total Pointer to a variable where the total memory size will be
130+
* stored.
131+
*/
132+
GGML_API GGML_CALL void ggml_backend_cann_get_device_memory(int32_t device,
133+
size_t* free,
134+
size_t* total);
135+
136+
/**
137+
* @brief Initializes resources required by the CANN backend.
138+
*/
139+
void ggml_cann_backend_init(void);
140+
141+
/**
142+
* @brief Frees resources used by the CANN backend.
143+
*/
144+
void ggml_cann_backend_free(void);
145+
146+
#ifdef __cplusplus
147+
}
148+
#endif

‎ggml/include/ggml.h

+3
Original file line numberDiff line numberDiff line change
@@ -753,6 +753,8 @@ extern "C" {
753753
GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1);
754754
GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
755755

756+
GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1);
757+
756758
// use this to compute the memory overhead of a tensor
757759
GGML_API size_t ggml_tensor_overhead(void);
758760

@@ -2397,6 +2399,7 @@ extern "C" {
23972399
GGML_API int ggml_cpu_has_rpc (void);
23982400
GGML_API int ggml_cpu_has_vsx (void);
23992401
GGML_API int ggml_cpu_has_matmul_int8(void);
2402+
GGML_API int ggml_cpu_has_cann (void);
24002403

24012404
//
24022405
// Internal types and functions exposed for tests and benchmarks

‎ggml/src/CMakeLists.txt

+69
Original file line numberDiff line numberDiff line change
@@ -766,6 +766,74 @@ if (GGML_CPU_HBM)
766766
target_link_libraries(ggml PUBLIC memkind)
767767
endif()
768768

769+
if (LLAMA_CANN)
770+
if ("cann${CANN_INSTALL_DIR}" STREQUAL "cann" AND DEFINED ENV{ASCEND_TOOLKIT_HOME})
771+
set(CANN_INSTALL_DIR $ENV{ASCEND_TOOLKIT_HOME})
772+
message(STATUS "CANN: updated CANN_INSTALL_DIR from ASCEND_TOOLKIT_HOME=$ENV{ASCEND_TOOLKIT_HOME}")
773+
endif()
774+
775+
if (CANN_INSTALL_DIR)
776+
# Only Support Linux.
777+
if (LLAMA_CANN)
778+
if (NOT UNIX)
779+
set(LLAMA_CANN OFF)
780+
message(WARNING "CANN: CANN toolkit supports unix but not ${CMAKE_SYSTEM_NAME}. Turning off LLAMA_CANN")
781+
endif()
782+
endif()
783+
784+
# Supported platforms: x86-64, arm64
785+
if (LLAMA_CANN)
786+
if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
787+
elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
788+
else()
789+
set(LLAMA_CANN OFF)
790+
message(WARNING "CANN: CANN toolkit supports x86-64 and arm64 but not ${CMAKE_SYSTEM_PROCESSOR}. Turning off LLAMA_CANN")
791+
endif()
792+
endif()
793+
794+
# Set header and libs
795+
if(LLAMA_CANN)
796+
set(CANN_INCLUDE_DIRS
797+
${CANN_INSTALL_DIR}/include
798+
${CANN_INSTALL_DIR}/include/aclnn
799+
${CANN_INSTALL_DIR}/acllib/include
800+
)
801+
802+
# TODO: find libs
803+
link_directories(
804+
${CANN_INSTALL_DIR}/lib64
805+
)
806+
807+
add_subdirectory(ggml-cann/kernels)
808+
list(APPEND CANN_LIBRARIES
809+
ascendcl
810+
nnopbase
811+
opapi
812+
acl_op_compiler
813+
ascendc_kernels
814+
)
815+
816+
set(GGML_HEADERS_CANN "../include/ggml-cann.h")
817+
file(GLOB GGML_SOURCES_CANN "ggml-cann/*.cpp")
818+
list(APPEND GGML_SOURCES_CANN "ggml-cann.cpp")
819+
820+
message(STATUS "CANN: CANN_INCLUDE_DIRS = ${CANN_INCLUDE_DIRS}")
821+
message(STATUS "CANN: CANN_LIBRARIES = ${CANN_LIBRARIES}")
822+
823+
set(GGML_EXTRA_LIBS ${GGML_EXTRA_LIBS} ${CANN_LIBRARIES} )
824+
set(GGML_EXTRA_INCLUDES ${GGML_EXTRA_INCLUDES} ${CANN_INCLUDE_DIRS})
825+
list(APPEND GGML_CDEF_PUBLIC GGML_USE_CANN)
826+
endif()
827+
else()
828+
set(LLAMA_CANN OFF)
829+
message(WARNING "CANN: Can't find CANN_INSTALL_DIR, do you forget to source set_var.sh. Turning off LLAMA_CANN")
830+
endif()
831+
832+
if(NOT LLAMA_CANN)
833+
message(WARNING "CANN: LLAMA_CANN is turned OFF, see above for details.")
834+
endif()
835+
endif()
836+
769837
function(get_flags CCID CCVER)
770838
set(C_FLAGS "")
771839
set(CXX_FLAGS "")
@@ -1180,6 +1248,7 @@ add_library(ggml
11801248
${GGML_SOURCES_ROCM} ${GGML_HEADERS_ROCM}
11811249
${GGML_SOURCES_BLAS} ${GGML_HEADERS_BLAS}
11821250
${GGML_SOURCES_LLAMAFILE} ${GGML_HEADERS_LLAMAFILE}
1251+
${GGML_SOURCES_CANN} ${GGML_HEADERS_CANN}
11831252
ggml-aarch64.c ggml-aarch64.h
11841253
)
11851254

‎ggml/src/ggml-backend.c

+5
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,11 @@ GGML_CALL static void ggml_backend_registry_init(void) {
445445
extern GGML_CALL void ggml_backend_kompute_reg_devices(void);
446446
ggml_backend_kompute_reg_devices();
447447
#endif
448+
449+
#ifdef GGML_USE_CANN
450+
extern GGML_CALL int ggml_backend_cann_reg_devices(void);
451+
ggml_backend_cann_reg_devices();
452+
#endif
448453
}
449454

450455
GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) {

0 commit comments

Comments
 (0)