From 6ada023b889f754e7a43defeaa8d2466a581a141 Mon Sep 17 00:00:00 2001 From: zhushuang Date: Wed, 8 Jul 2026 02:12:52 +0000 Subject: [PATCH 01/10] issue/1311 - feat: refactor InfiniCore cpu and cuda-like runtime to InfiniRT --- include/infiniop/handle.h | 2 + src/bridge/infini/rt.hpp | 49 +++++++++++++++ .../allocators/device_pinned_allocator.cc | 8 +-- .../context/allocators/host_allocator.cc | 2 - .../allocators/pinnable_block_allocator.cc | 13 ++-- .../allocators/stream_ordered_allocator.cc | 14 ++++- src/infinicore/context/context_impl.cc | 37 ++++++------ src/infinicore/context/runtime/runtime.cc | 54 +++++++++-------- src/infinicore/graph/graph.cc | 19 ++++-- src/infiniop/devices/handle.cc | 23 +++++-- test/infiniop/libinfiniop/liboperators.py | 24 ++++---- test/infiniop/libinfiniop/utils.py | 2 +- xmake.lua | 60 +++---------------- xmake/cpu.lua | 1 + xmake/nvidia.lua | 2 +- xmake/test.lua | 10 +++- 16 files changed, 188 insertions(+), 132 deletions(-) create mode 100644 src/bridge/infini/rt.hpp diff --git a/include/infiniop/handle.h b/include/infiniop/handle.h index da8689542..207ca2ce9 100644 --- a/include/infiniop/handle.h +++ b/include/infiniop/handle.h @@ -7,6 +7,8 @@ struct InfiniopHandle; typedef struct InfiniopHandle *infiniopHandle_t; +__INFINI_C __export infiniStatus_t infiniopSetRuntimeDevice(infiniDevice_t device, int device_id); + __INFINI_C __export infiniStatus_t infiniopCreateHandle(infiniopHandle_t *handle_ptr); __INFINI_C __export infiniStatus_t infiniopDestroyHandle(infiniopHandle_t handle); diff --git a/src/bridge/infini/rt.hpp b/src/bridge/infini/rt.hpp new file mode 100644 index 000000000..d53cc49ac --- /dev/null +++ b/src/bridge/infini/rt.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include "infinicore.h" +#include "infinirt.h" + +#include + +namespace infinicore::bridge::infini::rt { + +inline infiniStatus_t translate(::infini::rt::runtime::Error error) { + switch (error) { + case ::infini::rt::runtime::kSuccess: + return INFINI_STATUS_SUCCESS; + default: + return INFINI_STATUS_INTERNAL_ERROR; + } +} + +inline ::infini::rt::Device::Type translate(infiniDevice_t device) { + switch (device) { + case INFINI_DEVICE_CPU: + return ::infini::rt::Device::Type::kCpu; + case INFINI_DEVICE_NVIDIA: + return ::infini::rt::Device::Type::kNvidia; + default: + return ::infini::rt::Device::Type::kCount; + } +} + +inline infiniDevice_t translate(::infini::rt::Device::Type device) { + switch (device) { + case ::infini::rt::Device::Type::kCpu: + return INFINI_DEVICE_CPU; + case ::infini::rt::Device::Type::kNvidia: + return INFINI_DEVICE_NVIDIA; + default: + return INFINI_DEVICE_TYPE_COUNT; + } +} + +inline ::infini::rt::runtime::Stream translate(infinirtStream_t stream) { + return reinterpret_cast<::infini::rt::runtime::Stream>(stream); +} + +inline ::infini::rt::runtime::Stream *translate(infinirtStream_t *stream) { + return reinterpret_cast<::infini::rt::runtime::Stream *>(stream); +} + +} // namespace infinicore::bridge::infini::rt diff --git a/src/infinicore/context/allocators/device_pinned_allocator.cc b/src/infinicore/context/allocators/device_pinned_allocator.cc index 36ee599ff..a8764d5f6 100644 --- a/src/infinicore/context/allocators/device_pinned_allocator.cc +++ b/src/infinicore/context/allocators/device_pinned_allocator.cc @@ -1,6 +1,6 @@ #include "device_pinned_allocator.hpp" -#include +#include "../../../bridge/infini/rt.hpp" #include "../../utils.hpp" @@ -16,7 +16,7 @@ std::byte *DevicePinnedHostAllocator::allocate(size_t size) { return nullptr; } void *ptr; - INFINICORE_CHECK_ERROR(infinirtMallocHost(&ptr, size)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MallocHost(&ptr, size))); return (std::byte *)ptr; } @@ -25,7 +25,7 @@ void DevicePinnedHostAllocator::deallocate(std::byte *ptr) { return; } if (owner_ == context::getDevice()) { - INFINICORE_CHECK_ERROR(infinirtFreeHost(ptr)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::FreeHost(ptr))); gc(); } else { gc_queue_.push(ptr); @@ -35,7 +35,7 @@ void DevicePinnedHostAllocator::deallocate(std::byte *ptr) { void DevicePinnedHostAllocator::gc() { while (gc_queue_.empty() == false) { std::byte *p = gc_queue_.front(); - INFINICORE_CHECK_ERROR(infinirtFreeHost(p)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::FreeHost(p))); gc_queue_.pop(); } } diff --git a/src/infinicore/context/allocators/host_allocator.cc b/src/infinicore/context/allocators/host_allocator.cc index 41a55b453..0f1cc7072 100644 --- a/src/infinicore/context/allocators/host_allocator.cc +++ b/src/infinicore/context/allocators/host_allocator.cc @@ -1,7 +1,5 @@ #include "host_allocator.hpp" -#include - namespace infinicore { std::byte *HostAllocator::allocate(size_t size) { if (size == 0) { diff --git a/src/infinicore/context/allocators/pinnable_block_allocator.cc b/src/infinicore/context/allocators/pinnable_block_allocator.cc index 32e5c5e9b..3c70376f0 100644 --- a/src/infinicore/context/allocators/pinnable_block_allocator.cc +++ b/src/infinicore/context/allocators/pinnable_block_allocator.cc @@ -4,8 +4,9 @@ #include "../../utils.hpp" +#include "../../../bridge/infini/rt.hpp" + #include -#include #include namespace infinicore { @@ -74,7 +75,7 @@ std::byte *PinnableBlockAllocator::allocate(size_t size) { block->in_use = true; block->use_count = 1; - INFINICORE_CHECK_ERROR(infinirtMalloc(&block->ptr, block->size)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Malloc(&block->ptr, block->size))); all_blocks_[block->ptr] = block; return reinterpret_cast(block->ptr); @@ -101,7 +102,7 @@ std::byte *PinnableBlockAllocator::allocate(size_t size) { block->in_use = true; block->use_count = 1; - INFINICORE_CHECK_ERROR(infinirtMalloc(&block->ptr, block->size)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Malloc(&block->ptr, block->size))); large_blocks_.push_back(block); all_blocks_[block->ptr] = block; @@ -167,7 +168,7 @@ void PinnableBlockAllocator::trim() { for (auto &cls : size_classes_) { for (auto it = cls.free_blocks.begin(); it != cls.free_blocks.end();) { if (!(*it)->frozen) { - INFINICORE_CHECK_ERROR(infinirtFree((*it)->ptr)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Free((*it)->ptr))); all_blocks_.erase((*it)->ptr); it = cls.free_blocks.erase(it); } else { @@ -178,7 +179,7 @@ void PinnableBlockAllocator::trim() { // Free non-frozen large blocks for (auto it = large_blocks_.begin(); it != large_blocks_.end();) { if (!(*it)->frozen && !(*it)->in_use) { - INFINICORE_CHECK_ERROR(infinirtFree((*it)->ptr)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Free((*it)->ptr))); all_blocks_.erase((*it)->ptr); it = large_blocks_.erase(it); } else { @@ -192,7 +193,7 @@ PinnableBlockAllocator::~PinnableBlockAllocator() { std::lock_guard lock(mutex_); for (auto &p : all_blocks_) { if (p.second->ptr) { - infinirtFree(p.second->ptr); + (void)infini::rt::runtime::Free(p.second->ptr); } } all_blocks_.clear(); diff --git a/src/infinicore/context/allocators/stream_ordered_allocator.cc b/src/infinicore/context/allocators/stream_ordered_allocator.cc index 3bd0aaab1..00df7fb47 100644 --- a/src/infinicore/context/allocators/stream_ordered_allocator.cc +++ b/src/infinicore/context/allocators/stream_ordered_allocator.cc @@ -1,6 +1,6 @@ #include "stream_ordered_allocator.hpp" -#include +#include "../../../bridge/infini/rt.hpp" #include "../../utils.hpp" @@ -12,7 +12,11 @@ std::byte *StreamOrderedAllocator::allocate(size_t size) { return nullptr; } void *ptr = nullptr; - INFINICORE_CHECK_ERROR(infinirtMallocAsync(&ptr, size, context::getStream())); + if (device_.getType() != Device::Type::CPU) { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MallocAsync(&ptr, size, bridge::infini::rt::translate(context::getStream())))); + } else { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Malloc(&ptr, size))); + } return (std::byte *)ptr; } @@ -20,6 +24,10 @@ void StreamOrderedAllocator::deallocate(std::byte *ptr) { if (ptr == nullptr) { return; } - INFINICORE_CHECK_ERROR(infinirtFreeAsync(ptr, context::getStream())); + if (device_.getType() != Device::Type::CPU) { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::FreeAsync(ptr, bridge::infini::rt::translate(context::getStream())))); + } else { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Free(ptr))); + } } } // namespace infinicore diff --git a/src/infinicore/context/context_impl.cc b/src/infinicore/context/context_impl.cc index f126beb1d..06c85955c 100644 --- a/src/infinicore/context/context_impl.cc +++ b/src/infinicore/context/context_impl.cc @@ -1,4 +1,5 @@ #include "context_impl.hpp" +#include "../../bridge/infini/rt.hpp" #include "internal.hpp" #include "../utils.hpp" @@ -61,26 +62,28 @@ ContextImpl &ContextImpl::singleton() { } ContextImpl::ContextImpl() { - std::vector device_counter(static_cast(Device::Type::COUNT)); - INFINICORE_CHECK_ERROR(infinirtGetAllDeviceCount(device_counter.data())); - - // Reserve runtime slot for all devices. - runtime_table_[0].resize(device_counter[0]); - runtime_table_[0][0] = std::unique_ptr(new Runtime(Device(Device::Type::CPU, 0))); - - // Context will try to use the first non-cpu available device as the default runtime. - for (int i = int(Device::Type::COUNT) - 1; i > 0; i--) { - if (device_counter[i] > 0) { - runtime_table_[i].resize(device_counter[i]); - if (current_runtime_ == nullptr) { - runtime_table_[i][0] = std::unique_ptr(new Runtime(Device(Device::Type(i), 0))); - current_runtime_ = runtime_table_[i][0].get(); - } + std::vector device_counter(static_cast(Device::Type::COUNT), 0); + + infini::rt::set_runtime_device_type(bridge::infini::rt::translate(INFINI_DEVICE_CPU)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::GetDeviceCount(&device_counter[static_cast(Device::Type::CPU)]))); + + runtime_table_[static_cast(Device::Type::CPU)].resize(device_counter[static_cast(Device::Type::CPU)]); + if (device_counter[static_cast(Device::Type::CPU)] > 0) { + runtime_table_[static_cast(Device::Type::CPU)][0] = std::unique_ptr(new Runtime(Device(Device::Type::CPU, 0))); + } + + if constexpr (infini::rt::DeviceEnabled::value) { + infini::rt::set_runtime_device_type(bridge::infini::rt::translate(INFINI_DEVICE_NVIDIA)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::GetDeviceCount(&device_counter[static_cast(Device::Type::NVIDIA)]))); + runtime_table_[static_cast(Device::Type::NVIDIA)].resize(device_counter[static_cast(Device::Type::NVIDIA)]); + if (device_counter[static_cast(Device::Type::NVIDIA)] > 0) { + runtime_table_[static_cast(Device::Type::NVIDIA)][0] = std::unique_ptr(new Runtime(Device(Device::Type::NVIDIA, 0))); + current_runtime_ = runtime_table_[static_cast(Device::Type::NVIDIA)][0].get(); } } - if (current_runtime_ == nullptr) { - current_runtime_ = runtime_table_[0][0].get(); + if (current_runtime_ == nullptr && !runtime_table_[static_cast(Device::Type::CPU)].empty()) { + current_runtime_ = runtime_table_[static_cast(Device::Type::CPU)][0].get(); } } diff --git a/src/infinicore/context/runtime/runtime.cc b/src/infinicore/context/runtime/runtime.cc index 92ba0f737..783b018c2 100644 --- a/src/infinicore/context/runtime/runtime.cc +++ b/src/infinicore/context/runtime/runtime.cc @@ -1,4 +1,5 @@ #include "runtime.hpp" +#include "../../../bridge/infini/rt.hpp" #include "../../utils.hpp" @@ -10,7 +11,7 @@ namespace infinicore { Runtime::Runtime(Device device) : device_(device), graph_manager_(std::make_unique()) { activate(); - INFINICORE_CHECK_ERROR(infinirtStreamCreate(&stream_)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamCreate(bridge::infini::rt::translate(&stream_)))); INFINICORE_CHECK_ERROR(infiniopCreateHandle(&infiniop_handle_)); if (device_.getType() == Device::Type::CPU) { device_memory_allocator_ = std::make_unique(device); @@ -26,11 +27,14 @@ Runtime::~Runtime() { } device_memory_allocator_.reset(); infiniopDestroyHandle(infiniop_handle_); - infinirtStreamDestroy(stream_); + (void)infini::rt::runtime::StreamDestroy(bridge::infini::rt::translate(stream_)); } Runtime *Runtime::activate() { - INFINICORE_CHECK_ERROR(infinirtSetDevice((infiniDevice_t)device_.getType(), (int)device_.getIndex())); + auto rt_device = bridge::infini::rt::translate(static_cast(device_.getType())); + INFINICORE_ASSERT(rt_device != infini::rt::Device::Type::kCount); + infini::rt::set_runtime_device_type(rt_device); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::SetDevice(static_cast(device_.getIndex())))); return this; } @@ -47,11 +51,11 @@ infiniopHandle_t Runtime::infiniopHandle() const { } void Runtime::syncStream() { - INFINICORE_CHECK_ERROR(infinirtStreamSynchronize(stream_)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamSynchronize(bridge::infini::rt::translate(stream_)))); } void Runtime::syncDevice() { - INFINICORE_CHECK_ERROR(infinirtDeviceSynchronize()); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::DeviceSynchronize())); } void Runtime::trimMemory() { @@ -103,43 +107,47 @@ std::shared_ptr Runtime::reinstantiateBlob(std::shared_ptr blob) } void Runtime::memcpyH2D(void *dst, const void *src, size_t size, bool async) { - if (async) { - INFINICORE_CHECK_ERROR(infinirtMemcpyAsync(dst, src, size, INFINIRT_MEMCPY_H2D, stream_)); + if (async && device_.getType() != Device::Type::CPU) { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice, bridge::infini::rt::translate(stream_)))); } else { - INFINICORE_CHECK_ERROR(infinirtMemcpy(dst, src, size, INFINIRT_MEMCPY_H2D)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memcpy(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice))); } } void Runtime::memcpyD2H(void *dst, const void *src, size_t size) { - INFINICORE_CHECK_ERROR(infinirtMemcpy(dst, src, size, INFINIRT_MEMCPY_D2H)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memcpy(dst, src, size, infini::rt::runtime::kMemcpyDeviceToHost))); } void Runtime::memcpyD2D(void *dst, const void *src, size_t size, bool async) { - if (async) { - INFINICORE_CHECK_ERROR(infinirtMemcpyAsync(dst, src, size, INFINIRT_MEMCPY_D2D, stream_)); + if (async && device_.getType() != Device::Type::CPU) { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice, bridge::infini::rt::translate(stream_)))); } else { - INFINICORE_CHECK_ERROR(infinirtMemcpy(dst, src, size, INFINIRT_MEMCPY_D2D)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memcpy(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice))); } } void Runtime::setDeviceMemory(void *ptr, int value, size_t count) { - INFINICORE_CHECK_ERROR(infinirtMemset(ptr, value, count)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memset(ptr, value, count))); } void Runtime::setDeviceMemoryAsync(void *ptr, int value, size_t count, infinirtStream_t stream) { - INFINICORE_CHECK_ERROR(infinirtMemsetAsync(ptr, value, count, stream)); + if (device_.getType() != Device::Type::CPU) { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemsetAsync(ptr, value, count, bridge::infini::rt::translate(stream)))); + } else { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memset(ptr, value, count))); + } } // Timing method implementations infinirtEvent_t Runtime::createEvent() { infinirtEvent_t event; - INFINICORE_CHECK_ERROR(infinirtEventCreate(&event)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventCreate(&event))); return event; } infinirtEvent_t Runtime::createEventWithFlags(uint32_t flags) { infinirtEvent_t event; - INFINICORE_CHECK_ERROR(infinirtEventCreateWithFlags(&event, flags)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventCreateWithFlags(&event, flags))); return event; } @@ -147,26 +155,24 @@ void Runtime::recordEvent(infinirtEvent_t event, infinirtStream_t stream) { if (stream == nullptr) { stream = stream_; } - INFINICORE_CHECK_ERROR(infinirtEventRecord(event, stream)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventRecord(event, bridge::infini::rt::translate(stream)))); } bool Runtime::queryEvent(infinirtEvent_t event) { - infinirtEventStatus_t status; - INFINICORE_CHECK_ERROR(infinirtEventQuery(event, &status)); - return status == INFINIRT_EVENT_COMPLETE; + return bridge::infini::rt::translate(infini::rt::runtime::EventQuery(event)) == INFINI_STATUS_SUCCESS; } void Runtime::synchronizeEvent(infinirtEvent_t event) { - INFINICORE_CHECK_ERROR(infinirtEventSynchronize(event)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventSynchronize(event))); } void Runtime::destroyEvent(infinirtEvent_t event) { - INFINICORE_CHECK_ERROR(infinirtEventDestroy(event)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventDestroy(event))); } float Runtime::elapsedTime(infinirtEvent_t start, infinirtEvent_t end) { float ms; - INFINICORE_CHECK_ERROR(infinirtEventElapsedTime(&ms, start, end)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventElapsedTime(&ms, start, end))); return ms; } @@ -175,7 +181,7 @@ void Runtime::streamWaitEvent(infinirtStream_t stream, infinirtEvent_t event) { if (stream == nullptr) { stream = stream_; } - INFINICORE_CHECK_ERROR(infinirtStreamWaitEvent(stream, event)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamWaitEvent(bridge::infini::rt::translate(stream), event, 0))); } bool Runtime::isGraphRecording() const { diff --git a/src/infinicore/graph/graph.cc b/src/infinicore/graph/graph.cc index 3b8fc57e5..3511b10a1 100644 --- a/src/infinicore/graph/graph.cc +++ b/src/infinicore/graph/graph.cc @@ -2,7 +2,10 @@ #include "../utils.hpp" #include "infinicore/context/context.hpp" + +#ifdef USE_INFINIRT_GRAPH #include +#endif namespace infinicore::graph { @@ -31,6 +34,7 @@ DispatchableGraphOperator::~DispatchableGraphOperator() { * Graph * ========================= */ +#ifdef USE_INFINIRT_GRAPH struct Graph::DeviceGraph { infinirtGraph_t graph; infinirtGraphExec_t exec; @@ -54,17 +58,22 @@ struct Graph::DeviceGraph { INFINICORE_CHECK_ERROR(infinirtGraphLuanch(exec, context::getStream())); } }; +#else +struct Graph::DeviceGraph {}; +#endif Graph::Graph() { } void Graph::run() const { +#ifdef USE_INFINIRT_GRAPH if (device_graph_ != nullptr && device_graph_.get()->exec != nullptr) { device_graph_.get()->launch(); - } else { - for (auto &op : op_list_) { - op->run(); - } + return; + } +#endif + for (auto &op : op_list_) { + op->run(); } } @@ -73,6 +82,7 @@ void Graph::add_operator(std::shared_ptr op) { } void Graph::instantiate() { +#ifdef USE_INFINIRT_GRAPH // Reset device graph device_graph_ = std::make_unique(); @@ -112,6 +122,7 @@ void Graph::instantiate() { spdlog::warn("Fail to instantiate device graph: {}", std::string(device_graph_.get()->log_buffer.data())); } } +#endif } Graph::~Graph() = default; diff --git a/src/infiniop/devices/handle.cc b/src/infiniop/devices/handle.cc index b36d3b6bf..9ded0587b 100644 --- a/src/infiniop/devices/handle.cc +++ b/src/infiniop/devices/handle.cc @@ -1,6 +1,6 @@ #include "infiniop/handle.h" +#include "../../bridge/infini/rt.hpp" #include "../../utils.h" -#include "infinirt.h" #ifdef ENABLE_CPU_API #include "cpu/cpu_handle.h" @@ -24,14 +24,29 @@ #include "metax/metax_handle.h" #endif +__INFINI_C infiniStatus_t infiniopSetRuntimeDevice(infiniDevice_t device, int device_id) { + auto rt_device = infinicore::bridge::infini::rt::translate(device); + if (rt_device == infini::rt::Device::Type::kCount) { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } + + infini::rt::set_runtime_device_type(rt_device); + CHECK_STATUS(infinicore::bridge::infini::rt::translate(infini::rt::runtime::SetDevice(device_id))); + return INFINI_STATUS_SUCCESS; +} + __INFINI_C infiniStatus_t infiniopCreateHandle(infiniopHandle_t *handle_ptr) { if (handle_ptr == nullptr) { return INFINI_STATUS_NULL_POINTER; } - infiniDevice_t device; - int device_id; - CHECK_STATUS(infinirtGetDevice(&device, &device_id)); + infiniDevice_t device = infinicore::bridge::infini::rt::translate(infini::rt::runtime_device_type()); + if (device == INFINI_DEVICE_TYPE_COUNT) { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } + + int device_id = 0; + CHECK_STATUS(infinicore::bridge::infini::rt::translate(infini::rt::runtime::GetDevice(&device_id))); #define CREATE(CASE, NAMESPACE) \ case CASE: \ diff --git a/test/infiniop/libinfiniop/liboperators.py b/test/infiniop/libinfiniop/liboperators.py index ceffd5141..48ce04fe4 100644 --- a/test/infiniop/libinfiniop/liboperators.py +++ b/test/infiniop/libinfiniop/liboperators.py @@ -1,11 +1,12 @@ +import ctypes import os import platform -import ctypes -from ctypes import c_int, c_int64, c_uint64, POINTER +from ctypes import POINTER, c_int, c_int64, c_uint64 +from pathlib import Path + from .datatypes import * from .devices import * from .op_register import OpRegister -from pathlib import Path from .structs import * INFINI_ROOT = os.getenv("INFINI_ROOT") or str(Path.home() / ".infini") @@ -45,12 +46,12 @@ def find_library_in_ld_path(subdir, library_name): libop_path = find_library_in_ld_path("lib", "libinfiniop.so") librt_path = find_library_in_ld_path("lib", "libinfinirt.so") - assert ( - libop_path is not None - ), f"Cannot find infiniop.dll or libinfiniop.so. Check if INFINI_ROOT is set correctly." - assert ( - librt_path is not None - ), f"Cannot find infinirt.dll or libinfinirt.so. Check if INFINI_ROOT is set correctly." + assert libop_path is not None, ( + "Cannot find infiniop.dll or libinfiniop.so. Check if INFINI_ROOT is set correctly." + ) + assert librt_path is not None, ( + "Cannot find infinirt.dll or libinfinirt.so. Check if INFINI_ROOT is set correctly." + ) librt = ctypes.CDLL(librt_path) libop = ctypes.CDLL(libop_path) @@ -65,13 +66,12 @@ def find_library_in_ld_path(subdir, library_name): lib.infiniopCreateTensorDescriptor.restype = c_int lib.infiniopDestroyTensorDescriptor.argtypes = [infiniopTensorDescriptor_t] lib.infiniopDestroyTensorDescriptor.restype = c_int + lib.infiniopSetRuntimeDevice.argtypes = [c_int, c_int] + lib.infiniopSetRuntimeDevice.restype = c_int lib.infiniopCreateHandle.argtypes = [POINTER(infiniopHandle_t)] lib.infiniopCreateHandle.restype = c_int lib.infiniopDestroyHandle.argtypes = [infiniopHandle_t] lib.infiniopDestroyHandle.restype = c_int - lib.infinirtSetDevice.argtypes = [c_int, c_int] - lib.infinirtSetDevice.restype = c_int - OpRegister.register_lib(lib) return lib diff --git a/test/infiniop/libinfiniop/utils.py b/test/infiniop/libinfiniop/utils.py index c0560dbde..690425b54 100644 --- a/test/infiniop/libinfiniop/utils.py +++ b/test/infiniop/libinfiniop/utils.py @@ -730,7 +730,7 @@ def test_operator(device, test_func, test_cases, tensor_dtypes): to be passed to `test_func`. - tensor_dtypes (list): A list of tensor data types (e.g., `torch.float32`) to test. """ - LIBINFINIOP.infinirtSetDevice(device, ctypes.c_int(0)) + check_error(LIBINFINIOP.infiniopSetRuntimeDevice(device, ctypes.c_int(0))) handle = create_handle() tensor_dtypes = filter_tensor_dtypes_by_device(device, tensor_dtypes) try: diff --git a/xmake.lua b/xmake.lua index fcbcf423d..d98fe6e01 100644 --- a/xmake.lua +++ b/xmake.lua @@ -461,57 +461,13 @@ target("infini-utils") add_files("src/utils/*.cc") target_end() -target("infinirt") - set_kind("shared") - - if has_config("cpu") then - add_deps("infinirt-cpu") - end - if has_config("nv-gpu") then - add_deps("infinirt-nvidia") - end - if has_config("cambricon-mlu") then - add_deps("infinirt-cambricon") - end - if has_config("ascend-npu") then - add_deps("infinirt-ascend") - end - if has_config("metax-gpu") then - add_deps("infinirt-metax") - end - if has_config("moore-gpu") then - add_deps("infinirt-moore") - end - if has_config("iluvatar-gpu") then - add_deps("infinirt-iluvatar") - end - if has_config("ali-ppu") then - add_deps("infinirt-ali") - end - if has_config("qy-gpu") then - add_deps("infinirt-qy") - add_files("build/.objs/infinirt-qy/rules/qy.cuda/src/infinirt/cuda/*.cu.o", {public = true}) - end - if has_config("kunlun-xpu") then - add_deps("infinirt-kunlun") - end - if has_config("hygon-dcu") then - add_deps("infinirt-hygon") - end - set_languages("cxx17") - if not is_plat("windows") then - add_cxflags("-fPIC") - add_cxxflags("-fPIC") - add_ldflags("-fPIC", {force = true}) - end - set_installdir(os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini")) - add_files("src/infinirt/*.cc") - add_installfiles("include/infinirt.h", {prefixdir = "include"}) -target_end() - target("infiniop") set_kind("shared") - add_deps("infinirt") + local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") + add_includedirs(INFINI_ROOT.."/include", { public = true }) + add_linkdirs(INFINI_ROOT.."/lib") + add_links("infinirt") + add_rpathdirs(INFINI_ROOT.."/lib") local public_cuda_root = get_config("cuda") or os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") if public_cuda_root and public_cuda_root ~= "" then @@ -589,7 +545,6 @@ target_end() target("infiniccl") set_kind("shared") - add_deps("infinirt") if has_config("nv-gpu") then add_deps("infiniccl-nvidia") @@ -635,7 +590,7 @@ target_end() target("infinicore_c_api") set_kind("phony") - add_deps("infiniop", "infinirt", "infiniccl") + add_deps("infiniop", "infiniccl") after_build(function (target) print(YELLOW .. "[Congratulations!] Now you can install the libraries with \"xmake install\"" .. NC) end) target_end() @@ -650,7 +605,7 @@ target_end() target("infinicore_cpp_api") set_kind("shared") - add_deps("infiniop", "infinirt", "infiniccl") + add_deps("infiniop", "infiniccl") set_languages("cxx17") set_symbols("visibility") @@ -715,6 +670,7 @@ target("infinicore_cpp_api") add_linkdirs(INFINI_ROOT.."/lib") add_links("infiniop", "infinirt", "infiniccl") + add_rpathdirs(INFINI_ROOT.."/lib") if get_config("flash-attn") and get_config("flash-attn") ~= "" then add_installfiles("(builddir)/$(plat)/$(arch)/$(mode)/flash-attn*.so", {prefixdir = "lib"}) diff --git a/xmake/cpu.lua b/xmake/cpu.lua index e192fbbbd..a1974b823 100644 --- a/xmake/cpu.lua +++ b/xmake/cpu.lua @@ -26,6 +26,7 @@ target_end() target("infinirt-cpu") set_kind("static") + set_default(false) add_deps("infini-utils") on_install(function (target) end) diff --git a/xmake/nvidia.lua b/xmake/nvidia.lua index 282f5876f..6fbe90dfa 100644 --- a/xmake/nvidia.lua +++ b/xmake/nvidia.lua @@ -223,6 +223,7 @@ target_end() target("infinirt-nvidia") set_kind("static") + set_default(false) add_deps("infini-utils") on_install(function (target) end) @@ -246,7 +247,6 @@ target_end() target("infiniccl-nvidia") set_kind("static") - add_deps("infinirt") on_install(function (target) end) if has_config("ccl") then set_policy("build.cuda.devlink", true) diff --git a/xmake/test.lua b/xmake/test.lua index 64533163b..d4ef4abe7 100644 --- a/xmake/test.lua +++ b/xmake/test.lua @@ -54,14 +54,20 @@ target_end() target("infinirt-test") set_kind("binary") - add_deps("infinirt") + set_default(false) on_install(function (target) end) set_languages("cxx17") set_warnings("all", "error") + local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") + add_includedirs(INFINI_ROOT.."/include") + add_linkdirs(INFINI_ROOT.."/lib") + add_links("infinirt") + add_rpathdirs(INFINI_ROOT.."/lib") + add_files(os.projectdir().."/src/infinirt-test/*.cc") - set_installdir(os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini")) + set_installdir(INFINI_ROOT) target_end() target("infinicore-test") From 0e8698aa17d4d22700695d871a4a789dff327f91 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Wed, 24 Jun 2026 09:06:39 +0000 Subject: [PATCH 02/10] feat: add standalone infinirt graph bridge Route graph capture and replay through a dlopen-based standalone InfiniRT bridge when enabled. Package the standalone InfiniRT runtime as libinfiniops_infinirt.so and ensure libinfiniops.so records that private dependency. --- src/infinicore/graph/graph.cc | 83 +++-- .../graph/standalone_infinirt_graph_bridge.cc | 287 ++++++++++++++++++ .../standalone_infinirt_graph_bridge.hpp | 28 ++ xmake.lua | 93 +++--- 4 files changed, 431 insertions(+), 60 deletions(-) create mode 100644 src/infinicore/graph/standalone_infinirt_graph_bridge.cc create mode 100644 src/infinicore/graph/standalone_infinirt_graph_bridge.hpp diff --git a/src/infinicore/graph/graph.cc b/src/infinicore/graph/graph.cc index 3b8fc57e5..71638e147 100644 --- a/src/infinicore/graph/graph.cc +++ b/src/infinicore/graph/graph.cc @@ -2,6 +2,7 @@ #include "../utils.hpp" #include "infinicore/context/context.hpp" +#include "standalone_infinirt_graph_bridge.hpp" #include namespace infinicore::graph { @@ -32,9 +33,11 @@ DispatchableGraphOperator::~DispatchableGraphOperator() { * ========================= */ struct Graph::DeviceGraph { - infinirtGraph_t graph; - infinirtGraphExec_t exec; - infinirtGraphNode_t node; + infinirtGraph_t graph = nullptr; + infinirtGraphExec_t exec = nullptr; + infinirtGraphNode_t node = nullptr; + infinirtStream_t stream = nullptr; + bool standalone = false; std::vector log_buffer; DeviceGraph() { @@ -43,15 +46,30 @@ struct Graph::DeviceGraph { ~DeviceGraph() { if (exec) { - infinirtGraphExecDestroy(exec); + if (standalone) { + standalone_infinirt::graph_exec_destroy(exec); + } else { + infinirtGraphExecDestroy(exec); + } } if (graph) { - infinirtGraphDestroy(graph); + if (standalone) { + standalone_infinirt::graph_destroy(graph); + } else { + infinirtGraphDestroy(graph); + } + } + if (standalone && stream) { + standalone_infinirt::destroy_wrapped_stream(stream); } } void launch() { - INFINICORE_CHECK_ERROR(infinirtGraphLuanch(exec, context::getStream())); + if (standalone) { + INFINICORE_CHECK_ERROR(standalone_infinirt::graph_launch(exec, stream)); + } else { + INFINICORE_CHECK_ERROR(infinirtGraphLuanch(exec, context::getStream())); + } } }; @@ -75,6 +93,22 @@ void Graph::add_operator(std::shared_ptr op) { void Graph::instantiate() { // Reset device graph device_graph_ = std::make_unique(); + device_graph_->standalone = standalone_infinirt::available(); + device_graph_->stream = device_graph_->standalone + ? standalone_infinirt::wrap_stream(context::getDevice(), context::getStream()) + : context::getStream(); + if (device_graph_->standalone && device_graph_->stream == nullptr) { + spdlog::warn("Standalone InfiniRT graph bridge is enabled but failed to wrap the current stream. Falling back to eager execution."); + device_graph_.reset(); + return; + } + if (device_graph_->standalone) { + static bool logged_once = false; + if (!logged_once) { + logged_once = true; + spdlog::info("Using standalone InfiniRT graph bridge for graph capture and replay."); + } + } // warmup for (size_t iter = 0; iter < 5; ++iter) { @@ -82,35 +116,42 @@ void Graph::instantiate() { } infinicore::context::syncStream(); - if (infinirtStreamBeginCapture( - context::getStream(), - INFINIRT_STREAM_CAPTURE_MODE_RELAXED) - != INFINI_STATUS_SUCCESS) { + auto begin_status = device_graph_->standalone + ? standalone_infinirt::stream_begin_capture(device_graph_->stream, INFINIRT_STREAM_CAPTURE_MODE_RELAXED) + : infinirtStreamBeginCapture(context::getStream(), INFINIRT_STREAM_CAPTURE_MODE_RELAXED); + if (begin_status != INFINI_STATUS_SUCCESS) { + spdlog::warn("Fail to begin device graph capture."); + device_graph_.reset(); return; } // Run and record this->run(); - if (infinirtStreamEndCapture( - context::getStream(), - &device_graph_.get()->graph) - != INFINI_STATUS_SUCCESS) { + auto end_status = device_graph_->standalone + ? standalone_infinirt::stream_end_capture(device_graph_->stream, &device_graph_.get()->graph) + : infinirtStreamEndCapture(context::getStream(), &device_graph_.get()->graph); + if (end_status != INFINI_STATUS_SUCCESS) { + spdlog::warn("Fail to end device graph capture."); + device_graph_.reset(); return; } - if (infinirtGraphInstantiate( - &device_graph_.get()->exec, - device_graph_.get()->graph, - &device_graph_.get()->node, - device_graph_.get()->log_buffer.data(), - device_graph_.get()->log_buffer.size()) - != INFINI_STATUS_SUCCESS) { + auto instantiate_status = device_graph_->standalone + ? standalone_infinirt::graph_instantiate(&device_graph_.get()->exec, device_graph_.get()->graph) + : infinirtGraphInstantiate( + &device_graph_.get()->exec, + device_graph_.get()->graph, + &device_graph_.get()->node, + device_graph_.get()->log_buffer.data(), + device_graph_.get()->log_buffer.size()); + if (instantiate_status != INFINI_STATUS_SUCCESS) { static bool warned_once = false; if (!warned_once) { warned_once = true; spdlog::warn("Fail to instantiate device graph: {}", std::string(device_graph_.get()->log_buffer.data())); } + device_graph_.reset(); } } diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc new file mode 100644 index 000000000..db52a3975 --- /dev/null +++ b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc @@ -0,0 +1,287 @@ +#include "standalone_infinirt_graph_bridge.hpp" + +#ifdef USE_STANDALONE_INFINIRT_GRAPH + +#include "../utils.hpp" + +#include +#include +#include + +#include + +namespace infinicore::graph::standalone_infinirt { +namespace { + +using StreamWrapFn = infiniRtStatus_t (*)(infiniRtDevice_t, void *, infiniRtStream_t *); +using StreamDestroyFn = infiniRtStatus_t (*)(infiniRtStream_t); +using StreamBeginCaptureFn = infiniRtStatus_t (*)(infiniRtStream_t, infiniRtStreamCaptureMode_t); +using StreamEndCaptureFn = infiniRtStatus_t (*)(infiniRtStream_t, infiniRtGraph_t *); +using GraphDestroyFn = infiniRtStatus_t (*)(infiniRtGraph_t); +using GraphInstantiateFn = infiniRtStatus_t (*)(infiniRtGraphExec_t *, infiniRtGraph_t); +using GraphExecDestroyFn = infiniRtStatus_t (*)(infiniRtGraphExec_t); +using GraphLaunchFn = infiniRtStatus_t (*)(infiniRtGraphExec_t, infiniRtStream_t); + +struct Api { + void *handle = nullptr; + StreamWrapFn stream_wrap = nullptr; + StreamDestroyFn stream_destroy = nullptr; + StreamBeginCaptureFn stream_begin_capture = nullptr; + StreamEndCaptureFn stream_end_capture = nullptr; + GraphDestroyFn graph_destroy = nullptr; + GraphInstantiateFn graph_instantiate = nullptr; + GraphExecDestroyFn graph_exec_destroy = nullptr; + GraphLaunchFn graph_launch = nullptr; + bool loaded = false; +}; + +bool truthy_env(const char *name) { + auto value = std::getenv(name); + if (value == nullptr) { + return false; + } + std::string text{value}; + return text == "1" || text == "ON" || text == "on" || text == "true" || text == "TRUE"; +} + +std::string standalone_library_path() { + if (auto explicit_path = std::getenv("INFINIRT_GRAPH_LIBRARY")) { + return explicit_path; + } + if (auto root = std::getenv("INFINI_RT_ROOT")) { + auto lib = std::string(root) + "/lib/libinfinirt.so"; + void *handle = dlopen(lib.c_str(), RTLD_NOW | RTLD_LOCAL); + if (handle != nullptr) { + dlclose(handle); + return lib; + } + return std::string(root) + "/lib64/libinfinirt.so"; + } + return "libinfinirt.so"; +} + +template +bool load_symbol(void *handle, const char *name, T *symbol) { + *symbol = reinterpret_cast(dlsym(handle, name)); + return *symbol != nullptr; +} + +Api &api() { + static Api api_ = [] { + Api loaded{}; + const auto path = standalone_library_path(); + loaded.handle = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (loaded.handle == nullptr) { + spdlog::warn("Standalone InfiniRT graph bridge disabled: failed to load {}: {}", path, dlerror()); + return loaded; + } + + loaded.loaded = + load_symbol(loaded.handle, "infiniRtStreamWrap", &loaded.stream_wrap) + && load_symbol(loaded.handle, "infiniRtStreamDestroy", &loaded.stream_destroy) + && load_symbol(loaded.handle, "infiniRtStreamBeginCapture", &loaded.stream_begin_capture) + && load_symbol(loaded.handle, "infiniRtStreamEndCapture", &loaded.stream_end_capture) + && load_symbol(loaded.handle, "infiniRtGraphDestroy", &loaded.graph_destroy) + && load_symbol(loaded.handle, "infiniRtGraphInstantiate", &loaded.graph_instantiate) + && load_symbol(loaded.handle, "infiniRtGraphExecDestroy", &loaded.graph_exec_destroy) + && load_symbol(loaded.handle, "infiniRtGraphLaunch", &loaded.graph_launch); + + if (loaded.loaded) { + spdlog::info("Standalone InfiniRT graph bridge loaded: {}", path); + } else { + spdlog::warn("Standalone InfiniRT graph bridge disabled: required graph symbols missing in {}", path); + } + return loaded; + }(); + return api_; +} + +infiniRtDeviceType_t to_standalone_device_type(Device::Type type) { + switch (type) { + case Device::Type::CPU: + return INFINI_RT_DEVICE_CPU; + case Device::Type::NVIDIA: + return INFINI_RT_DEVICE_NVIDIA; + case Device::Type::CAMBRICON: + return INFINI_RT_DEVICE_CAMBRICON; + case Device::Type::ASCEND: + return INFINI_RT_DEVICE_ASCEND; + case Device::Type::METAX: + return INFINI_RT_DEVICE_METAX; + case Device::Type::MOORE: + return INFINI_RT_DEVICE_MOORE; + case Device::Type::ILUVATAR: + return INFINI_RT_DEVICE_ILUVATAR; + case Device::Type::KUNLUN: + return INFINI_RT_DEVICE_KUNLUN; + case Device::Type::HYGON: + return INFINI_RT_DEVICE_HYGON; + case Device::Type::QY: + return INFINI_RT_DEVICE_QY; + default: + return INFINI_RT_DEVICE_CPU; + } +} + +infiniStatus_t to_core_status(infiniRtStatus_t status) { + switch (status) { + case INFINI_RT_STATUS_SUCCESS: + return INFINI_STATUS_SUCCESS; + case INFINI_RT_STATUS_INVALID_ARGUMENT: + return INFINI_STATUS_BAD_PARAM; + case INFINI_RT_STATUS_UNSUPPORTED_DEVICE: + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + case INFINI_RT_STATUS_RUNTIME_ERROR: + default: + return INFINI_STATUS_INTERNAL_ERROR; + } +} + +infiniRtStreamCaptureMode_t to_standalone_capture_mode(infinirtStreamCaptureMode_t mode) { + switch (mode) { + case INFINIRT_STREAM_CAPTURE_MODE_GLOBAL: + return INFINI_RT_STREAM_CAPTURE_MODE_GLOBAL; + case INFINIRT_STREAM_CAPTURE_MODE_THREAD_LOCAL: + return INFINI_RT_STREAM_CAPTURE_MODE_THREAD_LOCAL; + case INFINIRT_STREAM_CAPTURE_MODE_RELAXED: + return INFINI_RT_STREAM_CAPTURE_MODE_RELAXED; + } + return INFINI_RT_STREAM_CAPTURE_MODE_RELAXED; +} + +infiniRtDevice_t to_standalone_device(const Device &device) { + return infiniRtDevice_t{ + to_standalone_device_type(device.getType()), + static_cast(device.getIndex()), + }; +} + +} // namespace + +bool enabled() { + return truthy_env("INFINICORE_USE_STANDALONE_INFINIRT_GRAPH"); +} + +bool available() { + return enabled() && api().loaded; +} + +infinirtStream_t wrap_stream(const Device &device, infinirtStream_t stream) { + auto &rt = api(); + if (!rt.loaded || stream == nullptr) { + return nullptr; + } + + auto standalone_device = to_standalone_device(device); + infiniRtStream_t wrapped = nullptr; + if (rt.stream_wrap(standalone_device, stream, &wrapped) != INFINI_RT_STATUS_SUCCESS) { + return nullptr; + } + return reinterpret_cast(wrapped); +} + +void destroy_wrapped_stream(infinirtStream_t stream) { + if (stream == nullptr || !api().loaded) { + return; + } + api().stream_destroy(reinterpret_cast(stream)); +} + +infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode) { + if (!api().loaded || stream == nullptr) { + return INFINI_STATUS_INTERNAL_ERROR; + } + return to_core_status(api().stream_begin_capture( + reinterpret_cast(stream), + to_standalone_capture_mode(mode))); +} + +infiniStatus_t stream_end_capture(infinirtStream_t stream, infinirtGraph_t *graph) { + if (!api().loaded || stream == nullptr || graph == nullptr) { + return INFINI_STATUS_INTERNAL_ERROR; + } + return to_core_status(api().stream_end_capture( + reinterpret_cast(stream), + reinterpret_cast(graph))); +} + +infiniStatus_t graph_destroy(infinirtGraph_t graph) { + if (!api().loaded || graph == nullptr) { + return INFINI_STATUS_INTERNAL_ERROR; + } + return to_core_status(api().graph_destroy(reinterpret_cast(graph))); +} + +infiniStatus_t graph_instantiate(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph) { + if (!api().loaded || graph_exec == nullptr || graph == nullptr) { + return INFINI_STATUS_INTERNAL_ERROR; + } + return to_core_status(api().graph_instantiate( + reinterpret_cast(graph_exec), + reinterpret_cast(graph))); +} + +infiniStatus_t graph_exec_destroy(infinirtGraphExec_t graph_exec) { + if (!api().loaded || graph_exec == nullptr) { + return INFINI_STATUS_INTERNAL_ERROR; + } + return to_core_status(api().graph_exec_destroy(reinterpret_cast(graph_exec))); +} + +infiniStatus_t graph_launch(infinirtGraphExec_t graph_exec, infinirtStream_t stream) { + if (!api().loaded || graph_exec == nullptr || stream == nullptr) { + return INFINI_STATUS_INTERNAL_ERROR; + } + return to_core_status(api().graph_launch( + reinterpret_cast(graph_exec), + reinterpret_cast(stream))); +} + +} // namespace infinicore::graph::standalone_infinirt + +#else + +namespace infinicore::graph::standalone_infinirt { + +bool enabled() { + return false; +} + +bool available() { + return false; +} + +infinirtStream_t wrap_stream(const Device &, infinirtStream_t) { + return nullptr; +} + +void destroy_wrapped_stream(infinirtStream_t) { +} + +infiniStatus_t stream_begin_capture(infinirtStream_t, infinirtStreamCaptureMode_t) { + return INFINI_STATUS_NOT_IMPLEMENTED; +} + +infiniStatus_t stream_end_capture(infinirtStream_t, infinirtGraph_t *) { + return INFINI_STATUS_NOT_IMPLEMENTED; +} + +infiniStatus_t graph_destroy(infinirtGraph_t) { + return INFINI_STATUS_NOT_IMPLEMENTED; +} + +infiniStatus_t graph_instantiate(infinirtGraphExec_t *, infinirtGraph_t) { + return INFINI_STATUS_NOT_IMPLEMENTED; +} + +infiniStatus_t graph_exec_destroy(infinirtGraphExec_t) { + return INFINI_STATUS_NOT_IMPLEMENTED; +} + +infiniStatus_t graph_launch(infinirtGraphExec_t, infinirtStream_t) { + return INFINI_STATUS_NOT_IMPLEMENTED; +} + +} // namespace infinicore::graph::standalone_infinirt + +#endif diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp b/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp new file mode 100644 index 000000000..0c58bff77 --- /dev/null +++ b/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include "infinicore/device.hpp" +#include + +namespace infinicore::graph::standalone_infinirt { + +bool enabled(); + +bool available(); + +infinirtStream_t wrap_stream(const Device &device, infinirtStream_t stream); + +void destroy_wrapped_stream(infinirtStream_t stream); + +infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode); + +infiniStatus_t stream_end_capture(infinirtStream_t stream, infinirtGraph_t *graph); + +infiniStatus_t graph_destroy(infinirtGraph_t graph); + +infiniStatus_t graph_instantiate(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph); + +infiniStatus_t graph_exec_destroy(infinirtGraphExec_t graph_exec); + +infiniStatus_t graph_launch(infinirtGraphExec_t graph_exec, infinirtStream_t stream); + +} // namespace infinicore::graph::standalone_infinirt diff --git a/xmake.lua b/xmake.lua index fcbcf423d..5762fdbfc 100644 --- a/xmake.lua +++ b/xmake.lua @@ -260,10 +260,20 @@ option("graph") set_description("Whether to use device graph instantiating feature, such as cuda graph for nvidia") option_end() -if has_config("graph") then +option("standalone-infinirt-graph") + set_default(false) + set_showmenu(true) + set_description("Whether to route graph lifecycle calls through an installed standalone InfiniRT") +option_end() + +if has_config("graph") or has_config("standalone-infinirt-graph") then add_defines("USE_INFINIRT_GRAPH") end +if has_config("standalone-infinirt-graph") then + add_defines("USE_STANDALONE_INFINIRT_GRAPH") +end + -- InfiniCCL option("ccl") @@ -327,6 +337,30 @@ end local infiniops_external_built = false +local function find_standalone_infinirt(infinirt_root, xmake_os) + local standalone_infinirt = path.join(infinirt_root, "lib", "libinfinirt.so") + if not xmake_os.isfile(standalone_infinirt) then + standalone_infinirt = path.join(infinirt_root, "lib64", "libinfinirt.so") + end + if not xmake_os.isfile(standalone_infinirt) then + raise("Standalone InfiniRT library not found under: " .. infinirt_root) + end + return standalone_infinirt +end + +local function patch_infiniops_private_infinirt(xmake_os, infiniops_lib, standalone_infinirt, private_infinirt) + local private_soname = path.filename(private_infinirt) + xmake_os.cp(standalone_infinirt, private_infinirt) + xmake_os.execv("patchelf", {"--set-soname", private_soname, private_infinirt}) + xmake_os.execv("patchelf", {"--replace-needed", standalone_infinirt, private_soname, infiniops_lib}) + xmake_os.execv("patchelf", {"--replace-needed", "libinfinirt.so", private_soname, infiniops_lib}) + + local needed = xmake_os.iorunv("patchelf", {"--print-needed", infiniops_lib}) + if not needed:find(private_soname, 1, true) then + xmake_os.execv("patchelf", {"--add-needed", private_soname, infiniops_lib}) + end +end + local function filter_infiniops_ops_for_backend(infiniops_ops) if not infiniops_ops or #infiniops_ops == 0 then return infiniops_ops @@ -408,20 +442,10 @@ local function build_infiniops_external(xmake_os) xmake_os.execv("cmake", {"--build", infiniops_builddir, "--target", "infiniops"}) xmake_os.execv("cmake", {"--install", infiniops_builddir, "--prefix", INFINI_ROOT}) if infinirt_root and infinirt_root ~= "" then - local standalone_infinirt = path.join(infinirt_root, "lib", "libinfinirt.so") - if not xmake_os.isfile(standalone_infinirt) then - standalone_infinirt = path.join(infinirt_root, "lib64", "libinfinirt.so") - end - if not xmake_os.isfile(standalone_infinirt) then - raise("Standalone InfiniRT library not found under: " .. infinirt_root) - end + local standalone_infinirt = find_standalone_infinirt(infinirt_root, xmake_os) local infiniops_lib = path.join(INFINI_ROOT, "lib", "libinfiniops.so") - local private_soname = "libinfiniops_infinirt.so" - local private_infinirt = path.join(INFINI_ROOT, "lib", private_soname) - xmake_os.cp(standalone_infinirt, private_infinirt) - xmake_os.execv("patchelf", {"--set-soname", private_soname, private_infinirt}) - xmake_os.execv("patchelf", {"--replace-needed", standalone_infinirt, private_soname, infiniops_lib}) - xmake_os.execv("patchelf", {"--replace-needed", "libinfinirt.so", private_soname, infiniops_lib}) + local private_infinirt = path.join(INFINI_ROOT, "lib", "libinfiniops_infinirt.so") + patch_infiniops_private_infinirt(xmake_os, infiniops_lib, standalone_infinirt, private_infinirt) end infiniops_external_built = true end @@ -669,6 +693,16 @@ target("infinicore_cpp_api") add_defines("CHAR_BIT=8", "INT_MIN=(-2147483647 - 1)", "INT_MAX=2147483647", "UINT_MAX=4294967295U") end add_includedirs(INFINI_ROOT.."/include", { public = true }) + local graph_infinirt_root = get_standalone_infinirt_root() + if has_config("standalone-infinirt-graph") then + if not graph_infinirt_root or graph_infinirt_root == "" then + raise("--standalone-infinirt-graph requires --infinirt-root or INFINI_RT_ROOT") + end + add_includedirs(graph_infinirt_root .. "/include") + if is_plat("linux") then + add_links("dl") + end + end if has_config("nv-gpu") then local cuda_root = os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") or get_config("cuda") or "/usr/local/cuda" add_includedirs(cuda_root .. "/include") @@ -696,19 +730,10 @@ target("infinicore_cpp_api") local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") local infinirt_root = get_standalone_infinirt_root() if infinirt_root and infinirt_root ~= "" then - local standalone_infinirt = path.join(infinirt_root, "lib", "libinfinirt.so") - if not os.isfile(standalone_infinirt) then - standalone_infinirt = path.join(infinirt_root, "lib64", "libinfinirt.so") - end - if not os.isfile(standalone_infinirt) then - raise("Standalone InfiniRT library not found under: " .. infinirt_root) - end + local standalone_infinirt = find_standalone_infinirt(infinirt_root, os) local infiniops_lib = path.join(INFINI_ROOT, "lib", "libinfiniops.so") - local private_soname = "libinfiniops_infinirt.so" - local private_infinirt = path.join(INFINI_ROOT, "lib", private_soname) - os.cp(standalone_infinirt, private_infinirt) - os.execv("patchelf", {"--set-soname", private_soname, private_infinirt}) - os.execv("patchelf", {"--replace-needed", "libinfinirt.so", private_soname, infiniops_lib}) + local private_infinirt = path.join(INFINI_ROOT, "lib", "libinfiniops_infinirt.so") + patch_infiniops_private_infinirt(os, infiniops_lib, standalone_infinirt, private_infinirt) end end) end @@ -967,19 +992,9 @@ target("_infinicore") end local infinirt_root = get_standalone_infinirt_root() if infinirt_root and infinirt_root ~= "" then - local standalone_infinirt = path.join(infinirt_root, "lib", "libinfinirt.so") - if not os.isfile(standalone_infinirt) then - standalone_infinirt = path.join(infinirt_root, "lib64", "libinfinirt.so") - end - if not os.isfile(standalone_infinirt) then - raise("Standalone InfiniRT library not found under: " .. infinirt_root) - end - local private_soname = "libinfiniops_infinirt.so" - local private_infinirt = path.join(INFINI_ROOT, "lib", private_soname) - os.cp(standalone_infinirt, private_infinirt) - os.execv("patchelf", {"--set-soname", private_soname, private_infinirt}) - os.execv("patchelf", {"--replace-needed", standalone_infinirt, private_soname, infiniops_lib}) - os.execv("patchelf", {"--replace-needed", "libinfinirt.so", private_soname, infiniops_lib}) + local standalone_infinirt = find_standalone_infinirt(infinirt_root, os) + local private_infinirt = path.join(INFINI_ROOT, "lib", "libinfiniops_infinirt.so") + patch_infiniops_private_infinirt(os, infiniops_lib, standalone_infinirt, private_infinirt) end os.mkdir(path.join(INFINI_ROOT, "lib")) if not infiniops_lib_installed then From ed3bca1ada3a7ea21201aa9a52cd88a9703e422a Mon Sep 17 00:00:00 2001 From: Jiacheng Huang Date: Fri, 3 Jul 2026 16:49:25 +0800 Subject: [PATCH 03/10] refactor: call standalone InfiniRT C++ graph API --- src/infinicore/graph/graph.cc | 43 ++-- .../graph/standalone_infinirt_graph_bridge.cc | 228 +++++------------- .../standalone_infinirt_graph_bridge.hpp | 6 +- xmake.lua | 3 - 4 files changed, 79 insertions(+), 201 deletions(-) diff --git a/src/infinicore/graph/graph.cc b/src/infinicore/graph/graph.cc index 71638e147..ab38f46e4 100644 --- a/src/infinicore/graph/graph.cc +++ b/src/infinicore/graph/graph.cc @@ -59,9 +59,6 @@ struct Graph::DeviceGraph { infinirtGraphDestroy(graph); } } - if (standalone && stream) { - standalone_infinirt::destroy_wrapped_stream(stream); - } } void launch() { @@ -93,16 +90,16 @@ void Graph::add_operator(std::shared_ptr op) { void Graph::instantiate() { // Reset device graph device_graph_ = std::make_unique(); - device_graph_->standalone = standalone_infinirt::available(); - device_graph_->stream = device_graph_->standalone - ? standalone_infinirt::wrap_stream(context::getDevice(), context::getStream()) - : context::getStream(); - if (device_graph_->standalone && device_graph_->stream == nullptr) { - spdlog::warn("Standalone InfiniRT graph bridge is enabled but failed to wrap the current stream. Falling back to eager execution."); - device_graph_.reset(); - return; - } + device_graph_->standalone = standalone_infinirt::available(context::getDevice()); + device_graph_->stream = context::getStream(); if (device_graph_->standalone) { + auto set_device_status = standalone_infinirt::set_device(context::getDevice()); + if (set_device_status != INFINI_STATUS_SUCCESS) { + spdlog::warn("Standalone InfiniRT graph bridge failed to select the current device. Falling back to eager execution."); + device_graph_.reset(); + return; + } + static bool logged_once = false; if (!logged_once) { logged_once = true; @@ -117,8 +114,8 @@ void Graph::instantiate() { infinicore::context::syncStream(); auto begin_status = device_graph_->standalone - ? standalone_infinirt::stream_begin_capture(device_graph_->stream, INFINIRT_STREAM_CAPTURE_MODE_RELAXED) - : infinirtStreamBeginCapture(context::getStream(), INFINIRT_STREAM_CAPTURE_MODE_RELAXED); + ? standalone_infinirt::stream_begin_capture(device_graph_->stream, INFINIRT_STREAM_CAPTURE_MODE_RELAXED) + : infinirtStreamBeginCapture(context::getStream(), INFINIRT_STREAM_CAPTURE_MODE_RELAXED); if (begin_status != INFINI_STATUS_SUCCESS) { spdlog::warn("Fail to begin device graph capture."); device_graph_.reset(); @@ -129,8 +126,8 @@ void Graph::instantiate() { this->run(); auto end_status = device_graph_->standalone - ? standalone_infinirt::stream_end_capture(device_graph_->stream, &device_graph_.get()->graph) - : infinirtStreamEndCapture(context::getStream(), &device_graph_.get()->graph); + ? standalone_infinirt::stream_end_capture(device_graph_->stream, &device_graph_.get()->graph) + : infinirtStreamEndCapture(context::getStream(), &device_graph_.get()->graph); if (end_status != INFINI_STATUS_SUCCESS) { spdlog::warn("Fail to end device graph capture."); device_graph_.reset(); @@ -138,13 +135,13 @@ void Graph::instantiate() { } auto instantiate_status = device_graph_->standalone - ? standalone_infinirt::graph_instantiate(&device_graph_.get()->exec, device_graph_.get()->graph) - : infinirtGraphInstantiate( - &device_graph_.get()->exec, - device_graph_.get()->graph, - &device_graph_.get()->node, - device_graph_.get()->log_buffer.data(), - device_graph_.get()->log_buffer.size()); + ? standalone_infinirt::graph_instantiate(&device_graph_.get()->exec, device_graph_.get()->graph) + : infinirtGraphInstantiate( + &device_graph_.get()->exec, + device_graph_.get()->graph, + &device_graph_.get()->node, + device_graph_.get()->log_buffer.data(), + device_graph_.get()->log_buffer.size()); if (instantiate_status != INFINI_STATUS_SUCCESS) { static bool warned_once = false; if (!warned_once) { diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc index db52a3975..be4c90940 100644 --- a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc +++ b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc @@ -2,38 +2,16 @@ #ifdef USE_STANDALONE_INFINIRT_GRAPH -#include "../utils.hpp" - #include -#include #include -#include +#include namespace infinicore::graph::standalone_infinirt { namespace { -using StreamWrapFn = infiniRtStatus_t (*)(infiniRtDevice_t, void *, infiniRtStream_t *); -using StreamDestroyFn = infiniRtStatus_t (*)(infiniRtStream_t); -using StreamBeginCaptureFn = infiniRtStatus_t (*)(infiniRtStream_t, infiniRtStreamCaptureMode_t); -using StreamEndCaptureFn = infiniRtStatus_t (*)(infiniRtStream_t, infiniRtGraph_t *); -using GraphDestroyFn = infiniRtStatus_t (*)(infiniRtGraph_t); -using GraphInstantiateFn = infiniRtStatus_t (*)(infiniRtGraphExec_t *, infiniRtGraph_t); -using GraphExecDestroyFn = infiniRtStatus_t (*)(infiniRtGraphExec_t); -using GraphLaunchFn = infiniRtStatus_t (*)(infiniRtGraphExec_t, infiniRtStream_t); - -struct Api { - void *handle = nullptr; - StreamWrapFn stream_wrap = nullptr; - StreamDestroyFn stream_destroy = nullptr; - StreamBeginCaptureFn stream_begin_capture = nullptr; - StreamEndCaptureFn stream_end_capture = nullptr; - GraphDestroyFn graph_destroy = nullptr; - GraphInstantiateFn graph_instantiate = nullptr; - GraphExecDestroyFn graph_exec_destroy = nullptr; - GraphLaunchFn graph_launch = nullptr; - bool loaded = false; -}; +using StandaloneDevice = infini::rt::Device; +using StandaloneRuntime = infini::rt::runtime::Runtime; bool truthy_env(const char *name) { auto value = std::getenv(name); @@ -44,116 +22,40 @@ bool truthy_env(const char *name) { return text == "1" || text == "ON" || text == "on" || text == "true" || text == "TRUE"; } -std::string standalone_library_path() { - if (auto explicit_path = std::getenv("INFINIRT_GRAPH_LIBRARY")) { - return explicit_path; - } - if (auto root = std::getenv("INFINI_RT_ROOT")) { - auto lib = std::string(root) + "/lib/libinfinirt.so"; - void *handle = dlopen(lib.c_str(), RTLD_NOW | RTLD_LOCAL); - if (handle != nullptr) { - dlclose(handle); - return lib; - } - return std::string(root) + "/lib64/libinfinirt.so"; - } - return "libinfinirt.so"; +bool supports_device(Device::Type type) { + return type == Device::Type::NVIDIA; } -template -bool load_symbol(void *handle, const char *name, T *symbol) { - *symbol = reinterpret_cast(dlsym(handle, name)); - return *symbol != nullptr; +template +infiniStatus_t to_core_status(Status status) { + return status == StandaloneRuntime::kSuccess + ? INFINI_STATUS_SUCCESS + : INFINI_STATUS_INTERNAL_ERROR; } -Api &api() { - static Api api_ = [] { - Api loaded{}; - const auto path = standalone_library_path(); - loaded.handle = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); - if (loaded.handle == nullptr) { - spdlog::warn("Standalone InfiniRT graph bridge disabled: failed to load {}: {}", path, dlerror()); - return loaded; - } - - loaded.loaded = - load_symbol(loaded.handle, "infiniRtStreamWrap", &loaded.stream_wrap) - && load_symbol(loaded.handle, "infiniRtStreamDestroy", &loaded.stream_destroy) - && load_symbol(loaded.handle, "infiniRtStreamBeginCapture", &loaded.stream_begin_capture) - && load_symbol(loaded.handle, "infiniRtStreamEndCapture", &loaded.stream_end_capture) - && load_symbol(loaded.handle, "infiniRtGraphDestroy", &loaded.graph_destroy) - && load_symbol(loaded.handle, "infiniRtGraphInstantiate", &loaded.graph_instantiate) - && load_symbol(loaded.handle, "infiniRtGraphExecDestroy", &loaded.graph_exec_destroy) - && load_symbol(loaded.handle, "infiniRtGraphLaunch", &loaded.graph_launch); - - if (loaded.loaded) { - spdlog::info("Standalone InfiniRT graph bridge loaded: {}", path); - } else { - spdlog::warn("Standalone InfiniRT graph bridge disabled: required graph symbols missing in {}", path); - } - return loaded; - }(); - return api_; +StandaloneRuntime::Stream to_standalone_stream(infinirtStream_t stream) { + return reinterpret_cast(stream); } -infiniRtDeviceType_t to_standalone_device_type(Device::Type type) { - switch (type) { - case Device::Type::CPU: - return INFINI_RT_DEVICE_CPU; - case Device::Type::NVIDIA: - return INFINI_RT_DEVICE_NVIDIA; - case Device::Type::CAMBRICON: - return INFINI_RT_DEVICE_CAMBRICON; - case Device::Type::ASCEND: - return INFINI_RT_DEVICE_ASCEND; - case Device::Type::METAX: - return INFINI_RT_DEVICE_METAX; - case Device::Type::MOORE: - return INFINI_RT_DEVICE_MOORE; - case Device::Type::ILUVATAR: - return INFINI_RT_DEVICE_ILUVATAR; - case Device::Type::KUNLUN: - return INFINI_RT_DEVICE_KUNLUN; - case Device::Type::HYGON: - return INFINI_RT_DEVICE_HYGON; - case Device::Type::QY: - return INFINI_RT_DEVICE_QY; - default: - return INFINI_RT_DEVICE_CPU; - } +StandaloneRuntime::Graph to_standalone_graph(infinirtGraph_t graph) { + return reinterpret_cast(graph); } -infiniStatus_t to_core_status(infiniRtStatus_t status) { - switch (status) { - case INFINI_RT_STATUS_SUCCESS: - return INFINI_STATUS_SUCCESS; - case INFINI_RT_STATUS_INVALID_ARGUMENT: - return INFINI_STATUS_BAD_PARAM; - case INFINI_RT_STATUS_UNSUPPORTED_DEVICE: - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - case INFINI_RT_STATUS_RUNTIME_ERROR: - default: - return INFINI_STATUS_INTERNAL_ERROR; - } +StandaloneRuntime::GraphExec to_standalone_graph_exec(infinirtGraphExec_t graph_exec) { + return reinterpret_cast(graph_exec); } -infiniRtStreamCaptureMode_t to_standalone_capture_mode(infinirtStreamCaptureMode_t mode) { +decltype(StandaloneRuntime::kStreamCaptureModeRelaxed) +to_standalone_capture_mode(infinirtStreamCaptureMode_t mode) { switch (mode) { case INFINIRT_STREAM_CAPTURE_MODE_GLOBAL: - return INFINI_RT_STREAM_CAPTURE_MODE_GLOBAL; + return StandaloneRuntime::kStreamCaptureModeGlobal; case INFINIRT_STREAM_CAPTURE_MODE_THREAD_LOCAL: - return INFINI_RT_STREAM_CAPTURE_MODE_THREAD_LOCAL; + return StandaloneRuntime::kStreamCaptureModeThreadLocal; case INFINIRT_STREAM_CAPTURE_MODE_RELAXED: - return INFINI_RT_STREAM_CAPTURE_MODE_RELAXED; + return StandaloneRuntime::kStreamCaptureModeRelaxed; } - return INFINI_RT_STREAM_CAPTURE_MODE_RELAXED; -} - -infiniRtDevice_t to_standalone_device(const Device &device) { - return infiniRtDevice_t{ - to_standalone_device_type(device.getType()), - static_cast(device.getIndex()), - }; + return StandaloneRuntime::kStreamCaptureModeRelaxed; } } // namespace @@ -162,79 +64,66 @@ bool enabled() { return truthy_env("INFINICORE_USE_STANDALONE_INFINIRT_GRAPH"); } -bool available() { - return enabled() && api().loaded; +bool available(const Device &device) { + return enabled() && supports_device(device.getType()); } -infinirtStream_t wrap_stream(const Device &device, infinirtStream_t stream) { - auto &rt = api(); - if (!rt.loaded || stream == nullptr) { - return nullptr; - } - - auto standalone_device = to_standalone_device(device); - infiniRtStream_t wrapped = nullptr; - if (rt.stream_wrap(standalone_device, stream, &wrapped) != INFINI_RT_STATUS_SUCCESS) { - return nullptr; - } - return reinterpret_cast(wrapped); -} - -void destroy_wrapped_stream(infinirtStream_t stream) { - if (stream == nullptr || !api().loaded) { - return; +infiniStatus_t set_device(const Device &device) { + if (!supports_device(device.getType())) { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; } - api().stream_destroy(reinterpret_cast(stream)); + return to_core_status(StandaloneRuntime::SetDevice(static_cast(device.getIndex()))); } infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode) { - if (!api().loaded || stream == nullptr) { - return INFINI_STATUS_INTERNAL_ERROR; + if (stream == nullptr) { + return INFINI_STATUS_NULL_POINTER; } - return to_core_status(api().stream_begin_capture( - reinterpret_cast(stream), + return to_core_status(StandaloneRuntime::StreamBeginCapture( + to_standalone_stream(stream), to_standalone_capture_mode(mode))); } infiniStatus_t stream_end_capture(infinirtStream_t stream, infinirtGraph_t *graph) { - if (!api().loaded || stream == nullptr || graph == nullptr) { - return INFINI_STATUS_INTERNAL_ERROR; + if (stream == nullptr || graph == nullptr) { + return INFINI_STATUS_NULL_POINTER; } - return to_core_status(api().stream_end_capture( - reinterpret_cast(stream), - reinterpret_cast(graph))); + return to_core_status(StandaloneRuntime::StreamEndCapture( + to_standalone_stream(stream), + reinterpret_cast(graph))); } infiniStatus_t graph_destroy(infinirtGraph_t graph) { - if (!api().loaded || graph == nullptr) { - return INFINI_STATUS_INTERNAL_ERROR; + if (graph == nullptr) { + return INFINI_STATUS_NULL_POINTER; } - return to_core_status(api().graph_destroy(reinterpret_cast(graph))); + return to_core_status(StandaloneRuntime::GraphDestroy(to_standalone_graph(graph))); } infiniStatus_t graph_instantiate(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph) { - if (!api().loaded || graph_exec == nullptr || graph == nullptr) { - return INFINI_STATUS_INTERNAL_ERROR; + if (graph_exec == nullptr || graph == nullptr) { + return INFINI_STATUS_NULL_POINTER; } - return to_core_status(api().graph_instantiate( - reinterpret_cast(graph_exec), - reinterpret_cast(graph))); + return to_core_status(StandaloneRuntime::GraphInstantiate( + reinterpret_cast(graph_exec), + to_standalone_graph(graph))); } infiniStatus_t graph_exec_destroy(infinirtGraphExec_t graph_exec) { - if (!api().loaded || graph_exec == nullptr) { - return INFINI_STATUS_INTERNAL_ERROR; + if (graph_exec == nullptr) { + return INFINI_STATUS_NULL_POINTER; } - return to_core_status(api().graph_exec_destroy(reinterpret_cast(graph_exec))); + return to_core_status(StandaloneRuntime::GraphExecDestroy( + to_standalone_graph_exec(graph_exec))); } infiniStatus_t graph_launch(infinirtGraphExec_t graph_exec, infinirtStream_t stream) { - if (!api().loaded || graph_exec == nullptr || stream == nullptr) { - return INFINI_STATUS_INTERNAL_ERROR; + if (graph_exec == nullptr || stream == nullptr) { + return INFINI_STATUS_NULL_POINTER; } - return to_core_status(api().graph_launch( - reinterpret_cast(graph_exec), - reinterpret_cast(stream))); + return to_core_status(StandaloneRuntime::GraphLaunch( + to_standalone_graph_exec(graph_exec), + to_standalone_stream(stream))); } } // namespace infinicore::graph::standalone_infinirt @@ -247,15 +136,12 @@ bool enabled() { return false; } -bool available() { +bool available(const Device &) { return false; } -infinirtStream_t wrap_stream(const Device &, infinirtStream_t) { - return nullptr; -} - -void destroy_wrapped_stream(infinirtStream_t) { +infiniStatus_t set_device(const Device &) { + return INFINI_STATUS_NOT_IMPLEMENTED; } infiniStatus_t stream_begin_capture(infinirtStream_t, infinirtStreamCaptureMode_t) { diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp b/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp index 0c58bff77..0ebbb0a6b 100644 --- a/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp +++ b/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp @@ -7,11 +7,9 @@ namespace infinicore::graph::standalone_infinirt { bool enabled(); -bool available(); +bool available(const Device &device); -infinirtStream_t wrap_stream(const Device &device, infinirtStream_t stream); - -void destroy_wrapped_stream(infinirtStream_t stream); +infiniStatus_t set_device(const Device &device); infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode); diff --git a/xmake.lua b/xmake.lua index 5762fdbfc..8648900ee 100644 --- a/xmake.lua +++ b/xmake.lua @@ -699,9 +699,6 @@ target("infinicore_cpp_api") raise("--standalone-infinirt-graph requires --infinirt-root or INFINI_RT_ROOT") end add_includedirs(graph_infinirt_root .. "/include") - if is_plat("linux") then - add_links("dl") - end end if has_config("nv-gpu") then local cuda_root = os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") or get_config("cuda") or "/usr/local/cuda" From 1c0b3bab68c4a561493f61128c48cb9e7217e458 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Tue, 30 Jun 2026 17:06:34 +0800 Subject: [PATCH 04/10] fix: support ascend paged graph runtime Synchronize RoPE cache copies before the cache is consumed by Ascend kernels. Rename paged attention block_idx locals to page_block_idx to avoid AscendC kernel symbol conflicts while preserving the same block table address calculation. Detect common Ascend driver library layouts before linking libascend_hal.so instead of relying on one hard-coded driver path. --- src/infinicore/nn/rope.cc | 14 ++++++++++++++ .../ascend/paged_attention_ascend_kernel.cpp | 8 ++++---- .../paged_attention_prefill_ascend_kernel.cpp | 8 ++++---- xmake/ascend.lua | 9 ++++++++- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/infinicore/nn/rope.cc b/src/infinicore/nn/rope.cc index 115e162e2..c8ab4f02d 100644 --- a/src/infinicore/nn/rope.cc +++ b/src/infinicore/nn/rope.cc @@ -1,6 +1,7 @@ #include "infinicore/nn/rope.hpp" #include "../../utils.h" #include "../utils.hpp" +#include "infinicore/context/context.hpp" #include "infinicore/ops/mrope.hpp" #include "infinicore/ops/rope.hpp" #include @@ -12,6 +13,16 @@ #include namespace infinicore::nn { +namespace { + +void sync_cache_copy(const Device &device) { + if (device.getType() == Device::Type::ASCEND) { + // Ascend H2D copies are async, and these host buffers leave scope after init. + infinicore::context::syncStream(); + } +} + +} // namespace RoPE::RoPE(size_t head_dim, size_t rotary_dim, @@ -95,6 +106,7 @@ void RoPE::initialize_cache() { auto cos_f32_cpu = Tensor::from_blob(cos_data.data(), {max_seq_len_, cache_dim}, DataType::F32, cpu_device); sin_cache_->copy_from(sin_f32_cpu); cos_cache_->copy_from(cos_f32_cpu); + sync_cache_copy(device_); } else if (dtype_ == DataType::BF16) { // Convert F32 to BF16 using the same conversion as Python's ml_dtypes.bfloat16 // This uses round-to-nearest-even (matching _f32_to_bf16 implementation) @@ -112,6 +124,7 @@ void RoPE::initialize_cache() { // copy_from handles cross-device copying to target device sin_cache_->copy_from(sin_bf16_cpu); cos_cache_->copy_from(cos_bf16_cpu); + sync_cache_copy(device_); } else if (dtype_ == DataType::F16) { // Convert F32 to F16 std::vector sin_f16_data(max_seq_len_ * cache_dim); @@ -127,6 +140,7 @@ void RoPE::initialize_cache() { sin_cache_->copy_from(sin_f16_cpu); cos_cache_->copy_from(cos_f16_cpu); + sync_cache_copy(device_); } else { throw std::runtime_error( "RoPE cache dtype conversion not yet supported for dtype: " diff --git a/src/infiniop/ops/paged_attention/ascend/paged_attention_ascend_kernel.cpp b/src/infiniop/ops/paged_attention/ascend/paged_attention_ascend_kernel.cpp index 0f47c2ff5..e6df4f50b 100644 --- a/src/infiniop/ops/paged_attention/ascend/paged_attention_ascend_kernel.cpp +++ b/src/infiniop/ops/paged_attention/ascend/paged_attention_ascend_kernel.cpp @@ -164,22 +164,22 @@ class PagedAttentionKernel { private: __aicore__ inline ptrdiff_t keyBase(size_t seq_idx, size_t kv_head_idx, size_t token_idx) { - const size_t block_idx = token_idx / _page_block_size; + const size_t page_block_idx = token_idx / _page_block_size; const size_t token_offset = token_idx % _page_block_size; const int64_t physical_block = loadIndex( _block_tables_gm, - static_cast(seq_idx) * _block_table_batch_stride + static_cast(block_idx)); + static_cast(seq_idx) * _block_table_batch_stride + static_cast(page_block_idx)); return static_cast(physical_block) * _k_batch_stride + static_cast(kv_head_idx) * _k_head_stride + static_cast(token_offset) * _k_row_stride; } __aicore__ inline ptrdiff_t valueBase(size_t seq_idx, size_t kv_head_idx, size_t token_idx) { - const size_t block_idx = token_idx / _page_block_size; + const size_t page_block_idx = token_idx / _page_block_size; const size_t token_offset = token_idx % _page_block_size; const int64_t physical_block = loadIndex( _block_tables_gm, - static_cast(seq_idx) * _block_table_batch_stride + static_cast(block_idx)); + static_cast(seq_idx) * _block_table_batch_stride + static_cast(page_block_idx)); return static_cast(physical_block) * _v_batch_stride + static_cast(kv_head_idx) * _v_head_stride + static_cast(token_offset) * _v_row_stride; diff --git a/src/infiniop/ops/paged_attention_prefill/ascend/paged_attention_prefill_ascend_kernel.cpp b/src/infiniop/ops/paged_attention_prefill/ascend/paged_attention_prefill_ascend_kernel.cpp index e1fce2785..db90b2056 100644 --- a/src/infiniop/ops/paged_attention_prefill/ascend/paged_attention_prefill_ascend_kernel.cpp +++ b/src/infiniop/ops/paged_attention_prefill/ascend/paged_attention_prefill_ascend_kernel.cpp @@ -197,22 +197,22 @@ class PagedAttentionPrefillKernel { } __aicore__ inline ptrdiff_t keyBase(size_t seq_idx, size_t kv_head_idx, size_t token_idx) { - const size_t block_idx = token_idx / _page_block_size; + const size_t page_block_idx = token_idx / _page_block_size; const size_t token_offset = token_idx % _page_block_size; const int64_t physical_block = loadPrefillIndex( _block_tables_gm, - static_cast(seq_idx) * _block_table_batch_stride + static_cast(block_idx)); + static_cast(seq_idx) * _block_table_batch_stride + static_cast(page_block_idx)); return static_cast(physical_block) * _k_batch_stride + static_cast(kv_head_idx) * _k_head_stride + static_cast(token_offset) * _k_row_stride; } __aicore__ inline ptrdiff_t valueBase(size_t seq_idx, size_t kv_head_idx, size_t token_idx) { - const size_t block_idx = token_idx / _page_block_size; + const size_t page_block_idx = token_idx / _page_block_size; const size_t token_offset = token_idx % _page_block_size; const int64_t physical_block = loadPrefillIndex( _block_tables_gm, - static_cast(seq_idx) * _block_table_batch_stride + static_cast(block_idx)); + static_cast(seq_idx) * _block_table_batch_stride + static_cast(page_block_idx)); return static_cast(physical_block) * _v_batch_stride + static_cast(kv_head_idx) * _v_head_stride + static_cast(token_offset) * _v_row_stride; diff --git a/xmake/ascend.lua b/xmake/ascend.lua index 6179cac13..07211b065 100644 --- a/xmake/ascend.lua +++ b/xmake/ascend.lua @@ -10,7 +10,14 @@ add_links("libascendcl.so") add_links("libnnopbase.so") add_links("libopapi.so") add_links("libruntime.so") -add_linkdirs(ASCEND_HOME .. "/../../driver/lib64/driver") +for _, driver_dir in ipairs({ + path.join(ASCEND_HOME, "../driver/lib64/driver"), + path.join(ASCEND_HOME, "../../driver/lib64/driver"), +}) do + if os.isdir(driver_dir) then + add_linkdirs(driver_dir) + end +end add_links("libascend_hal.so") local builddir = string.format( "%s/build/%s/%s/%s", From cf15b6a88f04984de13c664540f4dce90a16f010 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Mon, 6 Jul 2026 16:17:14 +0800 Subject: [PATCH 05/10] fix: support ascend standalone infinirt graph bridge --- .../graph/standalone_infinirt_graph_bridge.cc | 220 +++++++++++++++--- xmake.lua | 15 ++ xmake/ascend.lua | 23 +- 3 files changed, 217 insertions(+), 41 deletions(-) diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc index be4c90940..e74bcabde 100644 --- a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc +++ b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc @@ -2,6 +2,8 @@ #ifdef USE_STANDALONE_INFINIRT_GRAPH +#include "../utils.hpp" + #include #include @@ -11,7 +13,6 @@ namespace infinicore::graph::standalone_infinirt { namespace { using StandaloneDevice = infini::rt::Device; -using StandaloneRuntime = infini::rt::runtime::Runtime; bool truthy_env(const char *name) { auto value = std::getenv(name); @@ -22,40 +23,154 @@ bool truthy_env(const char *name) { return text == "1" || text == "ON" || text == "on" || text == "true" || text == "TRUE"; } +std::string standalone_library_path() { + if (auto explicit_path = std::getenv("INFINIRT_GRAPH_LIBRARY")) { + return explicit_path; + } + if (auto root = std::getenv("INFINI_RT_ROOT")) { + return std::string(root) + "/lib/libinfinirt.so"; + } + return "libinfinirt.so"; +} + +void log_loaded_once() { + static bool logged_once = false; + if (!logged_once) { + logged_once = true; + spdlog::info("Standalone InfiniRT graph bridge loaded: {}", standalone_library_path()); + } +} + +template +constexpr bool standalone_device_enabled() { + return infini::rt::DeviceEnabled::value; +} + bool supports_device(Device::Type type) { - return type == Device::Type::NVIDIA; + switch (type) { + case Device::Type::NVIDIA: + return standalone_device_enabled(); + case Device::Type::ASCEND: + return standalone_device_enabled(); + default: + return false; + } } -template +thread_local Device::Type current_device_type = Device::Type::CPU; + +template infiniStatus_t to_core_status(Status status) { - return status == StandaloneRuntime::kSuccess + return status == Runtime::kSuccess ? INFINI_STATUS_SUCCESS : INFINI_STATUS_INTERNAL_ERROR; } -StandaloneRuntime::Stream to_standalone_stream(infinirtStream_t stream) { - return reinterpret_cast(stream); +template +infiniStatus_t set_device_impl(int index) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + return to_core_status(Runtime::SetDevice(index)); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } +} + +template +infiniStatus_t stream_begin_capture_impl(infinirtStream_t stream, infinirtStreamCaptureMode_t mode) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + auto standalone_mode = Runtime::kStreamCaptureModeRelaxed; + switch (mode) { + case INFINIRT_STREAM_CAPTURE_MODE_GLOBAL: + standalone_mode = Runtime::kStreamCaptureModeGlobal; + break; + case INFINIRT_STREAM_CAPTURE_MODE_THREAD_LOCAL: + standalone_mode = Runtime::kStreamCaptureModeThreadLocal; + break; + case INFINIRT_STREAM_CAPTURE_MODE_RELAXED: + standalone_mode = Runtime::kStreamCaptureModeRelaxed; + break; + } + return to_core_status(Runtime::StreamBeginCapture( + reinterpret_cast(stream), + standalone_mode)); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } +} + +template +infiniStatus_t stream_end_capture_impl(infinirtStream_t stream, infinirtGraph_t *graph) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + return to_core_status(Runtime::StreamEndCapture( + reinterpret_cast(stream), + reinterpret_cast(graph))); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } +} + +template +infiniStatus_t graph_destroy_impl(infinirtGraph_t graph) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + return to_core_status(Runtime::GraphDestroy( + reinterpret_cast(graph))); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } +} + +template +infiniStatus_t graph_instantiate_impl(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + return to_core_status(Runtime::GraphInstantiate( + reinterpret_cast(graph_exec), + reinterpret_cast(graph))); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } } -StandaloneRuntime::Graph to_standalone_graph(infinirtGraph_t graph) { - return reinterpret_cast(graph); +template +infiniStatus_t graph_exec_destroy_impl(infinirtGraphExec_t graph_exec) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + return to_core_status(Runtime::GraphExecDestroy( + reinterpret_cast(graph_exec))); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } } -StandaloneRuntime::GraphExec to_standalone_graph_exec(infinirtGraphExec_t graph_exec) { - return reinterpret_cast(graph_exec); +template +infiniStatus_t graph_launch_impl(infinirtGraphExec_t graph_exec, infinirtStream_t stream) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + return to_core_status(Runtime::GraphLaunch( + reinterpret_cast(graph_exec), + reinterpret_cast(stream))); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } } -decltype(StandaloneRuntime::kStreamCaptureModeRelaxed) -to_standalone_capture_mode(infinirtStreamCaptureMode_t mode) { - switch (mode) { - case INFINIRT_STREAM_CAPTURE_MODE_GLOBAL: - return StandaloneRuntime::kStreamCaptureModeGlobal; - case INFINIRT_STREAM_CAPTURE_MODE_THREAD_LOCAL: - return StandaloneRuntime::kStreamCaptureModeThreadLocal; - case INFINIRT_STREAM_CAPTURE_MODE_RELAXED: - return StandaloneRuntime::kStreamCaptureModeRelaxed; +template +infiniStatus_t dispatch_current( + infiniStatus_t (*nvidia_fn)(Args...), + infiniStatus_t (*ascend_fn)(Args...), + Args... args) { + switch (current_device_type) { + case Device::Type::NVIDIA: + return nvidia_fn(args...); + case Device::Type::ASCEND: + return ascend_fn(args...); + default: + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; } - return StandaloneRuntime::kStreamCaptureModeRelaxed; } } // namespace @@ -65,65 +180,96 @@ bool enabled() { } bool available(const Device &device) { - return enabled() && supports_device(device.getType()); + if (!enabled() || !supports_device(device.getType())) { + return false; + } + log_loaded_once(); + return true; } infiniStatus_t set_device(const Device &device) { if (!supports_device(device.getType())) { return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; } - return to_core_status(StandaloneRuntime::SetDevice(static_cast(device.getIndex()))); + auto status = INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + switch (device.getType()) { + case Device::Type::NVIDIA: + status = set_device_impl(static_cast(device.getIndex())); + break; + case Device::Type::ASCEND: + status = set_device_impl(static_cast(device.getIndex())); + break; + default: + break; + } + if (status == INFINI_STATUS_SUCCESS) { + current_device_type = device.getType(); + } + return status; } infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode) { if (stream == nullptr) { return INFINI_STATUS_NULL_POINTER; } - return to_core_status(StandaloneRuntime::StreamBeginCapture( - to_standalone_stream(stream), - to_standalone_capture_mode(mode))); + return dispatch_current( + stream_begin_capture_impl, + stream_begin_capture_impl, + stream, + mode); } infiniStatus_t stream_end_capture(infinirtStream_t stream, infinirtGraph_t *graph) { if (stream == nullptr || graph == nullptr) { return INFINI_STATUS_NULL_POINTER; } - return to_core_status(StandaloneRuntime::StreamEndCapture( - to_standalone_stream(stream), - reinterpret_cast(graph))); + return dispatch_current( + stream_end_capture_impl, + stream_end_capture_impl, + stream, + graph); } infiniStatus_t graph_destroy(infinirtGraph_t graph) { if (graph == nullptr) { return INFINI_STATUS_NULL_POINTER; } - return to_core_status(StandaloneRuntime::GraphDestroy(to_standalone_graph(graph))); + return dispatch_current( + graph_destroy_impl, + graph_destroy_impl, + graph); } infiniStatus_t graph_instantiate(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph) { if (graph_exec == nullptr || graph == nullptr) { return INFINI_STATUS_NULL_POINTER; } - return to_core_status(StandaloneRuntime::GraphInstantiate( - reinterpret_cast(graph_exec), - to_standalone_graph(graph))); + return dispatch_current( + graph_instantiate_impl, + graph_instantiate_impl, + graph_exec, + graph); } infiniStatus_t graph_exec_destroy(infinirtGraphExec_t graph_exec) { if (graph_exec == nullptr) { return INFINI_STATUS_NULL_POINTER; } - return to_core_status(StandaloneRuntime::GraphExecDestroy( - to_standalone_graph_exec(graph_exec))); + return dispatch_current( + graph_exec_destroy_impl, + graph_exec_destroy_impl, + graph_exec); } infiniStatus_t graph_launch(infinirtGraphExec_t graph_exec, infinirtStream_t stream) { if (graph_exec == nullptr || stream == nullptr) { return INFINI_STATUS_NULL_POINTER; } - return to_core_status(StandaloneRuntime::GraphLaunch( - to_standalone_graph_exec(graph_exec), - to_standalone_stream(stream))); + return dispatch_current( + graph_launch_impl, + graph_launch_impl, + graph_exec, + stream); } } // namespace infinicore::graph::standalone_infinirt diff --git a/xmake.lua b/xmake.lua index 8648900ee..e0d23ad4f 100644 --- a/xmake.lua +++ b/xmake.lua @@ -699,6 +699,21 @@ target("infinicore_cpp_api") raise("--standalone-infinirt-graph requires --infinirt-root or INFINI_RT_ROOT") end add_includedirs(graph_infinirt_root .. "/include") + if has_config("ascend-npu") then + local ascend_home = os.getenv("ASCEND_HOME") or os.getenv("ASCEND_TOOLKIT_HOME") or os.getenv("ASCEND_HOME_PATH") + if ascend_home and ascend_home ~= "" then + for _, include_dir in ipairs({ + path.join(ascend_home, "include"), + path.join(ascend_home, "include/aclnn"), + path.join(ascend_home, "aarch64-linux/include"), + path.join(ascend_home, "aarch64-linux/include/aclnn"), + }) do + if os.isdir(include_dir) then + add_includedirs(include_dir) + end + end + end + end end if has_config("nv-gpu") then local cuda_root = os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") or get_config("cuda") or "/usr/local/cuda" diff --git a/xmake/ascend.lua b/xmake/ascend.lua index 07211b065..af82ab162 100644 --- a/xmake/ascend.lua +++ b/xmake/ascend.lua @@ -1,11 +1,26 @@ add_defines("ENABLE_ASCEND_API") -local ASCEND_HOME = os.getenv("ASCEND_HOME") or os.getenv("ASCEND_TOOLKIT_HOME") +local ASCEND_HOME = os.getenv("ASCEND_HOME") or os.getenv("ASCEND_TOOLKIT_HOME") or os.getenv("ASCEND_HOME_PATH") local SOC_VERSION = os.getenv("SOC_VERSION") -- Add include dirs -add_includedirs(ASCEND_HOME .. "/include") -add_includedirs(ASCEND_HOME .. "/include/aclnn") -add_linkdirs(ASCEND_HOME .. "/lib64") +for _, include_dir in ipairs({ + path.join(ASCEND_HOME, "include"), + path.join(ASCEND_HOME, "include/aclnn"), + path.join(ASCEND_HOME, "aarch64-linux/include"), + path.join(ASCEND_HOME, "aarch64-linux/include/aclnn"), +}) do + if os.isdir(include_dir) then + add_includedirs(include_dir) + end +end +for _, lib_dir in ipairs({ + path.join(ASCEND_HOME, "lib64"), + path.join(ASCEND_HOME, "aarch64-linux/lib64"), +}) do + if os.isdir(lib_dir) then + add_linkdirs(lib_dir) + end +end add_links("libascendcl.so") add_links("libnnopbase.so") add_links("libopapi.so") From 5691df0c6fa714f08fcbfb9f3d2d54dba668bd73 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Mon, 6 Jul 2026 16:58:07 +0800 Subject: [PATCH 06/10] fix: drop stale standalone InfiniRT C API bridge log The standalone bridge now calls the InfiniRT C++ runtime API directly, so remove the old loaded-library marker and keep the runtime log aligned with the actual path. --- src/infinicore/graph/graph.cc | 2 +- .../graph/standalone_infinirt_graph_bridge.cc | 26 +------------------ 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/src/infinicore/graph/graph.cc b/src/infinicore/graph/graph.cc index ab38f46e4..bd37260c6 100644 --- a/src/infinicore/graph/graph.cc +++ b/src/infinicore/graph/graph.cc @@ -103,7 +103,7 @@ void Graph::instantiate() { static bool logged_once = false; if (!logged_once) { logged_once = true; - spdlog::info("Using standalone InfiniRT graph bridge for graph capture and replay."); + spdlog::info("Using standalone InfiniRT C++ graph runtime API for graph capture and replay."); } } diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc index e74bcabde..7725d48c1 100644 --- a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc +++ b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc @@ -2,8 +2,6 @@ #ifdef USE_STANDALONE_INFINIRT_GRAPH -#include "../utils.hpp" - #include #include @@ -23,24 +21,6 @@ bool truthy_env(const char *name) { return text == "1" || text == "ON" || text == "on" || text == "true" || text == "TRUE"; } -std::string standalone_library_path() { - if (auto explicit_path = std::getenv("INFINIRT_GRAPH_LIBRARY")) { - return explicit_path; - } - if (auto root = std::getenv("INFINI_RT_ROOT")) { - return std::string(root) + "/lib/libinfinirt.so"; - } - return "libinfinirt.so"; -} - -void log_loaded_once() { - static bool logged_once = false; - if (!logged_once) { - logged_once = true; - spdlog::info("Standalone InfiniRT graph bridge loaded: {}", standalone_library_path()); - } -} - template constexpr bool standalone_device_enabled() { return infini::rt::DeviceEnabled::value; @@ -180,11 +160,7 @@ bool enabled() { } bool available(const Device &device) { - if (!enabled() || !supports_device(device.getType())) { - return false; - } - log_loaded_once(); - return true; + return enabled() && supports_device(device.getType()); } infiniStatus_t set_device(const Device &device) { From 44b7f1f9caa2ef5c875e34be42ffa77b6f4dd300 Mon Sep 17 00:00:00 2001 From: zhushuang Date: Mon, 6 Jul 2026 12:04:40 +0000 Subject: [PATCH 07/10] issue/1311 - feat: refactor infinicore cpu runtime with infinirt (cherry picked from commit 0287010376379e8e594eec034557a9c77f56bfc3) --- include/infiniop/handle.h | 2 + src/bridge/infini/rt.hpp | 40 +++++++++++++ .../allocators/device_pinned_allocator.cc | 8 +-- .../context/allocators/host_allocator.cc | 2 - .../allocators/pinnable_block_allocator.cc | 13 ++-- .../allocators/stream_ordered_allocator.cc | 14 ++++- src/infinicore/context/context_impl.cc | 37 ++++++------ src/infinicore/context/runtime/runtime.cc | 54 +++++++++-------- src/infinicore/graph/graph.cc | 19 ++++-- src/infiniop/devices/handle.cc | 23 +++++-- test/infiniop/libinfiniop/liboperators.py | 24 ++++---- test/infiniop/libinfiniop/utils.py | 19 +++--- xmake.lua | 60 +++---------------- xmake/cpu.lua | 1 + xmake/test.lua | 10 +++- 15 files changed, 185 insertions(+), 141 deletions(-) create mode 100644 src/bridge/infini/rt.hpp diff --git a/include/infiniop/handle.h b/include/infiniop/handle.h index da8689542..207ca2ce9 100644 --- a/include/infiniop/handle.h +++ b/include/infiniop/handle.h @@ -7,6 +7,8 @@ struct InfiniopHandle; typedef struct InfiniopHandle *infiniopHandle_t; +__INFINI_C __export infiniStatus_t infiniopSetRuntimeDevice(infiniDevice_t device, int device_id); + __INFINI_C __export infiniStatus_t infiniopCreateHandle(infiniopHandle_t *handle_ptr); __INFINI_C __export infiniStatus_t infiniopDestroyHandle(infiniopHandle_t handle); diff --git a/src/bridge/infini/rt.hpp b/src/bridge/infini/rt.hpp new file mode 100644 index 000000000..a79085cdc --- /dev/null +++ b/src/bridge/infini/rt.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include "infinicore.h" + +#include + +namespace infinicore::bridge::infini::rt { + +inline infiniStatus_t translate(::infini::rt::runtime::Error error) { + switch (error) { + case ::infini::rt::runtime::kSuccess: + return INFINI_STATUS_SUCCESS; + default: + return INFINI_STATUS_INTERNAL_ERROR; + } +} + +inline ::infini::rt::Device::Type translate(infiniDevice_t device) { + switch (device) { + case INFINI_DEVICE_CPU: + return ::infini::rt::Device::Type::kCpu; + case INFINI_DEVICE_NVIDIA: + return ::infini::rt::Device::Type::kNvidia; + default: + return ::infini::rt::Device::Type::kCount; + } +} + +inline infiniDevice_t translate(::infini::rt::Device::Type device) { + switch (device) { + case ::infini::rt::Device::Type::kCpu: + return INFINI_DEVICE_CPU; + case ::infini::rt::Device::Type::kNvidia: + return INFINI_DEVICE_NVIDIA; + default: + return INFINI_DEVICE_TYPE_COUNT; + } +} + +} // namespace infinicore::bridge::infini::rt diff --git a/src/infinicore/context/allocators/device_pinned_allocator.cc b/src/infinicore/context/allocators/device_pinned_allocator.cc index 36ee599ff..a8764d5f6 100644 --- a/src/infinicore/context/allocators/device_pinned_allocator.cc +++ b/src/infinicore/context/allocators/device_pinned_allocator.cc @@ -1,6 +1,6 @@ #include "device_pinned_allocator.hpp" -#include +#include "../../../bridge/infini/rt.hpp" #include "../../utils.hpp" @@ -16,7 +16,7 @@ std::byte *DevicePinnedHostAllocator::allocate(size_t size) { return nullptr; } void *ptr; - INFINICORE_CHECK_ERROR(infinirtMallocHost(&ptr, size)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MallocHost(&ptr, size))); return (std::byte *)ptr; } @@ -25,7 +25,7 @@ void DevicePinnedHostAllocator::deallocate(std::byte *ptr) { return; } if (owner_ == context::getDevice()) { - INFINICORE_CHECK_ERROR(infinirtFreeHost(ptr)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::FreeHost(ptr))); gc(); } else { gc_queue_.push(ptr); @@ -35,7 +35,7 @@ void DevicePinnedHostAllocator::deallocate(std::byte *ptr) { void DevicePinnedHostAllocator::gc() { while (gc_queue_.empty() == false) { std::byte *p = gc_queue_.front(); - INFINICORE_CHECK_ERROR(infinirtFreeHost(p)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::FreeHost(p))); gc_queue_.pop(); } } diff --git a/src/infinicore/context/allocators/host_allocator.cc b/src/infinicore/context/allocators/host_allocator.cc index 41a55b453..0f1cc7072 100644 --- a/src/infinicore/context/allocators/host_allocator.cc +++ b/src/infinicore/context/allocators/host_allocator.cc @@ -1,7 +1,5 @@ #include "host_allocator.hpp" -#include - namespace infinicore { std::byte *HostAllocator::allocate(size_t size) { if (size == 0) { diff --git a/src/infinicore/context/allocators/pinnable_block_allocator.cc b/src/infinicore/context/allocators/pinnable_block_allocator.cc index 32e5c5e9b..3c70376f0 100644 --- a/src/infinicore/context/allocators/pinnable_block_allocator.cc +++ b/src/infinicore/context/allocators/pinnable_block_allocator.cc @@ -4,8 +4,9 @@ #include "../../utils.hpp" +#include "../../../bridge/infini/rt.hpp" + #include -#include #include namespace infinicore { @@ -74,7 +75,7 @@ std::byte *PinnableBlockAllocator::allocate(size_t size) { block->in_use = true; block->use_count = 1; - INFINICORE_CHECK_ERROR(infinirtMalloc(&block->ptr, block->size)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Malloc(&block->ptr, block->size))); all_blocks_[block->ptr] = block; return reinterpret_cast(block->ptr); @@ -101,7 +102,7 @@ std::byte *PinnableBlockAllocator::allocate(size_t size) { block->in_use = true; block->use_count = 1; - INFINICORE_CHECK_ERROR(infinirtMalloc(&block->ptr, block->size)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Malloc(&block->ptr, block->size))); large_blocks_.push_back(block); all_blocks_[block->ptr] = block; @@ -167,7 +168,7 @@ void PinnableBlockAllocator::trim() { for (auto &cls : size_classes_) { for (auto it = cls.free_blocks.begin(); it != cls.free_blocks.end();) { if (!(*it)->frozen) { - INFINICORE_CHECK_ERROR(infinirtFree((*it)->ptr)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Free((*it)->ptr))); all_blocks_.erase((*it)->ptr); it = cls.free_blocks.erase(it); } else { @@ -178,7 +179,7 @@ void PinnableBlockAllocator::trim() { // Free non-frozen large blocks for (auto it = large_blocks_.begin(); it != large_blocks_.end();) { if (!(*it)->frozen && !(*it)->in_use) { - INFINICORE_CHECK_ERROR(infinirtFree((*it)->ptr)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Free((*it)->ptr))); all_blocks_.erase((*it)->ptr); it = large_blocks_.erase(it); } else { @@ -192,7 +193,7 @@ PinnableBlockAllocator::~PinnableBlockAllocator() { std::lock_guard lock(mutex_); for (auto &p : all_blocks_) { if (p.second->ptr) { - infinirtFree(p.second->ptr); + (void)infini::rt::runtime::Free(p.second->ptr); } } all_blocks_.clear(); diff --git a/src/infinicore/context/allocators/stream_ordered_allocator.cc b/src/infinicore/context/allocators/stream_ordered_allocator.cc index 3bd0aaab1..f2092aa93 100644 --- a/src/infinicore/context/allocators/stream_ordered_allocator.cc +++ b/src/infinicore/context/allocators/stream_ordered_allocator.cc @@ -1,6 +1,6 @@ #include "stream_ordered_allocator.hpp" -#include +#include "../../../bridge/infini/rt.hpp" #include "../../utils.hpp" @@ -12,7 +12,11 @@ std::byte *StreamOrderedAllocator::allocate(size_t size) { return nullptr; } void *ptr = nullptr; - INFINICORE_CHECK_ERROR(infinirtMallocAsync(&ptr, size, context::getStream())); + if (device_.getType() != Device::Type::CPU) { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MallocAsync(&ptr, size, context::getStream()))); + } else { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Malloc(&ptr, size))); + } return (std::byte *)ptr; } @@ -20,6 +24,10 @@ void StreamOrderedAllocator::deallocate(std::byte *ptr) { if (ptr == nullptr) { return; } - INFINICORE_CHECK_ERROR(infinirtFreeAsync(ptr, context::getStream())); + if (device_.getType() != Device::Type::CPU) { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::FreeAsync(ptr, context::getStream()))); + } else { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Free(ptr))); + } } } // namespace infinicore diff --git a/src/infinicore/context/context_impl.cc b/src/infinicore/context/context_impl.cc index f126beb1d..06c85955c 100644 --- a/src/infinicore/context/context_impl.cc +++ b/src/infinicore/context/context_impl.cc @@ -1,4 +1,5 @@ #include "context_impl.hpp" +#include "../../bridge/infini/rt.hpp" #include "internal.hpp" #include "../utils.hpp" @@ -61,26 +62,28 @@ ContextImpl &ContextImpl::singleton() { } ContextImpl::ContextImpl() { - std::vector device_counter(static_cast(Device::Type::COUNT)); - INFINICORE_CHECK_ERROR(infinirtGetAllDeviceCount(device_counter.data())); - - // Reserve runtime slot for all devices. - runtime_table_[0].resize(device_counter[0]); - runtime_table_[0][0] = std::unique_ptr(new Runtime(Device(Device::Type::CPU, 0))); - - // Context will try to use the first non-cpu available device as the default runtime. - for (int i = int(Device::Type::COUNT) - 1; i > 0; i--) { - if (device_counter[i] > 0) { - runtime_table_[i].resize(device_counter[i]); - if (current_runtime_ == nullptr) { - runtime_table_[i][0] = std::unique_ptr(new Runtime(Device(Device::Type(i), 0))); - current_runtime_ = runtime_table_[i][0].get(); - } + std::vector device_counter(static_cast(Device::Type::COUNT), 0); + + infini::rt::set_runtime_device_type(bridge::infini::rt::translate(INFINI_DEVICE_CPU)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::GetDeviceCount(&device_counter[static_cast(Device::Type::CPU)]))); + + runtime_table_[static_cast(Device::Type::CPU)].resize(device_counter[static_cast(Device::Type::CPU)]); + if (device_counter[static_cast(Device::Type::CPU)] > 0) { + runtime_table_[static_cast(Device::Type::CPU)][0] = std::unique_ptr(new Runtime(Device(Device::Type::CPU, 0))); + } + + if constexpr (infini::rt::DeviceEnabled::value) { + infini::rt::set_runtime_device_type(bridge::infini::rt::translate(INFINI_DEVICE_NVIDIA)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::GetDeviceCount(&device_counter[static_cast(Device::Type::NVIDIA)]))); + runtime_table_[static_cast(Device::Type::NVIDIA)].resize(device_counter[static_cast(Device::Type::NVIDIA)]); + if (device_counter[static_cast(Device::Type::NVIDIA)] > 0) { + runtime_table_[static_cast(Device::Type::NVIDIA)][0] = std::unique_ptr(new Runtime(Device(Device::Type::NVIDIA, 0))); + current_runtime_ = runtime_table_[static_cast(Device::Type::NVIDIA)][0].get(); } } - if (current_runtime_ == nullptr) { - current_runtime_ = runtime_table_[0][0].get(); + if (current_runtime_ == nullptr && !runtime_table_[static_cast(Device::Type::CPU)].empty()) { + current_runtime_ = runtime_table_[static_cast(Device::Type::CPU)][0].get(); } } diff --git a/src/infinicore/context/runtime/runtime.cc b/src/infinicore/context/runtime/runtime.cc index 92ba0f737..9c14c0207 100644 --- a/src/infinicore/context/runtime/runtime.cc +++ b/src/infinicore/context/runtime/runtime.cc @@ -1,4 +1,5 @@ #include "runtime.hpp" +#include "../../../bridge/infini/rt.hpp" #include "../../utils.hpp" @@ -10,7 +11,7 @@ namespace infinicore { Runtime::Runtime(Device device) : device_(device), graph_manager_(std::make_unique()) { activate(); - INFINICORE_CHECK_ERROR(infinirtStreamCreate(&stream_)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamCreate(&stream_))); INFINICORE_CHECK_ERROR(infiniopCreateHandle(&infiniop_handle_)); if (device_.getType() == Device::Type::CPU) { device_memory_allocator_ = std::make_unique(device); @@ -26,11 +27,14 @@ Runtime::~Runtime() { } device_memory_allocator_.reset(); infiniopDestroyHandle(infiniop_handle_); - infinirtStreamDestroy(stream_); + (void)infini::rt::runtime::StreamDestroy(stream_); } Runtime *Runtime::activate() { - INFINICORE_CHECK_ERROR(infinirtSetDevice((infiniDevice_t)device_.getType(), (int)device_.getIndex())); + auto rt_device = bridge::infini::rt::translate(static_cast(device_.getType())); + INFINICORE_ASSERT(rt_device != infini::rt::Device::Type::kCount); + infini::rt::set_runtime_device_type(rt_device); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::SetDevice(static_cast(device_.getIndex())))); return this; } @@ -47,11 +51,11 @@ infiniopHandle_t Runtime::infiniopHandle() const { } void Runtime::syncStream() { - INFINICORE_CHECK_ERROR(infinirtStreamSynchronize(stream_)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamSynchronize(stream_))); } void Runtime::syncDevice() { - INFINICORE_CHECK_ERROR(infinirtDeviceSynchronize()); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::DeviceSynchronize())); } void Runtime::trimMemory() { @@ -103,43 +107,47 @@ std::shared_ptr Runtime::reinstantiateBlob(std::shared_ptr blob) } void Runtime::memcpyH2D(void *dst, const void *src, size_t size, bool async) { - if (async) { - INFINICORE_CHECK_ERROR(infinirtMemcpyAsync(dst, src, size, INFINIRT_MEMCPY_H2D, stream_)); + if (async && device_.getType() != Device::Type::CPU) { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice, stream_))); } else { - INFINICORE_CHECK_ERROR(infinirtMemcpy(dst, src, size, INFINIRT_MEMCPY_H2D)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memcpy(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice))); } } void Runtime::memcpyD2H(void *dst, const void *src, size_t size) { - INFINICORE_CHECK_ERROR(infinirtMemcpy(dst, src, size, INFINIRT_MEMCPY_D2H)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memcpy(dst, src, size, infini::rt::runtime::kMemcpyDeviceToHost))); } void Runtime::memcpyD2D(void *dst, const void *src, size_t size, bool async) { - if (async) { - INFINICORE_CHECK_ERROR(infinirtMemcpyAsync(dst, src, size, INFINIRT_MEMCPY_D2D, stream_)); + if (async && device_.getType() != Device::Type::CPU) { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice, stream_))); } else { - INFINICORE_CHECK_ERROR(infinirtMemcpy(dst, src, size, INFINIRT_MEMCPY_D2D)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memcpy(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice))); } } void Runtime::setDeviceMemory(void *ptr, int value, size_t count) { - INFINICORE_CHECK_ERROR(infinirtMemset(ptr, value, count)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memset(ptr, value, count))); } void Runtime::setDeviceMemoryAsync(void *ptr, int value, size_t count, infinirtStream_t stream) { - INFINICORE_CHECK_ERROR(infinirtMemsetAsync(ptr, value, count, stream)); + if (device_.getType() != Device::Type::CPU) { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemsetAsync(ptr, value, count, stream))); + } else { + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memset(ptr, value, count))); + } } // Timing method implementations infinirtEvent_t Runtime::createEvent() { infinirtEvent_t event; - INFINICORE_CHECK_ERROR(infinirtEventCreate(&event)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventCreate(&event))); return event; } infinirtEvent_t Runtime::createEventWithFlags(uint32_t flags) { infinirtEvent_t event; - INFINICORE_CHECK_ERROR(infinirtEventCreateWithFlags(&event, flags)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventCreateWithFlags(&event, flags))); return event; } @@ -147,26 +155,24 @@ void Runtime::recordEvent(infinirtEvent_t event, infinirtStream_t stream) { if (stream == nullptr) { stream = stream_; } - INFINICORE_CHECK_ERROR(infinirtEventRecord(event, stream)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventRecord(event, stream))); } bool Runtime::queryEvent(infinirtEvent_t event) { - infinirtEventStatus_t status; - INFINICORE_CHECK_ERROR(infinirtEventQuery(event, &status)); - return status == INFINIRT_EVENT_COMPLETE; + return bridge::infini::rt::translate(infini::rt::runtime::EventQuery(event)) == INFINI_STATUS_SUCCESS; } void Runtime::synchronizeEvent(infinirtEvent_t event) { - INFINICORE_CHECK_ERROR(infinirtEventSynchronize(event)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventSynchronize(event))); } void Runtime::destroyEvent(infinirtEvent_t event) { - INFINICORE_CHECK_ERROR(infinirtEventDestroy(event)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventDestroy(event))); } float Runtime::elapsedTime(infinirtEvent_t start, infinirtEvent_t end) { float ms; - INFINICORE_CHECK_ERROR(infinirtEventElapsedTime(&ms, start, end)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventElapsedTime(&ms, start, end))); return ms; } @@ -175,7 +181,7 @@ void Runtime::streamWaitEvent(infinirtStream_t stream, infinirtEvent_t event) { if (stream == nullptr) { stream = stream_; } - INFINICORE_CHECK_ERROR(infinirtStreamWaitEvent(stream, event)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamWaitEvent(stream, event, 0))); } bool Runtime::isGraphRecording() const { diff --git a/src/infinicore/graph/graph.cc b/src/infinicore/graph/graph.cc index bd37260c6..0a09bf68e 100644 --- a/src/infinicore/graph/graph.cc +++ b/src/infinicore/graph/graph.cc @@ -2,8 +2,11 @@ #include "../utils.hpp" #include "infinicore/context/context.hpp" + +#ifdef USE_INFINIRT_GRAPH #include "standalone_infinirt_graph_bridge.hpp" #include +#endif namespace infinicore::graph { @@ -32,6 +35,7 @@ DispatchableGraphOperator::~DispatchableGraphOperator() { * Graph * ========================= */ +#ifdef USE_INFINIRT_GRAPH struct Graph::DeviceGraph { infinirtGraph_t graph = nullptr; infinirtGraphExec_t exec = nullptr; @@ -69,17 +73,22 @@ struct Graph::DeviceGraph { } } }; +#else +struct Graph::DeviceGraph {}; +#endif Graph::Graph() { } void Graph::run() const { +#ifdef USE_INFINIRT_GRAPH if (device_graph_ != nullptr && device_graph_.get()->exec != nullptr) { device_graph_.get()->launch(); - } else { - for (auto &op : op_list_) { - op->run(); - } + return; + } +#endif + for (auto &op : op_list_) { + op->run(); } } @@ -88,6 +97,7 @@ void Graph::add_operator(std::shared_ptr op) { } void Graph::instantiate() { +#ifdef USE_INFINIRT_GRAPH // Reset device graph device_graph_ = std::make_unique(); device_graph_->standalone = standalone_infinirt::available(context::getDevice()); @@ -150,6 +160,7 @@ void Graph::instantiate() { } device_graph_.reset(); } +#endif } Graph::~Graph() = default; diff --git a/src/infiniop/devices/handle.cc b/src/infiniop/devices/handle.cc index b36d3b6bf..9ded0587b 100644 --- a/src/infiniop/devices/handle.cc +++ b/src/infiniop/devices/handle.cc @@ -1,6 +1,6 @@ #include "infiniop/handle.h" +#include "../../bridge/infini/rt.hpp" #include "../../utils.h" -#include "infinirt.h" #ifdef ENABLE_CPU_API #include "cpu/cpu_handle.h" @@ -24,14 +24,29 @@ #include "metax/metax_handle.h" #endif +__INFINI_C infiniStatus_t infiniopSetRuntimeDevice(infiniDevice_t device, int device_id) { + auto rt_device = infinicore::bridge::infini::rt::translate(device); + if (rt_device == infini::rt::Device::Type::kCount) { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } + + infini::rt::set_runtime_device_type(rt_device); + CHECK_STATUS(infinicore::bridge::infini::rt::translate(infini::rt::runtime::SetDevice(device_id))); + return INFINI_STATUS_SUCCESS; +} + __INFINI_C infiniStatus_t infiniopCreateHandle(infiniopHandle_t *handle_ptr) { if (handle_ptr == nullptr) { return INFINI_STATUS_NULL_POINTER; } - infiniDevice_t device; - int device_id; - CHECK_STATUS(infinirtGetDevice(&device, &device_id)); + infiniDevice_t device = infinicore::bridge::infini::rt::translate(infini::rt::runtime_device_type()); + if (device == INFINI_DEVICE_TYPE_COUNT) { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } + + int device_id = 0; + CHECK_STATUS(infinicore::bridge::infini::rt::translate(infini::rt::runtime::GetDevice(&device_id))); #define CREATE(CASE, NAMESPACE) \ case CASE: \ diff --git a/test/infiniop/libinfiniop/liboperators.py b/test/infiniop/libinfiniop/liboperators.py index ceffd5141..48ce04fe4 100644 --- a/test/infiniop/libinfiniop/liboperators.py +++ b/test/infiniop/libinfiniop/liboperators.py @@ -1,11 +1,12 @@ +import ctypes import os import platform -import ctypes -from ctypes import c_int, c_int64, c_uint64, POINTER +from ctypes import POINTER, c_int, c_int64, c_uint64 +from pathlib import Path + from .datatypes import * from .devices import * from .op_register import OpRegister -from pathlib import Path from .structs import * INFINI_ROOT = os.getenv("INFINI_ROOT") or str(Path.home() / ".infini") @@ -45,12 +46,12 @@ def find_library_in_ld_path(subdir, library_name): libop_path = find_library_in_ld_path("lib", "libinfiniop.so") librt_path = find_library_in_ld_path("lib", "libinfinirt.so") - assert ( - libop_path is not None - ), f"Cannot find infiniop.dll or libinfiniop.so. Check if INFINI_ROOT is set correctly." - assert ( - librt_path is not None - ), f"Cannot find infinirt.dll or libinfinirt.so. Check if INFINI_ROOT is set correctly." + assert libop_path is not None, ( + "Cannot find infiniop.dll or libinfiniop.so. Check if INFINI_ROOT is set correctly." + ) + assert librt_path is not None, ( + "Cannot find infinirt.dll or libinfinirt.so. Check if INFINI_ROOT is set correctly." + ) librt = ctypes.CDLL(librt_path) libop = ctypes.CDLL(libop_path) @@ -65,13 +66,12 @@ def find_library_in_ld_path(subdir, library_name): lib.infiniopCreateTensorDescriptor.restype = c_int lib.infiniopDestroyTensorDescriptor.argtypes = [infiniopTensorDescriptor_t] lib.infiniopDestroyTensorDescriptor.restype = c_int + lib.infiniopSetRuntimeDevice.argtypes = [c_int, c_int] + lib.infiniopSetRuntimeDevice.restype = c_int lib.infiniopCreateHandle.argtypes = [POINTER(infiniopHandle_t)] lib.infiniopCreateHandle.restype = c_int lib.infiniopDestroyHandle.argtypes = [infiniopHandle_t] lib.infiniopDestroyHandle.restype = c_int - lib.infinirtSetDevice.argtypes = [c_int, c_int] - lib.infinirtSetDevice.restype = c_int - OpRegister.register_lib(lib) return lib diff --git a/test/infiniop/libinfiniop/utils.py b/test/infiniop/libinfiniop/utils.py index c0560dbde..8b0b1c173 100644 --- a/test/infiniop/libinfiniop/utils.py +++ b/test/infiniop/libinfiniop/utils.py @@ -1,10 +1,12 @@ -from typing import Sequence -import torch import ctypes +from typing import Sequence + import numpy as np +import torch + from .datatypes import * from .devices import * -from .liboperators import infiniopTensorDescriptor_t, LIBINFINIOP, infiniopHandle_t +from .liboperators import LIBINFINIOP, infiniopHandle_t, infiniopTensorDescriptor_t def check_error(status): @@ -602,9 +604,10 @@ def print_discrepancy( if actual.shape != expected.shape: raise ValueError("Tensors must have the same shape to compare.") - import torch import sys + import torch + is_terminal = sys.stdout.isatty() actual = actual.to("cpu") @@ -730,7 +733,7 @@ def test_operator(device, test_func, test_cases, tensor_dtypes): to be passed to `test_func`. - tensor_dtypes (list): A list of tensor data types (e.g., `torch.float32`) to test. """ - LIBINFINIOP.infinirtSetDevice(device, ctypes.c_int(0)) + check_error(LIBINFINIOP.infiniopSetRuntimeDevice(device, ctypes.c_int(0))) handle = create_handle() tensor_dtypes = filter_tensor_dtypes_by_device(device, tensor_dtypes) try: @@ -768,12 +771,9 @@ def get_test_devices(args): if args.qy: devices_to_test.append(InfiniDeviceEnum.QY) if args.cambricon: - import torch_mlu - devices_to_test.append(InfiniDeviceEnum.CAMBRICON) if args.ascend: import torch - import torch_npu torch.npu.set_device(0) # Ascend NPU needs explicit device initialization devices_to_test.append(InfiniDeviceEnum.ASCEND) @@ -783,12 +783,9 @@ def get_test_devices(args): devices_to_test.append(InfiniDeviceEnum.METAX) if args.moore: import torch - import torch_musa devices_to_test.append(InfiniDeviceEnum.MOORE) if args.kunlun: - import torch_xmlir - devices_to_test.append(InfiniDeviceEnum.KUNLUN) if args.hygon: import torch diff --git a/xmake.lua b/xmake.lua index e0d23ad4f..642e908f9 100644 --- a/xmake.lua +++ b/xmake.lua @@ -485,57 +485,13 @@ target("infini-utils") add_files("src/utils/*.cc") target_end() -target("infinirt") - set_kind("shared") - - if has_config("cpu") then - add_deps("infinirt-cpu") - end - if has_config("nv-gpu") then - add_deps("infinirt-nvidia") - end - if has_config("cambricon-mlu") then - add_deps("infinirt-cambricon") - end - if has_config("ascend-npu") then - add_deps("infinirt-ascend") - end - if has_config("metax-gpu") then - add_deps("infinirt-metax") - end - if has_config("moore-gpu") then - add_deps("infinirt-moore") - end - if has_config("iluvatar-gpu") then - add_deps("infinirt-iluvatar") - end - if has_config("ali-ppu") then - add_deps("infinirt-ali") - end - if has_config("qy-gpu") then - add_deps("infinirt-qy") - add_files("build/.objs/infinirt-qy/rules/qy.cuda/src/infinirt/cuda/*.cu.o", {public = true}) - end - if has_config("kunlun-xpu") then - add_deps("infinirt-kunlun") - end - if has_config("hygon-dcu") then - add_deps("infinirt-hygon") - end - set_languages("cxx17") - if not is_plat("windows") then - add_cxflags("-fPIC") - add_cxxflags("-fPIC") - add_ldflags("-fPIC", {force = true}) - end - set_installdir(os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini")) - add_files("src/infinirt/*.cc") - add_installfiles("include/infinirt.h", {prefixdir = "include"}) -target_end() - target("infiniop") set_kind("shared") - add_deps("infinirt") + local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") + add_includedirs(INFINI_ROOT.."/include", { public = true }) + add_linkdirs(INFINI_ROOT.."/lib") + add_links("infinirt") + add_rpathdirs(INFINI_ROOT.."/lib") local public_cuda_root = get_config("cuda") or os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") if public_cuda_root and public_cuda_root ~= "" then @@ -613,7 +569,6 @@ target_end() target("infiniccl") set_kind("shared") - add_deps("infinirt") if has_config("nv-gpu") then add_deps("infiniccl-nvidia") @@ -659,7 +614,7 @@ target_end() target("infinicore_c_api") set_kind("phony") - add_deps("infiniop", "infinirt", "infiniccl") + add_deps("infiniop", "infiniccl") after_build(function (target) print(YELLOW .. "[Congratulations!] Now you can install the libraries with \"xmake install\"" .. NC) end) target_end() @@ -674,7 +629,7 @@ target_end() target("infinicore_cpp_api") set_kind("shared") - add_deps("infiniop", "infinirt", "infiniccl") + add_deps("infiniop", "infiniccl") set_languages("cxx17") set_symbols("visibility") @@ -752,6 +707,7 @@ target("infinicore_cpp_api") add_linkdirs(INFINI_ROOT.."/lib") add_links("infiniop", "infinirt", "infiniccl") + add_rpathdirs(INFINI_ROOT.."/lib") if get_config("flash-attn") and get_config("flash-attn") ~= "" then add_installfiles("(builddir)/$(plat)/$(arch)/$(mode)/flash-attn*.so", {prefixdir = "lib"}) diff --git a/xmake/cpu.lua b/xmake/cpu.lua index e192fbbbd..a1974b823 100644 --- a/xmake/cpu.lua +++ b/xmake/cpu.lua @@ -26,6 +26,7 @@ target_end() target("infinirt-cpu") set_kind("static") + set_default(false) add_deps("infini-utils") on_install(function (target) end) diff --git a/xmake/test.lua b/xmake/test.lua index 64533163b..d4ef4abe7 100644 --- a/xmake/test.lua +++ b/xmake/test.lua @@ -54,14 +54,20 @@ target_end() target("infinirt-test") set_kind("binary") - add_deps("infinirt") + set_default(false) on_install(function (target) end) set_languages("cxx17") set_warnings("all", "error") + local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") + add_includedirs(INFINI_ROOT.."/include") + add_linkdirs(INFINI_ROOT.."/lib") + add_links("infinirt") + add_rpathdirs(INFINI_ROOT.."/lib") + add_files(os.projectdir().."/src/infinirt-test/*.cc") - set_installdir(os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini")) + set_installdir(INFINI_ROOT) target_end() target("infinicore-test") From 2438773d074790ad6b4f8037f78aaacf62283bfa Mon Sep 17 00:00:00 2001 From: gongchensu Date: Wed, 8 Jul 2026 11:26:32 +0800 Subject: [PATCH 08/10] build: link Core against installed InfiniRT Use --infinirt-root or INFINI_RT_ROOT as the installed InfiniRT prefix for Core targets, install the runtime handle header with Core, and remove stale dependencies on the old in-tree infinirt target. The standalone graph build switch is removed because graph capture is now controlled by USE_INFINIRT_GRAPH and routed through the InfiniRT runtime API directly. --- xmake.lua | 124 ++++++++++++++++++++++----------------------- xmake/ali.lua | 1 - xmake/ascend.lua | 1 - xmake/bang.lua | 1 - xmake/hygon.lua | 1 - xmake/iluvatar.lua | 1 - xmake/kunlun.lua | 1 - xmake/metax.lua | 1 - xmake/moore.lua | 1 - xmake/nvidia.lua | 1 - xmake/qy.lua | 1 - 11 files changed, 60 insertions(+), 74 deletions(-) diff --git a/xmake.lua b/xmake.lua index 642e908f9..960ad5dcd 100644 --- a/xmake.lua +++ b/xmake.lua @@ -260,20 +260,10 @@ option("graph") set_description("Whether to use device graph instantiating feature, such as cuda graph for nvidia") option_end() -option("standalone-infinirt-graph") - set_default(false) - set_showmenu(true) - set_description("Whether to route graph lifecycle calls through an installed standalone InfiniRT") -option_end() - -if has_config("graph") or has_config("standalone-infinirt-graph") then +if has_config("graph") then add_defines("USE_INFINIRT_GRAPH") end -if has_config("standalone-infinirt-graph") then - add_defines("USE_STANDALONE_INFINIRT_GRAPH") -end - -- InfiniCCL option("ccl") @@ -302,10 +292,10 @@ option_end() option("infinirt-root") set_default("") set_showmenu(true) - set_description("Path to an installed standalone InfiniRT prefix used by --infiniops") + set_description("Path to an installed InfiniRT prefix") option_end() -local function get_standalone_infinirt_root() +local function get_infinirt_root() local infinirt_root = get_config("infinirt-root") if not infinirt_root or infinirt_root == "" then infinirt_root = os.getenv("INFINI_RT_ROOT") @@ -337,22 +327,52 @@ end local infiniops_external_built = false -local function find_standalone_infinirt(infinirt_root, xmake_os) - local standalone_infinirt = path.join(infinirt_root, "lib", "libinfinirt.so") - if not xmake_os.isfile(standalone_infinirt) then - standalone_infinirt = path.join(infinirt_root, "lib64", "libinfinirt.so") +local function find_infinirt_library(infinirt_root, xmake_os) + local infinirt_library = path.join(infinirt_root, "lib", "libinfinirt.so") + if not xmake_os.isfile(infinirt_library) then + infinirt_library = path.join(infinirt_root, "lib64", "libinfinirt.so") end - if not xmake_os.isfile(standalone_infinirt) then - raise("Standalone InfiniRT library not found under: " .. infinirt_root) + if not xmake_os.isfile(infinirt_library) then + raise("InfiniRT library not found under: " .. infinirt_root) end - return standalone_infinirt + return infinirt_library end -local function patch_infiniops_private_infinirt(xmake_os, infiniops_lib, standalone_infinirt, private_infinirt) +local function add_external_infinirt() + local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") + local infinirt_root = get_infinirt_root() + if infinirt_root and infinirt_root ~= "" then + add_includedirs(path.join(infinirt_root, "include"), { public = true }) + add_linkdirs(path.join(infinirt_root, "lib"), path.join(infinirt_root, "lib64")) + add_rpathdirs(path.join(infinirt_root, "lib"), path.join(infinirt_root, "lib64")) + else + add_includedirs(INFINI_ROOT .. "/include", { public = true }) + add_linkdirs(INFINI_ROOT .. "/lib") + add_rpathdirs(INFINI_ROOT .. "/lib") + end + if has_config("ascend-npu") then + local ascend_home = os.getenv("ASCEND_HOME") or os.getenv("ASCEND_TOOLKIT_HOME") or os.getenv("ASCEND_HOME_PATH") + if ascend_home and ascend_home ~= "" then + for _, include_dir in ipairs({ + path.join(ascend_home, "include"), + path.join(ascend_home, "include/aclnn"), + path.join(ascend_home, "aarch64-linux/include"), + path.join(ascend_home, "aarch64-linux/include/aclnn"), + }) do + if os.isdir(include_dir) then + add_includedirs(include_dir, { public = true }) + end + end + end + end + add_links("infinirt") +end + +local function patch_infiniops_private_infinirt(xmake_os, infiniops_lib, infinirt_library, private_infinirt) local private_soname = path.filename(private_infinirt) - xmake_os.cp(standalone_infinirt, private_infinirt) + xmake_os.cp(infinirt_library, private_infinirt) xmake_os.execv("patchelf", {"--set-soname", private_soname, private_infinirt}) - xmake_os.execv("patchelf", {"--replace-needed", standalone_infinirt, private_soname, infiniops_lib}) + xmake_os.execv("patchelf", {"--replace-needed", infinirt_library, private_soname, infiniops_lib}) xmake_os.execv("patchelf", {"--replace-needed", "libinfinirt.so", private_soname, infiniops_lib}) local needed = xmake_os.iorunv("patchelf", {"--print-needed", infiniops_lib}) @@ -410,7 +430,7 @@ local function build_infiniops_external(xmake_os) local infiniops_root = path.absolute(get_config("infiniops-root") or "submodules/InfiniOps", os.projectdir()) local infiniops_builddir = path.join(infiniops_root, "build") local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") - local infinirt_root = get_standalone_infinirt_root() + local infinirt_root = get_infinirt_root() local cmake_config_args = { "-S", infiniops_root, "-B", infiniops_builddir, @@ -442,10 +462,10 @@ local function build_infiniops_external(xmake_os) xmake_os.execv("cmake", {"--build", infiniops_builddir, "--target", "infiniops"}) xmake_os.execv("cmake", {"--install", infiniops_builddir, "--prefix", INFINI_ROOT}) if infinirt_root and infinirt_root ~= "" then - local standalone_infinirt = find_standalone_infinirt(infinirt_root, xmake_os) + local infinirt_library = find_infinirt_library(infinirt_root, xmake_os) local infiniops_lib = path.join(INFINI_ROOT, "lib", "libinfiniops.so") local private_infinirt = path.join(INFINI_ROOT, "lib", "libinfiniops_infinirt.so") - patch_infiniops_private_infinirt(xmake_os, infiniops_lib, standalone_infinirt, private_infinirt) + patch_infiniops_private_infinirt(xmake_os, infiniops_lib, infinirt_library, private_infinirt) end infiniops_external_built = true end @@ -487,11 +507,7 @@ target_end() target("infiniop") set_kind("shared") - local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") - add_includedirs(INFINI_ROOT.."/include", { public = true }) - add_linkdirs(INFINI_ROOT.."/lib") - add_links("infinirt") - add_rpathdirs(INFINI_ROOT.."/lib") + add_external_infinirt() local public_cuda_root = get_config("cuda") or os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") if public_cuda_root and public_cuda_root ~= "" then @@ -565,6 +581,7 @@ target("infiniop") add_installfiles("include/infiniop/*.h", {prefixdir = "include/infiniop"}) add_installfiles("include/infiniop.h", {prefixdir = "include"}) add_installfiles("include/infinicore.h", {prefixdir = "include"}) + add_installfiles("include/infinirt.h", {prefixdir = "include"}) target_end() target("infiniccl") @@ -648,28 +665,7 @@ target("infinicore_cpp_api") add_defines("CHAR_BIT=8", "INT_MIN=(-2147483647 - 1)", "INT_MAX=2147483647", "UINT_MAX=4294967295U") end add_includedirs(INFINI_ROOT.."/include", { public = true }) - local graph_infinirt_root = get_standalone_infinirt_root() - if has_config("standalone-infinirt-graph") then - if not graph_infinirt_root or graph_infinirt_root == "" then - raise("--standalone-infinirt-graph requires --infinirt-root or INFINI_RT_ROOT") - end - add_includedirs(graph_infinirt_root .. "/include") - if has_config("ascend-npu") then - local ascend_home = os.getenv("ASCEND_HOME") or os.getenv("ASCEND_TOOLKIT_HOME") or os.getenv("ASCEND_HOME_PATH") - if ascend_home and ascend_home ~= "" then - for _, include_dir in ipairs({ - path.join(ascend_home, "include"), - path.join(ascend_home, "include/aclnn"), - path.join(ascend_home, "aarch64-linux/include"), - path.join(ascend_home, "aarch64-linux/include/aclnn"), - }) do - if os.isdir(include_dir) then - add_includedirs(include_dir) - end - end - end - end - end + add_external_infinirt() if has_config("nv-gpu") then local cuda_root = os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") or get_config("cuda") or "/usr/local/cuda" add_includedirs(cuda_root .. "/include") @@ -680,7 +676,7 @@ target("infinicore_cpp_api") raise("InfiniOps root not found: " .. infiniops_root) end get_infiniops_backend_cmake_arg() - local infinirt_root = get_standalone_infinirt_root() + local infinirt_root = get_infinirt_root() if infinirt_root and infinirt_root ~= "" then add_includedirs(infinirt_root .. "/include") add_linkdirs(infinirt_root .. "/lib", infinirt_root .. "/lib64") @@ -695,19 +691,18 @@ target("infinicore_cpp_api") end) after_install(function (target) local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") - local infinirt_root = get_standalone_infinirt_root() + local infinirt_root = get_infinirt_root() if infinirt_root and infinirt_root ~= "" then - local standalone_infinirt = find_standalone_infinirt(infinirt_root, os) + local infinirt_library = find_infinirt_library(infinirt_root, os) local infiniops_lib = path.join(INFINI_ROOT, "lib", "libinfiniops.so") local private_infinirt = path.join(INFINI_ROOT, "lib", "libinfiniops_infinirt.so") - patch_infiniops_private_infinirt(os, infiniops_lib, standalone_infinirt, private_infinirt) + patch_infiniops_private_infinirt(os, infiniops_lib, infinirt_library, private_infinirt) end end) end add_linkdirs(INFINI_ROOT.."/lib") - add_links("infiniop", "infinirt", "infiniccl") - add_rpathdirs(INFINI_ROOT.."/lib") + add_links("infiniop", "infiniccl") if get_config("flash-attn") and get_config("flash-attn") ~= "" then add_installfiles("(builddir)/$(plat)/$(arch)/$(mode)/flash-attn*.so", {prefixdir = "lib"}) @@ -936,14 +931,15 @@ target("_infinicore") set_kind("shared") local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") add_includedirs(INFINI_ROOT.."/include", { public = true }) + add_external_infinirt() add_linkdirs(INFINI_ROOT.."/lib") - add_links("infiniop", "infinirt", "infiniccl") + add_links("infiniop", "infiniccl") add_files("src/infinicore/pybind11/**.cc") if has_config("infiniops") then - local infinirt_root = get_standalone_infinirt_root() + local infinirt_root = get_infinirt_root() if infinirt_root and infinirt_root ~= "" then add_includedirs(infinirt_root .. "/include") add_linkdirs(infinirt_root .. "/lib", infinirt_root .. "/lib64") @@ -958,11 +954,11 @@ target("_infinicore") infiniops_lib = path.join(infiniops_root, "build", "src", "libinfiniops.so") infiniops_lib_installed = false end - local infinirt_root = get_standalone_infinirt_root() + local infinirt_root = get_infinirt_root() if infinirt_root and infinirt_root ~= "" then - local standalone_infinirt = find_standalone_infinirt(infinirt_root, os) + local infinirt_library = find_infinirt_library(infinirt_root, os) local private_infinirt = path.join(INFINI_ROOT, "lib", "libinfiniops_infinirt.so") - patch_infiniops_private_infinirt(os, infiniops_lib, standalone_infinirt, private_infinirt) + patch_infiniops_private_infinirt(os, infiniops_lib, infinirt_library, private_infinirt) end os.mkdir(path.join(INFINI_ROOT, "lib")) if not infiniops_lib_installed then diff --git a/xmake/ali.lua b/xmake/ali.lua index 940650d67..550f7e0fc 100644 --- a/xmake/ali.lua +++ b/xmake/ali.lua @@ -104,7 +104,6 @@ target_end() target("infiniccl-ali") set_kind("static") - add_deps("infinirt") on_install(function (target) end) if has_config("ccl") then set_policy("build.cuda.devlink", true) diff --git a/xmake/ascend.lua b/xmake/ascend.lua index af82ab162..f8b2fe9ac 100644 --- a/xmake/ascend.lua +++ b/xmake/ascend.lua @@ -91,7 +91,6 @@ target_end() target("infiniccl-ascend") set_kind("static") - add_deps("infinirt") add_deps("infini-utils") set_warnings("all", "error") set_languages("cxx17") diff --git a/xmake/bang.lua b/xmake/bang.lua index ffa85ef6d..0be267c94 100644 --- a/xmake/bang.lua +++ b/xmake/bang.lua @@ -65,7 +65,6 @@ target_end() target("infiniccl-cambricon") set_kind("static") - add_deps("infinirt") add_deps("infini-utils") set_warnings("all", "error") set_languages("cxx17") diff --git a/xmake/hygon.lua b/xmake/hygon.lua index 94ae1e7ad..e5c86b863 100644 --- a/xmake/hygon.lua +++ b/xmake/hygon.lua @@ -197,7 +197,6 @@ target_end() target("infiniccl-hygon") set_kind("static") - add_deps("infinirt") on_install(function (target) end) if has_config("ccl") then diff --git a/xmake/iluvatar.lua b/xmake/iluvatar.lua index 34a913a11..5f69fd56e 100644 --- a/xmake/iluvatar.lua +++ b/xmake/iluvatar.lua @@ -88,7 +88,6 @@ target_end() target("infiniccl-iluvatar") set_kind("static") - add_deps("infinirt") on_install(function (target) end) if has_config("ccl") then diff --git a/xmake/kunlun.lua b/xmake/kunlun.lua index 84ba14082..4f3a40db2 100644 --- a/xmake/kunlun.lua +++ b/xmake/kunlun.lua @@ -108,7 +108,6 @@ target_end() target("infiniccl-kunlun") set_kind("static") - add_deps("infinirt") add_deps("infini-utils") set_warnings("all", "error") set_languages("cxx17") diff --git a/xmake/metax.lua b/xmake/metax.lua index 85407ed1b..3e700fff2 100644 --- a/xmake/metax.lua +++ b/xmake/metax.lua @@ -158,7 +158,6 @@ target_end() target("infiniccl-metax") set_kind("static") - add_deps("infinirt") on_install(function (target) end) set_warnings("all", "error") if not is_plat("windows") then diff --git a/xmake/moore.lua b/xmake/moore.lua index d34dd24f7..6e8800be8 100644 --- a/xmake/moore.lua +++ b/xmake/moore.lua @@ -81,7 +81,6 @@ target_end() target("infiniccl-moore") set_kind("static") - add_deps("infinirt") on_install(function (target) end) set_warnings("all", "error") if not is_plat("windows") then diff --git a/xmake/nvidia.lua b/xmake/nvidia.lua index 282f5876f..aca0dad66 100644 --- a/xmake/nvidia.lua +++ b/xmake/nvidia.lua @@ -246,7 +246,6 @@ target_end() target("infiniccl-nvidia") set_kind("static") - add_deps("infinirt") on_install(function (target) end) if has_config("ccl") then set_policy("build.cuda.devlink", true) diff --git a/xmake/qy.lua b/xmake/qy.lua index 35d53d6ef..82cf92d6b 100644 --- a/xmake/qy.lua +++ b/xmake/qy.lua @@ -174,7 +174,6 @@ target_end() target("infiniccl-qy") set_kind("static") - add_deps("infinirt") on_install(function (target) end) if has_config("ccl") then add_rules("qy.cuda", {override = true}) From 04434fc46d69c7775b83b2165bb542acade47505 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Wed, 8 Jul 2026 11:26:45 +0800 Subject: [PATCH 09/10] fix: adapt InfiniRT runtime bridge for Ascend Extend the Core-to-InfiniRT device translation table for Ascend and the other runtime-backed devices. Add explicit stream conversion helpers because Core still stores streams as infinirtStream_t while the InfiniRT C++ runtime API takes infini::rt::runtime::Stream. Include infinirt.h here so the bridge sees the runtime handle typedefs. Use those conversions for stream creation, async allocation, memcpy, memset, event recording, and stream wait calls. --- src/bridge/infini/rt.hpp | 42 ++++++++++++++++++- .../allocators/stream_ordered_allocator.cc | 9 +++- src/infinicore/context/context_impl.cc | 10 +++++ src/infinicore/context/runtime/runtime.cc | 18 ++++---- 4 files changed, 68 insertions(+), 11 deletions(-) diff --git a/src/bridge/infini/rt.hpp b/src/bridge/infini/rt.hpp index a79085cdc..3b4b350bd 100644 --- a/src/bridge/infini/rt.hpp +++ b/src/bridge/infini/rt.hpp @@ -1,6 +1,6 @@ #pragma once -#include "infinicore.h" +#include "infinirt.h" #include @@ -21,6 +21,22 @@ inline ::infini::rt::Device::Type translate(infiniDevice_t device) { return ::infini::rt::Device::Type::kCpu; case INFINI_DEVICE_NVIDIA: return ::infini::rt::Device::Type::kNvidia; + case INFINI_DEVICE_CAMBRICON: + return ::infini::rt::Device::Type::kCambricon; + case INFINI_DEVICE_ASCEND: + return ::infini::rt::Device::Type::kAscend; + case INFINI_DEVICE_METAX: + return ::infini::rt::Device::Type::kMetax; + case INFINI_DEVICE_MOORE: + return ::infini::rt::Device::Type::kMoore; + case INFINI_DEVICE_ILUVATAR: + return ::infini::rt::Device::Type::kIluvatar; + case INFINI_DEVICE_KUNLUN: + return ::infini::rt::Device::Type::kKunlun; + case INFINI_DEVICE_HYGON: + return ::infini::rt::Device::Type::kHygon; + case INFINI_DEVICE_QY: + return ::infini::rt::Device::Type::kQy; default: return ::infini::rt::Device::Type::kCount; } @@ -32,9 +48,33 @@ inline infiniDevice_t translate(::infini::rt::Device::Type device) { return INFINI_DEVICE_CPU; case ::infini::rt::Device::Type::kNvidia: return INFINI_DEVICE_NVIDIA; + case ::infini::rt::Device::Type::kCambricon: + return INFINI_DEVICE_CAMBRICON; + case ::infini::rt::Device::Type::kAscend: + return INFINI_DEVICE_ASCEND; + case ::infini::rt::Device::Type::kMetax: + return INFINI_DEVICE_METAX; + case ::infini::rt::Device::Type::kMoore: + return INFINI_DEVICE_MOORE; + case ::infini::rt::Device::Type::kIluvatar: + return INFINI_DEVICE_ILUVATAR; + case ::infini::rt::Device::Type::kKunlun: + return INFINI_DEVICE_KUNLUN; + case ::infini::rt::Device::Type::kHygon: + return INFINI_DEVICE_HYGON; + case ::infini::rt::Device::Type::kQy: + return INFINI_DEVICE_QY; default: return INFINI_DEVICE_TYPE_COUNT; } } +inline ::infini::rt::runtime::Stream to_rt_stream(infinirtStream_t stream) { + return reinterpret_cast<::infini::rt::runtime::Stream>(stream); +} + +inline infinirtStream_t to_core_stream(::infini::rt::runtime::Stream stream) { + return reinterpret_cast(stream); +} + } // namespace infinicore::bridge::infini::rt diff --git a/src/infinicore/context/allocators/stream_ordered_allocator.cc b/src/infinicore/context/allocators/stream_ordered_allocator.cc index f2092aa93..06ce4a30d 100644 --- a/src/infinicore/context/allocators/stream_ordered_allocator.cc +++ b/src/infinicore/context/allocators/stream_ordered_allocator.cc @@ -13,7 +13,10 @@ std::byte *StreamOrderedAllocator::allocate(size_t size) { } void *ptr = nullptr; if (device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MallocAsync(&ptr, size, context::getStream()))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MallocAsync( + &ptr, + size, + bridge::infini::rt::to_rt_stream(context::getStream())))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Malloc(&ptr, size))); } @@ -25,7 +28,9 @@ void StreamOrderedAllocator::deallocate(std::byte *ptr) { return; } if (device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::FreeAsync(ptr, context::getStream()))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::FreeAsync( + ptr, + bridge::infini::rt::to_rt_stream(context::getStream())))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Free(ptr))); } diff --git a/src/infinicore/context/context_impl.cc b/src/infinicore/context/context_impl.cc index 06c85955c..64d10dcfc 100644 --- a/src/infinicore/context/context_impl.cc +++ b/src/infinicore/context/context_impl.cc @@ -82,6 +82,16 @@ ContextImpl::ContextImpl() { } } + if constexpr (infini::rt::DeviceEnabled::value) { + infini::rt::set_runtime_device_type(bridge::infini::rt::translate(INFINI_DEVICE_ASCEND)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::GetDeviceCount(&device_counter[static_cast(Device::Type::ASCEND)]))); + runtime_table_[static_cast(Device::Type::ASCEND)].resize(device_counter[static_cast(Device::Type::ASCEND)]); + if (device_counter[static_cast(Device::Type::ASCEND)] > 0) { + runtime_table_[static_cast(Device::Type::ASCEND)][0] = std::unique_ptr(new Runtime(Device(Device::Type::ASCEND, 0))); + current_runtime_ = runtime_table_[static_cast(Device::Type::ASCEND)][0].get(); + } + } + if (current_runtime_ == nullptr && !runtime_table_[static_cast(Device::Type::CPU)].empty()) { current_runtime_ = runtime_table_[static_cast(Device::Type::CPU)][0].get(); } diff --git a/src/infinicore/context/runtime/runtime.cc b/src/infinicore/context/runtime/runtime.cc index 9c14c0207..18424d036 100644 --- a/src/infinicore/context/runtime/runtime.cc +++ b/src/infinicore/context/runtime/runtime.cc @@ -11,7 +11,9 @@ namespace infinicore { Runtime::Runtime(Device device) : device_(device), graph_manager_(std::make_unique()) { activate(); - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamCreate(&stream_))); + infini::rt::runtime::Stream stream = nullptr; + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamCreate(&stream))); + stream_ = bridge::infini::rt::to_core_stream(stream); INFINICORE_CHECK_ERROR(infiniopCreateHandle(&infiniop_handle_)); if (device_.getType() == Device::Type::CPU) { device_memory_allocator_ = std::make_unique(device); @@ -27,7 +29,7 @@ Runtime::~Runtime() { } device_memory_allocator_.reset(); infiniopDestroyHandle(infiniop_handle_); - (void)infini::rt::runtime::StreamDestroy(stream_); + (void)infini::rt::runtime::StreamDestroy(bridge::infini::rt::to_rt_stream(stream_)); } Runtime *Runtime::activate() { @@ -51,7 +53,7 @@ infiniopHandle_t Runtime::infiniopHandle() const { } void Runtime::syncStream() { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamSynchronize(stream_))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamSynchronize(bridge::infini::rt::to_rt_stream(stream_)))); } void Runtime::syncDevice() { @@ -108,7 +110,7 @@ std::shared_ptr Runtime::reinstantiateBlob(std::shared_ptr blob) void Runtime::memcpyH2D(void *dst, const void *src, size_t size, bool async) { if (async && device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice, stream_))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice, bridge::infini::rt::to_rt_stream(stream_)))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memcpy(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice))); } @@ -120,7 +122,7 @@ void Runtime::memcpyD2H(void *dst, const void *src, size_t size) { void Runtime::memcpyD2D(void *dst, const void *src, size_t size, bool async) { if (async && device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice, stream_))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice, bridge::infini::rt::to_rt_stream(stream_)))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memcpy(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice))); } @@ -132,7 +134,7 @@ void Runtime::setDeviceMemory(void *ptr, int value, size_t count) { void Runtime::setDeviceMemoryAsync(void *ptr, int value, size_t count, infinirtStream_t stream) { if (device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemsetAsync(ptr, value, count, stream))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemsetAsync(ptr, value, count, bridge::infini::rt::to_rt_stream(stream)))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memset(ptr, value, count))); } @@ -155,7 +157,7 @@ void Runtime::recordEvent(infinirtEvent_t event, infinirtStream_t stream) { if (stream == nullptr) { stream = stream_; } - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventRecord(event, stream))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventRecord(event, bridge::infini::rt::to_rt_stream(stream)))); } bool Runtime::queryEvent(infinirtEvent_t event) { @@ -181,7 +183,7 @@ void Runtime::streamWaitEvent(infinirtStream_t stream, infinirtEvent_t event) { if (stream == nullptr) { stream = stream_; } - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamWaitEvent(stream, event, 0))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamWaitEvent(bridge::infini::rt::to_rt_stream(stream), event, 0))); } bool Runtime::isGraphRecording() const { From 1e5e8a768c41043fd3c6ab5921054c6a3ca7f8d5 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Wed, 8 Jul 2026 11:27:00 +0800 Subject: [PATCH 10/10] feat: route graph capture through InfiniRT runtime Use the InfiniRT C++ runtime graph API directly for begin/end capture, instantiate, launch, and graph destruction. Drop the old standalone bridge and environment-variable-gated path so graph mode uses the installed InfiniRT runtime whenever USE_INFINIRT_GRAPH is enabled. --- src/infinicore/graph/graph.cc | 98 +++--- .../graph/standalone_infinirt_graph_bridge.cc | 295 ------------------ .../standalone_infinirt_graph_bridge.hpp | 26 -- 3 files changed, 40 insertions(+), 379 deletions(-) delete mode 100644 src/infinicore/graph/standalone_infinirt_graph_bridge.cc delete mode 100644 src/infinicore/graph/standalone_infinirt_graph_bridge.hpp diff --git a/src/infinicore/graph/graph.cc b/src/infinicore/graph/graph.cc index 0a09bf68e..441148f8e 100644 --- a/src/infinicore/graph/graph.cc +++ b/src/infinicore/graph/graph.cc @@ -1,15 +1,19 @@ #include "graph_manager.hpp" +#include "../../bridge/infini/rt.hpp" #include "../utils.hpp" #include "infinicore/context/context.hpp" #ifdef USE_INFINIRT_GRAPH -#include "standalone_infinirt_graph_bridge.hpp" -#include +#include #endif namespace infinicore::graph { +#ifdef USE_INFINIRT_GRAPH +namespace rt_runtime = ::infini::rt::runtime; +#endif + /* ========================= * GraphTensor * ========================= */ @@ -37,40 +41,29 @@ DispatchableGraphOperator::~DispatchableGraphOperator() { #ifdef USE_INFINIRT_GRAPH struct Graph::DeviceGraph { - infinirtGraph_t graph = nullptr; - infinirtGraphExec_t exec = nullptr; - infinirtGraphNode_t node = nullptr; - infinirtStream_t stream = nullptr; - bool standalone = false; - std::vector log_buffer; - - DeviceGraph() { - log_buffer.resize(4 * 1024); - } + rt_runtime::Graph graph = nullptr; + rt_runtime::GraphExec exec = nullptr; + rt_runtime::Stream stream = nullptr; + ::infini::rt::Device::Type device_type = ::infini::rt::Device::Type::kCount; + int device_index = 0; ~DeviceGraph() { + if (device_type != ::infini::rt::Device::Type::kCount) { + ::infini::rt::set_runtime_device_type(device_type); + (void)rt_runtime::SetDevice(device_index); + } if (exec) { - if (standalone) { - standalone_infinirt::graph_exec_destroy(exec); - } else { - infinirtGraphExecDestroy(exec); - } + (void)rt_runtime::GraphExecDestroy(exec); } if (graph) { - if (standalone) { - standalone_infinirt::graph_destroy(graph); - } else { - infinirtGraphDestroy(graph); - } + (void)rt_runtime::GraphDestroy(graph); } } void launch() { - if (standalone) { - INFINICORE_CHECK_ERROR(standalone_infinirt::graph_launch(exec, stream)); - } else { - INFINICORE_CHECK_ERROR(infinirtGraphLuanch(exec, context::getStream())); - } + ::infini::rt::set_runtime_device_type(device_type); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(rt_runtime::SetDevice(device_index))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(rt_runtime::GraphLaunch(exec, stream))); } }; #else @@ -100,22 +93,10 @@ void Graph::instantiate() { #ifdef USE_INFINIRT_GRAPH // Reset device graph device_graph_ = std::make_unique(); - device_graph_->standalone = standalone_infinirt::available(context::getDevice()); - device_graph_->stream = context::getStream(); - if (device_graph_->standalone) { - auto set_device_status = standalone_infinirt::set_device(context::getDevice()); - if (set_device_status != INFINI_STATUS_SUCCESS) { - spdlog::warn("Standalone InfiniRT graph bridge failed to select the current device. Falling back to eager execution."); - device_graph_.reset(); - return; - } - - static bool logged_once = false; - if (!logged_once) { - logged_once = true; - spdlog::info("Using standalone InfiniRT C++ graph runtime API for graph capture and replay."); - } - } + auto current_device = context::getDevice(); + device_graph_->device_type = bridge::infini::rt::translate(static_cast(current_device.getType())); + device_graph_->device_index = static_cast(current_device.getIndex()); + device_graph_->stream = bridge::infini::rt::to_rt_stream(context::getStream()); // warmup for (size_t iter = 0; iter < 5; ++iter) { @@ -123,9 +104,9 @@ void Graph::instantiate() { } infinicore::context::syncStream(); - auto begin_status = device_graph_->standalone - ? standalone_infinirt::stream_begin_capture(device_graph_->stream, INFINIRT_STREAM_CAPTURE_MODE_RELAXED) - : infinirtStreamBeginCapture(context::getStream(), INFINIRT_STREAM_CAPTURE_MODE_RELAXED); + auto begin_status = bridge::infini::rt::translate(rt_runtime::StreamBeginCapture( + device_graph_->stream, + rt_runtime::StreamCaptureMode::kStreamCaptureModeRelaxed)); if (begin_status != INFINI_STATUS_SUCCESS) { spdlog::warn("Fail to begin device graph capture."); device_graph_.reset(); @@ -135,30 +116,31 @@ void Graph::instantiate() { // Run and record this->run(); - auto end_status = device_graph_->standalone - ? standalone_infinirt::stream_end_capture(device_graph_->stream, &device_graph_.get()->graph) - : infinirtStreamEndCapture(context::getStream(), &device_graph_.get()->graph); + auto end_status = bridge::infini::rt::translate(rt_runtime::StreamEndCapture( + device_graph_->stream, + &device_graph_->graph)); if (end_status != INFINI_STATUS_SUCCESS) { spdlog::warn("Fail to end device graph capture."); device_graph_.reset(); return; } - auto instantiate_status = device_graph_->standalone - ? standalone_infinirt::graph_instantiate(&device_graph_.get()->exec, device_graph_.get()->graph) - : infinirtGraphInstantiate( - &device_graph_.get()->exec, - device_graph_.get()->graph, - &device_graph_.get()->node, - device_graph_.get()->log_buffer.data(), - device_graph_.get()->log_buffer.size()); + auto instantiate_status = bridge::infini::rt::translate(rt_runtime::GraphInstantiate( + &device_graph_->exec, + device_graph_->graph)); if (instantiate_status != INFINI_STATUS_SUCCESS) { static bool warned_once = false; if (!warned_once) { warned_once = true; - spdlog::warn("Fail to instantiate device graph: {}", std::string(device_graph_.get()->log_buffer.data())); + spdlog::warn("Fail to instantiate device graph."); } device_graph_.reset(); + return; + } + static bool logged_once = false; + if (!logged_once) { + logged_once = true; + spdlog::info("Using InfiniRT C++ graph runtime API for graph capture and replay."); } #endif } diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc deleted file mode 100644 index 7725d48c1..000000000 --- a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc +++ /dev/null @@ -1,295 +0,0 @@ -#include "standalone_infinirt_graph_bridge.hpp" - -#ifdef USE_STANDALONE_INFINIRT_GRAPH - -#include -#include - -#include - -namespace infinicore::graph::standalone_infinirt { -namespace { - -using StandaloneDevice = infini::rt::Device; - -bool truthy_env(const char *name) { - auto value = std::getenv(name); - if (value == nullptr) { - return false; - } - std::string text{value}; - return text == "1" || text == "ON" || text == "on" || text == "true" || text == "TRUE"; -} - -template -constexpr bool standalone_device_enabled() { - return infini::rt::DeviceEnabled::value; -} - -bool supports_device(Device::Type type) { - switch (type) { - case Device::Type::NVIDIA: - return standalone_device_enabled(); - case Device::Type::ASCEND: - return standalone_device_enabled(); - default: - return false; - } -} - -thread_local Device::Type current_device_type = Device::Type::CPU; - -template -infiniStatus_t to_core_status(Status status) { - return status == Runtime::kSuccess - ? INFINI_STATUS_SUCCESS - : INFINI_STATUS_INTERNAL_ERROR; -} - -template -infiniStatus_t set_device_impl(int index) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - return to_core_status(Runtime::SetDevice(index)); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t stream_begin_capture_impl(infinirtStream_t stream, infinirtStreamCaptureMode_t mode) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - auto standalone_mode = Runtime::kStreamCaptureModeRelaxed; - switch (mode) { - case INFINIRT_STREAM_CAPTURE_MODE_GLOBAL: - standalone_mode = Runtime::kStreamCaptureModeGlobal; - break; - case INFINIRT_STREAM_CAPTURE_MODE_THREAD_LOCAL: - standalone_mode = Runtime::kStreamCaptureModeThreadLocal; - break; - case INFINIRT_STREAM_CAPTURE_MODE_RELAXED: - standalone_mode = Runtime::kStreamCaptureModeRelaxed; - break; - } - return to_core_status(Runtime::StreamBeginCapture( - reinterpret_cast(stream), - standalone_mode)); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t stream_end_capture_impl(infinirtStream_t stream, infinirtGraph_t *graph) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - return to_core_status(Runtime::StreamEndCapture( - reinterpret_cast(stream), - reinterpret_cast(graph))); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t graph_destroy_impl(infinirtGraph_t graph) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - return to_core_status(Runtime::GraphDestroy( - reinterpret_cast(graph))); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t graph_instantiate_impl(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - return to_core_status(Runtime::GraphInstantiate( - reinterpret_cast(graph_exec), - reinterpret_cast(graph))); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t graph_exec_destroy_impl(infinirtGraphExec_t graph_exec) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - return to_core_status(Runtime::GraphExecDestroy( - reinterpret_cast(graph_exec))); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t graph_launch_impl(infinirtGraphExec_t graph_exec, infinirtStream_t stream) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - return to_core_status(Runtime::GraphLaunch( - reinterpret_cast(graph_exec), - reinterpret_cast(stream))); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t dispatch_current( - infiniStatus_t (*nvidia_fn)(Args...), - infiniStatus_t (*ascend_fn)(Args...), - Args... args) { - switch (current_device_type) { - case Device::Type::NVIDIA: - return nvidia_fn(args...); - case Device::Type::ASCEND: - return ascend_fn(args...); - default: - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -} // namespace - -bool enabled() { - return truthy_env("INFINICORE_USE_STANDALONE_INFINIRT_GRAPH"); -} - -bool available(const Device &device) { - return enabled() && supports_device(device.getType()); -} - -infiniStatus_t set_device(const Device &device) { - if (!supports_device(device.getType())) { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } - auto status = INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - switch (device.getType()) { - case Device::Type::NVIDIA: - status = set_device_impl(static_cast(device.getIndex())); - break; - case Device::Type::ASCEND: - status = set_device_impl(static_cast(device.getIndex())); - break; - default: - break; - } - if (status == INFINI_STATUS_SUCCESS) { - current_device_type = device.getType(); - } - return status; -} - -infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode) { - if (stream == nullptr) { - return INFINI_STATUS_NULL_POINTER; - } - return dispatch_current( - stream_begin_capture_impl, - stream_begin_capture_impl, - stream, - mode); -} - -infiniStatus_t stream_end_capture(infinirtStream_t stream, infinirtGraph_t *graph) { - if (stream == nullptr || graph == nullptr) { - return INFINI_STATUS_NULL_POINTER; - } - return dispatch_current( - stream_end_capture_impl, - stream_end_capture_impl, - stream, - graph); -} - -infiniStatus_t graph_destroy(infinirtGraph_t graph) { - if (graph == nullptr) { - return INFINI_STATUS_NULL_POINTER; - } - return dispatch_current( - graph_destroy_impl, - graph_destroy_impl, - graph); -} - -infiniStatus_t graph_instantiate(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph) { - if (graph_exec == nullptr || graph == nullptr) { - return INFINI_STATUS_NULL_POINTER; - } - return dispatch_current( - graph_instantiate_impl, - graph_instantiate_impl, - graph_exec, - graph); -} - -infiniStatus_t graph_exec_destroy(infinirtGraphExec_t graph_exec) { - if (graph_exec == nullptr) { - return INFINI_STATUS_NULL_POINTER; - } - return dispatch_current( - graph_exec_destroy_impl, - graph_exec_destroy_impl, - graph_exec); -} - -infiniStatus_t graph_launch(infinirtGraphExec_t graph_exec, infinirtStream_t stream) { - if (graph_exec == nullptr || stream == nullptr) { - return INFINI_STATUS_NULL_POINTER; - } - return dispatch_current( - graph_launch_impl, - graph_launch_impl, - graph_exec, - stream); -} - -} // namespace infinicore::graph::standalone_infinirt - -#else - -namespace infinicore::graph::standalone_infinirt { - -bool enabled() { - return false; -} - -bool available(const Device &) { - return false; -} - -infiniStatus_t set_device(const Device &) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -infiniStatus_t stream_begin_capture(infinirtStream_t, infinirtStreamCaptureMode_t) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -infiniStatus_t stream_end_capture(infinirtStream_t, infinirtGraph_t *) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -infiniStatus_t graph_destroy(infinirtGraph_t) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -infiniStatus_t graph_instantiate(infinirtGraphExec_t *, infinirtGraph_t) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -infiniStatus_t graph_exec_destroy(infinirtGraphExec_t) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -infiniStatus_t graph_launch(infinirtGraphExec_t, infinirtStream_t) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -} // namespace infinicore::graph::standalone_infinirt - -#endif diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp b/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp deleted file mode 100644 index 0ebbb0a6b..000000000 --- a/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include "infinicore/device.hpp" -#include - -namespace infinicore::graph::standalone_infinirt { - -bool enabled(); - -bool available(const Device &device); - -infiniStatus_t set_device(const Device &device); - -infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode); - -infiniStatus_t stream_end_capture(infinirtStream_t stream, infinirtGraph_t *graph); - -infiniStatus_t graph_destroy(infinirtGraph_t graph); - -infiniStatus_t graph_instantiate(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph); - -infiniStatus_t graph_exec_destroy(infinirtGraphExec_t graph_exec); - -infiniStatus_t graph_launch(infinirtGraphExec_t graph_exec, infinirtStream_t stream); - -} // namespace infinicore::graph::standalone_infinirt