[C++][GPU][FlightRPC] CudaMemoryManager::ViewBufferFrom() creates non-CudaBuffer views on HMM-enabled systems

Describe the bug, including details regarding any error messages, version, and platform.

GrpcCudaDataTest.TestDoGet, GrpcCudaDataTest.TestDoPut and GrpcCudaDataTest.TestDoExchange in arrow-flight-test fail on a system where HMM (Heterogeneous Memory Management) is enabled:

[ RUN      ] GrpcCudaDataTest.TestDoGet
unknown file: Failure
C++ exception with description "std::bad_cast" thrown in the test body.

[  FAILED  ] GrpcCudaDataTest.TestDoGet (125 ms)
[ RUN      ] GrpcCudaDataTest.TestDoPut
.../cpp/src/arrow/flight/test_definitions.cc:1368: Failure
Failed
'do_put_result.writer->WriteRecordBatch(*cuda_batch)' failed with Unknown error: Flight RPC failed with message: Unexpected error in RPC handling. ...
[  FAILED  ] GrpcCudaDataTest.TestDoPut (7 ms)
[ RUN      ] GrpcCudaDataTest.TestDoExchange
.../cpp/src/arrow/flight/test_definitions.cc:1408: Failure
Failed
'_error_or_value132.status()' failed with Unknown error: Flight RPC failed with message: Unexpected error in RPC handling. ...
[  FAILED  ] GrpcCudaDataTest.TestDoExchange (4 ms)

Backtrace of the std::bad_cast in TestDoGet:

#0  __cxa_throw ()
#1  arrow::cuda::CudaBufferReader::CudaBufferReader(std::shared_ptr<arrow::Buffer> const&)
#2  arrow::cuda::CudaMemoryManager::GetBufferReader(std::shared_ptr<arrow::Buffer>)
#3  arrow::Buffer::GetReader(std::shared_ptr<arrow::Buffer>)
#4  arrow::ipc::StreamDecoderInternal::OnRecordBatchMessageDecoded(...)
...
#8  arrow::flight::ClientStreamReader::Next()
#9  arrow::flight::CudaDataTest::TestDoGet()

CudaMemoryManager::ViewBufferFrom() creates a CPU-on-device view when CudaContext::GetDeviceAddress() succeeds:

arrow/cpp/src/arrow/gpu/cuda_context.cc

Lines 499 to 508 in 9e5d697

    
           Result<std::shared_ptr<Buffer>> CudaMemoryManager::ViewBufferFrom( 
        
               const std::shared_ptr<Buffer>& buf, const std::shared_ptr<MemoryManager>& from) { 
        
             if (from->is_cpu()) { 
        
               // CPU-on-device view 
        
               ARROW_ASSIGN_OR_RAISE(auto to_context, cuda_device()->GetContext()); 
        
               ARROW_ASSIGN_OR_RAISE(auto address, GetDeviceAddress(buf->data(), to_context)); 
        
               return std::make_shared<Buffer>(address, buf->size(), shared_from_this(), buf); 
        
             } 
        
             return nullptr; 
        
           }

CudaContext::GetDeviceAddress() uses cuPointerGetAttribute(CU_POINTER_ATTRIBUTE_DEVICE_POINTER):

arrow/cpp/src/arrow/gpu/cuda_context.cc

Lines 748 to 755 in 9e5d697

    
           Result<uintptr_t> CudaContext::GetDeviceAddress(uintptr_t addr) { 
        
             ContextSaver set_temporary(*this); 
        
             CUdeviceptr ptr; 
        
             CU_RETURN_NOT_OK("cuPointerGetAttribute", 
        
                              cuPointerGetAttribute(&ptr, CU_POINTER_ATTRIBUTE_DEVICE_POINTER, 
        
                                                    static_cast<CUdeviceptr>(addr))); 
        
             return static_cast<uintptr_t>(ptr); 
        
           }

The view is a plain arrow::Buffer on the CUDA device, not a CudaBuffer.

Without HMM, cuPointerGetAttribute() succeeds only for page-locked host memory registered to CUDA. With HMM, it also succeeds for pageable host memory allocated by malloc() and so on. So Buffer::ViewOrCopy() returns a view instead of a copy for normal CPU buffers.

Flight client uses Buffer::ViewOrCopy(data->body, memory_manager_). The result is passed to CudaBufferReader via Buffer::GetReader(). CudaBufferReader requires a CudaBuffer. So it throws std::bad_cast:

arrow/cpp/src/arrow/gpu/cuda_memory.cc

Lines 221 to 229 in 9e5d697

    
           CudaBufferReader::CudaBufferReader(const std::shared_ptr<Buffer>& buffer) 
        
               : address_(buffer->address()), size_(buffer->size()), position_(0), is_open_(true) { 
        
             auto maybe_buffer = CudaBuffer::FromBuffer(buffer); 
        
             if (ARROW_PREDICT_FALSE(!maybe_buffer.ok())) { 
        
               throw std::bad_cast(); 
        
             } 
        
             buffer_ = *std::move(maybe_buffer); 
        
             context_ = buffer_->context(); 
        
           }

Component(s)

C++, GPU, FlightRPC

Navigation Menu

Repository navigation

[C++][GPU][FlightRPC] CudaMemoryManager::ViewBufferFrom() creates non-CudaBuffer views on HMM-enabled systems #55360

Describe the bug, including details regarding any error messages, version, and platform.

Component(s)

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

	Result<std::shared_ptr<Buffer>> CudaMemoryManager::ViewBufferFrom(
	const std::shared_ptr<Buffer>& buf, const std::shared_ptr<MemoryManager>& from) {
	if (from->is_cpu()) {
	// CPU-on-device view
	ARROW_ASSIGN_OR_RAISE(auto to_context, cuda_device()->GetContext());
	ARROW_ASSIGN_OR_RAISE(auto address, GetDeviceAddress(buf->data(), to_context));
	return std::make_shared<Buffer>(address, buf->size(), shared_from_this(), buf);
	}
	return nullptr;
	}

	Result<uintptr_t> CudaContext::GetDeviceAddress(uintptr_t addr) {
	ContextSaver set_temporary(*this);
	CUdeviceptr ptr;
	CU_RETURN_NOT_OK("cuPointerGetAttribute",
	cuPointerGetAttribute(&ptr, CU_POINTER_ATTRIBUTE_DEVICE_POINTER,
	static_cast<CUdeviceptr>(addr)));
	return static_cast<uintptr_t>(ptr);
	}

	CudaBufferReader::CudaBufferReader(const std::shared_ptr<Buffer>& buffer)
	: address_(buffer->address()), size_(buffer->size()), position_(0), is_open_(true) {
	auto maybe_buffer = CudaBuffer::FromBuffer(buffer);
	if (ARROW_PREDICT_FALSE(!maybe_buffer.ok())) {
	throw std::bad_cast();
	}
	buffer_ = *std::move(maybe_buffer);
	context_ = buffer_->context();
	}

[C++][GPU][FlightRPC] CudaMemoryManager::ViewBufferFrom() creates non-CudaBuffer views on HMM-enabled systems #55360

Description

Describe the bug, including details regarding any error messages, version, and platform.

Component(s)

Activity

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions