PI queue mapping on to CUstream objects. More...

#include <cuda/pi_cuda.hpp>

Collaboration diagram for _pi_queue:

[legend]

Classes
struct	CommandBatch

struct	pi_queue_group_t

Public Types
using	native_type = CUstream

using	native_type = hipStream_t

using	queue_type = _pi_device::queue_group_info_t::type

typedef struct _pi_queue::CommandBatch	command_batch

Public Member Functions
	_pi_queue (std::vector< CUstream > &&compute_streams, std::vector< CUstream > &&transfer_streams, _pi_context context, _pi_device device, pi_queue_properties properties, unsigned int flags, bool backend_owns=true)

	~_pi_queue ()

native_type	get_next_compute_stream (pi_uint32 *stream_token=nullptr)

native_type	get_next_compute_stream (pi_uint32 num_events_in_wait_list, const pi_event event_wait_list, _pi_stream_guard &guard, pi_uint32 stream_token=nullptr)

native_type	get_next_transfer_stream ()

native_type	get ()

bool	has_been_synchronized (pi_uint32 stream_token)

bool	can_reuse_stream (pi_uint32 stream_token)

template<typename T >
void	for_each_stream (T &&f)

template<typename T >
void	sync_streams (T &&f)

_pi_context *	get_context () const

_pi_device *	get_device () const

pi_uint32	increment_reference_count () noexcept

pi_uint32	decrement_reference_count () noexcept

pi_uint32	get_reference_count () const noexcept

pi_uint32	get_next_event_id () noexcept

bool	backend_has_ownership () const noexcept

	_pi_queue (pi_context ContextArg, cm_support::CmQueue *CmQueueArg)

	_pi_queue (hipStream_t stream, _pi_context context, _pi_device device, pi_queue_properties properties)

	~_pi_queue ()

native_type	get () const noexcept

_pi_context *	get_context () const

pi_uint32	increment_reference_count () noexcept

pi_uint32	decrement_reference_count () noexcept

pi_uint32	get_reference_count () const noexcept

pi_uint32	get_next_event_id () noexcept

	_pi_queue (std::vector< ze_command_queue_handle_t > &ComputeQueues, std::vector< ze_command_queue_handle_t > &CopyQueues, pi_context Context, pi_device Device, bool OwnZeCommandQueue, pi_queue_properties Properties=0)

pi_result	synchronize ()

pi_queue_group_t &	getQueueGroup (bool UseCopyEngine)

bool	useCopyEngine (bool PreferCopyEngine=true) const

void	CaptureIndirectAccesses ()

bool	isBatchingAllowed (bool IsCopy) const

bool	isInOrderQueue () const

void	adjustBatchSizeForFullBatch (bool IsCopy)

void	adjustBatchSizeForPartialBatch (bool IsCopy)

pi_result	resetCommandList (pi_command_list_ptr_t CommandList, bool MakeAvailable, std::vector< _pi_event * > &EventListToCleanup)

bool	hasOpenCommandList (bool IsCopy) const

pi_result	executeCommandList (pi_command_list_ptr_t CommandList, bool IsBlocking=false, bool OKToBatchCommand=false)

pi_result	executeOpenCommandList (bool IsCopy)

pi_command_list_ptr_t	eventOpenCommandList (pi_event Event)

pi_result	executeAllOpenCommandLists ()

Public Attributes
std::vector< native_type >	compute_streams_

std::vector< native_type >	transfer_streams_

std::vector< bool >	delay_compute_

_pi_context *	context_

_pi_device *	device_

pi_queue_properties	properties_

std::atomic_uint32_t	refCount_

std::atomic_uint32_t	eventCount_

std::atomic_uint32_t	compute_stream_idx_

std::atomic_uint32_t	transfer_stream_idx_

unsigned int	num_compute_streams_

unsigned int	num_transfer_streams_

unsigned int	last_sync_compute_streams_

unsigned int	last_sync_transfer_streams_

unsigned int	flags_

std::mutex	compute_stream_sync_mutex_

std::mutex	compute_stream_mutex_

std::mutex	transfer_stream_mutex_

bool	has_ownership_

pi_context	Context = nullptr

cm_support::CmQueue *	CmQueuePtr = nullptr

native_type	stream_

pi_queue_group_t	ComputeQueueGroup {this, queue_type::Compute}

pi_queue_group_t	CopyQueueGroup {this, queue_type::MainCopy}

const pi_context	Context

const pi_device	Device

pi_event	LastCommandEvent = nullptr

std::vector< pi_kernel >	KernelsToBeSubmitted

bool	OwnZeCommandQueue

pi_command_list_map_t	CommandListMap

command_batch	ComputeCommandBatch

command_batch	CopyCommandBatch

pi_queue_properties	Properties

pi_uint32	RefCountExternal {1}

bool	Healthy {true}

Static Public Attributes
static constexpr int	default_num_compute_streams = 128

static constexpr int	default_num_transfer_streams = 64

Detailed Description

PI queue mapping on to CUstream objects.

PI queue mapping on to hipStream_t objects.

Definition at line 392 of file pi_cuda.hpp.

Member Typedef Documentation

◆ command_batch

typedef struct _pi_queue::CommandBatch _pi_queue::command_batch

◆ native_type [1/2]

using _pi_queue::native_type = hipStream_t

Definition at line 372 of file pi_hip.hpp.

◆ native_type [2/2]

using _pi_queue::native_type = CUstream

Definition at line 393 of file pi_cuda.hpp.

◆ queue_type

using _pi_queue::queue_type = _pi_device::queue_group_info_t::type

Definition at line 806 of file pi_level_zero.hpp.

Constructor & Destructor Documentation

◆ _pi_queue() [1/4]

_pi_queue::_pi_queue	(	std::vector< CUstream > &&	compute_streams,
		std::vector< CUstream > &&	transfer_streams,
		_pi_context *	context,
		_pi_device *	device,
		pi_queue_properties	properties,
		unsigned int	flags,
		bool	backend_owns = `true`
	)

inline

Definition at line 424 of file pi_cuda.hpp.

Referenced by piextQueueCreateWithNativeHandle(), and piQueueCreate().

◆ ~_pi_queue() [1/2]

_pi_queue::~_pi_queue ( )

inline

Definition at line 440 of file pi_cuda.hpp.

References context_, cuda_piContextRelease(), cuda_piDeviceRelease(), and device_.

◆ _pi_queue() [2/4]

_pi_queue::_pi_queue	(	pi_context	ContextArg,
		cm_support::CmQueue *	CmQueueArg
	)

inline

Definition at line 109 of file pi_esimd_emulator.hpp.

◆ _pi_queue() [3/4]

_pi_queue::_pi_queue	(	hipStream_t	stream,
		_pi_context *	context,
		_pi_device *	device,
		pi_queue_properties	properties
	)

inline

Definition at line 381 of file pi_hip.hpp.

◆ ~_pi_queue() [2/2]

_pi_queue::~_pi_queue ( )

inline

Definition at line 389 of file pi_hip.hpp.

References context_, device_, hip_piContextRelease(), and hip_piDeviceRelease().

◆ _pi_queue() [4/4]

_pi_queue::_pi_queue	(	std::vector< ze_command_queue_handle_t > &	ComputeQueues,
		std::vector< ze_command_queue_handle_t > &	CopyQueues,
		pi_context	Context,
		pi_device	Device,
		bool	OwnZeCommandQueue,
		pi_queue_properties	Properties = `0`
	)

Definition at line 1055 of file pi_level_zero.cpp.

References Context.

Member Function Documentation

◆ adjustBatchSizeForFullBatch()

void _pi_queue::adjustBatchSizeForFullBatch ( bool IsCopy )

Definition at line 1316 of file pi_level_zero.cpp.

References ComputeCommandBatch, CopyCommandBatch, CommandListBatchConfig::dynamic(), CommandListBatchConfig::DynamicSizeMax, CommandListBatchConfig::DynamicSizeStep, _pi_queue::CommandBatch::NumTimesClosedEarly, CommandListBatchConfig::NumTimesClosedEarlyThreshold, _pi_queue::CommandBatch::NumTimesClosedFull, CommandListBatchConfig::NumTimesClosedFullThreshold, _pi_queue::CommandBatch::QueueBatchSize, ZeCommandListBatchComputeConfig, ZeCommandListBatchConfig(), and ZeCommandListBatchCopyConfig.

Referenced by executeCommandList().

◆ adjustBatchSizeForPartialBatch()

void _pi_queue::adjustBatchSizeForPartialBatch ( bool IsCopy )

Definition at line 1343 of file pi_level_zero.cpp.

References ComputeCommandBatch, CopyCommandBatch, CommandListBatchConfig::dynamic(), _pi_queue::CommandBatch::NumTimesClosedEarly, _pi_queue::CommandBatch::NumTimesClosedFull, _pi_queue::CommandBatch::OpenCommandList, _pi_queue::CommandBatch::QueueBatchSize, ZeCommandListBatchComputeConfig, ZeCommandListBatchConfig(), and ZeCommandListBatchCopyConfig.

Referenced by executeOpenCommandList().

◆ backend_has_ownership()

bool _pi_queue::backend_has_ownership ( ) const

inlinenoexcept

Definition at line 583 of file pi_cuda.hpp.

References has_ownership_.

Referenced by cuda_piQueueRelease().

◆ can_reuse_stream()

bool _pi_queue::can_reuse_stream ( pi_uint32 stream_token )

inline

Definition at line 466 of file pi_cuda.hpp.

References compute_stream_idx_, compute_streams_, has_been_synchronized(), and max().

Referenced by get_next_compute_stream().

◆ CaptureIndirectAccesses()

void _pi_queue::CaptureIndirectAccesses ( )

Definition at line 1369 of file pi_level_zero.cpp.

References _pi_platform::Contexts, Device, KernelsToBeSubmitted, and _pi_device::Platform.

Referenced by executeCommandList(), and piEnqueueKernelLaunch().

◆ decrement_reference_count() [1/2]

pi_uint32 _pi_queue::decrement_reference_count ( )

inlinenoexcept

Definition at line 400 of file pi_hip.hpp.

References refCount_.

◆ decrement_reference_count() [2/2]

pi_uint32 _pi_queue::decrement_reference_count ( )

inlinenoexcept

Definition at line 577 of file pi_cuda.hpp.

References refCount_.

Referenced by cuda_piQueueRelease(), and hip_piQueueRelease().

◆ eventOpenCommandList()

pi_command_list_ptr_t _pi_queue::eventOpenCommandList ( pi_event Event )

Definition at line 1675 of file pi_level_zero.cpp.

References CommandListMap, ComputeCommandBatch, CopyCommandBatch, hasOpenCommandList(), and _pi_queue::CommandBatch::OpenCommandList.

Referenced by _pi_ze_event_list_t::createAndRetainPiZeEventList().

◆ executeAllOpenCommandLists()

pi_result _pi_queue::executeAllOpenCommandLists ( )

inline

Definition at line 996 of file pi_level_zero.hpp.

Referenced by _pi_ze_event_list_t::createAndRetainPiZeEventList(), piQueueFinish(), and piQueueRelease().

◆ executeCommandList()

pi_result _pi_queue::executeCommandList	(	pi_command_list_ptr_t	CommandList,
		bool	IsBlocking = `false`,
		bool	OKToBatchCommand = `false`
	)

Definition at line 1393 of file pi_level_zero.cpp.

References adjustBatchSizeForFullBatch(), CaptureIndirectAccesses(), _pi_event::CleanedUp, CommandListMap, ComputeCommandBatch, _pi_platform::ContextsMutex, CopyCommandBatch, createEventAndAssociateQueue(), Device, cl::sycl::detail::pi::die(), CommandListBatchConfig::dynamic(), hasOpenCommandList(), Healthy, isBatchingAllowed(), LastCommandEvent, _pi_queue::CommandBatch::OpenCommandList, PI_CALL, PI_COMMAND_TYPE_USER, piEventRelease(), piEventRetain(), _pi_device::Platform, PrintPiTrace, _pi_queue::CommandBatch::QueueBatchSize, synchronize(), ZE_CALL, ZE_CALL_NOCHECK, ZeCommandListBatchComputeConfig, ZeCommandListBatchConfig(), ZeCommandListBatchCopyConfig, and _pi_event::ZeEvent.

Referenced by enqueueMemCopyHelper(), enqueueMemCopyRectHelper(), enqueueMemFillHelper(), enqueueMemImageCommandHelper(), executeOpenCommandList(), piEnqueueEventsWait(), piEnqueueEventsWaitWithBarrier(), piEnqueueKernelLaunch(), piEnqueueMemBufferMap(), piEnqueueMemUnmap(), piextUSMEnqueueMemAdvise(), and piextUSMEnqueuePrefetch().

◆ executeOpenCommandList()

pi_result _pi_queue::executeOpenCommandList ( bool IsCopy )

Definition at line 1700 of file pi_level_zero.cpp.

References adjustBatchSizeForPartialBatch(), CommandListMap, ComputeCommandBatch, CopyCommandBatch, executeCommandList(), hasOpenCommandList(), and _pi_queue::CommandBatch::OpenCommandList.

Referenced by _pi_ze_event_list_t::createAndRetainPiZeEventList(), and _pi_context::getAvailableCommandList().

◆ for_each_stream()

template<typename T >

void _pi_queue::for_each_stream ( T && f )

inline

Definition at line 488 of file pi_cuda.hpp.

References compute_stream_mutex_, compute_streams_, num_compute_streams_, num_transfer_streams_, transfer_stream_mutex_, and transfer_streams_.

Referenced by cuda_piQueueRelease().

◆ get() [1/2]

native_type _pi_queue::get ( )

inline

Definition at line 456 of file pi_cuda.hpp.

References get_next_compute_stream().

Referenced by hip_piEnqueueKernelLaunch(), hip_piEnqueueMemBufferCopy(), hip_piEnqueueMemBufferCopyRect(), hip_piEnqueueMemBufferFill(), hip_piEnqueueMemBufferRead(), hip_piEnqueueMemBufferReadRect(), hip_piEnqueueMemBufferWrite(), hip_piEnqueueMemBufferWriteRect(), hip_piEnqueueMemImageCopy(), hip_piEnqueueMemImageRead(), and hip_piEnqueueMemImageWrite().

◆ get() [2/2]

native_type _pi_queue::get ( ) const

inlinenoexcept

Definition at line 394 of file pi_hip.hpp.

References stream_.

◆ get_context() [1/2]

_pi_context* _pi_queue::get_context ( ) const

inline

Definition at line 396 of file pi_hip.hpp.

References context_.

◆ get_context() [2/2]

_pi_context* _pi_queue::get_context ( ) const

inline

Definition at line 571 of file pi_cuda.hpp.

References context_.

◆ get_device()

_pi_device* _pi_queue::get_device ( ) const

inline

Definition at line 573 of file pi_cuda.hpp.

References device_.

Referenced by cuda_piEnqueueKernelLaunch().

◆ get_next_compute_stream() [1/2]

CUstream _pi_queue::get_next_compute_stream ( pi_uint32 * stream_token = nullptr )

Definition at line 380 of file pi_cuda.cpp.

References compute_stream_idx_, compute_stream_mutex_, compute_streams_, delay_compute_, flags_, and num_compute_streams_.

Referenced by cuda_piEnqueueEventsWaitWithBarrier(), cuda_piEnqueueKernelLaunch(), get(), get_next_compute_stream(), and get_next_transfer_stream().

◆ get_next_compute_stream() [2/2]

CUstream _pi_queue::get_next_compute_stream	(	pi_uint32	num_events_in_wait_list,
		const pi_event *	event_wait_list,
		_pi_stream_guard &	guard,
		pi_uint32 *	stream_token = `nullptr`
	)