commands.hpp

Go to the documentation of this file.

//==-------------- commands.hpp - SYCL standard header file ----------------==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#pragma once
 
#include <atomic>
#include <cstdint>
#include <deque>
#include <memory>
#include <set>
#include <unordered_set>
#include <vector>
 
#include <CL/sycl/access/access.hpp>
#include <CL/sycl/detail/accessor_impl.hpp>
#include <CL/sycl/detail/cg.hpp>
#include <detail/program_manager/program_manager.hpp>
 
__SYCL_INLINE_NAMESPACE(cl) {
namespace sycl {
namespace detail {
 
class queue_impl;
class event_impl;
class context_impl;
class DispatchHostTask;
 
using QueueImplPtr = std::shared_ptr<detail::queue_impl>;
using EventImplPtr = std::shared_ptr<detail::event_impl>;
using ContextImplPtr = std::shared_ptr<detail::context_impl>;
using StreamImplPtr = std::shared_ptr<detail::stream_impl>;
 
class Command;
class AllocaCommand;
class AllocaCommandBase;
class ReleaseCommand;
class ExecCGCommand;
class EmptyCommand;
 
enum BlockingT { NON_BLOCKING = 0, BLOCKING };
 
struct EnqueueResultT {
  enum ResultT {
    SyclEnqueueReady,
    SyclEnqueueSuccess,
    SyclEnqueueBlocked,
    SyclEnqueueFailed
  };
  EnqueueResultT(ResultT Result = SyclEnqueueSuccess, Command *Cmd = nullptr,
                 cl_int ErrCode = CL_SUCCESS)
      : MResult(Result), MCmd(Cmd), MErrCode(ErrCode) {}
  ResultT MResult;
  Command *MCmd;
  cl_int MErrCode;
};
 
struct DepDesc {
  DepDesc(Command *DepCommand, const Requirement *Req,
          AllocaCommandBase *AllocaCmd)
      : MDepCommand(DepCommand), MDepRequirement(Req), MAllocaCmd(AllocaCmd) {}
 
  friend bool operator<(const DepDesc &Lhs, const DepDesc &Rhs) {
    return std::tie(Lhs.MDepRequirement, Lhs.MDepCommand) <
           std::tie(Rhs.MDepRequirement, Rhs.MDepCommand);
  }
 
  Command *MDepCommand = nullptr;
  const Requirement *MDepRequirement = nullptr;
  AllocaCommandBase *MAllocaCmd = nullptr;
};
 
class Command {
public:
  enum CommandType {
    RUN_CG,
    COPY_MEMORY,
    ALLOCA,
    ALLOCA_SUB_BUF,
    RELEASE,
    MAP_MEM_OBJ,
    UNMAP_MEM_OBJ,
    UPDATE_REQUIREMENT,
    EMPTY_TASK,
    HOST_TASK
  };
 
  Command(CommandType Type, QueueImplPtr Queue);
 
  [[nodiscard]] Command *addDep(DepDesc NewDep,
                                std::vector<Command *> &ToCleanUp);
 
  [[nodiscard]] Command *addDep(EventImplPtr Event,
                                std::vector<Command *> &ToCleanUp);
 
  void addUser(Command *NewUser) { MUsers.insert(NewUser); }
 
  CommandType getType() const { return MType; }
 
  virtual bool enqueue(EnqueueResultT &EnqueueResult, BlockingT Blocking,
                       std::vector<Command *> &ToCleanUp);
 
  bool isFinished();
 
  bool isSuccessfullyEnqueued() const {
    return MEnqueueStatus == EnqueueResultT::SyclEnqueueSuccess;
  }
 
  bool isEnqueueBlocked() const {
    return MEnqueueStatus == EnqueueResultT::SyclEnqueueBlocked;
  }
 
  const QueueImplPtr &getQueue() const { return MQueue; }
 
  const QueueImplPtr &getSubmittedQueue() const { return MSubmittedQueue; }
 
  const EventImplPtr &getEvent() const { return MEvent; }
 
  // Methods needed to support SYCL instrumentation
 
  void emitInstrumentationDataProxy();
  virtual void emitInstrumentationData() = 0;
  void resolveReleaseDependencies(std::set<Command *> &list);
  void emitEdgeEventForCommandDependence(Command *Cmd, void *ObjAddr,
                                         const std::string &Prefix,
                                         bool IsCommand);
  void emitEdgeEventForEventDependence(Command *Cmd, RT::PiEvent &EventAddr);
  void emitEnqueuedEventSignal(RT::PiEvent &PiEventAddr);
  uint64_t makeTraceEventProlog(void *MAddress);
  void makeTraceEventEpilog();
  void emitInstrumentation(uint16_t Type, const char *Txt = nullptr);
 
  // End Methods needed to support SYCL instrumentation
 
  virtual void printDot(std::ostream &Stream) const = 0;
 
  virtual const Requirement *getRequirement() const {
    assert(false && "Internal Error. The command has no stored requirement");
    return nullptr;
  }
 
  virtual ~Command() = default;
 
  const char *getBlockReason() const;
 
  virtual const ContextImplPtr &getWorkerContext() const;
 
  virtual const QueueImplPtr &getWorkerQueue() const;
 
  virtual bool producesPiEvent() const;
 
  virtual bool supportsPostEnqueueCleanup() const;
 
protected:
  QueueImplPtr MQueue;
  QueueImplPtr MSubmittedQueue;
  EventImplPtr MEvent;
 
  std::vector<EventImplPtr> &MPreparedDepsEvents;
  std::vector<EventImplPtr> &MPreparedHostDepsEvents;
 
  void waitForEvents(QueueImplPtr Queue, std::vector<EventImplPtr> &RawEvents,
                     RT::PiEvent &Event);
 
  void waitForPreparedHostEvents() const;
 
  [[nodiscard]] Command *processDepEvent(EventImplPtr DepEvent,
                                         const DepDesc &Dep,
                                         std::vector<Command *> &ToCleanUp);
 
  virtual cl_int enqueueImp() = 0;
 
  CommandType MType;
  std::mutex MEnqueueMtx;
 
  friend class DispatchHostTask;
 
public:
  const std::vector<EventImplPtr> &getPreparedHostDepsEvents() const {
    return MPreparedHostDepsEvents;
  }
 
  std::vector<DepDesc> MDeps;
  std::unordered_set<Command *> MUsers;
  bool MIsBlockable = false;
  unsigned MLeafCounter = 0;
 
  struct Marks {
    bool MVisited = false;
    bool MToBeDeleted = false;
  };
  Marks MMarks;
 
  enum class BlockReason : int { HostAccessor = 0, HostTask };
 
  // Only have reasonable value while MIsBlockable is true
  BlockReason MBlockReason;
 
  std::atomic<EnqueueResultT::ResultT> MEnqueueStatus;
 
  // All member variable defined here  are needed for the SYCL instrumentation
  // layer. Do not guard these variables below with XPTI_ENABLE_INSTRUMENTATION
  // to ensure we have the same object layout when the macro in the library and
  // SYCL app are not the same.
 
  void *MTraceEvent = nullptr;
  int32_t MStreamID = -1;
  void *MAddress = nullptr;
  std::string MAddressString;
  std::string MCommandNodeType;
  std::string MCommandName;
  bool MTraceEventPrologComplete = false;
  bool MFirstInstance = false;
  uint64_t MInstanceID = 0;
 
  // This flag allows to control whether host event should be set complete
  // after successfull enqueue of command. Event is considered as host event if
  // either it's is_host() return true or there is no backend representation
  // of event (i.e. getHandleRef() return reference to nullptr value).
  // By default the flag is set to true due to most of host operations are
  // synchronous. The only asynchronous operation currently is host-task.
  bool MShouldCompleteEventIfPossible = true;
 
  bool MPostEnqueueCleanup = false;
};
 
class EmptyCommand : public Command {
public:
  EmptyCommand(QueueImplPtr Queue);
 
  void printDot(std::ostream &Stream) const final;
  const Requirement *getRequirement() const final { return &MRequirements[0]; }
  void addRequirement(Command *DepCmd, AllocaCommandBase *AllocaCmd,
                      const Requirement *Req);
 
  void emitInstrumentationData() override;
 
  bool producesPiEvent() const final;
 
private:
  cl_int enqueueImp() final;
 
  // Employing deque here as it allows to push_back/emplace_back without
  // invalidation of pointer or reference to stored data item regardless of
  // iterator invalidation.
  std::deque<Requirement> MRequirements;
};
 
class ReleaseCommand : public Command {
public:
  ReleaseCommand(QueueImplPtr Queue, AllocaCommandBase *AllocaCmd);
 
  void printDot(std::ostream &Stream) const final;
  void emitInstrumentationData() override;
  bool producesPiEvent() const final;
  bool supportsPostEnqueueCleanup() const final;
 
private:
  cl_int enqueueImp() final;
 
  AllocaCommandBase *MAllocaCmd = nullptr;
};
 
class AllocaCommandBase : public Command {
public:
  AllocaCommandBase(CommandType Type, QueueImplPtr Queue, Requirement Req,
                    AllocaCommandBase *LinkedAllocaCmd);
 
  ReleaseCommand *getReleaseCmd() { return &MReleaseCmd; }
 
  SYCLMemObjI *getSYCLMemObj() const { return MRequirement.MSYCLMemObj; }
 
  virtual void *getMemAllocation() const = 0;
 
  const Requirement *getRequirement() const final { return &MRequirement; }
 
  void emitInstrumentationData() override;
 
  bool producesPiEvent() const final;
 
  bool supportsPostEnqueueCleanup() const final;
 
  void *MMemAllocation = nullptr;
 
  AllocaCommandBase *MLinkedAllocaCmd = nullptr;
  bool MIsActive = true;
 
  bool MIsLeaderAlloca = true;
 
protected:
  Requirement MRequirement;
  ReleaseCommand MReleaseCmd;
};
 
class AllocaCommand : public AllocaCommandBase {
public:
  AllocaCommand(QueueImplPtr Queue, Requirement Req,
                bool InitFromUserData = true,
                AllocaCommandBase *LinkedAllocaCmd = nullptr);
 
  void *getMemAllocation() const final { return MMemAllocation; }
  void printDot(std::ostream &Stream) const final;
  void emitInstrumentationData() override;
 
private:
  cl_int enqueueImp() final;
 
  bool MInitFromUserData = false;
};
 
class AllocaSubBufCommand : public AllocaCommandBase {
public:
  AllocaSubBufCommand(QueueImplPtr Queue, Requirement Req,
                      AllocaCommandBase *ParentAlloca,
                      std::vector<Command *> &ToEnqueue,
                      std::vector<Command *> &ToCleanUp);
 
  void *getMemAllocation() const final;
  void printDot(std::ostream &Stream) const final;
  AllocaCommandBase *getParentAlloca() { return MParentAlloca; }
  void emitInstrumentationData() override;
 
private:
  cl_int enqueueImp() final;
 
  AllocaCommandBase *MParentAlloca = nullptr;
};
 
class MapMemObject : public Command {
public:
  MapMemObject(AllocaCommandBase *SrcAllocaCmd, Requirement Req, void **DstPtr,
               QueueImplPtr Queue, access::mode MapMode);
 
  void printDot(std::ostream &Stream) const final;
  const Requirement *getRequirement() const final { return &MSrcReq; }
  void emitInstrumentationData() override;
 
private:
  cl_int enqueueImp() final;
 
  AllocaCommandBase *MSrcAllocaCmd = nullptr;
  Requirement MSrcReq;
  void **MDstPtr = nullptr;
  access::mode MMapMode;
};
 
class UnMapMemObject : public Command {
public:
  UnMapMemObject(AllocaCommandBase *DstAllocaCmd, Requirement Req,
                 void **SrcPtr, QueueImplPtr Queue);
 
  void printDot(std::ostream &Stream) const final;
  const Requirement *getRequirement() const final { return &MDstReq; }
  void emitInstrumentationData() override;
  bool producesPiEvent() const final;
 
private:
  cl_int enqueueImp() final;
 
  AllocaCommandBase *MDstAllocaCmd = nullptr;
  Requirement MDstReq;
  void **MSrcPtr = nullptr;
};
 
class MemCpyCommand : public Command {
public:
  MemCpyCommand(Requirement SrcReq, AllocaCommandBase *SrcAllocaCmd,
                Requirement DstReq, AllocaCommandBase *DstAllocaCmd,
                QueueImplPtr SrcQueue, QueueImplPtr DstQueue);
 
  void printDot(std::ostream &Stream) const final;
  const Requirement *getRequirement() const final { return &MDstReq; }
  void emitInstrumentationData() final;
  const ContextImplPtr &getWorkerContext() const final;
  const QueueImplPtr &getWorkerQueue() const final;
  bool producesPiEvent() const final;
 
private:
  cl_int enqueueImp() final;
 
  QueueImplPtr MSrcQueue;
  Requirement MSrcReq;
  AllocaCommandBase *MSrcAllocaCmd = nullptr;
  Requirement MDstReq;
  AllocaCommandBase *MDstAllocaCmd = nullptr;
};
 
class MemCpyCommandHost : public Command {
public:
  MemCpyCommandHost(Requirement SrcReq, AllocaCommandBase *SrcAllocaCmd,
                    Requirement DstReq, void **DstPtr, QueueImplPtr SrcQueue,
                    QueueImplPtr DstQueue);
 
  void printDot(std::ostream &Stream) const final;
  const Requirement *getRequirement() const final { return &MDstReq; }
  void emitInstrumentationData() final;
  const ContextImplPtr &getWorkerContext() const final;
  const QueueImplPtr &getWorkerQueue() const final;
 
private:
  cl_int enqueueImp() final;
 
  QueueImplPtr MSrcQueue;
  Requirement MSrcReq;
  AllocaCommandBase *MSrcAllocaCmd = nullptr;
  Requirement MDstReq;
  void **MDstPtr = nullptr;
};
 
cl_int enqueueImpKernel(
    const QueueImplPtr &Queue, NDRDescT &NDRDesc, std::vector<ArgDesc> &Args,
    const std::shared_ptr<detail::kernel_bundle_impl> &KernelBundleImplPtr,
    const std::shared_ptr<detail::kernel_impl> &MSyclKernel,
    const std::string &KernelName, const detail::OSModuleHandle &OSModuleHandle,
    std::vector<RT::PiEvent> &RawEvents, RT::PiEvent *OutEvent,
    const std::function<void *(Requirement *Req)> &getMemAllocationFunc);
 
class ExecCGCommand : public Command {
public:
  ExecCGCommand(std::unique_ptr<detail::CG> CommandGroup, QueueImplPtr Queue);
 
  std::vector<StreamImplPtr> getStreams() const;
 
  void clearStreams();
 
  void printDot(std::ostream &Stream) const final;
  void emitInstrumentationData() final;
 
  detail::CG &getCG() const { return *MCommandGroup; }
 
  // MEmptyCmd is only employed if this command refers to host-task.
  // The mechanism of lookup for single EmptyCommand amongst users of
  // host-task-representing command is unreliable. This unreliability roots in
  // the cleanup process.
  EmptyCommand *MEmptyCmd = nullptr;
 
  // This function is only usable for native kernel to prevent access to free'd
  // memory in DispatchNativeKernel.
  // TODO remove when native kernel support is terminated.
  void releaseCG() {
    MCommandGroup.release();
  }
 
  bool producesPiEvent() const final;
 
  bool supportsPostEnqueueCleanup() const final;
 
private:
  cl_int enqueueImp() final;
 
  AllocaCommandBase *getAllocaForReq(Requirement *Req);
 
  std::unique_ptr<detail::CG> MCommandGroup;
 
  friend class Command;
};
 
class UpdateHostRequirementCommand : public Command {
public:
  UpdateHostRequirementCommand(QueueImplPtr Queue, Requirement Req,
                               AllocaCommandBase *SrcAllocaCmd, void **DstPtr);
 
  void printDot(std::ostream &Stream) const final;
  const Requirement *getRequirement() const final { return &MDstReq; }
  void emitInstrumentationData() final;
 
private:
  cl_int enqueueImp() final;
 
  AllocaCommandBase *MSrcAllocaCmd = nullptr;
  Requirement MDstReq;
  void **MDstPtr = nullptr;
};
 
} // namespace detail
} // namespace sycl
} // __SYCL_INLINE_NAMESPACE(cl)