scheduler.hpp

Go to the documentation of this file.

//==-------------- scheduler.hpp - SYCL standard header file ---------------==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#pragma once
 
#include <detail/cg.hpp>
#include <detail/scheduler/commands.hpp>
#include <detail/scheduler/leaves_collection.hpp>
#include <detail/sycl_mem_obj_i.hpp>
 
#include <cstddef>
#include <memory>
#include <queue>
#include <set>
#include <shared_mutex>
#include <unordered_map>
#include <unordered_set>
#include <vector>
 
 
// For testing purposes
class MockScheduler;
 
namespace sycl {
inline namespace _V1 {
namespace ext::oneapi::experimental::detail {
class exec_graph_impl;
class node_impl;
} // namespace ext::oneapi::experimental::detail
namespace detail {
class queue_impl;
class event_impl;
class context_impl;
class DispatchHostTask;
 
using ContextImplPtr = std::shared_ptr<detail::context_impl>;
using EventImplPtr = std::shared_ptr<detail::event_impl>;
using QueueImplPtr = std::shared_ptr<detail::queue_impl>;
using StreamImplPtr = std::shared_ptr<detail::stream_impl>;
 
using QueueIdT = std::hash<std::shared_ptr<detail::queue_impl>>::result_type;
using CommandPtr = std::unique_ptr<Command>;
using FusionList = std::unique_ptr<KernelFusionCommand>;
using FusionMap = std::unordered_map<QueueIdT, FusionList>;
 
struct MemObjRecord {
  MemObjRecord(ContextImplPtr Ctx, std::size_t LeafLimit,
               LeavesCollection::AllocateDependencyF AllocateDependency)
      : MReadLeaves{this, LeafLimit, AllocateDependency},
        MWriteLeaves{this, LeafLimit, AllocateDependency}, MCurContext{Ctx} {}
  // Contains all allocation commands for the memory object.
  std::vector<AllocaCommandBase *> MAllocaCommands;
 
  // Contains latest read only commands working with memory object.
  LeavesCollection MReadLeaves;
 
  // Contains latest write commands working with memory object.
  LeavesCollection MWriteLeaves;
 
  // The context which has the latest state of the memory object.
  ContextImplPtr MCurContext;
 
  // The mode this object can be accessed from the host (host_accessor).
  // Valid only if the current usage is on host.
  access::mode MHostAccess = access::mode::read_write;
 
  // The flag indicates that the content of the memory object was/will be
  // modified. Used while deciding if copy back needed.
  bool MMemModified = false;
};
 
//
//    +----------+     +----------+     +----------+
//    |          |     |          |     |          |
//    | Allocate |<----| Execute  |<----| Execute  |
//    |          |     |          |     |          |
//    +----------+     +----------+     +----------+
//
//                             +----------+
//                             |          |
//                             |    D     |
//                             |          |
//                             +----------+
//                            /            \
//                           /              \
//                          v                v
//                      +----------+     +----------+
//                      |          |     |          |
//                      |     B    |     |    C     |
//                      |          |     |          |
//                      +----------+     +----------+
//                             \            /
//                              \          /
//                               v        v
//                              +----------+
//                              |          |
//                              |    A     |
//                              |          |
//                              +----------+
class Scheduler {
public:
  EventImplPtr
  addCG(std::unique_ptr<detail::CG> CommandGroup, const QueueImplPtr &Queue,
        bool EventNeeded,
        sycl::detail::pi::PiExtCommandBuffer CommandBuffer = nullptr,
        const std::vector<sycl::detail::pi::PiExtSyncPoint> &Dependencies = {});
 
  EventImplPtr addCopyBack(Requirement *Req);
 
 
  void waitForEvent(const EventImplPtr &Event, bool *Success = nullptr);
 
  bool removeMemoryObject(detail::SYCLMemObjI *MemObj, bool StrictLock = true);
 
  EventImplPtr addHostAccessor(Requirement *Req);
 
  void releaseHostAccessor(Requirement *Req);
 
  static Scheduler &getInstance();
  static bool isInstanceAlive();
 
  static MemObjRecord *getMemObjRecord(const Requirement *const Req);
 
  void deferMemObjRelease(const std::shared_ptr<detail::SYCLMemObjI> &MemObj);
 
  void startFusion(QueueImplPtr Queue);
 
  void cleanUpCmdFusion(sycl::detail::queue_impl *Queue);
 
  void cancelFusion(QueueImplPtr Queue);
 
  EventImplPtr completeFusion(QueueImplPtr Queue, const property_list &);
  sycl::detail::pi::PiKernel completeSpecConstMaterialization(
      QueueImplPtr Queue, const RTDeviceBinaryImage *BinImage,
      const std::string &KernelName, std::vector<unsigned char> &SpecConstBlob);
 
  bool isInFusionMode(QueueIdT Queue);
 
  void releaseResources(BlockingT Blocking = BlockingT::BLOCKING);
  bool isDeferredMemObjectsEmpty();
 
  void enqueueCommandForCG(EventImplPtr NewEvent,
                           std::vector<Command *> &AuxilaryCmds,
                           BlockingT Blocking = NON_BLOCKING);
 
  EventImplPtr addCommandGraphUpdate(
      ext::oneapi::experimental::detail::exec_graph_impl *Graph,
      std::vector<std::shared_ptr<ext::oneapi::experimental::detail::node_impl>>
          Nodes,
      const QueueImplPtr &Queue, std::vector<Requirement *> Requirements,
      std::vector<detail::EventImplPtr> &Events);
 
  static bool
  areEventsSafeForSchedulerBypass(const std::vector<sycl::event> &DepEvents,
                                  ContextImplPtr Context);
  static bool
  areEventsSafeForSchedulerBypass(const std::vector<EventImplPtr> &DepEvents,
                                  ContextImplPtr Context);
 
protected:
  using RWLockT = std::shared_timed_mutex;
  using ReadLockT = std::shared_lock<RWLockT>;
  using WriteLockT = std::unique_lock<RWLockT>;
 
  WriteLockT acquireWriteLock() {
#ifdef _WIN32
    WriteLockT Lock(MGraphLock, std::defer_lock);
    while (!Lock.try_lock_for(std::chrono::milliseconds(10))) {
      // Without yield while loop acts like endless while loop and occupies the
      // whole CPU when multiple command groups are created in multiple host
      // threads
      std::this_thread::yield();
    }
#else
    WriteLockT Lock(MGraphLock);
    // It is a deadlock on UNIX in implementation of lock and lock_shared, if
    // try_lock in the loop above will be executed, so using a single lock here
#endif // _WIN32
    return Lock;
  }
 
  WriteLockT acquireFusionWriteLock() {
#ifdef _WIN32
    WriteLockT Lock(MFusionMapLock, std::defer_lock);
    while (!Lock.try_lock_for(std::chrono::milliseconds(10))) {
      // Without yield while loop acts like endless while loop and occupies the
      // whole CPU when multiple command groups are created in multiple host
      // threads
      std::this_thread::yield();
    }
#else
    WriteLockT Lock(MFusionMapLock);
    // It is a deadlock on UNIX in implementation of lock and lock_shared, if
    // try_lock in the loop above will be executed, so using a single lock here
#endif // _WIN32
    return Lock;
  }
 
  ReadLockT acquireReadLock() { return ReadLockT{MGraphLock}; }
 
  ReadLockT acquireFusionReadLock() { return ReadLockT{MFusionMapLock}; }
 
  void cleanupCommands(const std::vector<Command *> &Cmds);
 
  void NotifyHostTaskCompletion(Command *Cmd);
 
  static void enqueueLeavesOfReqUnlocked(const Requirement *const Req,
                                         ReadLockT &GraphReadLock,
                                         std::vector<Command *> &ToCleanUp);
 
  static void
  enqueueUnblockedCommands(const std::vector<EventImplPtr> &CmdsToEnqueue,
                           ReadLockT &GraphReadLock,
                           std::vector<Command *> &ToCleanUp);
 
  // May lock graph with read and write modes during execution.
  void cleanupDeferredMemObjects(BlockingT Blocking);
 
  // POD struct to convey some additional information from GraphBuilder::addCG
  // to the Scheduler to support kernel fusion.
  struct GraphBuildResult {
    Command *NewCmd;
    EventImplPtr NewEvent;
    bool ShouldEnqueue;
  };
 
  void takeAuxiliaryResources(const EventImplPtr &Dst, const EventImplPtr &Src);
  void registerAuxiliaryResources(
      EventImplPtr &Event, std::vector<std::shared_ptr<const void>> Resources);
  void cleanupAuxiliaryResources(BlockingT Blocking);
 
  class GraphBuilder {
  public:
    GraphBuilder();
 
    GraphBuildResult addCG(
        std::unique_ptr<detail::CG> CommandGroup, const QueueImplPtr &Queue,
        std::vector<Command *> &ToEnqueue, bool EventNeeded,
        sycl::detail::pi::PiExtCommandBuffer CommandBuffer = nullptr,
        const std::vector<sycl::detail::pi::PiExtSyncPoint> &Dependencies = {});
 
    Command *addCGUpdateHost(std::unique_ptr<detail::CG> CommandGroup,
                             std::vector<Command *> &ToEnqueue);
 
    Command *addCopyBack(Requirement *Req, std::vector<Command *> &ToEnqueue);
 
    Command *addHostAccessor(Requirement *Req,
                             std::vector<Command *> &ToEnqueue);
 
    void optimize();
 
    void optimize(const EventImplPtr &Event);
 
    void cleanupCommand(Command *Cmd, bool AllowUnsubmitted = false);
 
    void rescheduleCommand(Command *Cmd, const QueueImplPtr &Queue);
 
    MemObjRecord *getMemObjRecord(SYCLMemObjI *MemObject);
 
    MemObjRecord *getOrInsertMemObjRecord(const QueueImplPtr &Queue,
                                          const Requirement *Req);
 
    void decrementLeafCountersForRecord(MemObjRecord *Record);
 
    void cleanupCommandsForRecord(MemObjRecord *Record);
 
    void removeRecordForMemObj(SYCLMemObjI *MemObject);
 
    void addNodeToLeaves(MemObjRecord *Record, Command *Cmd,
                         access::mode AccessMode,
                         std::vector<Command *> &ToEnqueue);
 
    void updateLeaves(const std::set<Command *> &Cmds, MemObjRecord *Record,
                      access::mode AccessMode,
                      std::vector<Command *> &ToCleanUp);
 
    Command *connectDepEvent(Command *const Cmd, const EventImplPtr &DepEvent,
                             const DepDesc &Dep,
                             std::vector<Command *> &ToCleanUp);
 
    void startFusion(QueueImplPtr Queue);
 
    void cleanUpCmdFusion(sycl::detail::queue_impl *Queue);
 
    void cancelFusion(QueueImplPtr Queue, std::vector<Command *> &ToEnqueue);
 
    EventImplPtr completeFusion(QueueImplPtr Queue,
                                std::vector<Command *> &ToEnqueue,
                                const property_list &);
 
    bool isInFusionMode(QueueIdT queue);
 
    Command *addCommandGraphUpdate(
        ext::oneapi::experimental::detail::exec_graph_impl *Graph,
        std::vector<
            std::shared_ptr<ext::oneapi::experimental::detail::node_impl>>
            Nodes,
        const QueueImplPtr &Queue, std::vector<Requirement *> Requirements,
        std::vector<detail::EventImplPtr> &Events,
        std::vector<Command *> &ToEnqueue);
 
    std::vector<SYCLMemObjI *> MMemObjs;
 
  private:
    Command *insertMemoryMove(MemObjRecord *Record, Requirement *Req,
                              const QueueImplPtr &Queue,
                              std::vector<Command *> &ToEnqueue);
 
    // Inserts commands required to remap the memory object to its current host
    // context so that the required access mode becomes valid.
    Command *remapMemoryObject(MemObjRecord *Record, Requirement *Req,
                               AllocaCommandBase *HostAllocaCmd,
                               std::vector<Command *> &ToEnqueue);
 
    UpdateHostRequirementCommand *
    insertUpdateHostReqCmd(MemObjRecord *Record, Requirement *Req,
                           const QueueImplPtr &Queue,
                           std::vector<Command *> &ToEnqueue);
 
    std::set<Command *> findDepsForReq(MemObjRecord *Record,
                                       const Requirement *Req,
                                       const ContextImplPtr &Context);
 
    EmptyCommand *addEmptyCmd(Command *Cmd,
                              const std::vector<Requirement *> &Req,
                              Command::BlockReason Reason,
                              std::vector<Command *> &ToEnqueue);
 
    void createGraphForCommand(Command *NewCmd, CG &CG, bool isInteropTask,
                               std::vector<Requirement *> &Reqs,
                               const std::vector<detail::EventImplPtr> &Events,
                               QueueImplPtr Queue,
                               std::vector<Command *> &ToEnqueue);
 
  protected:
    DepDesc findDepForRecord(Command *Cmd, MemObjRecord *Record);
 
    AllocaCommandBase *findAllocaForReq(MemObjRecord *Record,
                                        const Requirement *Req,
                                        const ContextImplPtr &Context,
                                        bool AllowConst = true);
 
    friend class Command;
 
  private:
    friend class ::MockScheduler;
 
    AllocaCommandBase *
    getOrCreateAllocaForReq(MemObjRecord *Record, const Requirement *Req,
                            const QueueImplPtr &Queue,
                            std::vector<Command *> &ToEnqueue);
 
    void markModifiedIfWrite(MemObjRecord *Record, Requirement *Req);
 
    FusionMap::iterator findFusionList(QueueIdT Id) {
      return MFusionMap.find(Id);
    }
 
    void removeNodeFromGraph(Command *Node, std::vector<Command *> &ToEnqueue);
 
    std::queue<Command *> MCmdsToVisit;
    std::vector<Command *> MVisitedCmds;
 
    FusionMap MFusionMap;
 
    void printGraphAsDot(const char *ModeName);
    enum PrintOptions {
      BeforeAddCG = 0,
      AfterAddCG,
      BeforeAddCopyBack,
      AfterAddCopyBack,
      BeforeAddHostAcc,
      AfterAddHostAcc,
      AfterFusionComplete,
      AfterFusionCancel,
      Size
    };
    std::array<bool, PrintOptions::Size> MPrintOptionsArray{false};
  };
 
  class GraphProcessor {
  public:
    static void waitForEvent(const EventImplPtr &Event,
                             ReadLockT &GraphReadLock,
                             std::vector<Command *> &ToCleanUp,
                             bool LockTheLock = true, bool *Success = nullptr);
 
    static bool enqueueCommand(Command *Cmd, ReadLockT &GraphReadLock,
                               EnqueueResultT &EnqueueResult,
                               std::vector<Command *> &ToCleanUp,
                               Command *RootCommand,
                               BlockingT Blocking = NON_BLOCKING);
 
    static bool handleBlockingCmd(Command *Cmd, EnqueueResultT &EnqueueResult,
                                  Command *RootCommand, BlockingT Blocking);
  };
 
  void waitForRecordToFinish(MemObjRecord *Record, ReadLockT &GraphReadLock);
  bool checkLeavesCompletion(MemObjRecord *Record);
 
  GraphBuilder MGraphBuilder;
  RWLockT MGraphLock;
  RWLockT MFusionMapLock;
 
  std::vector<Command *> MDeferredCleanupCommands;
  std::mutex MDeferredCleanupMutex;
 
  std::vector<std::shared_ptr<SYCLMemObjI>> MDeferredMemObjRelease;
  std::mutex MDeferredMemReleaseMutex;
 
  std::unordered_map<EventImplPtr, std::vector<std::shared_ptr<const void>>>
      MAuxiliaryResources;
  std::mutex MAuxiliaryResourcesMutex;
 
  friend class Command;
  friend class DispatchHostTask;
  friend class queue_impl;
  friend class event_impl;
  friend class ::MockScheduler;
 
private:
  static void printFusionWarning(const std::string &Message);
 
  static KernelFusionCommand *isPartOfActiveFusion(Command *Cmd);
};
 
} // namespace detail
} // namespace _V1
} // namespace sycl