DPC++ Runtime
Runtime libraries for oneAPI DPC++
kernel_program_cache.hpp
Go to the documentation of this file.
1 //==--- kernel_program_cache.hpp - Cache for kernel and program -*- C++-*---==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #pragma once
10 
11 #include <detail/platform_impl.hpp>
12 #include <sycl/detail/common.hpp>
13 #include <sycl/detail/locked.hpp>
14 #include <sycl/detail/os_util.hpp>
15 #include <sycl/detail/pi.hpp>
16 #include <sycl/detail/util.hpp>
17 
18 #include <atomic>
19 #include <condition_variable>
20 #include <map>
21 #include <mutex>
22 #include <type_traits>
23 
24 // For testing purposes
25 class MockKernelProgramCache;
26 
27 namespace sycl {
29 namespace detail {
30 class context_impl;
32 public:
35  struct BuildError {
36  std::string Msg;
38 
39  bool isFilledIn() const { return !Msg.empty(); }
40  };
41 
43  enum BuildState { BS_InProgress, BS_Done, BS_Failed };
44 
49  template <typename T> struct BuildResult {
50  std::atomic<T *> Ptr;
51  std::atomic<BuildState> State;
53 
64  std::condition_variable MBuildCV;
66  std::mutex MBuildResultMutex;
67 
68  BuildResult(T *P, BuildState S) : Ptr{P}, State{S}, Error{"", 0} {}
69  };
70 
71  using PiProgramT = std::remove_pointer<RT::PiProgram>::type;
72  using PiProgramPtrT = std::atomic<PiProgramT *>;
74  using ProgramCacheKeyT = std::pair<std::pair<SerializedObj, std::uintptr_t>,
75  std::pair<RT::PiDevice, std::string>>;
76  using CommonProgramKeyT = std::pair<std::uintptr_t, RT::PiDevice>;
77 
78  struct ProgramCache {
79  std::map<ProgramCacheKeyT, ProgramWithBuildStateT> Cache;
80  std::multimap<CommonProgramKeyT, ProgramCacheKeyT> KeyMap;
81 
82  size_t size() const noexcept { return Cache.size(); }
83  };
84 
86 
87  using PiKernelT = std::remove_pointer<RT::PiKernel>::type;
88 
89  using PiKernelPtrT = std::atomic<PiKernelT *>;
91  using KernelByNameT = std::map<std::string, KernelWithBuildStateT>;
92  using KernelCacheT = std::map<RT::PiProgram, KernelByNameT>;
93 
94  using KernelFastCacheKeyT =
95  std::tuple<SerializedObj, OSModuleHandle, RT::PiDevice, std::string,
96  std::string>;
97  using KernelFastCacheValT =
98  std::tuple<RT::PiKernel, std::mutex *, RT::PiProgram>;
99  using KernelFastCacheT = std::map<KernelFastCacheKeyT, KernelFastCacheValT>;
100 
102 
103  void setContextPtr(const ContextPtr &AContext) { MParentContext = AContext; }
104 
106  return {MCachedPrograms, MProgramCacheMutex};
107  }
108 
110  return {MKernelsPerProgramCache, MKernelsPerProgramCacheMutex};
111  }
112 
113  std::pair<ProgramWithBuildStateT *, bool>
115  auto LockedCache = acquireCachedPrograms();
116  auto &ProgCache = LockedCache.get();
117  auto Inserted = ProgCache.Cache.emplace(
118  std::piecewise_construct, std::forward_as_tuple(CacheKey),
119  std::forward_as_tuple(nullptr, BS_InProgress));
120  if (Inserted.second) {
121  // Save reference between the common key and the full key.
122  CommonProgramKeyT CommonKey =
123  std::make_pair(CacheKey.first.second, CacheKey.second.first);
124  ProgCache.KeyMap.emplace(std::piecewise_construct,
125  std::forward_as_tuple(CommonKey),
126  std::forward_as_tuple(CacheKey));
127  }
128  return std::make_pair(&Inserted.first->second, Inserted.second);
129  }
130 
131  std::pair<KernelWithBuildStateT *, bool>
132  getOrInsertKernel(RT::PiProgram Program, const std::string &KernelName) {
133  auto LockedCache = acquireKernelsPerProgramCache();
134  auto &Cache = LockedCache.get()[Program];
135  auto Inserted = Cache.emplace(
136  std::piecewise_construct, std::forward_as_tuple(KernelName),
137  std::forward_as_tuple(nullptr, BS_InProgress));
138  return std::make_pair(&Inserted.first->second, Inserted.second);
139  }
140 
141  template <typename T, class Predicate>
142  void waitUntilBuilt(BuildResult<T> &BR, Predicate Pred) const {
143  std::unique_lock<std::mutex> Lock(BR.MBuildResultMutex);
144 
145  BR.MBuildCV.wait(Lock, Pred);
146  }
147 
148  template <typename ExceptionT, typename RetT>
150  // Any thread which will find nullptr in cache will wait until the pointer
151  // is not null anymore.
152  waitUntilBuilt(*BuildResult, [BuildResult]() {
153  int State = BuildResult->State.load();
154  return State == BuildState::BS_Done || State == BuildState::BS_Failed;
155  });
156 
157  if (BuildResult->Error.isFilledIn()) {
158  const BuildError &Error = BuildResult->Error;
159  throw ExceptionT(Error.Msg, Error.Code);
160  }
161 
162  return BuildResult->Ptr.load();
163  }
164 
165  template <typename T> void notifyAllBuild(BuildResult<T> &BR) const {
166  BR.MBuildCV.notify_all();
167  }
168 
169  template <typename KeyT>
171  std::unique_lock<std::mutex> Lock(MKernelFastCacheMutex);
172  auto It = MKernelFastCache.find(CacheKey);
173  if (It != MKernelFastCache.end()) {
174  return It->second;
175  }
176  return std::make_tuple(nullptr, nullptr, nullptr);
177  }
178 
179  template <typename KeyT, typename ValT>
180  void saveKernel(KeyT &&CacheKey, ValT &&CacheVal) {
181  std::unique_lock<std::mutex> Lock(MKernelFastCacheMutex);
182  // if no insertion took place, thus some other thread has already inserted
183  // smth in the cache
184  MKernelFastCache.emplace(CacheKey, CacheVal);
185  }
186 
190  void reset() {
191  MCachedPrograms = ProgramCache{};
192  MKernelsPerProgramCache = KernelCacheT{};
193  MKernelFastCache = KernelFastCacheT{};
194  }
195 
196 private:
197  std::mutex MProgramCacheMutex;
198  std::mutex MKernelsPerProgramCacheMutex;
199 
200  ProgramCache MCachedPrograms;
201  KernelCacheT MKernelsPerProgramCache;
202  ContextPtr MParentContext;
203 
204  std::mutex MKernelFastCacheMutex;
205  KernelFastCacheT MKernelFastCache;
206  friend class ::MockKernelProgramCache;
207 };
208 } // namespace detail
209 } // __SYCL_INLINE_VER_NAMESPACE(_V1)
210 } // namespace sycl
sycl::_V1::detail::KernelProgramCache::ProgramCache::Cache
std::map< ProgramCacheKeyT, ProgramWithBuildStateT > Cache
Definition: kernel_program_cache.hpp:79
sycl::_V1::detail::KernelProgramCache::acquireCachedPrograms
Locked< ProgramCache > acquireCachedPrograms()
Definition: kernel_program_cache.hpp:105
sycl::_V1::detail::KernelProgramCache::KernelByNameT
std::map< std::string, KernelWithBuildStateT > KernelByNameT
Definition: kernel_program_cache.hpp:91
sycl::_V1::detail::KernelProgramCache::KernelFastCacheKeyT
std::tuple< SerializedObj, OSModuleHandle, RT::PiDevice, std::string, std::string > KernelFastCacheKeyT
Definition: kernel_program_cache.hpp:96
__SYCL_INLINE_VER_NAMESPACE
#define __SYCL_INLINE_VER_NAMESPACE(X)
Definition: defines_elementary.hpp:11
sycl::_V1::detail::KernelProgramCache::BuildError
Denotes build error data.
Definition: kernel_program_cache.hpp:35
sycl::_V1::detail::KernelProgramCache::BuildResult::Error
BuildError Error
Definition: kernel_program_cache.hpp:52
sycl::_V1::detail::KernelProgramCache::waitUntilBuilt
RetT * waitUntilBuilt(BuildResult< RetT > *BuildResult)
Definition: kernel_program_cache.hpp:149
sycl::_V1::detail::KernelProgramCache::PiProgramPtrT
std::atomic< PiProgramT * > PiProgramPtrT
Definition: kernel_program_cache.hpp:72
sycl::_V1::detail::pi::PiDevice
::pi_device PiDevice
Definition: pi.hpp:124
os_util.hpp
sycl::_V1::detail::KernelProgramCache::saveKernel
void saveKernel(KeyT &&CacheKey, ValT &&CacheVal)
Definition: kernel_program_cache.hpp:180
sycl
---— Error handling, matching OpenCL plugin semantics.
Definition: access.hpp:14
sycl::_V1::detail::KernelProgramCache::PiKernelT
std::remove_pointer< RT::PiKernel >::type PiKernelT
Definition: kernel_program_cache.hpp:87
sycl::_V1::detail::make_tuple
constexpr tuple< Ts... > make_tuple(Ts... Args)
Definition: tuple.hpp:36
pi.hpp
sycl::_V1::detail::KernelProgramCache::BuildState
BuildState
Denotes the state of a build.
Definition: kernel_program_cache.hpp:43
sycl::_V1::detail::KernelProgramCache::BuildResult
Denotes pointer to some entity with its general state and build error.
Definition: kernel_program_cache.hpp:49
sycl::_V1::detail::KernelProgramCache::BuildError::isFilledIn
bool isFilledIn() const
Definition: kernel_program_cache.hpp:39
sycl::_V1::detail::KernelProgramCache::BuildResult::Ptr
std::atomic< T * > Ptr
Definition: kernel_program_cache.hpp:50
sycl::_V1::detail::KernelProgramCache::PiProgramT
std::remove_pointer< RT::PiProgram >::type PiProgramT
Definition: kernel_program_cache.hpp:71
sycl::_V1::detail::KernelProgramCache::acquireKernelsPerProgramCache
Locked< KernelCacheT > acquireKernelsPerProgramCache()
Definition: kernel_program_cache.hpp:109
platform_impl.hpp
sycl::_V1::detail::KernelProgramCache::BuildResult::BuildResult
BuildResult(T *P, BuildState S)
Definition: kernel_program_cache.hpp:68
sycl::_V1::detail::KernelProgramCache::CommonProgramKeyT
std::pair< std::uintptr_t, RT::PiDevice > CommonProgramKeyT
Definition: kernel_program_cache.hpp:76
sycl::_V1::detail::KernelProgramCache::ProgramCacheKeyT
std::pair< std::pair< SerializedObj, std::uintptr_t >, std::pair< RT::PiDevice, std::string > > ProgramCacheKeyT
Definition: kernel_program_cache.hpp:75
sycl::_V1::detail::pi::PiProgram
::pi_program PiProgram
Definition: pi.hpp:130
sycl::_V1::detail::KernelProgramCache::setContextPtr
void setContextPtr(const ContextPtr &AContext)
Definition: kernel_program_cache.hpp:103
common.hpp
sycl::_V1::detail::KernelProgramCache::ProgramCache::KeyMap
std::multimap< CommonProgramKeyT, ProgramCacheKeyT > KeyMap
Definition: kernel_program_cache.hpp:80
sycl::_V1::detail::KernelProgramCache::ProgramCache::size
size_t size() const noexcept
Definition: kernel_program_cache.hpp:82
sycl::_V1::detail::KernelProgramCache::notifyAllBuild
void notifyAllBuild(BuildResult< T > &BR) const
Definition: kernel_program_cache.hpp:165
sycl::_V1::detail::KernelProgramCache::getOrInsertKernel
std::pair< KernelWithBuildStateT *, bool > getOrInsertKernel(RT::PiProgram Program, const std::string &KernelName)
Definition: kernel_program_cache.hpp:132
sycl::_V1::detail::OSModuleHandle
intptr_t OSModuleHandle
Uniquely identifies an operating system module (executable or a dynamic library)
Definition: os_util.hpp:48
sycl::_V1::detail::KernelProgramCache::BuildResult::MBuildResultMutex
std::mutex MBuildResultMutex
A mutex to be employed along with MBuildCV.
Definition: kernel_program_cache.hpp:66
sycl::_V1::detail::KernelProgramCache::KernelFastCacheT
std::map< KernelFastCacheKeyT, KernelFastCacheValT > KernelFastCacheT
Definition: kernel_program_cache.hpp:99
sycl::_V1::detail::KernelProgramCache::BuildError::Code
pi_int32 Code
Definition: kernel_program_cache.hpp:37
sycl::_V1::detail::KernelProgramCache::KernelCacheT
std::map< RT::PiProgram, KernelByNameT > KernelCacheT
Definition: kernel_program_cache.hpp:92
sycl::_V1::detail::KernelProgramCache::reset
void reset()
Clears cache state.
Definition: kernel_program_cache.hpp:190
sycl::_V1::detail::KernelProgramCache::ProgramCache
Definition: kernel_program_cache.hpp:78
sycl::_V1::detail::Locked
Represents a reference to value with appropriate lock acquired.
Definition: locked.hpp:23
sycl::_V1::detail::KernelProgramCache::BuildResult::State
std::atomic< BuildState > State
Definition: kernel_program_cache.hpp:51
util.hpp
sycl::_V1::detail::KernelProgramCache::PiKernelPtrT
std::atomic< PiKernelT * > PiKernelPtrT
Definition: kernel_program_cache.hpp:89
sycl::_V1::detail::SerializedObj
std::vector< unsigned char > SerializedObj
Definition: util.hpp:68
sycl::_V1::detail::KernelProgramCache::getOrInsertProgram
std::pair< ProgramWithBuildStateT *, bool > getOrInsertProgram(const ProgramCacheKeyT &CacheKey)
Definition: kernel_program_cache.hpp:114
sycl::_V1::detail::KernelProgramCache::waitUntilBuilt
void waitUntilBuilt(BuildResult< T > &BR, Predicate Pred) const
Definition: kernel_program_cache.hpp:142
sycl::_V1::detail::KernelProgramCache
Definition: kernel_program_cache.hpp:31
sycl::_V1::detail::context_impl
Definition: context_impl.hpp:33
pi_int32
int32_t pi_int32
Definition: pi.h:128
sycl::_V1::detail::KernelProgramCache::tryToGetKernelFast
KernelFastCacheValT tryToGetKernelFast(KeyT &&CacheKey)
Definition: kernel_program_cache.hpp:170
locked.hpp
sycl::_V1::detail::KernelProgramCache::KernelFastCacheValT
std::tuple< RT::PiKernel, std::mutex *, RT::PiProgram > KernelFastCacheValT
Definition: kernel_program_cache.hpp:98
sycl::_V1::detail::KernelProgramCache::BuildError::Msg
std::string Msg
Definition: kernel_program_cache.hpp:36
sycl::_V1::detail::KernelProgramCache::BuildResult::MBuildCV
std::condition_variable MBuildCV
Condition variable to signal that build result is ready.
Definition: kernel_program_cache.hpp:64