DPC++ Runtime
Runtime libraries for oneAPI DPC++
usm_allocator.cpp
Go to the documentation of this file.
1 //===---------- usm_allocator.cpp - Allocator for USM memory --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include <algorithm>
10 #include <array>
11 #include <bitset>
12 #include <cassert>
13 #include <cctype>
14 #include <iomanip>
15 #include <list>
16 #include <memory>
17 #include <mutex>
18 #include <shared_mutex>
19 #include <string>
20 #include <unordered_map>
21 #include <utility>
22 #include <vector>
23 
24 #include "usm_allocator.hpp"
26 #include <sycl/detail/spinlock.hpp>
27 
28 // USM allocations are a minimum of 4KB/64KB/2MB even when a smaller size is
29 // requested. The implementation distinguishes between allocations of size
30 // ChunkCutOff = (minimum-alloc-size / 2) and those that are larger.
31 // Allocation requests smaller than ChunkCutoff use chunks taken from a single
32 // USM allocation. Thus, for example, for a 64KB minimum allocation size,
33 // and 8-byte allocations, only 1 in ~8000 requests results in a new
34 // USM allocation. Freeing results only in a chunk of a larger allocation
35 // to be marked as available and no real return to the system.
36 // An allocation is returned to the system only when all
37 // chunks in the larger allocation are freed by the program.
38 // Allocations larger than ChunkCutOff use a separate USM allocation for each
39 // request. These are subject to "pooling". That is, when such an allocation is
40 // freed by the program it is retained in a pool. The pool is available for
41 // future allocations, which means there are fewer actual USM
42 // allocations/deallocations.
43 
44 namespace settings {
45 
46 constexpr auto operator""_B(unsigned long long x) -> size_t { return x; }
47 constexpr auto operator""_KB(unsigned long long x) -> size_t {
48  return x * 1024;
49 }
50 constexpr auto operator""_MB(unsigned long long x) -> size_t {
51  return x * 1024 * 1024;
52 }
53 constexpr auto operator""_GB(unsigned long long x) -> size_t {
54  return x * 1024 * 1024 * 1024;
55 }
56 
57 // The largest size which is allocated via the allocator.
58 // Allocations with size > CutOff bypass the USM allocator and
59 // go directly to the runtime.
60 static constexpr size_t CutOff = (size_t)1 << 31; // 2GB
61 
62 // Protects the capacity checking of the pool.
63 static sycl::detail::SpinLock PoolLock;
64 
65 static class SetLimits {
66 public:
67  // String names of memory types for printing in limits traces.
68  static constexpr const char *MemTypeNames[MemType::All] = {
69  "Host", "Device", "Shared", "SharedReadOnly"};
70 
71  // Minimum allocation size that will be requested from the system.
72  // By default this is the minimum allocation size of each memory type.
73  size_t SlabMinSize[MemType::All] = {};
74 
75  // Allocations up to this limit will be subject to chunking/pooling
77 
78  // When pooling, each bucket will hold a max of 4 unfreed slabs
79  size_t Capacity[MemType::All] = {};
80 
81  // Holds the minimum bucket size valid for allocation of a memory type.
83 
84  // Maximum memory left unfreed in pool
85  size_t MaxPoolSize = 16_MB;
86 
87  size_t CurPoolSize = 0;
88  size_t CurPoolSizes[MemType::All] = {};
89 
90  size_t EnableBuffers = 1;
91 
92  // Whether to print pool usage statistics
93  int PoolTrace = 0;
94 
96  // Buckets for Host use a minimum of the cache line size of 64 bytes.
97  // This prevents two separate allocations residing in the same cache line.
98  // Buckets for Device and Shared allocations will use starting size of 512.
99  // This is because memory compression on newer GPUs makes the
100  // minimum granularity 512 bytes instead of 64.
105 
106  // Initialize default pool settings.
108  Capacity[MemType::Host] = 4;
109  SlabMinSize[MemType::Host] = 64_KB;
110 
113  SlabMinSize[MemType::Device] = 64_KB;
114 
115  // Disable pooling of shared USM allocations.
119 
120  // Allow pooling of shared allocations that are only modified on host.
124 
125  // Parse optional parameters of this form:
126  // SYCL_PI_LEVEL_ZERO_USM_ALLOCATOR=[EnableBuffers][;[MaxPoolSize][;memtypelimits]...]
127  // memtypelimits: [<memtype>:]<limits>
128  // memtype: host|device|shared
129  // limits: [MaxPoolableSize][,[Capacity][,SlabMinSize]]
130  //
131  // Without a memory type, the limits are applied to each memory type.
132  // Parameters are for each context, except MaxPoolSize, which is overall
133  // pool size for all contexts.
134  // Duplicate specifications will result in the right-most taking effect.
135  //
136  // EnableBuffers: Apply chunking/pooling to SYCL buffers.
137  // Default 1.
138  // MaxPoolSize: Limit on overall unfreed memory.
139  // Default 16MB.
140  // MaxPoolableSize: Maximum allocation size subject to chunking/pooling.
141  // Default 2MB host, 4MB device and 0 shared.
142  // Capacity: Maximum number of unfreed allocations in each bucket.
143  // Default 4.
144  // SlabMinSize: Minimum allocation size requested from USM.
145  // Default 64KB host and device, 2MB shared.
146  //
147  // Example of usage:
148  // SYCL_PI_LEVEL_ZERO_USM_ALLOCATOR=1;32M;host:1M,4,64K;device:1M,4,64K;shared:0,0,2M
149 
150  auto GetValue = [=](std::string &Param, size_t Length, size_t &Setting) {
151  size_t Multiplier = 1;
152  if (tolower(Param[Length - 1]) == 'k') {
153  Length--;
154  Multiplier = 1_KB;
155  }
156  if (tolower(Param[Length - 1]) == 'm') {
157  Length--;
158  Multiplier = 1_MB;
159  }
160  if (tolower(Param[Length - 1]) == 'g') {
161  Length--;
162  Multiplier = 1_GB;
163  }
164  std::string TheNumber = Param.substr(0, Length);
165  if (TheNumber.find_first_not_of("0123456789") == std::string::npos)
166  Setting = std::stoi(TheNumber) * Multiplier;
167  };
168 
169  auto ParamParser = [=](std::string &Params, size_t &Setting,
170  bool &ParamWasSet) {
171  bool More;
172  if (Params.size() == 0) {
173  ParamWasSet = false;
174  return false;
175  }
176  size_t Pos = Params.find(',');
177  if (Pos != std::string::npos) {
178  if (Pos > 0) {
179  GetValue(Params, Pos, Setting);
180  ParamWasSet = true;
181  }
182  Params.erase(0, Pos + 1);
183  More = true;
184  } else {
185  GetValue(Params, Params.size(), Setting);
186  ParamWasSet = true;
187  More = false;
188  }
189  return More;
190  };
191 
192  auto MemParser = [=](std::string &Params, MemType M) {
193  bool ParamWasSet;
194  MemType LM = M;
195  if (M == MemType::All)
196  LM = MemType::Host;
197 
198  bool More = ParamParser(Params, MaxPoolableSize[LM], ParamWasSet);
199  if (ParamWasSet && M == MemType::All) {
202  }
203  if (More) {
204  More = ParamParser(Params, Capacity[LM], ParamWasSet);
205  if (ParamWasSet && M == MemType::All) {
208  }
209  }
210  if (More) {
211  ParamParser(Params, SlabMinSize[LM], ParamWasSet);
212  if (ParamWasSet && M == MemType::All) {
215  }
216  }
217  };
218 
219  auto MemTypeParser = [=](std::string &Params) {
220  int Pos = 0;
221  MemType M = MemType::All;
222  if (Params.compare(0, 5, "host:") == 0) {
223  Pos = 5;
224  M = MemType::Host;
225  } else if (Params.compare(0, 7, "device:") == 0) {
226  Pos = 7;
227  M = MemType::Device;
228  } else if (Params.compare(0, 7, "shared:") == 0) {
229  Pos = 7;
230  M = MemType::Shared;
231  } else if (Params.compare(0, 17, "read_only_shared:") == 0) {
232  Pos = 17;
234  }
235  if (Pos > 0)
236  Params.erase(0, Pos);
237  MemParser(Params, M);
238  };
239 
240  // Update pool settings if specified in environment.
241  char *PoolParams = getenv("SYCL_PI_LEVEL_ZERO_USM_ALLOCATOR");
242  if (PoolParams != nullptr) {
243  std::string Params(PoolParams);
244  size_t Pos = Params.find(';');
245  if (Pos != std::string::npos) {
246  if (Pos > 0) {
247  GetValue(Params, Pos, EnableBuffers);
248  }
249  Params.erase(0, Pos + 1);
250  size_t Pos = Params.find(';');
251  if (Pos != std::string::npos) {
252  if (Pos > 0) {
253  GetValue(Params, Pos, MaxPoolSize);
254  }
255  Params.erase(0, Pos + 1);
256  do {
257  size_t Pos = Params.find(';');
258  if (Pos != std::string::npos) {
259  if (Pos > 0) {
260  std::string MemParams = Params.substr(0, Pos);
261  MemTypeParser(MemParams);
262  }
263  Params.erase(0, Pos + 1);
264  if (Params.size() == 0)
265  break;
266  } else {
267  MemTypeParser(Params);
268  break;
269  }
270  } while (true);
271  } else {
272  GetValue(Params, Params.size(), MaxPoolSize);
273  }
274  } else {
275  GetValue(Params, Params.size(), EnableBuffers);
276  }
277  }
278 
279  char *PoolTraceVal = getenv("SYCL_PI_LEVEL_ZERO_USM_ALLOCATOR_TRACE");
280  if (PoolTraceVal != nullptr) {
281  PoolTrace = std::atoi(PoolTraceVal);
282  }
283  if (PoolTrace < 1)
284  return;
285 
286  std::cout << "USM Pool Settings (Built-in or Adjusted by Environment "
287  "Variable)"
288  << std::endl;
289 
290  std::cout << std::setw(15) << "Parameter" << std::setw(12) << "Host"
291  << std::setw(12) << "Device" << std::setw(12) << "Shared RW"
292  << std::setw(12) << "Shared RO" << std::endl;
293  std::cout << std::setw(15) << "SlabMinSize" << std::setw(12)
297  << SlabMinSize[MemType::SharedReadOnly] << std::endl;
298  std::cout << std::setw(15) << "MaxPoolableSize" << std::setw(12)
302  << MaxPoolableSize[MemType::SharedReadOnly] << std::endl;
303  std::cout << std::setw(15) << "Capacity" << std::setw(12)
304  << Capacity[MemType::Host] << std::setw(12)
307  << Capacity[MemType::SharedReadOnly] << std::endl;
308  std::cout << std::setw(15) << "MaxPoolSize" << std::setw(12) << MaxPoolSize
309  << std::endl;
310  std::cout << std::setw(15) << "EnableBuffers" << std::setw(12)
311  << EnableBuffers << std::endl
312  << std::endl;
313  }
314 } USMSettings;
315 } // namespace settings
316 
317 using namespace settings;
318 
319 // Aligns the pointer down to the specified alignment
320 // (e.g. returns 8 for Size = 13, Alignment = 8)
321 static void *AlignPtrDown(void *Ptr, const size_t Alignment) {
322  return reinterpret_cast<void *>((reinterpret_cast<size_t>(Ptr)) &
323  (~(Alignment - 1)));
324 }
325 
326 // Aligns the pointer up to the specified alignment
327 // (e.g. returns 16 for Size = 13, Alignment = 8)
328 static void *AlignPtrUp(void *Ptr, const size_t Alignment) {
329  void *AlignedPtr = AlignPtrDown(Ptr, Alignment);
330  // Special case when the pointer is already aligned
331  if (Ptr == AlignedPtr) {
332  return Ptr;
333  }
334  return static_cast<char *>(AlignedPtr) + Alignment;
335 }
336 
337 // Aligns the value up to the specified alignment
338 // (e.g. returns 16 for Size = 13, Alignment = 8)
339 static size_t AlignUp(size_t Val, size_t Alignment) {
340  assert(Alignment > 0);
341  return (Val + Alignment - 1) & (~(Alignment - 1));
342 }
343 
344 class Bucket;
345 
346 // Represents the allocated memory block of size 'SlabMinSize'
347 // Internally, it splits the memory block into chunks. The number of
348 // chunks depends of the size of a Bucket which created the Slab.
349 // The chunks
350 // Note: Bucket's method are responsible for thread safety of Slab access,
351 // so no locking happens here.
352 class Slab {
353 
354  // Pointer to the allocated memory of SlabMinSize bytes
355  void *MemPtr;
356 
357  // Represents the current state of each chunk:
358  // if the bit is set then the chunk is allocated
359  // the chunk is free for allocation otherwise
360  std::vector<bool> Chunks;
361 
362  // Total number of allocated chunks at the moment.
363  size_t NumAllocated = 0;
364 
365  // The bucket which the slab belongs to
366  Bucket &bucket;
367 
368  using ListIter = std::list<std::unique_ptr<Slab>>::iterator;
369 
370  // Store iterator to the corresponding node in avail/unavail list
371  // to achieve O(1) removal
372  ListIter SlabListIter;
373 
374  // Hints where to start search for free chunk in a slab
375  size_t FirstFreeChunkIdx = 0;
376 
377  // Return the index of the first available chunk, -1 otherwize
378  size_t FindFirstAvailableChunkIdx() const;
379 
380  // Register/Unregister the slab in the global slab address map.
381  void regSlab(Slab &);
382  void unregSlab(Slab &);
383  static void regSlabByAddr(void *, Slab &);
384  static void unregSlabByAddr(void *, Slab &);
385 
386 public:
387  Slab(Bucket &);
388  ~Slab();
389 
390  void setIterator(ListIter It) { SlabListIter = It; }
391  ListIter getIterator() const { return SlabListIter; }
392 
393  size_t getNumAllocated() const { return NumAllocated; }
394 
395  // Get pointer to allocation that is one piece of this slab.
396  void *getChunk();
397 
398  // Get pointer to allocation that is this entire slab.
399  void *getSlab();
400 
401  void *getPtr() const { return MemPtr; }
402  void *getEnd() const;
403 
404  size_t getChunkSize() const;
405  size_t getNumChunks() const { return Chunks.size(); }
406 
407  bool hasAvail();
408 
409  Bucket &getBucket();
410  const Bucket &getBucket() const;
411 
412  void freeChunk(void *Ptr);
413 };
414 
415 class Bucket {
416  const size_t Size;
417 
418  // List of slabs which have at least 1 available chunk.
419  std::list<std::unique_ptr<Slab>> AvailableSlabs;
420 
421  // List of slabs with 0 available chunk.
422  std::list<std::unique_ptr<Slab>> UnavailableSlabs;
423 
424  // Protects the bucket and all the corresponding slabs
425  std::mutex BucketLock;
426 
427  // Reference to the allocator context, used access memory allocation
428  // routines, slab map and etc.
429  USMAllocContext::USMAllocImpl &OwnAllocCtx;
430 
431  // For buckets used in chunked mode, a counter of slabs in the pool.
432  // For allocations that use an entire slab each, the entries in the Available
433  // list are entries in the pool.Each slab is available for a new
434  // allocation.The size of the Available list is the size of the pool.
435  // For allocations that use slabs in chunked mode, slabs will be in the
436  // Available list if any one or more of their chunks is free.The entire slab
437  // is not necessarily free, just some chunks in the slab are free. To
438  // implement pooling we will allow one slab in the Available list to be
439  // entirely empty. Normally such a slab would have been freed from USM. But
440  // now we don't, and treat this slab as "in the pool".
441  // When a slab becomes entirely free we have to decide whether to return it to
442  // USM or keep it allocated. A simple check for size of the Available list is
443  // not sufficient to check whether any slab has been pooled yet.We would have
444  // to traverse the entire Available listand check if any of them is entirely
445  // free. Instead we keep a counter of entirely empty slabs within the
446  // Available list to speed up the process of checking if a slab in this bucket
447  // is already pooled.
448  size_t chunkedSlabsInPool;
449 
450  // Statistics
451  size_t allocPoolCount;
452  size_t freeCount;
453  size_t currSlabsInUse;
454  size_t currSlabsInPool;
455  size_t maxSlabsInPool;
456 
457 public:
458  // Statistics
459  size_t allocCount;
461 
462  Bucket(size_t Sz, USMAllocContext::USMAllocImpl &AllocCtx)
463  : Size{Sz}, OwnAllocCtx{AllocCtx}, chunkedSlabsInPool(0),
464  allocPoolCount(0), freeCount(0), currSlabsInUse(0), currSlabsInPool(0),
465  maxSlabsInPool(0), allocCount(0), maxSlabsInUse(0) {}
466 
467  // Get pointer to allocation that is one piece of an available slab in this
468  // bucket.
469  void *getChunk(bool &FromPool);
470 
471  // Get pointer to allocation that is a full slab in this bucket.
472  void *getSlab(bool &FromPool);
473 
474  // Return the allocation size of this bucket.
475  size_t getSize() const { return Size; }
476 
477  // Free an allocation that is one piece of a slab in this bucket.
478  void freeChunk(void *Ptr, Slab &Slab, bool &ToPool);
479 
480  // Free an allocation that is a full slab in this bucket.
481  void freeSlab(Slab &Slab, bool &ToPool);
482 
483  SystemMemory &getMemHandle();
484 
485  MemType getMemType();
486 
488 
489  // Check whether an allocation to be freed can be placed in the pool.
490  bool CanPool(bool &ToPool);
491 
492  // The minimum allocation size for any slab.
493  size_t SlabMinSize();
494 
495  // The allocation size for a slab in this bucket.
496  size_t SlabAllocSize();
497 
498  // The minimum size of a chunk from this bucket's slabs.
499  size_t ChunkCutOff();
500 
501  // The number of slabs in this bucket that can be in the pool.
502  size_t Capacity();
503 
504  // The maximum allocation size subject to pooling.
505  size_t MaxPoolableSize();
506 
507  // Update allocation count
508  void countAlloc(bool FromPool);
509 
510  // Update free count
511  void countFree();
512 
513  // Update statistics of Available/Unavailable
514  void updateStats(int InUse, int InPool);
515 
516  // Print bucket statistics
517  void printStats(bool &TitlePrinted, MemType MT);
518 
519 private:
520  void onFreeChunk(Slab &, bool &ToPool);
521 
522  // Update statistics of pool usage, and indicate that an allocation was made
523  // from the pool.
524  void decrementPool(bool &FromPool);
525 
526  // Get a slab to be used for chunked allocations.
527  decltype(AvailableSlabs.begin()) getAvailSlab(bool &FromPool);
528 
529  // Get a slab that will be used as a whole for a single allocation.
530  decltype(AvailableSlabs.begin()) getAvailFullSlab(bool &FromPool);
531 };
532 
534  // It's important for the map to be destroyed last after buckets and their
535  // slabs This is because slab's destructor removes the object from the map.
536  std::unordered_multimap<void *, Slab &> KnownSlabs;
537  std::shared_timed_mutex KnownSlabsMapLock;
538 
539  // Handle to the memory allocation routine
540  std::unique_ptr<SystemMemory> MemHandle;
541 
542  // Store as unique_ptrs since Bucket is not Movable(because of std::mutex)
543  std::vector<std::unique_ptr<Bucket>> Buckets;
544 
545 public:
546  USMAllocImpl(std::unique_ptr<SystemMemory> SystemMemHandle)
547  : MemHandle{std::move(SystemMemHandle)} {
548 
549  // Generate buckets sized such as: 64, 96, 128, 192, ..., CutOff.
550  // Powers of 2 and the value halfway between the powers of 2.
551  auto Size1 = USMSettings.MinBucketSize[MemHandle->getMemType()];
552  auto Size2 = Size1 + Size1 / 2;
553  for (; Size2 < CutOff; Size1 *= 2, Size2 *= 2) {
554  Buckets.push_back(std::make_unique<Bucket>(Size1, *this));
555  Buckets.push_back(std::make_unique<Bucket>(Size2, *this));
556  }
557  Buckets.push_back(std::make_unique<Bucket>(CutOff, *this));
558  }
559 
560  void *allocate(size_t Size, size_t Alignment, bool &FromPool);
561  void *allocate(size_t Size, bool &FromPool);
562  void deallocate(void *Ptr, bool &ToPool, bool OwnZeMemHandle);
563 
564  SystemMemory &getMemHandle() { return *MemHandle; }
565 
566  std::shared_timed_mutex &getKnownSlabsMapLock() { return KnownSlabsMapLock; }
567  std::unordered_multimap<void *, Slab &> &getKnownSlabs() {
568  return KnownSlabs;
569  }
570 
571  size_t SlabMinSize() {
572  return USMSettings.SlabMinSize[(*MemHandle).getMemType()];
573  };
574 
575  void printStats(bool &TitlePrinted, size_t &HighBucketSize,
576  size_t &HighPeakSlabsInUse, MemType MT);
577 
578 private:
579  Bucket &findBucket(size_t Size);
580 };
581 
582 bool operator==(const Slab &Lhs, const Slab &Rhs) {
583  return Lhs.getPtr() == Rhs.getPtr();
584 }
585 
586 std::ostream &operator<<(std::ostream &Os, const Slab &Slab) {
587  Os << "Slab<" << Slab.getPtr() << ", " << Slab.getEnd() << ", "
588  << Slab.getBucket().getSize() << ">";
589  return Os;
590 }
591 
593  : // In case bucket size is not a multiple of SlabMinSize, we would have
594  // some padding at the end of the slab.
595  Chunks(Bkt.SlabMinSize() / Bkt.getSize()), NumAllocated{0},
596  bucket(Bkt), SlabListIter{}, FirstFreeChunkIdx{0} {
597  auto SlabSize = Bkt.SlabAllocSize();
598  MemPtr = Bkt.getMemHandle().allocate(SlabSize);
599  regSlab(*this);
600 }
601 
603  unregSlab(*this);
604  bucket.getMemHandle().deallocate(MemPtr, true /* OwnZeMemHandle */);
605 }
606 
607 // Return the index of the first available chunk, -1 otherwize
608 size_t Slab::FindFirstAvailableChunkIdx() const {
609  // Use the first free chunk index as a hint for the search.
610  auto It = std::find_if(Chunks.begin() + FirstFreeChunkIdx, Chunks.end(),
611  [](auto x) { return !x; });
612  if (It != Chunks.end()) {
613  return It - Chunks.begin();
614  }
615 
616  return static_cast<size_t>(-1);
617 }
618 
619 void *Slab::getChunk() {
620  // assert(NumAllocated != Chunks.size());
621 
622  const size_t ChunkIdx = FindFirstAvailableChunkIdx();
623  // Free chunk must exist, otherwise we would have allocated another slab
624  assert(ChunkIdx != (static_cast<size_t>(-1)));
625 
626  void *const FreeChunk =
627  (static_cast<uint8_t *>(getPtr())) + ChunkIdx * getChunkSize();
628  Chunks[ChunkIdx] = true;
629  NumAllocated += 1;
630 
631  // Use the found index as the next hint
632  FirstFreeChunkIdx = ChunkIdx;
633 
634  return FreeChunk;
635 }
636 
637 void *Slab::getSlab() { return getPtr(); }
638 
639 Bucket &Slab::getBucket() { return bucket; }
640 const Bucket &Slab::getBucket() const { return bucket; }
641 
642 size_t Slab::getChunkSize() const { return bucket.getSize(); }
643 
644 void Slab::regSlabByAddr(void *Addr, Slab &Slab) {
646  auto &Map = Slab.getBucket().getUsmAllocCtx().getKnownSlabs();
647 
648  std::lock_guard<std::shared_timed_mutex> Lg(Lock);
649  Map.insert({Addr, Slab});
650 }
651 
652 void Slab::unregSlabByAddr(void *Addr, Slab &Slab) {
654  auto &Map = Slab.getBucket().getUsmAllocCtx().getKnownSlabs();
655 
656  std::lock_guard<std::shared_timed_mutex> Lg(Lock);
657 
658  auto Slabs = Map.equal_range(Addr);
659  // At least the must get the current slab from the map.
660  assert(Slabs.first != Slabs.second && "Slab is not found");
661 
662  for (auto It = Slabs.first; It != Slabs.second; ++It) {
663  if (It->second == Slab) {
664  Map.erase(It);
665  return;
666  }
667  }
668 
669  assert(false && "Slab is not found");
670 }
671 
672 void Slab::regSlab(Slab &Slab) {
673  void *StartAddr = AlignPtrDown(Slab.getPtr(), bucket.SlabMinSize());
674  void *EndAddr = static_cast<char *>(StartAddr) + bucket.SlabMinSize();
675 
676  regSlabByAddr(StartAddr, Slab);
677  regSlabByAddr(EndAddr, Slab);
678 }
679 
680 void Slab::unregSlab(Slab &Slab) {
681  void *StartAddr = AlignPtrDown(Slab.getPtr(), bucket.SlabMinSize());
682  void *EndAddr = static_cast<char *>(StartAddr) + bucket.SlabMinSize();
683 
684  unregSlabByAddr(StartAddr, Slab);
685  unregSlabByAddr(EndAddr, Slab);
686 }
687 
688 void Slab::freeChunk(void *Ptr) {
689  // This method should be called through bucket(since we might remove the slab
690  // as a result), therefore all locks are done on that level.
691 
692  // Make sure that we're in the right slab
693  assert(Ptr >= getPtr() && Ptr < getEnd());
694 
695  // Even if the pointer p was previously aligned, it's still inside the
696  // corresponding chunk, so we get the correct index here.
697  auto ChunkIdx =
698  (static_cast<char *>(Ptr) - static_cast<char *>(MemPtr)) / getChunkSize();
699 
700  // Make sure that the chunk was allocated
701  assert(Chunks[ChunkIdx] && "double free detected");
702 
703  Chunks[ChunkIdx] = false;
704  NumAllocated -= 1;
705 
706  if (ChunkIdx < FirstFreeChunkIdx)
707  FirstFreeChunkIdx = ChunkIdx;
708 }
709 
710 void *Slab::getEnd() const {
711  return static_cast<char *>(getPtr()) + bucket.SlabMinSize();
712 }
713 
714 bool Slab::hasAvail() { return NumAllocated != getNumChunks(); }
715 
716 // If a slab was available in the pool then note that the current pooled
717 // size has reduced by the size of a slab in this bucket.
718 void Bucket::decrementPool(bool &FromPool) {
719  FromPool = true;
720  updateStats(1, -1);
722 }
723 
724 auto Bucket::getAvailFullSlab(bool &FromPool)
725  -> decltype(AvailableSlabs.begin()) {
726  // Return a slab that will be used for a single allocation.
727  if (AvailableSlabs.size() == 0) {
728  auto It = AvailableSlabs.insert(AvailableSlabs.begin(),
729  std::make_unique<Slab>(*this));
730  (*It)->setIterator(It);
731  FromPool = false;
732  updateStats(1, 0);
733  } else {
734  decrementPool(FromPool);
735  }
736 
737  return AvailableSlabs.begin();
738 }
739 
740 void *Bucket::getSlab(bool &FromPool) {
741  std::lock_guard<std::mutex> Lg(BucketLock);
742 
743  auto SlabIt = getAvailFullSlab(FromPool);
744  auto *FreeSlab = (*SlabIt)->getSlab();
745  auto It =
746  UnavailableSlabs.insert(UnavailableSlabs.begin(), std::move(*SlabIt));
747  AvailableSlabs.erase(SlabIt);
748  (*It)->setIterator(It);
749  return FreeSlab;
750 }
751 
752 void Bucket::freeSlab(Slab &Slab, bool &ToPool) {
753  std::lock_guard<std::mutex> Lg(BucketLock);
754  auto SlabIter = Slab.getIterator();
755  assert(SlabIter != UnavailableSlabs.end());
756  if (CanPool(ToPool)) {
757  auto It =
758  AvailableSlabs.insert(AvailableSlabs.begin(), std::move(*SlabIter));
759  UnavailableSlabs.erase(SlabIter);
760  (*It)->setIterator(It);
761  } else {
762  UnavailableSlabs.erase(SlabIter);
763  }
764 }
765 
766 auto Bucket::getAvailSlab(bool &FromPool) -> decltype(AvailableSlabs.begin()) {
767 
768  if (AvailableSlabs.size() == 0) {
769  auto It = AvailableSlabs.insert(AvailableSlabs.begin(),
770  std::make_unique<Slab>(*this));
771  (*It)->setIterator(It);
772 
773  updateStats(1, 0);
774  FromPool = false;
775  } else {
776  if ((*(AvailableSlabs.begin()))->getNumAllocated() == 0) {
777  // If this was an empty slab, it was in the pool.
778  // Now it is no longer in the pool, so update count.
779  --chunkedSlabsInPool;
780  decrementPool(FromPool);
781  } else {
782  // Allocation from existing slab is treated as from pool for statistics.
783  FromPool = true;
784  }
785  }
786 
787  return AvailableSlabs.begin();
788 }
789 
790 void *Bucket::getChunk(bool &FromPool) {
791  std::lock_guard<std::mutex> Lg(BucketLock);
792 
793  auto SlabIt = getAvailSlab(FromPool);
794  auto *FreeChunk = (*SlabIt)->getChunk();
795 
796  // If the slab is full, move it to unavailable slabs and update its iterator
797  if (!((*SlabIt)->hasAvail())) {
798  auto It =
799  UnavailableSlabs.insert(UnavailableSlabs.begin(), std::move(*SlabIt));
800  AvailableSlabs.erase(SlabIt);
801  (*It)->setIterator(It);
802  }
803 
804  return FreeChunk;
805 }
806 
807 void Bucket::freeChunk(void *Ptr, Slab &Slab, bool &ToPool) {
808  std::lock_guard<std::mutex> Lg(BucketLock);
809 
810  Slab.freeChunk(Ptr);
811 
812  onFreeChunk(Slab, ToPool);
813 }
814 
815 // The lock must be acquired before calling this method
816 void Bucket::onFreeChunk(Slab &Slab, bool &ToPool) {
817  ToPool = true;
818 
819  // In case if the slab was previously full and now has 1 available
820  // chunk, it should be moved to the list of available slabs
821  if (Slab.getNumAllocated() == (Slab.getNumChunks() - 1)) {
822  auto SlabIter = Slab.getIterator();
823  assert(SlabIter != UnavailableSlabs.end());
824 
825  auto It =
826  AvailableSlabs.insert(AvailableSlabs.begin(), std::move(*SlabIter));
827  UnavailableSlabs.erase(SlabIter);
828 
829  (*It)->setIterator(It);
830  }
831 
832  // Check if slab is empty, and pool it if we can.
833  if (Slab.getNumAllocated() == 0) {
834  // The slab is now empty.
835  // If pool has capacity then put the slab in the pool.
836  // The ToPool parameter indicates whether the Slab will be put in the pool
837  // or freed from USM.
838  if (!CanPool(ToPool)) {
839  // Note: since the slab is stored as unique_ptr, just remove it from
840  // the list to destroy the object.
841  auto It = Slab.getIterator();
842  assert(It != AvailableSlabs.end());
843  AvailableSlabs.erase(It);
844  }
845  }
846 }
847 
848 bool Bucket::CanPool(bool &ToPool) {
849  std::lock_guard<sycl::detail::SpinLock> Lock{PoolLock};
850  size_t NewFreeSlabsInBucket;
851  // Check if this bucket is used in chunked form or as full slabs.
852  bool chunkedBucket = getSize() <= ChunkCutOff();
853  if (chunkedBucket)
854  NewFreeSlabsInBucket = chunkedSlabsInPool + 1;
855  else
856  NewFreeSlabsInBucket = AvailableSlabs.size() + 1;
857  if (Capacity() >= NewFreeSlabsInBucket) {
858  size_t NewPoolSize = USMSettings.CurPoolSize + SlabAllocSize();
859  if (USMSettings.MaxPoolSize >= NewPoolSize) {
860  USMSettings.CurPoolSize = NewPoolSize;
861  if (chunkedBucket)
862  ++chunkedSlabsInPool;
863 
864  updateStats(-1, 1);
865  ToPool = true;
866  return true;
867  }
868  }
869  updateStats(-1, 0);
870  ToPool = false;
871  return false;
872 }
873 
874 SystemMemory &Bucket::getMemHandle() { return OwnAllocCtx.getMemHandle(); }
875 
877 
879 
880 size_t Bucket::SlabAllocSize() { return std::max(getSize(), SlabMinSize()); }
881 
883  // For buckets used in chunked mode, just one slab in pool is sufficient.
884  // For larger buckets, the capacity could be more and is adjustable.
885  if (getSize() <= ChunkCutOff())
886  return 1;
887  else
888  return USMSettings.Capacity[getMemType()];
889 }
890 
893 }
894 
895 size_t Bucket::ChunkCutOff() { return SlabMinSize() / 2; }
896 
897 void Bucket::countAlloc(bool FromPool) {
898  ++allocCount;
899  if (FromPool)
900  ++allocPoolCount;
901 }
902 
903 void Bucket::countFree() { ++freeCount; }
904 
905 void Bucket::updateStats(int InUse, int InPool) {
906  if (USMSettings.PoolTrace == 0)
907  return;
908  currSlabsInUse += InUse;
909  maxSlabsInUse = std::max(currSlabsInUse, maxSlabsInUse);
910  currSlabsInPool += InPool;
911  maxSlabsInPool = std::max(currSlabsInPool, maxSlabsInPool);
912  // Increment or decrement current pool sizes based on whether
913  // slab was added to or removed from pool.
915 }
916 
917 void Bucket::printStats(bool &TitlePrinted, MemType MT) {
918  if (allocCount) {
919  if (!TitlePrinted) {
920  auto Label = USMSettings.MemTypeNames[MT];
921  std::cout << Label << " memory statistics\n";
922  std::cout << std::setw(14) << "Bucket Size" << std::setw(12) << "Allocs"
923  << std::setw(12) << "Frees" << std::setw(18)
924  << "Allocs from Pool" << std::setw(20) << "Peak Slabs in Use"
925  << std::setw(21) << "Peak Slabs in Pool" << std::endl;
926  TitlePrinted = true;
927  }
928  std::cout << std::setw(14) << getSize() << std::setw(12) << allocCount
929  << std::setw(12) << freeCount << std::setw(18) << allocPoolCount
930  << std::setw(20) << maxSlabsInUse << std::setw(21)
931  << maxSlabsInPool << std::endl;
932  }
933 }
934 
935 // SystemMemory &Bucket::getMemHandle() { return OwnAllocCtx.getMemHandle(); }
936 
937 void *USMAllocContext::USMAllocImpl::allocate(size_t Size, bool &FromPool) {
938  void *Ptr;
939 
940  if (Size == 0)
941  return nullptr;
942 
943  FromPool = false;
944  if (Size > USMSettings.MaxPoolableSize[getMemHandle().getMemType()]) {
945  return getMemHandle().allocate(Size);
946  }
947 
948  auto &Bucket = findBucket(Size);
949 
950  if (Size > Bucket.ChunkCutOff())
951  Ptr = Bucket.getSlab(FromPool);
952  else
953  Ptr = Bucket.getChunk(FromPool);
954 
955  if (USMSettings.PoolTrace > 1)
956  Bucket.countAlloc(FromPool);
957 
958  return Ptr;
959 }
960 
961 void *USMAllocContext::USMAllocImpl::allocate(size_t Size, size_t Alignment,
962  bool &FromPool) {
963  void *Ptr;
964 
965  if (Size == 0)
966  return nullptr;
967 
968  if (Alignment <= 1)
969  return allocate(Size, FromPool);
970 
971  size_t AlignedSize = (Size > 1) ? AlignUp(Size, Alignment) : Alignment;
972 
973  // Check if requested allocation size is within pooling limit.
974  // If not, just request aligned pointer from the system.
975  FromPool = false;
976  if (AlignedSize > USMSettings.MaxPoolableSize[getMemHandle().getMemType()]) {
977  return getMemHandle().allocate(Size, Alignment);
978  }
979 
980  auto &Bucket = findBucket(AlignedSize);
981 
982  if (AlignedSize > Bucket.ChunkCutOff()) {
983  Ptr = Bucket.getSlab(FromPool);
984  } else {
985  Ptr = Bucket.getChunk(FromPool);
986  }
987 
988  if (USMSettings.PoolTrace > 1)
989  Bucket.countAlloc(FromPool);
990 
991  return AlignPtrUp(Ptr, Alignment);
992 }
993 
994 Bucket &USMAllocContext::USMAllocImpl::findBucket(size_t Size) {
995  assert(Size <= CutOff && "Unexpected size");
996 
997  auto It = std::find_if(
998  Buckets.begin(), Buckets.end(),
999  [Size](const auto &BucketPtr) { return BucketPtr->getSize() >= Size; });
1000 
1001  assert((It != Buckets.end()) && "Bucket should always exist");
1002 
1003  return *(*It);
1004 }
1005 
1006 void USMAllocContext::USMAllocImpl::deallocate(void *Ptr, bool &ToPool,
1007  bool OwnZeMemHandle) {
1008  auto *SlabPtr = AlignPtrDown(Ptr, SlabMinSize());
1009 
1010  // Lock the map on read
1011  std::shared_lock<std::shared_timed_mutex> Lk(getKnownSlabsMapLock());
1012 
1013  ToPool = false;
1014  auto Slabs = getKnownSlabs().equal_range(SlabPtr);
1015  if (Slabs.first == Slabs.second) {
1016  Lk.unlock();
1017  getMemHandle().deallocate(Ptr, OwnZeMemHandle);
1018  return;
1019  }
1020 
1021  for (auto It = Slabs.first; It != Slabs.second; ++It) {
1022  // The slab object won't be deleted until it's removed from the map which is
1023  // protected by the lock, so it's safe to access it here.
1024  auto &Slab = It->second;
1025  if (Ptr >= Slab.getPtr() && Ptr < Slab.getEnd()) {
1026  // Unlock the map before freeing the chunk, it may be locked on write
1027  // there
1028  Lk.unlock();
1029  auto &Bucket = Slab.getBucket();
1030 
1031  if (USMSettings.PoolTrace > 1)
1032  Bucket.countFree();
1033 
1034  if (Bucket.getSize() <= Bucket.ChunkCutOff()) {
1035  Bucket.freeChunk(Ptr, Slab, ToPool);
1036  } else {
1037  Bucket.freeSlab(Slab, ToPool);
1038  }
1039 
1040  return;
1041  }
1042  }
1043 
1044  Lk.unlock();
1045  // There is a rare case when we have a pointer from system allocation next
1046  // to some slab with an entry in the map. So we find a slab
1047  // but the range checks fail.
1048  getMemHandle().deallocate(Ptr, OwnZeMemHandle);
1049 }
1050 
1051 USMAllocContext::USMAllocContext(std::unique_ptr<SystemMemory> MemHandle)
1052  : pImpl(std::make_unique<USMAllocImpl>(std::move(MemHandle))) {}
1053 
1054 void *USMAllocContext::allocate(size_t size) {
1055  // For full-slab allocations indicates whether slab is from Pool.
1056  bool FromPool;
1057  auto Ptr = pImpl->allocate(size, FromPool);
1058 
1059  if (USMSettings.PoolTrace > 2) {
1060  auto MT = pImpl->getMemHandle().getMemType();
1061  std::cout << "Allocated " << std::setw(8) << size << " "
1062  << USMSettings.MemTypeNames[MT] << " USM bytes from "
1063  << (FromPool ? "Pool" : "USM") << " ->" << Ptr << std::endl;
1064  }
1065  return Ptr;
1066 }
1067 
1068 void *USMAllocContext::allocate(size_t size, size_t alignment) {
1069  bool FromPool;
1070  auto Ptr = pImpl->allocate(size, alignment, FromPool);
1071 
1072  if (USMSettings.PoolTrace > 2) {
1073  auto MT = pImpl->getMemHandle().getMemType();
1074  std::cout << "Allocated " << std::setw(8) << size << " "
1075  << USMSettings.MemTypeNames[MT] << " USM bytes aligned at "
1076  << alignment << " from " << (FromPool ? "Pool" : "USM") << " ->"
1077  << Ptr << std::endl;
1078  }
1079  return Ptr;
1080 }
1081 
1082 void USMAllocContext::deallocate(void *ptr, bool OwnZeMemHandle) {
1083  bool ToPool;
1084  pImpl->deallocate(ptr, ToPool, OwnZeMemHandle);
1085 
1086  if (USMSettings.PoolTrace > 2) {
1087  auto MT = pImpl->getMemHandle().getMemType();
1088  std::cout << "Freed " << USMSettings.MemTypeNames[MT] << " USM " << ptr
1089  << " to " << (ToPool ? "Pool" : "USM")
1090  << ", Current total pool size " << USMSettings.CurPoolSize
1091  << ", Current pool sizes ["
1096  }
1097  return;
1098 }
1099 
1100 // Define destructor for use with unique_ptr
1102  bool TitlePrinted = false;
1103  size_t HighBucketSize;
1104  size_t HighPeakSlabsInUse;
1105  if (USMSettings.PoolTrace > 1) {
1106  MemType MT = pImpl->getMemHandle().getMemType();
1107  pImpl->printStats(TitlePrinted, HighBucketSize, HighPeakSlabsInUse, MT);
1108  if (TitlePrinted) {
1109  std::cout << "Current Pool Size " << USMSettings.CurPoolSize << std::endl;
1110  const char *Label = USMSettings.MemTypeNames[MT];
1111  std::cout << "Suggested Setting: SYCL_PI_LEVEL_ZERO_USM_ALLOCATOR=;"
1112  << std::string(1, tolower(*Label)) << std::string(Label + 1)
1113  << ":" << HighBucketSize << "," << HighPeakSlabsInUse << ",64K"
1114  << std::endl;
1115  }
1116  }
1117 }
1118 
1120  size_t &HighBucketSize,
1121  size_t &HighPeakSlabsInUse,
1122  MemType MT) {
1123  HighBucketSize = 0;
1124  HighPeakSlabsInUse = 0;
1125  for (auto &B : Buckets) {
1126  (*B).printStats(TitlePrinted, MT);
1127  HighPeakSlabsInUse = std::max((*B).maxSlabsInUse, HighPeakSlabsInUse);
1128  if ((*B).allocCount)
1129  HighBucketSize = std::max((*B).SlabAllocSize(), HighBucketSize);
1130  }
1131 }
1132 
USMAllocContext::USMAllocImpl::getKnownSlabs
std::unordered_multimap< void *, Slab & > & getKnownSlabs()
Definition: usm_allocator.cpp:567
operator<<
std::ostream & operator<<(std::ostream &Os, const Slab &Slab)
Definition: usm_allocator.cpp:586
Bucket::ChunkCutOff
size_t ChunkCutOff()
Definition: usm_allocator.cpp:895
USMAllocContext::USMAllocImpl::getMemHandle
SystemMemory & getMemHandle()
Definition: usm_allocator.cpp:564
Slab::Slab
Slab(Bucket &)
Definition: usm_allocator.cpp:592
settings::SetLimits
Definition: usm_allocator.cpp:65
Slab::getPtr
void * getPtr() const
Definition: usm_allocator.cpp:401
Slab
Definition: usm_allocator.cpp:352
Bucket::getSize
size_t getSize() const
Definition: usm_allocator.cpp:475
Bucket::getMemType
MemType getMemType()
Definition: usm_allocator.cpp:876
SystemMemory::allocate
virtual void * allocate(size_t size)=0
Bucket::getChunk
void * getChunk(bool &FromPool)
Definition: usm_allocator.cpp:790
SystemMemory
Definition: usm_allocator.hpp:17
Slab::getEnd
void * getEnd() const
Definition: usm_allocator.cpp:710
settings::SetLimits::EnableBuffers
size_t EnableBuffers
Definition: usm_allocator.cpp:90
settings::SetLimits::SetLimits
SetLimits()
Definition: usm_allocator.cpp:95
Slab::getNumChunks
size_t getNumChunks() const
Definition: usm_allocator.cpp:405
Bucket::MaxPoolableSize
size_t MaxPoolableSize()
Definition: usm_allocator.cpp:891
Bucket::freeSlab
void freeSlab(Slab &Slab, bool &ToPool)
Definition: usm_allocator.cpp:752
Bucket::Capacity
size_t Capacity()
Definition: usm_allocator.cpp:882
Slab::getSlab
void * getSlab()
Definition: usm_allocator.cpp:637
usm_allocator.hpp
MemType
MemType
Definition: usm_allocator.hpp:14
AlignPtrDown
static void * AlignPtrDown(void *Ptr, const size_t Alignment)
Definition: usm_allocator.cpp:321
enableBufferPooling
bool enableBufferPooling()
Definition: usm_allocator.cpp:1133
operator==
bool operator==(const Slab &Lhs, const Slab &Rhs)
Definition: usm_allocator.cpp:582
Bucket
Definition: usm_allocator.cpp:415
Bucket::Bucket
Bucket(size_t Sz, USMAllocContext::USMAllocImpl &AllocCtx)
Definition: usm_allocator.cpp:462
USMAllocContext::USMAllocContext
USMAllocContext(std::unique_ptr< SystemMemory > memHandle)
Definition: usm_allocator.cpp:1051
Bucket::maxSlabsInUse
size_t maxSlabsInUse
Definition: usm_allocator.cpp:460
settings::SetLimits::MinBucketSize
size_t MinBucketSize[MemType::All]
Definition: usm_allocator.cpp:82
USMAllocContext::allocate
void * allocate(size_t size)
Definition: usm_allocator.cpp:1054
Slab::getChunk
void * getChunk()
Definition: usm_allocator.cpp:619
settings::SetLimits::SlabMinSize
size_t SlabMinSize[MemType::All]
Definition: usm_allocator.cpp:73
USMAllocContext::USMAllocImpl::allocate
void * allocate(size_t Size, size_t Alignment, bool &FromPool)
Definition: usm_allocator.cpp:961
Shared
@ Shared
Definition: usm_allocator.hpp:14
Slab::setIterator
void setIterator(ListIter It)
Definition: usm_allocator.cpp:390
USMAllocContext::USMAllocImpl
Definition: usm_allocator.cpp:533
SystemMemory::deallocate
virtual void deallocate(void *ptr, bool OwnZeMemHandle)=0
USMAllocContext::deallocate
void deallocate(void *ptr, bool OwnZeMemHandle)
Definition: usm_allocator.cpp:1082
Device
@ Device
Definition: usm_allocator.hpp:14
Bucket::allocCount
size_t allocCount
Definition: usm_allocator.cpp:459
AlignUp
static size_t AlignUp(size_t Val, size_t Alignment)
Definition: usm_allocator.cpp:339
spinlock.hpp
Bucket::countFree
void countFree()
Definition: usm_allocator.cpp:903
Host
@ Host
Definition: usm_allocator.hpp:14
Bucket::CanPool
bool CanPool(bool &ToPool)
Definition: usm_allocator.cpp:848
settings::SetLimits::PoolTrace
int PoolTrace
Definition: usm_allocator.cpp:93
USMAllocContext::~USMAllocContext
~USMAllocContext()
Definition: usm_allocator.cpp:1101
Bucket::SlabAllocSize
size_t SlabAllocSize()
Definition: usm_allocator.cpp:880
settings::CutOff
static constexpr size_t CutOff
Definition: usm_allocator.cpp:60
Bucket::SlabMinSize
size_t SlabMinSize()
Definition: usm_allocator.cpp:878
Bucket::getUsmAllocCtx
USMAllocContext::USMAllocImpl & getUsmAllocCtx()
Definition: usm_allocator.cpp:487
sycl::_V1::setw
__width_manipulator__ setw(int Width)
Definition: stream.hpp:734
iostream_proxy.hpp
settings::SetLimits::CurPoolSizes
size_t CurPoolSizes[MemType::All]
Definition: usm_allocator.cpp:88
Slab::getChunkSize
size_t getChunkSize() const
Definition: usm_allocator.cpp:642
settings
Definition: usm_allocator.cpp:44
Slab::~Slab
~Slab()
Definition: usm_allocator.cpp:602
USMAllocContext::USMAllocImpl::SlabMinSize
size_t SlabMinSize()
Definition: usm_allocator.cpp:571
Bucket::updateStats
void updateStats(int InUse, int InPool)
Definition: usm_allocator.cpp:905
Slab::hasAvail
bool hasAvail()
Definition: usm_allocator.cpp:714
std
Definition: accessor.hpp:3071
SystemMemory::getMemType
virtual MemType getMemType()=0
Bucket::countAlloc
void countAlloc(bool FromPool)
Definition: usm_allocator.cpp:897
Bucket::getMemHandle
SystemMemory & getMemHandle()
Definition: usm_allocator.cpp:874
Slab::getBucket
Bucket & getBucket()
Definition: usm_allocator.cpp:639
settings::PoolLock
static sycl::detail::SpinLock PoolLock
Definition: usm_allocator.cpp:63
settings::SetLimits::CurPoolSize
size_t CurPoolSize
Definition: usm_allocator.cpp:87
USMAllocContext
Definition: usm_allocator.hpp:26
settings::SetLimits::MaxPoolSize
size_t MaxPoolSize
Definition: usm_allocator.cpp:85
Bucket::freeChunk
void freeChunk(void *Ptr, Slab &Slab, bool &ToPool)
Definition: usm_allocator.cpp:807
settings::USMSettings
static class settings::SetLimits USMSettings
AlignPtrUp
static void * AlignPtrUp(void *Ptr, const size_t Alignment)
Definition: usm_allocator.cpp:328
USMAllocContext::USMAllocImpl::USMAllocImpl
USMAllocImpl(std::unique_ptr< SystemMemory > SystemMemHandle)
Definition: usm_allocator.cpp:546
Slab::getIterator
ListIter getIterator() const
Definition: usm_allocator.cpp:391
Bucket::printStats
void printStats(bool &TitlePrinted, MemType MT)
Definition: usm_allocator.cpp:917
USMAllocContext::USMAllocImpl::deallocate
void deallocate(void *Ptr, bool &ToPool, bool OwnZeMemHandle)
Definition: usm_allocator.cpp:1006
std::cout
__SYCL_EXTERN_STREAM_ATTRS ostream cout
Linked to standard output.
Bucket::getSlab
void * getSlab(bool &FromPool)
Definition: usm_allocator.cpp:740
All
@ All
Definition: usm_allocator.hpp:14
Slab::freeChunk
void freeChunk(void *Ptr)
Definition: usm_allocator.cpp:688
settings::SetLimits::Capacity
size_t Capacity[MemType::All]
Definition: usm_allocator.cpp:79
settings::SetLimits::MaxPoolableSize
size_t MaxPoolableSize[MemType::All]
Definition: usm_allocator.cpp:76
settings::SetLimits::MemTypeNames
static constexpr const char * MemTypeNames[MemType::All]
Definition: usm_allocator.cpp:68
Slab::getNumAllocated
size_t getNumAllocated() const
Definition: usm_allocator.cpp:393
USMAllocContext::USMAllocImpl::printStats
void printStats(bool &TitlePrinted, size_t &HighBucketSize, size_t &HighPeakSlabsInUse, MemType MT)
Definition: usm_allocator.cpp:1119
SharedReadOnly
@ SharedReadOnly
Definition: usm_allocator.hpp:14
USMAllocContext::USMAllocImpl::getKnownSlabsMapLock
std::shared_timed_mutex & getKnownSlabsMapLock()
Definition: usm_allocator.cpp:566