DPC++ Runtime
Runtime libraries for oneAPI DPC++
usm_allocator.cpp
Go to the documentation of this file.
1 //===---------- usm_allocator.cpp - Allocator for USM memory --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include <algorithm>
10 #include <array>
11 #include <bitset>
12 #include <cassert>
13 #include <cctype>
14 #include <iomanip>
15 #include <iostream>
16 #include <list>
17 #include <memory>
18 #include <mutex>
19 #include <shared_mutex>
20 #include <string>
21 #include <unordered_map>
22 #include <utility>
23 #include <vector>
24 
25 #include "usm_allocator.hpp"
27 #include <iostream>
28 
29 // USM allocations are a minimum of 4KB/64KB/2MB even when a smaller size is
30 // requested. The implementation distinguishes between allocations of size
31 // ChunkCutOff = (minimum-alloc-size / 2) and those that are larger.
32 // Allocation requests smaller than ChunkCutoff use chunks taken from a single
33 // USM allocation. Thus, for example, for a 64KB minimum allocation size,
34 // and 8-byte allocations, only 1 in ~8000 requests results in a new
35 // USM allocation. Freeing results only in a chunk of a larger allocation
36 // to be marked as available and no real return to the system.
37 // An allocation is returned to the system only when all
38 // chunks in the larger allocation are freed by the program.
39 // Allocations larger than ChunkCutOff use a separate USM allocation for each
40 // request. These are subject to "pooling". That is, when such an allocation is
41 // freed by the program it is retained in a pool. The pool is available for
42 // future allocations, which means there are fewer actual USM
43 // allocations/deallocations.
44 
45 namespace settings {
46 
47 constexpr auto operator""_B(unsigned long long x) -> size_t { return x; }
48 constexpr auto operator""_KB(unsigned long long x) -> size_t {
49  return x * 1024;
50 }
51 constexpr auto operator""_MB(unsigned long long x) -> size_t {
52  return x * 1024 * 1024;
53 }
54 constexpr auto operator""_GB(unsigned long long x) -> size_t {
55  return x * 1024 * 1024 * 1024;
56 }
57 
58 // Buckets for Host use a minimum of the cache line size of 64 bytes.
59 // This prevents two separate allocations residing in the same cache line.
60 // Buckets for Device and Shared allocations will use starting size of 512.
61 // This is because memory compression on newer GPUs makes the
62 // minimum granularity 512 bytes instead of 64.
63 static constexpr size_t MinBucketSize[SystemMemory::All] = {64, 512, 512, 512};
64 
65 // The largest size which is allocated via the allocator.
66 // Allocations with size > CutOff bypass the USM allocator and
67 // go directly to the runtime.
68 static constexpr size_t CutOff = (size_t)1 << 31; // 2GB
69 
70 // Protects the capacity checking of the pool.
71 static sycl::detail::SpinLock PoolLock;
72 
73 static class SetLimits {
74 public:
75  // Minimum allocation size that will be requested from the system.
76  // By default this is the minimum allocation size of each memory type.
78 
79  // Allocations up to this limit will be subject to chunking/pooling
81 
82  // When pooling, each bucket will hold a max of 4 unfreed slabs
84 
85  // Maximum memory left unfreed in pool
86  size_t MaxPoolSize = 16_MB;
87 
88  size_t CurPoolSize = 0;
89  size_t CurPoolSizes[SystemMemory::All] = {0, 0, 0, 0};
90 
91  size_t EnableBuffers = 1;
92 
93  // Whether to print pool usage statistics
94  int PoolTrace = 0;
95 
97  // Initialize default pool settings.
101 
105 
106  // Disable pooling of shared USM allocations.
110 
111  // Allow pooling of shared allocations that are only modified on host.
115 
116  // Parse optional parameters of this form:
117  // SYCL_PI_LEVEL_ZERO_USM_ALLOCATOR=[EnableBuffers][;[MaxPoolSize][;memtypelimits]...]
118  // memtypelimits: [<memtype>:]<limits>
119  // memtype: host|device|shared
120  // limits: [MaxPoolableSize][,[Capacity][,SlabMinSize]]
121  //
122  // Without a memory type, the limits are applied to each memory type.
123  // Parameters are for each context, except MaxPoolSize, which is overall
124  // pool size for all contexts.
125  // Duplicate specifications will result in the right-most taking effect.
126  //
127  // EnableBuffers: Apply chunking/pooling to SYCL buffers.
128  // Default 1.
129  // MaxPoolSize: Limit on overall unfreed memory.
130  // Default 16MB.
131  // MaxPoolableSize: Maximum allocation size subject to chunking/pooling.
132  // Default 2MB host, 4MB device and 0 shared.
133  // Capacity: Maximum number of unfreed allocations in each bucket.
134  // Default 4.
135  // SlabMinSize: Minimum allocation size requested from USM.
136  // Default 64KB host and device, 2MB shared.
137  //
138  // Example of usage:
139  // SYCL_PI_LEVEL_ZERO_USM_ALLOCATOR=1;32M;host:1M,4,64K;device:1M,4,64K;shared:0,0,2M
140 
141  auto GetValue = [=](std::string &Param, size_t Length, size_t &Setting) {
142  size_t Multiplier = 1;
143  if (tolower(Param[Length - 1]) == 'k') {
144  Length--;
145  Multiplier = 1_KB;
146  }
147  if (tolower(Param[Length - 1]) == 'm') {
148  Length--;
149  Multiplier = 1_MB;
150  }
151  if (tolower(Param[Length - 1]) == 'g') {
152  Length--;
153  Multiplier = 1_GB;
154  }
155  std::string TheNumber = Param.substr(0, Length);
156  if (TheNumber.find_first_not_of("0123456789") == std::string::npos)
157  Setting = std::stoi(TheNumber) * Multiplier;
158  };
159 
160  auto ParamParser = [=](std::string &Params, size_t &Setting,
161  bool &ParamWasSet) {
162  bool More;
163  if (Params.size() == 0) {
164  ParamWasSet = false;
165  return false;
166  }
167  size_t Pos = Params.find(',');
168  if (Pos != std::string::npos) {
169  if (Pos > 0) {
170  GetValue(Params, Pos, Setting);
171  ParamWasSet = true;
172  }
173  Params.erase(0, Pos + 1);
174  More = true;
175  } else {
176  GetValue(Params, Params.size(), Setting);
177  ParamWasSet = true;
178  More = false;
179  }
180  return More;
181  };
182 
183  auto MemParser = [=](std::string &Params, SystemMemory::MemType M) {
184  bool ParamWasSet;
185  SystemMemory::MemType LM = M;
186  if (M == SystemMemory::All)
187  LM = SystemMemory::Host;
188 
189  bool More = ParamParser(Params, MaxPoolableSize[LM], ParamWasSet);
190  if (ParamWasSet && M == SystemMemory::All) {
194  }
195  if (More) {
196  More = ParamParser(Params, Capacity[LM], ParamWasSet);
197  if (ParamWasSet && M == SystemMemory::All) {
200  }
201  }
202  if (More) {
203  ParamParser(Params, SlabMinSize[LM], ParamWasSet);
204  if (ParamWasSet && M == SystemMemory::All) {
208  }
209  }
210  };
211 
212  auto MemTypeParser = [=](std::string &Params) {
213  int Pos = 0;
215  if (Params.compare(0, 5, "host:") == 0) {
216  Pos = 5;
217  M = SystemMemory::Host;
218  } else if (Params.compare(0, 7, "device:") == 0) {
219  Pos = 7;
221  } else if (Params.compare(0, 7, "shared:") == 0) {
222  Pos = 7;
224  }
225  if (Pos > 0)
226  Params.erase(0, Pos);
227  MemParser(Params, M);
228  };
229 
230  // Update pool settings if specified in environment.
231  char *PoolParams = getenv("SYCL_PI_LEVEL_ZERO_USM_ALLOCATOR");
232  if (PoolParams != nullptr) {
233  std::string Params(PoolParams);
234  size_t Pos = Params.find(';');
235  if (Pos != std::string::npos) {
236  if (Pos > 0) {
237  GetValue(Params, Pos, EnableBuffers);
238  }
239  Params.erase(0, Pos + 1);
240  size_t Pos = Params.find(';');
241  if (Pos != std::string::npos) {
242  if (Pos > 0) {
243  GetValue(Params, Pos, MaxPoolSize);
244  }
245  Params.erase(0, Pos + 1);
246  do {
247  size_t Pos = Params.find(';');
248  if (Pos != std::string::npos) {
249  if (Pos > 0) {
250  std::string MemParams = Params.substr(0, Pos);
251  MemTypeParser(MemParams);
252  }
253  Params.erase(0, Pos + 1);
254  if (Params.size() == 0)
255  break;
256  } else {
257  MemTypeParser(Params);
258  break;
259  }
260  } while (true);
261  } else {
262  GetValue(Params, Params.size(), MaxPoolSize);
263  }
264  } else {
265  GetValue(Params, Params.size(), EnableBuffers);
266  }
267  }
268 
269  char *PoolTraceVal = getenv("SYCL_PI_LEVEL_ZERO_USM_ALLOCATOR_TRACE");
270  if (PoolTraceVal != nullptr) {
271  PoolTrace = std::atoi(PoolTraceVal);
272  }
273  if (PoolTrace < 1)
274  return;
275 
276  std::cout << "USM Pool Settings (Built-in or Adjusted by Environment "
277  "Variable)\n";
278 
279  std::cout << std::setw(15) << "Parameter" << std::setw(12) << "Host"
280  << std::setw(12) << "Device" << std::setw(12) << "Shared"
281  << std::endl;
282  std::cout << std::setw(15) << "SlabMinSize" << std::setw(12)
283  << SlabMinSize[0] << std::setw(12) << SlabMinSize[1]
284  << std::setw(12) << SlabMinSize[2] << std::endl;
285  std::cout << std::setw(15) << "MaxPoolableSize" << std::setw(12)
286  << MaxPoolableSize[0] << std::setw(12) << MaxPoolableSize[1]
287  << std::setw(12) << MaxPoolableSize[2] << std::endl;
288  std::cout << std::setw(15) << "Capacity" << std::setw(12) << Capacity[0]
289  << std::setw(12) << Capacity[1] << std::setw(12) << Capacity[2]
290  << std::endl;
291  std::cout << std::setw(15) << "MaxPoolSize" << std::setw(12) << MaxPoolSize
292  << std::endl;
293  std::cout << std::setw(15) << "EnableBuffers" << std::setw(12)
294  << EnableBuffers << std::endl
295  << std::endl;
296  }
297 } USMSettings;
298 } // namespace settings
299 
300 using namespace settings;
301 
302 static const char *MemTypeNames[SystemMemory::All] = {
303  "Host", "Device", "Shared", "SharedReadOnly"};
304 
305 // Aligns the pointer down to the specified alignment
306 // (e.g. returns 8 for Size = 13, Alignment = 8)
307 static void *AlignPtrDown(void *Ptr, const size_t Alignment) {
308  return reinterpret_cast<void *>((reinterpret_cast<size_t>(Ptr)) &
309  (~(Alignment - 1)));
310 }
311 
312 // Aligns the pointer up to the specified alignment
313 // (e.g. returns 16 for Size = 13, Alignment = 8)
314 static void *AlignPtrUp(void *Ptr, const size_t Alignment) {
315  void *AlignedPtr = AlignPtrDown(Ptr, Alignment);
316  // Special case when the pointer is already aligned
317  if (Ptr == AlignedPtr) {
318  return Ptr;
319  }
320  return static_cast<char *>(AlignedPtr) + Alignment;
321 }
322 
323 // Aligns the value up to the specified alignment
324 // (e.g. returns 16 for Size = 13, Alignment = 8)
325 static size_t AlignUp(size_t Val, size_t Alignment) {
326  assert(Alignment > 0);
327  return (Val + Alignment - 1) & (~(Alignment - 1));
328 }
329 
330 class Bucket;
331 
332 // Represents the allocated memory block of size 'SlabMinSize'
333 // Internally, it splits the memory block into chunks. The number of
334 // chunks depends of the size of a Bucket which created the Slab.
335 // The chunks
336 // Note: Bucket's method are responsible for thread safety of Slab access,
337 // so no locking happens here.
338 class Slab {
339 
340  // Pointer to the allocated memory of SlabMinSize bytes
341  void *MemPtr;
342 
343  // Represents the current state of each chunk:
344  // if the bit is set then the chunk is allocated
345  // the chunk is free for allocation otherwise
346  std::vector<bool> Chunks;
347 
348  // Total number of allocated chunks at the moment.
349  size_t NumAllocated = 0;
350 
351  // The bucket which the slab belongs to
352  Bucket &bucket;
353 
354  using ListIter = std::list<std::unique_ptr<Slab>>::iterator;
355 
356  // Store iterator to the corresponding node in avail/unavail list
357  // to achieve O(1) removal
358  ListIter SlabListIter;
359 
360  // Hints where to start search for free chunk in a slab
361  size_t FirstFreeChunkIdx = 0;
362 
363  // Return the index of the first available chunk, -1 otherwize
364  size_t FindFirstAvailableChunkIdx() const;
365 
366  // Register/Unregister the slab in the global slab address map.
367  void regSlab(Slab &);
368  void unregSlab(Slab &);
369  static void regSlabByAddr(void *, Slab &);
370  static void unregSlabByAddr(void *, Slab &);
371 
372 public:
373  Slab(Bucket &);
374  ~Slab();
375 
376  void setIterator(ListIter It) { SlabListIter = It; }
377  ListIter getIterator() const { return SlabListIter; }
378 
379  size_t getNumAllocated() const { return NumAllocated; }
380 
381  // Get pointer to allocation that is one piece of this slab.
382  void *getChunk();
383 
384  // Get pointer to allocation that is this entire slab.
385  void *getSlab();
386 
387  void *getPtr() const { return MemPtr; }
388  void *getEnd() const;
389 
390  size_t getChunkSize() const;
391  size_t getNumChunks() const { return Chunks.size(); }
392 
393  bool hasAvail();
394 
395  Bucket &getBucket();
396  const Bucket &getBucket() const;
397 
398  void freeChunk(void *Ptr);
399 };
400 
401 class Bucket {
402  const size_t Size;
403 
404  // List of slabs which have at least 1 available chunk.
405  std::list<std::unique_ptr<Slab>> AvailableSlabs;
406 
407  // List of slabs with 0 available chunk.
408  std::list<std::unique_ptr<Slab>> UnavailableSlabs;
409 
410  // Protects the bucket and all the corresponding slabs
411  std::mutex BucketLock;
412 
413  // Reference to the allocator context, used access memory allocation
414  // routines, slab map and etc.
415  USMAllocContext::USMAllocImpl &OwnAllocCtx;
416 
417  // For buckets used in chunked mode, a counter of slabs in the pool.
418  // For allocations that use an entire slab each, the entries in the Available
419  // list are entries in the pool.Each slab is available for a new
420  // allocation.The size of the Available list is the size of the pool.
421  // For allocations that use slabs in chunked mode, slabs will be in the
422  // Available list if any one or more of their chunks is free.The entire slab
423  // is not necessarily free, just some chunks in the slab are free. To
424  // implement pooling we will allow one slab in the Available list to be
425  // entirely empty. Normally such a slab would have been freed from USM. But
426  // now we don't, and treat this slab as "in the pool".
427  // When a slab becomes entirely free we have to decide whether to return it to
428  // USM or keep it allocated. A simple check for size of the Available list is
429  // not sufficient to check whether any slab has been pooled yet.We would have
430  // to traverse the entire Available listand check if any of them is entirely
431  // free. Instead we keep a counter of entirely empty slabs within the
432  // Available list to speed up the process of checking if a slab in this bucket
433  // is already pooled.
434  size_t chunkedSlabsInPool;
435 
436  // Statistics
437  size_t allocPoolCount;
438  size_t freeCount;
439  size_t currSlabsInUse;
440  size_t currSlabsInPool;
441  size_t maxSlabsInPool;
442 
443 public:
444  // Statistics
445  size_t allocCount;
447 
448  Bucket(size_t Sz, USMAllocContext::USMAllocImpl &AllocCtx)
449  : Size{Sz}, OwnAllocCtx{AllocCtx}, chunkedSlabsInPool(0),
450  allocPoolCount(0), freeCount(0), currSlabsInUse(0), currSlabsInPool(0),
451  maxSlabsInPool(0), allocCount(0), maxSlabsInUse(0) {}
452 
453  // Get pointer to allocation that is one piece of an available slab in this
454  // bucket.
455  void *getChunk(bool &FromPool);
456 
457  // Get pointer to allocation that is a full slab in this bucket.
458  void *getSlab(bool &FromPool);
459 
460  // Return the allocation size of this bucket.
461  size_t getSize() const { return Size; }
462 
463  // Free an allocation that is one piece of a slab in this bucket.
464  void freeChunk(void *Ptr, Slab &Slab, bool &ToPool);
465 
466  // Free an allocation that is a full slab in this bucket.
467  void freeSlab(Slab &Slab, bool &ToPool);
468 
469  SystemMemory &getMemHandle();
470 
471  SystemMemory::MemType getMemType();
472 
474 
475  // Check whether an allocation to be freed can be placed in the pool.
476  bool CanPool(bool &ToPool);
477 
478  // The minimum allocation size for any slab.
479  size_t SlabMinSize();
480 
481  // The allocation size for a slab in this bucket.
482  size_t SlabAllocSize();
483 
484  // The minimum size of a chunk from this bucket's slabs.
485  size_t ChunkCutOff();
486 
487  // The number of slabs in this bucket that can be in the pool.
488  size_t Capacity();
489 
490  // The maximum allocation size subject to pooling.
491  size_t MaxPoolableSize();
492 
493  // Update allocation count
494  void countAlloc(bool FromPool);
495 
496  // Update free count
497  void countFree();
498 
499  // Update statistics of Available/Unavailable
500  void updateStats(int InUse, int InPool);
501 
502  // Print bucket statistics
503  void printStats(bool &TitlePrinted, SystemMemory::MemType MT);
504 
505 private:
506  void onFreeChunk(Slab &, bool &ToPool);
507 
508  // Update statistics of pool usage, and indicate that an allocation was made
509  // from the pool.
510  void decrementPool(bool &FromPool);
511 
512  // Get a slab to be used for chunked allocations.
513  decltype(AvailableSlabs.begin()) getAvailSlab(bool &FromPool);
514 
515  // Get a slab that will be used as a whole for a single allocation.
516  decltype(AvailableSlabs.begin()) getAvailFullSlab(bool &FromPool);
517 };
518 
520  // It's important for the map to be destroyed last after buckets and their
521  // slabs This is because slab's destructor removes the object from the map.
522  std::unordered_multimap<void *, Slab &> KnownSlabs;
523  std::shared_timed_mutex KnownSlabsMapLock;
524 
525  // Handle to the memory allocation routine
526  std::unique_ptr<SystemMemory> MemHandle;
527 
528  // Store as unique_ptrs since Bucket is not Movable(because of std::mutex)
529  std::vector<std::unique_ptr<Bucket>> Buckets;
530 
531 public:
532  USMAllocImpl(std::unique_ptr<SystemMemory> SystemMemHandle)
533  : MemHandle{std::move(SystemMemHandle)} {
534 
535  // Generate buckets sized such as: 64, 96, 128, 192, ..., CutOff.
536  // Powers of 2 and the value halfway between the powers of 2.
537  auto Size1 = MinBucketSize[MemHandle->getMemType()];
538  auto Size2 = Size1 + Size1 / 2;
539  for (; Size2 < CutOff; Size1 *= 2, Size2 *= 2) {
540  Buckets.push_back(std::make_unique<Bucket>(Size1, *this));
541  Buckets.push_back(std::make_unique<Bucket>(Size2, *this));
542  }
543  Buckets.push_back(std::make_unique<Bucket>(CutOff, *this));
544  }
545 
546  void *allocate(size_t Size, size_t Alignment, bool &FromPool);
547  void *allocate(size_t Size, bool &FromPool);
548  void deallocate(void *Ptr, bool &ToPool, bool OwnZeMemHandle);
549 
550  SystemMemory &getMemHandle() { return *MemHandle; }
551 
552  std::shared_timed_mutex &getKnownSlabsMapLock() { return KnownSlabsMapLock; }
553  std::unordered_multimap<void *, Slab &> &getKnownSlabs() {
554  return KnownSlabs;
555  }
556 
557  size_t SlabMinSize() {
558  return USMSettings.SlabMinSize[(*MemHandle).getMemType()];
559  };
560 
561  void printStats(bool &TitlePrinted, size_t &HighBucketSize,
562  size_t &HighPeakSlabsInUse, SystemMemory::MemType MT);
563 
564 private:
565  Bucket &findBucket(size_t Size);
566 };
567 
568 bool operator==(const Slab &Lhs, const Slab &Rhs) {
569  return Lhs.getPtr() == Rhs.getPtr();
570 }
571 
572 std::ostream &operator<<(std::ostream &Os, const Slab &Slab) {
573  Os << "Slab<" << Slab.getPtr() << ", " << Slab.getEnd() << ", "
574  << Slab.getBucket().getSize() << ">";
575  return Os;
576 }
577 
579  : // In case bucket size is not a multiple of SlabMinSize, we would have
580  // some padding at the end of the slab.
581  Chunks(Bkt.SlabMinSize() / Bkt.getSize()), NumAllocated{0},
582  bucket(Bkt), SlabListIter{}, FirstFreeChunkIdx{0} {
583  auto SlabSize = Bkt.SlabAllocSize();
584  MemPtr = Bkt.getMemHandle().allocate(SlabSize);
585  regSlab(*this);
586 }
587 
589  unregSlab(*this);
590  bucket.getMemHandle().deallocate(MemPtr, true /* OwnZeMemHandle */);
591 }
592 
593 // Return the index of the first available chunk, -1 otherwize
594 size_t Slab::FindFirstAvailableChunkIdx() const {
595  // Use the first free chunk index as a hint for the search.
596  auto It = std::find_if(Chunks.begin() + FirstFreeChunkIdx, Chunks.end(),
597  [](auto x) { return !x; });
598  if (It != Chunks.end()) {
599  return It - Chunks.begin();
600  }
601 
602  return static_cast<size_t>(-1);
603 }
604 
605 void *Slab::getChunk() {
606  // assert(NumAllocated != Chunks.size());
607 
608  const size_t ChunkIdx = FindFirstAvailableChunkIdx();
609  // Free chunk must exist, otherwise we would have allocated another slab
610  assert(ChunkIdx != (static_cast<size_t>(-1)));
611 
612  void *const FreeChunk =
613  (static_cast<uint8_t *>(getPtr())) + ChunkIdx * getChunkSize();
614  Chunks[ChunkIdx] = true;
615  NumAllocated += 1;
616 
617  // Use the found index as the next hint
618  FirstFreeChunkIdx = ChunkIdx;
619 
620  return FreeChunk;
621 }
622 
623 void *Slab::getSlab() { return getPtr(); }
624 
625 Bucket &Slab::getBucket() { return bucket; }
626 const Bucket &Slab::getBucket() const { return bucket; }
627 
628 size_t Slab::getChunkSize() const { return bucket.getSize(); }
629 
630 void Slab::regSlabByAddr(void *Addr, Slab &Slab) {
632  auto &Map = Slab.getBucket().getUsmAllocCtx().getKnownSlabs();
633 
634  std::lock_guard<std::shared_timed_mutex> Lg(Lock);
635  Map.insert({Addr, Slab});
636 }
637 
638 void Slab::unregSlabByAddr(void *Addr, Slab &Slab) {
640  auto &Map = Slab.getBucket().getUsmAllocCtx().getKnownSlabs();
641 
642  std::lock_guard<std::shared_timed_mutex> Lg(Lock);
643 
644  auto Slabs = Map.equal_range(Addr);
645  // At least the must get the current slab from the map.
646  assert(Slabs.first != Slabs.second && "Slab is not found");
647 
648  for (auto It = Slabs.first; It != Slabs.second; ++It) {
649  if (It->second == Slab) {
650  Map.erase(It);
651  return;
652  }
653  }
654 
655  assert(false && "Slab is not found");
656 }
657 
658 void Slab::regSlab(Slab &Slab) {
659  void *StartAddr = AlignPtrDown(Slab.getPtr(), bucket.SlabMinSize());
660  void *EndAddr = static_cast<char *>(StartAddr) + bucket.SlabMinSize();
661 
662  regSlabByAddr(StartAddr, Slab);
663  regSlabByAddr(EndAddr, Slab);
664 }
665 
666 void Slab::unregSlab(Slab &Slab) {
667  void *StartAddr = AlignPtrDown(Slab.getPtr(), bucket.SlabMinSize());
668  void *EndAddr = static_cast<char *>(StartAddr) + bucket.SlabMinSize();
669 
670  unregSlabByAddr(StartAddr, Slab);
671  unregSlabByAddr(EndAddr, Slab);
672 }
673 
674 void Slab::freeChunk(void *Ptr) {
675  // This method should be called through bucket(since we might remove the slab
676  // as a result), therefore all locks are done on that level.
677 
678  // Make sure that we're in the right slab
679  assert(Ptr >= getPtr() && Ptr < getEnd());
680 
681  // Even if the pointer p was previously aligned, it's still inside the
682  // corresponding chunk, so we get the correct index here.
683  auto ChunkIdx =
684  (static_cast<char *>(Ptr) - static_cast<char *>(MemPtr)) / getChunkSize();
685 
686  // Make sure that the chunk was allocated
687  assert(Chunks[ChunkIdx] && "double free detected");
688 
689  Chunks[ChunkIdx] = false;
690  NumAllocated -= 1;
691 
692  if (ChunkIdx < FirstFreeChunkIdx)
693  FirstFreeChunkIdx = ChunkIdx;
694 }
695 
696 void *Slab::getEnd() const {
697  return static_cast<char *>(getPtr()) + bucket.SlabMinSize();
698 }
699 
700 bool Slab::hasAvail() { return NumAllocated != getNumChunks(); }
701 
702 // If a slab was available in the pool then note that the current pooled
703 // size has reduced by the size of a slab in this bucket.
704 void Bucket::decrementPool(bool &FromPool) {
705  FromPool = true;
706  updateStats(1, -1);
708 }
709 
710 auto Bucket::getAvailFullSlab(bool &FromPool)
711  -> decltype(AvailableSlabs.begin()) {
712  // Return a slab that will be used for a single allocation.
713  if (AvailableSlabs.size() == 0) {
714  auto It = AvailableSlabs.insert(AvailableSlabs.begin(),
715  std::make_unique<Slab>(*this));
716  (*It)->setIterator(It);
717  FromPool = false;
718  updateStats(1, 0);
719  } else {
720  decrementPool(FromPool);
721  }
722 
723  return AvailableSlabs.begin();
724 }
725 
726 void *Bucket::getSlab(bool &FromPool) {
727  std::lock_guard<std::mutex> Lg(BucketLock);
728 
729  auto SlabIt = getAvailFullSlab(FromPool);
730  auto *FreeSlab = (*SlabIt)->getSlab();
731  auto It =
732  UnavailableSlabs.insert(UnavailableSlabs.begin(), std::move(*SlabIt));
733  AvailableSlabs.erase(SlabIt);
734  (*It)->setIterator(It);
735  return FreeSlab;
736 }
737 
738 void Bucket::freeSlab(Slab &Slab, bool &ToPool) {
739  std::lock_guard<std::mutex> Lg(BucketLock);
740  auto SlabIter = Slab.getIterator();
741  assert(SlabIter != UnavailableSlabs.end());
742  if (CanPool(ToPool)) {
743  auto It =
744  AvailableSlabs.insert(AvailableSlabs.begin(), std::move(*SlabIter));
745  UnavailableSlabs.erase(SlabIter);
746  (*It)->setIterator(It);
747  } else {
748  UnavailableSlabs.erase(SlabIter);
749  }
750 }
751 
752 auto Bucket::getAvailSlab(bool &FromPool) -> decltype(AvailableSlabs.begin()) {
753 
754  if (AvailableSlabs.size() == 0) {
755  auto It = AvailableSlabs.insert(AvailableSlabs.begin(),
756  std::make_unique<Slab>(*this));
757  (*It)->setIterator(It);
758 
759  updateStats(1, 0);
760  FromPool = false;
761  } else {
762  if ((*(AvailableSlabs.begin()))->getNumAllocated() == 0) {
763  // If this was an empty slab, it was in the pool.
764  // Now it is no longer in the pool, so update count.
765  --chunkedSlabsInPool;
766  decrementPool(FromPool);
767  } else {
768  // Allocation from existing slab is treated as from pool for statistics.
769  FromPool = true;
770  }
771  }
772 
773  return AvailableSlabs.begin();
774 }
775 
776 void *Bucket::getChunk(bool &FromPool) {
777  std::lock_guard<std::mutex> Lg(BucketLock);
778 
779  auto SlabIt = getAvailSlab(FromPool);
780  auto *FreeChunk = (*SlabIt)->getChunk();
781 
782  // If the slab is full, move it to unavailable slabs and update its iterator
783  if (!((*SlabIt)->hasAvail())) {
784  auto It =
785  UnavailableSlabs.insert(UnavailableSlabs.begin(), std::move(*SlabIt));
786  AvailableSlabs.erase(SlabIt);
787  (*It)->setIterator(It);
788  }
789 
790  return FreeChunk;
791 }
792 
793 void Bucket::freeChunk(void *Ptr, Slab &Slab, bool &ToPool) {
794  std::lock_guard<std::mutex> Lg(BucketLock);
795 
796  Slab.freeChunk(Ptr);
797 
798  onFreeChunk(Slab, ToPool);
799 }
800 
801 // The lock must be acquired before calling this method
802 void Bucket::onFreeChunk(Slab &Slab, bool &ToPool) {
803  ToPool = true;
804 
805  // In case if the slab was previously full and now has 1 available
806  // chunk, it should be moved to the list of available slabs
807  if (Slab.getNumAllocated() == (Slab.getNumChunks() - 1)) {
808  auto SlabIter = Slab.getIterator();
809  assert(SlabIter != UnavailableSlabs.end());
810 
811  auto It =
812  AvailableSlabs.insert(AvailableSlabs.begin(), std::move(*SlabIter));
813  UnavailableSlabs.erase(SlabIter);
814 
815  (*It)->setIterator(It);
816  }
817 
818  // Check if slab is empty, and pool it if we can.
819  if (Slab.getNumAllocated() == 0) {
820  // The slab is now empty.
821  // If pool has capacity then put the slab in the pool.
822  // The ToPool parameter indicates whether the Slab will be put in the pool
823  // or freed from USM.
824  if (!CanPool(ToPool)) {
825  // Note: since the slab is stored as unique_ptr, just remove it from
826  // the list to destroy the object.
827  auto It = Slab.getIterator();
828  assert(It != AvailableSlabs.end());
829  AvailableSlabs.erase(It);
830  }
831  }
832 }
833 
834 bool Bucket::CanPool(bool &ToPool) {
835  std::lock_guard<sycl::detail::SpinLock> Lock{PoolLock};
836  size_t NewFreeSlabsInBucket;
837  // Check if this bucket is used in chunked form or as full slabs.
838  bool chunkedBucket = getSize() <= ChunkCutOff();
839  if (chunkedBucket)
840  NewFreeSlabsInBucket = chunkedSlabsInPool + 1;
841  else
842  NewFreeSlabsInBucket = AvailableSlabs.size() + 1;
843  if (Capacity() >= NewFreeSlabsInBucket) {
844  size_t NewPoolSize = USMSettings.CurPoolSize + SlabAllocSize();
845  if (USMSettings.MaxPoolSize >= NewPoolSize) {
846  USMSettings.CurPoolSize = NewPoolSize;
847  if (chunkedBucket)
848  ++chunkedSlabsInPool;
849 
850  updateStats(-1, 1);
851  ToPool = true;
852  return true;
853  }
854  }
855  updateStats(-1, 0);
856  ToPool = false;
857  return false;
858 }
859 
860 SystemMemory &Bucket::getMemHandle() { return OwnAllocCtx.getMemHandle(); }
861 
863  return getMemHandle().getMemType();
864 }
865 
867 
868 size_t Bucket::SlabAllocSize() { return std::max(getSize(), SlabMinSize()); }
869 
871  // For buckets used in chunked mode, just one slab in pool is sufficient.
872  // For larger buckets, the capacity could be more and is adjustable.
873  if (getSize() <= ChunkCutOff())
874  return 1;
875  else
876  return USMSettings.Capacity[getMemType()];
877 }
878 
881 }
882 
883 size_t Bucket::ChunkCutOff() { return SlabMinSize() / 2; }
884 
885 void Bucket::countAlloc(bool FromPool) {
886  ++allocCount;
887  if (FromPool)
888  ++allocPoolCount;
889 }
890 
891 void Bucket::countFree() { ++freeCount; }
892 
893 void Bucket::updateStats(int InUse, int InPool) {
894  if (USMSettings.PoolTrace == 0)
895  return;
896  currSlabsInUse += InUse;
897  maxSlabsInUse = std::max(currSlabsInUse, maxSlabsInUse);
898  currSlabsInPool += InPool;
899  maxSlabsInPool = std::max(currSlabsInPool, maxSlabsInPool);
900  // Increment or decrement current pool sizes based on whether
901  // slab was added to or removed from pool.
903 }
904 
905 void Bucket::printStats(bool &TitlePrinted, SystemMemory::MemType MT) {
906  if (allocCount) {
907  if (!TitlePrinted) {
908  auto Label = MemTypeNames[MT];
909  std::cout << Label << " memory statistics\n";
910  std::cout << std::setw(14) << "Bucket Size" << std::setw(12) << "Allocs"
911  << std::setw(12) << "Frees" << std::setw(18)
912  << "Allocs from Pool" << std::setw(20) << "Peak Slabs in Use"
913  << std::setw(21) << "Peak Slabs in Pool" << std::endl;
914  TitlePrinted = true;
915  }
916  std::cout << std::setw(14) << getSize() << std::setw(12) << allocCount
917  << std::setw(12) << freeCount << std::setw(18) << allocPoolCount
918  << std::setw(20) << maxSlabsInUse << std::setw(21)
919  << maxSlabsInPool << std::endl;
920  }
921 }
922 
923 // SystemMemory &Bucket::getMemHandle() { return OwnAllocCtx.getMemHandle(); }
924 
925 void *USMAllocContext::USMAllocImpl::allocate(size_t Size, bool &FromPool) {
926  void *Ptr;
927 
928  if (Size == 0)
929  return nullptr;
930 
931  FromPool = false;
932  if (Size > USMSettings.MaxPoolableSize[getMemHandle().getMemType()]) {
933  return getMemHandle().allocate(Size);
934  }
935 
936  auto &Bucket = findBucket(Size);
937 
938  if (Size > Bucket.ChunkCutOff())
939  Ptr = Bucket.getSlab(FromPool);
940  else
941  Ptr = Bucket.getChunk(FromPool);
942 
943  if (USMSettings.PoolTrace > 1)
944  Bucket.countAlloc(FromPool);
945 
946  return Ptr;
947 }
948 
949 void *USMAllocContext::USMAllocImpl::allocate(size_t Size, size_t Alignment,
950  bool &FromPool) {
951  void *Ptr;
952 
953  if (Size == 0)
954  return nullptr;
955 
956  if (Alignment <= 1)
957  return allocate(Size, FromPool);
958 
959  size_t AlignedSize = (Size > 1) ? AlignUp(Size, Alignment) : Alignment;
960 
961  // Check if requested allocation size is within pooling limit.
962  // If not, just request aligned pointer from the system.
963  FromPool = false;
964  if (AlignedSize > USMSettings.MaxPoolableSize[getMemHandle().getMemType()]) {
965  return getMemHandle().allocate(Size, Alignment);
966  }
967 
968  auto &Bucket = findBucket(AlignedSize);
969 
970  if (AlignedSize > Bucket.ChunkCutOff()) {
971  Ptr = Bucket.getSlab(FromPool);
972  } else {
973  Ptr = Bucket.getChunk(FromPool);
974  }
975 
976  if (USMSettings.PoolTrace > 1)
977  Bucket.countAlloc(FromPool);
978 
979  return AlignPtrUp(Ptr, Alignment);
980 }
981 
982 Bucket &USMAllocContext::USMAllocImpl::findBucket(size_t Size) {
983  assert(Size <= CutOff && "Unexpected size");
984 
985  auto It = std::find_if(
986  Buckets.begin(), Buckets.end(),
987  [Size](const auto &BucketPtr) { return BucketPtr->getSize() >= Size; });
988 
989  assert((It != Buckets.end()) && "Bucket should always exist");
990 
991  return *(*It);
992 }
993 
994 void USMAllocContext::USMAllocImpl::deallocate(void *Ptr, bool &ToPool,
995  bool OwnZeMemHandle) {
996  auto *SlabPtr = AlignPtrDown(Ptr, SlabMinSize());
997 
998  // Lock the map on read
999  std::shared_lock<std::shared_timed_mutex> Lk(getKnownSlabsMapLock());
1000 
1001  ToPool = false;
1002  auto Slabs = getKnownSlabs().equal_range(SlabPtr);
1003  if (Slabs.first == Slabs.second) {
1004  Lk.unlock();
1005  getMemHandle().deallocate(Ptr, OwnZeMemHandle);
1006  return;
1007  }
1008 
1009  for (auto It = Slabs.first; It != Slabs.second; ++It) {
1010  // The slab object won't be deleted until it's removed from the map which is
1011  // protected by the lock, so it's safe to access it here.
1012  auto &Slab = It->second;
1013  if (Ptr >= Slab.getPtr() && Ptr < Slab.getEnd()) {
1014  // Unlock the map before freeing the chunk, it may be locked on write
1015  // there
1016  Lk.unlock();
1017  auto &Bucket = Slab.getBucket();
1018 
1019  if (USMSettings.PoolTrace > 1)
1020  Bucket.countFree();
1021 
1022  if (Bucket.getSize() <= Bucket.ChunkCutOff()) {
1023  Bucket.freeChunk(Ptr, Slab, ToPool);
1024  } else {
1025  Bucket.freeSlab(Slab, ToPool);
1026  }
1027 
1028  return;
1029  }
1030  }
1031 
1032  Lk.unlock();
1033  // There is a rare case when we have a pointer from system allocation next
1034  // to some slab with an entry in the map. So we find a slab
1035  // but the range checks fail.
1036  getMemHandle().deallocate(Ptr, OwnZeMemHandle);
1037 }
1038 
1039 USMAllocContext::USMAllocContext(std::unique_ptr<SystemMemory> MemHandle)
1040  : pImpl(std::make_unique<USMAllocImpl>(std::move(MemHandle))) {}
1041 
1042 void *USMAllocContext::allocate(size_t size) {
1043  // For full-slab allocations indicates whether slab is from Pool.
1044  bool FromPool;
1045  auto Ptr = pImpl->allocate(size, FromPool);
1046 
1047  if (USMSettings.PoolTrace > 2) {
1048  auto MT = pImpl->getMemHandle().getMemType();
1049  std::cout << "Allocated " << std::setw(8) << size << " " << MemTypeNames[MT]
1050  << " USM bytes from " << (FromPool ? "Pool" : "USM") << " ->"
1051  << Ptr << std::endl;
1052  }
1053  return Ptr;
1054 }
1055 
1056 void *USMAllocContext::allocate(size_t size, size_t alignment) {
1057  bool FromPool;
1058  auto Ptr = pImpl->allocate(size, alignment, FromPool);
1059 
1060  if (USMSettings.PoolTrace > 2) {
1061  auto MT = pImpl->getMemHandle().getMemType();
1062  std::cout << "Allocated " << std::setw(8) << size << " " << MemTypeNames[MT]
1063  << " USM bytes aligned at " << alignment << " from "
1064  << (FromPool ? "Pool" : "USM") << " ->" << Ptr << std::endl;
1065  }
1066  return Ptr;
1067 }
1068 
1069 void USMAllocContext::deallocate(void *ptr, bool OwnZeMemHandle) {
1070  bool ToPool;
1071  pImpl->deallocate(ptr, ToPool, OwnZeMemHandle);
1072 
1073  if (USMSettings.PoolTrace > 2) {
1074  auto MT = pImpl->getMemHandle().getMemType();
1075  std::cout << "Freed " << MemTypeNames[MT] << " USM " << ptr << " to "
1076  << (ToPool ? "Pool" : "USM") << ", Current total pool size "
1077  << USMSettings.CurPoolSize << ", Current pool sizes ["
1081  }
1082  return;
1083 }
1084 
1085 // Define destructor for use with unique_ptr
1087  bool TitlePrinted = false;
1088  size_t HighBucketSize;
1089  size_t HighPeakSlabsInUse;
1090  if (USMSettings.PoolTrace > 1) {
1091  SystemMemory::MemType MT = pImpl->getMemHandle().getMemType();
1092  pImpl->printStats(TitlePrinted, HighBucketSize, HighPeakSlabsInUse, MT);
1093  if (TitlePrinted) {
1094  std::cout << "Current Pool Size " << USMSettings.CurPoolSize << std::endl;
1095  const char *Label = MemTypeNames[MT];
1096  std::cout << "Suggested Setting: SYCL_PI_LEVEL_ZERO_USM_ALLOCATOR=;"
1097  << std::string(1, tolower(*Label)) << std::string(Label + 1)
1098  << ":" << HighBucketSize << "," << HighPeakSlabsInUse << ",64K"
1099  << std::endl;
1100  }
1101  }
1102 }
1103 
1105  size_t &HighBucketSize,
1106  size_t &HighPeakSlabsInUse,
1107  SystemMemory::MemType MT) {
1108  HighBucketSize = 0;
1109  HighPeakSlabsInUse = 0;
1110  for (auto &B : Buckets) {
1111  (*B).printStats(TitlePrinted, MT);
1112  HighPeakSlabsInUse = std::max((*B).maxSlabsInUse, HighPeakSlabsInUse);
1113  if ((*B).allocCount)
1114  HighBucketSize = std::max((*B).SlabAllocSize(), HighBucketSize);
1115  }
1116 }
1117 
SystemMemory::SharedReadOnly
@ SharedReadOnly
Definition: usm_allocator.hpp:17
USMAllocContext::USMAllocImpl::getKnownSlabs
std::unordered_multimap< void *, Slab & > & getKnownSlabs()
Definition: usm_allocator.cpp:553
operator<<
std::ostream & operator<<(std::ostream &Os, const Slab &Slab)
Definition: usm_allocator.cpp:572
Bucket::ChunkCutOff
size_t ChunkCutOff()
Definition: usm_allocator.cpp:883
USMAllocContext::USMAllocImpl::getMemHandle
SystemMemory & getMemHandle()
Definition: usm_allocator.cpp:550
Slab::Slab
Slab(Bucket &)
Definition: usm_allocator.cpp:578
settings::SetLimits
Definition: usm_allocator.cpp:73
Slab::getPtr
void * getPtr() const
Definition: usm_allocator.cpp:387
settings::MinBucketSize
static constexpr size_t MinBucketSize[SystemMemory::All]
Definition: usm_allocator.cpp:63
Slab
Definition: usm_allocator.cpp:338
Bucket::getSize
size_t getSize() const
Definition: usm_allocator.cpp:461
SystemMemory::allocate
virtual void * allocate(size_t size)=0
SystemMemory::Host
@ Host
Definition: usm_allocator.hpp:17
Bucket::getChunk
void * getChunk(bool &FromPool)
Definition: usm_allocator.cpp:776
SystemMemory
Definition: usm_allocator.hpp:15
Slab::getEnd
void * getEnd() const
Definition: usm_allocator.cpp:696
settings::SetLimits::EnableBuffers
size_t EnableBuffers
Definition: usm_allocator.cpp:91
settings::SetLimits::SlabMinSize
size_t SlabMinSize[SystemMemory::All]
Definition: usm_allocator.cpp:77
SystemMemory::Device
@ Device
Definition: usm_allocator.hpp:17
settings::SetLimits::SetLimits
SetLimits()
Definition: usm_allocator.cpp:96
Slab::getNumChunks
size_t getNumChunks() const
Definition: usm_allocator.cpp:391
Bucket::MaxPoolableSize
size_t MaxPoolableSize()
Definition: usm_allocator.cpp:879
Bucket::freeSlab
void freeSlab(Slab &Slab, bool &ToPool)
Definition: usm_allocator.cpp:738
Bucket::Capacity
size_t Capacity()
Definition: usm_allocator.cpp:870
Slab::getSlab
void * getSlab()
Definition: usm_allocator.cpp:623
usm_allocator.hpp
AlignPtrDown
static void * AlignPtrDown(void *Ptr, const size_t Alignment)
Definition: usm_allocator.cpp:307
enableBufferPooling
bool enableBufferPooling()
Definition: usm_allocator.cpp:1118
operator==
bool operator==(const Slab &Lhs, const Slab &Rhs)
Definition: usm_allocator.cpp:568
Bucket::getMemType
SystemMemory::MemType getMemType()
Definition: usm_allocator.cpp:862
Bucket
Definition: usm_allocator.cpp:401
Bucket::Bucket
Bucket(size_t Sz, USMAllocContext::USMAllocImpl &AllocCtx)
Definition: usm_allocator.cpp:448
SystemMemory::MemType
MemType
Definition: usm_allocator.hpp:17
USMAllocContext::USMAllocContext
USMAllocContext(std::unique_ptr< SystemMemory > memHandle)
Definition: usm_allocator.cpp:1039
Bucket::maxSlabsInUse
size_t maxSlabsInUse
Definition: usm_allocator.cpp:446
USMAllocContext::allocate
void * allocate(size_t size)
Definition: usm_allocator.cpp:1042
Slab::getChunk
void * getChunk()
Definition: usm_allocator.cpp:605
SystemMemory::Shared
@ Shared
Definition: usm_allocator.hpp:17
USMAllocContext::USMAllocImpl::allocate
void * allocate(size_t Size, size_t Alignment, bool &FromPool)
Definition: usm_allocator.cpp:949
MemTypeNames
static const char * MemTypeNames[SystemMemory::All]
Definition: usm_allocator.cpp:302
SystemMemory::All
@ All
Definition: usm_allocator.hpp:17
Slab::setIterator
void setIterator(ListIter It)
Definition: usm_allocator.cpp:376
settings::SetLimits::CurPoolSizes
size_t CurPoolSizes[SystemMemory::All]
Definition: usm_allocator.cpp:89
USMAllocContext::USMAllocImpl
Definition: usm_allocator.cpp:519
SystemMemory::deallocate
virtual void deallocate(void *ptr, bool OwnZeMemHandle)=0
USMAllocContext::deallocate
void deallocate(void *ptr, bool OwnZeMemHandle)
Definition: usm_allocator.cpp:1069
Bucket::allocCount
size_t allocCount
Definition: usm_allocator.cpp:445
AlignUp
static size_t AlignUp(size_t Val, size_t Alignment)
Definition: usm_allocator.cpp:325
spinlock.hpp
Bucket::countFree
void countFree()
Definition: usm_allocator.cpp:891
Bucket::CanPool
bool CanPool(bool &ToPool)
Definition: usm_allocator.cpp:834
settings::SetLimits::MaxPoolableSize
size_t MaxPoolableSize[SystemMemory::All]
Definition: usm_allocator.cpp:80
settings::SetLimits::PoolTrace
int PoolTrace
Definition: usm_allocator.cpp:94
USMAllocContext::~USMAllocContext
~USMAllocContext()
Definition: usm_allocator.cpp:1086
Bucket::SlabAllocSize
size_t SlabAllocSize()
Definition: usm_allocator.cpp:868
settings::CutOff
static constexpr size_t CutOff
Definition: usm_allocator.cpp:68
Bucket::SlabMinSize
size_t SlabMinSize()
Definition: usm_allocator.cpp:866
Bucket::getUsmAllocCtx
USMAllocContext::USMAllocImpl & getUsmAllocCtx()
Definition: usm_allocator.cpp:473
Slab::getChunkSize
size_t getChunkSize() const
Definition: usm_allocator.cpp:628
settings
Definition: usm_allocator.cpp:45
Slab::~Slab
~Slab()
Definition: usm_allocator.cpp:588
USMAllocContext::USMAllocImpl::SlabMinSize
size_t SlabMinSize()
Definition: usm_allocator.cpp:557
Bucket::updateStats
void updateStats(int InUse, int InPool)
Definition: usm_allocator.cpp:893
USMAllocContext::USMAllocImpl::printStats
void printStats(bool &TitlePrinted, size_t &HighBucketSize, size_t &HighPeakSlabsInUse, SystemMemory::MemType MT)
Definition: usm_allocator.cpp:1104
Slab::hasAvail
bool hasAvail()
Definition: usm_allocator.cpp:700
std
Definition: accessor.hpp:2616
SystemMemory::getMemType
virtual MemType getMemType()=0
Bucket::countAlloc
void countAlloc(bool FromPool)
Definition: usm_allocator.cpp:885
Bucket::getMemHandle
SystemMemory & getMemHandle()
Definition: usm_allocator.cpp:860
Slab::getBucket
Bucket & getBucket()
Definition: usm_allocator.cpp:625
settings::PoolLock
static sycl::detail::SpinLock PoolLock
Definition: usm_allocator.cpp:71
settings::SetLimits::CurPoolSize
size_t CurPoolSize
Definition: usm_allocator.cpp:88
USMAllocContext
Definition: usm_allocator.hpp:25
cl::sycl::setw
__width_manipulator__ setw(int Width)
Definition: stream.hpp:735
settings::SetLimits::MaxPoolSize
size_t MaxPoolSize
Definition: usm_allocator.cpp:86
Bucket::freeChunk
void freeChunk(void *Ptr, Slab &Slab, bool &ToPool)
Definition: usm_allocator.cpp:793
settings::USMSettings
static class settings::SetLimits USMSettings
AlignPtrUp
static void * AlignPtrUp(void *Ptr, const size_t Alignment)
Definition: usm_allocator.cpp:314
USMAllocContext::USMAllocImpl::USMAllocImpl
USMAllocImpl(std::unique_ptr< SystemMemory > SystemMemHandle)
Definition: usm_allocator.cpp:532
Slab::getIterator
ListIter getIterator() const
Definition: usm_allocator.cpp:377
USMAllocContext::USMAllocImpl::deallocate
void deallocate(void *Ptr, bool &ToPool, bool OwnZeMemHandle)
Definition: usm_allocator.cpp:994
Bucket::printStats
void printStats(bool &TitlePrinted, SystemMemory::MemType MT)
Definition: usm_allocator.cpp:905
Bucket::getSlab
void * getSlab(bool &FromPool)
Definition: usm_allocator.cpp:726
Slab::freeChunk
void freeChunk(void *Ptr)
Definition: usm_allocator.cpp:674
Slab::getNumAllocated
size_t getNumAllocated() const
Definition: usm_allocator.cpp:379
settings::SetLimits::Capacity
size_t Capacity[SystemMemory::All]
Definition: usm_allocator.cpp:83
USMAllocContext::USMAllocImpl::getKnownSlabsMapLock
std::shared_timed_mutex & getKnownSlabsMapLock()
Definition: usm_allocator.cpp:552