DPC++ Runtime
Runtime libraries for oneAPI Data Parallel C++
usm_allocator.cpp
Go to the documentation of this file.
1 //===---------- usm_allocator.cpp - Allocator for USM memory --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include <algorithm>
10 #include <array>
11 #include <bitset>
12 #include <cassert>
13 #include <iostream>
14 #include <list>
15 #include <memory>
16 #include <mutex>
17 #include <shared_mutex>
18 #include <string>
19 #include <unordered_map>
20 #include <utility>
21 #include <vector>
22 
23 #include "usm_allocator.hpp"
25 #include <iostream>
26 
27 // USM allocations are a mimimum of 64KB in size even when a smaller size is
28 // requested. The implementation distinguishes between allocations of size
29 // ChunkCutOff (32KB) and those that are larger.
30 // Allocation requests smaller than ChunkCutoff use chunks taken from a single
31 // 64KB USM allocation. Thus, for example, for 8-byte allocations, only 1 in
32 // ~8000 requests results in a new USM allocation. Freeing results only in a
33 // chunk of a larger 64KB allocation to be marked as available and no real
34 // return to the system. An allocation is returned to the system only when all
35 // chunks in a 64KB allocation are freed by the program.
36 // Allocations larger than ChunkCutOff use a separate USM allocation for each
37 // request. These are subject to "pooling". That is, when such an allocation is
38 // freed by the program it is retained in a pool. The pool is available for
39 // future allocations, which means there are fewer actual USM
40 // allocations/deallocations.
41 
42 namespace settings {
43 // Minimum allocation size that will be requested from the system.
44 static constexpr size_t SlabMinSize = 64 * 1024; // 64KB
45 
46 // Allocations <= ChunkCutOff will use chunks from individual slabs.
47 // Allocations > ChunkCutOff will be rounded up to a multiple of
48 // SlabMinSize and allocated to occupy the whole slab.
49 static constexpr size_t ChunkCutOff = SlabMinSize / 2;
50 // The largest size which is allocated via the allocator.
51 // Allocations with size > CutOff bypass the USM allocator and
52 // go directly to the runtime.
53 static constexpr size_t CutOff = (size_t)1 << 31; // 2GB
54 
55 // Unfortunately we cannot deduce the size of the array, so every change
56 // to the number of buckets should be reflected here.
57 using BucketsArrayType = std::array<size_t, 53>;
58 
59 // Generates a list of bucket sizes used by the allocator.
61 
62 // In order to make bucket sizes constexpr simply write
63 // them all. There are some restrictions that doesn't
64 // allow to write this in a nicer way.
65 
66 // Simple helper to compute power of 2
67 #define P(n) (1ULL << n)
68 
69  BucketsArrayType Sizes = {32, 48,
70  64, 96,
71  128, 192,
72  P(8), P(8) + P(7),
73  P(9), P(9) + P(8),
74  P(10), P(10) + P(9),
75  P(11), P(11) + P(10),
76  P(12), P(12) + P(11),
77  P(13), P(13) + P(12),
78  P(14), P(14) + P(13),
79  P(15), P(15) + P(14),
80  P(16), P(16) + P(15),
81  P(17), P(17) + P(16),
82  P(18), P(18) + P(17),
83  P(19), P(19) + P(18),
84  P(20), P(20) + P(19),
85  P(21), P(21) + P(20),
86  P(22), P(22) + P(21),
87  P(23), P(23) + P(22),
88  P(24), P(24) + P(23),
89  P(25), P(25) + P(24),
90  P(26), P(26) + P(25),
91  P(27), P(27) + P(26),
92  P(28), P(28) + P(27),
93  P(29), P(29) + P(28),
94  P(30), P(30) + P(29),
95  CutOff};
96 #undef P
97 
98  return Sizes;
99 }
100 
102 
103 // The implementation expects that SlabMinSize is 2^n
104 static_assert((SlabMinSize & (SlabMinSize - 1)) == 0,
105  "SlabMinSize must be a power of 2");
106 
107 // Protects the capacity checking of the pool.
108 static sycl::detail::SpinLock PoolLock;
109 
110 static class SetLimits {
111 public:
112  size_t MaxPoolableSize = 1;
113  size_t Capacity = 4;
114  size_t MaxPoolSize = 256;
115  size_t CurPoolSize = 0;
116 
118  // Parse optional parameters of this form (applicable to each context):
119  // SYCL_PI_LEVEL_ZERO_USM_ALLOCATOR_SETTINGS=[<MaxPoolableSize>][,[<Capacity>][,[<MaxPoolSize>]]]
120  // MaxPoolableSize: Maximum poolable allocation size, specified in MB.
121  // Default 1MB.
122  // Capacity: Number of pooled allocations in each bucket.
123  // Default 4.
124  // MaxPoolSize: Maximum size of pool, specified in MB.
125  // Default 256MB.
126 
127  char *PoolParams = getenv("SYCL_PI_LEVEL_ZERO_USM_ALLOCATOR");
128  if (PoolParams != nullptr) {
129  std::string Params(PoolParams);
130  size_t Pos = Params.find(',');
131  if (Pos != std::string::npos) {
132  if (Pos > 0)
133  MaxPoolableSize = std::stoi(Params.substr(0, Pos));
134  Params.erase(0, Pos + 1);
135  Pos = Params.find(',');
136  if (Pos != std::string::npos) {
137  if (Pos > 0)
138  Capacity = std::stoi(Params.substr(0, Pos));
139  Params.erase(0, Pos + 1);
140  if (Pos != std::string::npos)
141  MaxPoolSize = std::stoi(Params);
142  } else {
143  Capacity = std::stoi(Params);
144  }
145  } else
146  MaxPoolableSize = std::stoi(Params);
147  }
148  MaxPoolableSize *= (1 << 20);
149  MaxPoolSize *= (1 << 20);
150  }
152 } // namespace settings
153 
154 // Aligns the pointer down to the specified alignment
155 // (e.g. returns 8 for Size = 13, Alignment = 8)
156 static void *AlignPtrDown(void *Ptr, const size_t Alignment) {
157  return reinterpret_cast<void *>((reinterpret_cast<size_t>(Ptr)) &
158  (~(Alignment - 1)));
159 }
160 
161 // Aligns the pointer up to the specified alignment
162 // (e.g. returns 16 for Size = 13, Alignment = 8)
163 static void *AlignPtrUp(void *Ptr, const size_t Alignment) {
164  void *AlignedPtr = AlignPtrDown(Ptr, Alignment);
165  // Special case when the pointer is already aligned
166  if (Ptr == AlignedPtr) {
167  return Ptr;
168  }
169  return static_cast<char *>(AlignedPtr) + Alignment;
170 }
171 
172 // Aligns the value up to the specified alignment
173 // (e.g. returns 16 for Size = 13, Alignment = 8)
174 static size_t AlignUp(size_t Val, size_t Alignment) {
175  assert(Alignment > 0);
176  return (Val + Alignment - 1) & (~(Alignment - 1));
177 }
178 
179 class Bucket;
180 
181 // Represents the allocated memory block of size 'settings::SlabMinSize'
182 // Internally, it splits the memory block into chunks. The number of
183 // chunks depends of the size of a Bucket which created the Slab.
184 // The chunks
185 // Note: Bucket's method are responsible for thread safety of Slab access,
186 // so no locking happens here.
187 class Slab {
188 
189  // Pointer to the allocated memory of SlabMinSize bytes
190  void *MemPtr;
191 
192  // Represents the current state of each chunk:
193  // if the bit is set then the chunk is allocated
194  // the chunk is free for allocation otherwise
195  std::vector<bool> Chunks;
196 
197  // Total number of allocated chunks at the moment.
198  size_t NumAllocated = 0;
199 
200  // The bucket which the slab belongs to
201  Bucket &bucket;
202 
203  using ListIter = std::list<std::unique_ptr<Slab>>::iterator;
204 
205  // Store iterator to the corresponding node in avail/unavail list
206  // to achieve O(1) removal
207  ListIter SlabListIter;
208 
209  // Hints where to start search for free chunk in a slab
210  size_t FirstFreeChunkIdx = 0;
211 
212  // Return the index of the first available chunk, -1 otherwize
213  size_t FindFirstAvailableChunkIdx() const;
214 
215  // Register/Unregister the slab in the global slab address map.
216  static void regSlab(Slab &);
217  static void unregSlab(Slab &);
218  static void regSlabByAddr(void *, Slab &);
219  static void unregSlabByAddr(void *, Slab &);
220 
221 public:
222  Slab(Bucket &);
223  ~Slab();
224 
225  void setIterator(ListIter It) { SlabListIter = It; }
226  ListIter getIterator() const { return SlabListIter; }
227 
228  size_t getNumAllocated() const { return NumAllocated; }
229 
230  // Get pointer to allocation that is one piece of this slab.
231  void *getChunk();
232 
233  // Get pointer to allocation that is this entire slab.
234  void *getSlab();
235 
236  void *getPtr() const { return MemPtr; }
237  void *getEnd() const {
238  return static_cast<char *>(getPtr()) + settings::SlabMinSize;
239  }
240 
241  size_t getChunkSize() const;
242  size_t getNumChunks() const { return Chunks.size(); }
243 
244  bool hasAvail();
245 
246  Bucket &getBucket();
247  const Bucket &getBucket() const;
248 
249  void freeChunk(void *Ptr);
250 };
251 
252 class Bucket {
253  const size_t Size;
254 
255  // List of slabs which have at least 1 available chunk.
256  std::list<std::unique_ptr<Slab>> AvailableSlabs;
257 
258  // List of slabs with 0 available chunk.
259  std::list<std::unique_ptr<Slab>> UnavailableSlabs;
260 
261  // Protects the bucket and all the corresponding slabs
262  std::mutex BucketLock;
263 
264  // Reference to the allocator context, used access memory allocation
265  // routines, slab map and etc.
266  USMAllocContext::USMAllocImpl &OwnAllocCtx;
267 
268 public:
269  Bucket(size_t Sz, USMAllocContext::USMAllocImpl &AllocCtx)
270  : Size{Sz}, OwnAllocCtx{AllocCtx} {}
271 
272  // Get pointer to allocation that is one piece of an available slab in this
273  // bucket.
274  void *getChunk();
275 
276  // Get pointer to allocation that is a full slab in this bucket.
277  void *getSlab();
278 
279  size_t getSize() const { return Size; }
280 
281  // Free an allocation that is one piece of a slab in this bucket.
282  void freeChunk(void *Ptr, Slab &Slab);
283 
284  // Free an allocation that is a full slab in this bucket.
285  void freeSlab(Slab &Slab);
286 
289 
290  // Check whether an allocation to be freed can be placed in the pool.
291  bool CanPool();
292 
293 private:
294  void onFreeChunk(Slab &);
295 
296  // Get a slab to be used for chunked allocations.
297  // These slabs are used for allocations <= ChunkCutOff and not pooled.
298  decltype(AvailableSlabs.begin()) getAvailSlab();
299 
300  // Get a slab that will be used as a whole for a single allocation.
301  // These slabs are > ChunkCutOff in size and pooled.
302  decltype(AvailableSlabs.begin()) getAvailFullSlab();
303 };
304 
306  // It's important for the map to be destroyed last after buckets and their
307  // slabs This is because slab's destructor removes the object from the map.
308  std::unordered_multimap<void *, Slab &> KnownSlabs;
309  std::shared_timed_mutex KnownSlabsMapLock;
310 
311  // Handle to the memory allocation routine
312  std::unique_ptr<SystemMemory> MemHandle;
313 
314  // Store as unique_ptrs since Bucket is not Movable(because of std::mutex)
315  std::vector<std::unique_ptr<Bucket>> Buckets;
316 
317 public:
318  USMAllocImpl(std::unique_ptr<SystemMemory> SystemMemHandle)
319  : MemHandle{std::move(SystemMemHandle)} {
320 
321  Buckets.reserve(settings::BucketSizes.size());
322 
323  for (auto &&Size : settings::BucketSizes) {
324  Buckets.emplace_back(std::make_unique<Bucket>(Size, *this));
325  }
326  }
327 
328  void *allocate(size_t Size, size_t Alignment);
329  void *allocate(size_t Size);
330  void deallocate(void *Ptr);
331 
332  SystemMemory &getMemHandle() { return *MemHandle; }
333 
334  std::shared_timed_mutex &getKnownSlabsMapLock() { return KnownSlabsMapLock; }
335  std::unordered_multimap<void *, Slab &> &getKnownSlabs() {
336  return KnownSlabs;
337  }
338 
339 private:
340  Bucket &findBucket(size_t Size);
341 };
342 
343 bool operator==(const Slab &Lhs, const Slab &Rhs) {
344  return Lhs.getPtr() == Rhs.getPtr();
345 }
346 
347 std::ostream &operator<<(std::ostream &Os, const Slab &Slab) {
348  Os << "Slab<" << Slab.getPtr() << ", " << Slab.getEnd() << ", "
349  << Slab.getBucket().getSize() << ">";
350  return Os;
351 }
352 
354  : // In case bucket size is not a multiple of SlabMinSize, we would have
355  // some padding at the end of the slab.
356  Chunks(settings::SlabMinSize / Bkt.getSize()), NumAllocated{0},
357  bucket(Bkt), SlabListIter{}, FirstFreeChunkIdx{0} {
358  size_t SlabAllocSize = Bkt.getSize();
359  if (SlabAllocSize < settings::SlabMinSize)
360  SlabAllocSize = settings::SlabMinSize;
361  MemPtr = Bkt.getMemHandle().allocate(SlabAllocSize);
362  regSlab(*this);
363 }
364 
366  unregSlab(*this);
367  bucket.getMemHandle().deallocate(MemPtr);
368 }
369 
370 // Return the index of the first available chunk, -1 otherwize
371 size_t Slab::FindFirstAvailableChunkIdx() const {
372  // Use the first free chunk index as a hint for the search.
373  auto It = std::find_if(Chunks.begin() + FirstFreeChunkIdx, Chunks.end(),
374  [](auto x) { return !x; });
375  if (It != Chunks.end()) {
376  return It - Chunks.begin();
377  }
378 
379  return static_cast<size_t>(-1);
380 }
381 
382 void *Slab::getChunk() {
383  assert(NumAllocated != Chunks.size());
384 
385  const size_t ChunkIdx = FindFirstAvailableChunkIdx();
386  // Free chunk must exist, otherwise we would have allocated another slab
387  assert(ChunkIdx != (static_cast<size_t>(-1)));
388 
389  void *const FreeChunk =
390  (static_cast<uint8_t *>(getPtr())) + ChunkIdx * getChunkSize();
391  Chunks[ChunkIdx] = true;
392  NumAllocated += 1;
393 
394  // Use the found index as the next hint
395  FirstFreeChunkIdx = ChunkIdx;
396 
397  return FreeChunk;
398 }
399 
400 void *Slab::getSlab() { return getPtr(); }
401 
402 Bucket &Slab::getBucket() { return bucket; }
403 const Bucket &Slab::getBucket() const { return bucket; }
404 
405 size_t Slab::getChunkSize() const { return bucket.getSize(); }
406 
407 void Slab::regSlabByAddr(void *Addr, Slab &Slab) {
409  auto &Map = Slab.getBucket().getUsmAllocCtx().getKnownSlabs();
410 
411  std::lock_guard<std::shared_timed_mutex> Lg(Lock);
412  Map.insert({Addr, Slab});
413 }
414 
415 void Slab::unregSlabByAddr(void *Addr, Slab &Slab) {
417  auto &Map = Slab.getBucket().getUsmAllocCtx().getKnownSlabs();
418 
419  std::lock_guard<std::shared_timed_mutex> Lg(Lock);
420 
421  auto Slabs = Map.equal_range(Addr);
422  // At least the must get the current slab from the map.
423  assert(Slabs.first != Slabs.second && "Slab is not found");
424 
425  for (auto It = Slabs.first; It != Slabs.second; ++It) {
426  if (It->second == Slab) {
427  Map.erase(It);
428  return;
429  }
430  }
431 
432  assert(false && "Slab is not found");
433 }
434 
435 void Slab::regSlab(Slab &Slab) {
436  void *StartAddr = AlignPtrDown(Slab.getPtr(), settings::SlabMinSize);
437  void *EndAddr = static_cast<char *>(StartAddr) + settings::SlabMinSize;
438 
439  regSlabByAddr(StartAddr, Slab);
440  regSlabByAddr(EndAddr, Slab);
441 }
442 
443 void Slab::unregSlab(Slab &Slab) {
444  void *StartAddr = AlignPtrDown(Slab.getPtr(), settings::SlabMinSize);
445  void *EndAddr = static_cast<char *>(StartAddr) + settings::SlabMinSize;
446 
447  unregSlabByAddr(StartAddr, Slab);
448  unregSlabByAddr(EndAddr, Slab);
449 }
450 
451 void Slab::freeChunk(void *Ptr) {
452  // This method should be called through bucket(since we might remove the slab
453  // as a result), therefore all locks are done on that level.
454 
455  // Make sure that we're in the right slab
456  assert(Ptr >= getPtr() && Ptr < getEnd());
457 
458  // Even if the pointer p was previously aligned, it's still inside the
459  // corresponding chunk, so we get the correct index here.
460  auto ChunkIdx =
461  (static_cast<char *>(Ptr) - static_cast<char *>(MemPtr)) / getChunkSize();
462 
463  // Make sure that the chunk was allocated
464  assert(Chunks[ChunkIdx] && "double free detected");
465 
466  Chunks[ChunkIdx] = false;
467  NumAllocated -= 1;
468 
469  if (ChunkIdx < FirstFreeChunkIdx)
470  FirstFreeChunkIdx = ChunkIdx;
471 }
472 
473 bool Slab::hasAvail() { return NumAllocated != getNumChunks(); }
474 
475 auto Bucket::getAvailFullSlab() -> decltype(AvailableSlabs.begin()) {
476  // Return a slab that will be used for a single allocation.
477  if (AvailableSlabs.size() == 0) {
478  auto It = AvailableSlabs.insert(AvailableSlabs.begin(),
479  std::make_unique<Slab>(*this));
480  (*It)->setIterator(It);
481  } else {
482  // If a slab was available in the pool then note that the current pooled
483  // size has reduced by the size of this slab.
485  }
486 
487  return AvailableSlabs.begin();
488 }
489 
491  std::lock_guard<std::mutex> Lg(BucketLock);
492 
493  auto SlabIt = getAvailFullSlab();
494  auto *FreeSlab = (*SlabIt)->getSlab();
495  auto It =
496  UnavailableSlabs.insert(UnavailableSlabs.begin(), std::move(*SlabIt));
497  AvailableSlabs.erase(SlabIt);
498  (*It)->setIterator(It);
499  return FreeSlab;
500 }
501 
503  std::lock_guard<std::mutex> Lg(BucketLock);
504  auto SlabIter = Slab.getIterator();
505  assert(SlabIter != UnavailableSlabs.end());
506  if (CanPool()) {
507  auto It =
508  AvailableSlabs.insert(AvailableSlabs.begin(), std::move(*SlabIter));
509  UnavailableSlabs.erase(SlabIter);
510  (*It)->setIterator(It);
511  } else {
512  UnavailableSlabs.erase(SlabIter);
513  }
514 }
515 
516 auto Bucket::getAvailSlab() -> decltype(AvailableSlabs.begin()) {
517  if (AvailableSlabs.size() == 0) {
518  auto It = AvailableSlabs.insert(AvailableSlabs.begin(),
519  std::make_unique<Slab>(*this));
520  (*It)->setIterator(It);
521  }
522 
523  return AvailableSlabs.begin();
524 }
525 
527  std::lock_guard<std::mutex> Lg(BucketLock);
528 
529  auto SlabIt = getAvailSlab();
530  auto *FreeChunk = (*SlabIt)->getChunk();
531 
532  // If the slab is full, move it to unavailable slabs and update its iterator
533  if (!((*SlabIt)->hasAvail())) {
534  auto It =
535  UnavailableSlabs.insert(UnavailableSlabs.begin(), std::move(*SlabIt));
536  AvailableSlabs.erase(SlabIt);
537  (*It)->setIterator(It);
538  }
539 
540  return FreeChunk;
541 }
542 
543 void Bucket::freeChunk(void *Ptr, Slab &Slab) {
544  std::lock_guard<std::mutex> Lg(BucketLock);
545 
546  Slab.freeChunk(Ptr);
547 
548  onFreeChunk(Slab);
549 }
550 
551 // The lock must be acquired before calling this method
552 void Bucket::onFreeChunk(Slab &Slab) {
553  // In case if the slab was previously full and now has 1 available
554  // chunk, it should be moved to the list of available slabs
555  if (Slab.getNumAllocated() == (Slab.getNumChunks() - 1)) {
556  auto SlabIter = Slab.getIterator();
557  assert(SlabIter != UnavailableSlabs.end());
558 
559  auto It =
560  AvailableSlabs.insert(AvailableSlabs.begin(), std::move(*SlabIter));
561  UnavailableSlabs.erase(SlabIter);
562 
563  (*It)->setIterator(It);
564  }
565 
566  // If slab has no chunks allocated we could pool it if capacity is available
567  // or release it to the system.
568  if (Slab.getNumAllocated() == 0) {
569  // Pool has no space so release it.
570  if (!CanPool()) {
571  // Remove the slab when all the chunks from it are deallocated
572  // Note: since the slab is stored as unique_ptr, just remove it from
573  // the list to remove the list to destroy the object
574  auto It = Slab.getIterator();
575  assert(It != AvailableSlabs.end());
576 
577  AvailableSlabs.erase(It);
578  }
579  }
580 }
581 
583  std::lock_guard<sycl::detail::SpinLock> Lock{settings::PoolLock};
584  size_t NewFreeSlabsInBucket = AvailableSlabs.size() + 1;
585  if (settings::USMPoolSettings.Capacity >= NewFreeSlabsInBucket) {
586  size_t NewPoolSize = settings::USMPoolSettings.CurPoolSize + Size;
587  if (settings::USMPoolSettings.MaxPoolSize >= NewPoolSize) {
589  return true;
590  }
591  }
592  return false;
593 }
594 
595 SystemMemory &Bucket::getMemHandle() { return OwnAllocCtx.getMemHandle(); }
596 
598  if (Size == 0)
599  return nullptr;
600 
601  if (Size > settings::USMPoolSettings.MaxPoolableSize) {
602  return getMemHandle().allocate(Size);
603  }
604 
605  auto &Bucket = findBucket(Size);
606  if (Size > settings::ChunkCutOff) {
607  return Bucket.getSlab();
608  }
609 
610  return Bucket.getChunk();
611 }
612 
613 void *USMAllocContext::USMAllocImpl::allocate(size_t Size, size_t Alignment) {
614  if (Size == 0)
615  return nullptr;
616 
617  if (Alignment <= 1)
618  return allocate(Size);
619 
620  size_t AlignedSize = (Size > 1) ? AlignUp(Size, Alignment) : Alignment;
621 
622  // Check if requested allocation size is within pooling limit.
623  // If not, just request aligned pointer from the system.
624  if (AlignedSize > settings::USMPoolSettings.MaxPoolableSize) {
625  return getMemHandle().allocate(Size, Alignment);
626  }
627 
628  void *Ptr;
629  auto &Bucket = findBucket(AlignedSize);
630  if (AlignedSize > settings::ChunkCutOff) {
631  Ptr = Bucket.getSlab();
632  } else {
633  Ptr = Bucket.getChunk();
634  }
635  return AlignPtrUp(Ptr, Alignment);
636 }
637 
638 Bucket &USMAllocContext::USMAllocImpl::findBucket(size_t Size) {
639  assert(Size <= settings::CutOff && "Unexpected size");
640 
641  auto It = std::find_if(
642  Buckets.begin(), Buckets.end(),
643  [Size](const auto &BucketPtr) { return BucketPtr->getSize() >= Size; });
644 
645  assert((It != Buckets.end()) && "Bucket should always exist");
646 
647  return *(*It);
648 }
649 
651  auto *SlabPtr = AlignPtrDown(Ptr, settings::SlabMinSize);
652 
653  // Lock the map on read
654  std::shared_lock<std::shared_timed_mutex> Lk(getKnownSlabsMapLock());
655 
656  auto Slabs = getKnownSlabs().equal_range(SlabPtr);
657  if (Slabs.first == Slabs.second) {
658  Lk.unlock();
659  getMemHandle().deallocate(Ptr);
660  return;
661  }
662 
663  for (auto It = Slabs.first; It != Slabs.second; ++It) {
664  // The slab object won't be deleted until it's removed from the map which is
665  // protected by the lock, so it's safe to access it here.
666  auto &Slab = It->second;
667  if (Ptr >= Slab.getPtr() && Ptr < Slab.getEnd()) {
668  // Unlock the map before freeing the chunk, it may be locked on write
669  // there
670  Lk.unlock();
671  auto &Bucket = Slab.getBucket();
673  Bucket.freeChunk(Ptr, Slab);
674  } else {
676  }
677  return;
678  }
679  }
680 
681  Lk.unlock();
682  // There is a rare case when we have a pointer from system allocation next
683  // to some slab with an entry in the map. So we find a slab
684  // but the range checks fail.
685  getMemHandle().deallocate(Ptr);
686 }
687 
688 USMAllocContext::USMAllocContext(std::unique_ptr<SystemMemory> MemHandle)
689  : pImpl(std::make_unique<USMAllocImpl>(std::move(MemHandle))) {}
690 
691 void *USMAllocContext::allocate(size_t size) { return pImpl->allocate(size); }
692 
693 void *USMAllocContext::allocate(size_t size, size_t alignment) {
694  return pImpl->allocate(size, alignment);
695 }
696 
697 void USMAllocContext::deallocate(void *ptr) { return pImpl->deallocate(ptr); }
698 
699 // Define destructor for its usage with unique_ptr
USMAllocContext::USMAllocImpl::getKnownSlabs
std::unordered_multimap< void *, Slab & > & getKnownSlabs()
Definition: usm_allocator.cpp:335
operator<<
std::ostream & operator<<(std::ostream &Os, const Slab &Slab)
Definition: usm_allocator.cpp:347
USMAllocContext::USMAllocImpl::getMemHandle
SystemMemory & getMemHandle()
Definition: usm_allocator.cpp:332
Slab::Slab
Slab(Bucket &)
Definition: usm_allocator.cpp:353
settings::SetLimits
Definition: usm_allocator.cpp:110
Slab::getPtr
void * getPtr() const
Definition: usm_allocator.cpp:236
Slab
Definition: usm_allocator.cpp:187
Bucket::getSize
size_t getSize() const
Definition: usm_allocator.cpp:279
settings::SlabMinSize
static constexpr size_t SlabMinSize
Definition: usm_allocator.cpp:44
Slab::getEnd
void * getEnd() const
Definition: usm_allocator.cpp:237
SystemMemory::allocate
virtual void * allocate(size_t size)=0
SystemMemory
Definition: usm_allocator.hpp:15
settings::generateBucketSizes
static constexpr BucketsArrayType generateBucketSizes()
Definition: usm_allocator.cpp:60
settings::USMPoolSettings
static class settings::SetLimits USMPoolSettings
settings::SetLimits::SetLimits
SetLimits()
Definition: usm_allocator.cpp:117
USMAllocContext::USMAllocImpl::deallocate
void deallocate(void *Ptr)
Definition: usm_allocator.cpp:650
Slab::getNumChunks
size_t getNumChunks() const
Definition: usm_allocator.cpp:242
Slab::getSlab
void * getSlab()
Definition: usm_allocator.cpp:400
usm_allocator.hpp
AlignPtrDown
static void * AlignPtrDown(void *Ptr, const size_t Alignment)
Definition: usm_allocator.cpp:156
USMAllocContext::USMAllocImpl::allocate
void * allocate(size_t Size, size_t Alignment)
Definition: usm_allocator.cpp:613
operator==
bool operator==(const Slab &Lhs, const Slab &Rhs)
Definition: usm_allocator.cpp:343
Bucket::freeChunk
void freeChunk(void *Ptr, Slab &Slab)
Definition: usm_allocator.cpp:543
Bucket
Definition: usm_allocator.cpp:252
USMAllocContext::deallocate
void deallocate(void *ptr)
Definition: usm_allocator.cpp:697
Bucket::Bucket
Bucket(size_t Sz, USMAllocContext::USMAllocImpl &AllocCtx)
Definition: usm_allocator.cpp:269
USMAllocContext::USMAllocContext
USMAllocContext(std::unique_ptr< SystemMemory > memHandle)
Definition: usm_allocator.cpp:688
USMAllocContext::allocate
void * allocate(size_t size)
Definition: usm_allocator.cpp:691
Slab::getChunk
void * getChunk()
Definition: usm_allocator.cpp:382
Slab::setIterator
void setIterator(ListIter It)
Definition: usm_allocator.cpp:225
Bucket::freeSlab
void freeSlab(Slab &Slab)
Definition: usm_allocator.cpp:502
USMAllocContext::USMAllocImpl
Definition: usm_allocator.cpp:305
Bucket::CanPool
bool CanPool()
Definition: usm_allocator.cpp:582
AlignUp
static size_t AlignUp(size_t Val, size_t Alignment)
Definition: usm_allocator.cpp:174
spinlock.hpp
settings::ChunkCutOff
static constexpr size_t ChunkCutOff
Definition: usm_allocator.cpp:49
USMAllocContext::~USMAllocContext
~USMAllocContext()
settings::CutOff
static constexpr size_t CutOff
Definition: usm_allocator.cpp:53
Bucket::getUsmAllocCtx
USMAllocContext::USMAllocImpl & getUsmAllocCtx()
Definition: usm_allocator.cpp:288
settings::BucketsArrayType
std::array< size_t, 53 > BucketsArrayType
Definition: usm_allocator.cpp:57
Slab::getChunkSize
size_t getChunkSize() const
Definition: usm_allocator.cpp:405
settings
Definition: usm_allocator.cpp:42
Slab::~Slab
~Slab()
Definition: usm_allocator.cpp:365
Slab::hasAvail
bool hasAvail()
Definition: usm_allocator.cpp:473
std
Definition: accessor.hpp:2356
Bucket::getMemHandle
SystemMemory & getMemHandle()
Definition: usm_allocator.cpp:595
Slab::getBucket
Bucket & getBucket()
Definition: usm_allocator.cpp:402
settings::PoolLock
static sycl::detail::SpinLock PoolLock
Definition: usm_allocator.cpp:105
settings::SetLimits::CurPoolSize
size_t CurPoolSize
Definition: usm_allocator.cpp:115
USMAllocContext
Definition: usm_allocator.hpp:23
settings::SetLimits::MaxPoolSize
size_t MaxPoolSize
Definition: usm_allocator.cpp:114
P
#define P(n)
SystemMemory::deallocate
virtual void deallocate(void *ptr)=0
AlignPtrUp
static void * AlignPtrUp(void *Ptr, const size_t Alignment)
Definition: usm_allocator.cpp:163
USMAllocContext::USMAllocImpl::USMAllocImpl
USMAllocImpl(std::unique_ptr< SystemMemory > SystemMemHandle)
Definition: usm_allocator.cpp:318
Slab::getIterator
ListIter getIterator() const
Definition: usm_allocator.cpp:226
Bucket::getSlab
void * getSlab()
Definition: usm_allocator.cpp:490
Bucket::getChunk
void * getChunk()
Definition: usm_allocator.cpp:526
Slab::freeChunk
void freeChunk(void *Ptr)
Definition: usm_allocator.cpp:451
Slab::getNumAllocated
size_t getNumAllocated() const
Definition: usm_allocator.cpp:228
settings::BucketSizes
static constexpr BucketsArrayType BucketSizes
Definition: usm_allocator.cpp:101
settings::SetLimits::Capacity
size_t Capacity
Definition: usm_allocator.cpp:113
USMAllocContext::USMAllocImpl::getKnownSlabsMapLock
std::shared_timed_mutex & getKnownSlabsMapLock()
Definition: usm_allocator.cpp:334
settings::SetLimits::MaxPoolableSize
size_t MaxPoolableSize
Definition: usm_allocator.cpp:112