@@ -88,6 +88,9 @@ struct FirstSerialArena {
8888// used.
8989class PROTOBUF_EXPORT SerialArena {
9090 public:
91+ static constexpr size_t kBlockHeaderSize =
92+ ArenaAlignDefault::Ceil (sizeof (ArenaBlock));
93+
9194 void CleanupList ();
9295 size_t FreeStringBlocks () {
9396 // On the active block delete all strings skipping the unused instances.
@@ -273,6 +276,24 @@ class PROTOBUF_EXPORT SerialArena {
273276 std::vector<void *> PeekCleanupListForTesting ();
274277
275278 private:
279+ friend class ThreadSafeArena ;
280+
281+ // See comments for cached_blocks_.
282+ struct CachedBlock {
283+ // Simple linked list.
284+ CachedBlock* next;
285+ };
286+
287+ static constexpr ptrdiff_t kPrefetchForwardsDegree = ABSL_CACHELINE_SIZE * 16 ;
288+ static constexpr ptrdiff_t kPrefetchBackwardsDegree = ABSL_CACHELINE_SIZE * 6 ;
289+
290+ // Constructor is private as only New() should be used.
291+ inline SerialArena (ArenaBlock* b, ThreadSafeArena& parent);
292+
293+ // Constructors to handle the first SerialArena.
294+ inline explicit SerialArena (ThreadSafeArena& parent);
295+ inline SerialArena (FirstSerialArena, ArenaBlock* b, ThreadSafeArena& parent);
296+
276297 bool MaybeAllocateString (void *& p);
277298 ABSL_ATTRIBUTE_RETURNS_NONNULL void * AllocateFromStringBlockFallback ();
278299
@@ -288,9 +309,6 @@ class PROTOBUF_EXPORT SerialArena {
288309 cleanup::CreateNode (tag, limit_, elem, destructor);
289310 }
290311
291- static constexpr ptrdiff_t kPrefetchForwardsDegree = ABSL_CACHELINE_SIZE * 16 ;
292- static constexpr ptrdiff_t kPrefetchBackwardsDegree = ABSL_CACHELINE_SIZE * 6 ;
293-
294312 // Prefetch the next kPrefetchForwardsDegree bytes after `prefetch_ptr_` and
295313 // up to `prefetch_limit_`, if `next` is within kPrefetchForwardsDegree bytes
296314 // of `prefetch_ptr_`.
@@ -335,9 +353,6 @@ class PROTOBUF_EXPORT SerialArena {
335353 }
336354 }
337355
338- private:
339- friend class ThreadSafeArena ;
340-
341356 // Creates a new SerialArena inside mem using the remaining memory as for
342357 // future allocations.
343358 // The `parent` arena must outlive the serial arena, which is guaranteed
@@ -362,6 +377,29 @@ class PROTOBUF_EXPORT SerialArena {
362377 std::memory_order_relaxed);
363378 }
364379
380+ // Helper getters/setters to handle relaxed operations on atomic variables.
381+ ArenaBlock* head () { return head_.load (std::memory_order_relaxed); }
382+ const ArenaBlock* head () const {
383+ return head_.load (std::memory_order_relaxed);
384+ }
385+
386+ char * ptr () { return ptr_.load (std::memory_order_relaxed); }
387+ const char * ptr () const { return ptr_.load (std::memory_order_relaxed); }
388+ void set_ptr (char * ptr) { return ptr_.store (ptr, std::memory_order_relaxed); }
389+ PROTOBUF_ALWAYS_INLINE void set_range (char * ptr, char * limit) {
390+ set_ptr (ptr);
391+ prefetch_ptr_ = ptr;
392+ limit_ = limit;
393+ prefetch_limit_ = limit;
394+ }
395+
396+ void * AllocateAlignedFallback (size_t n);
397+ void * AllocateAlignedWithCleanupFallback (size_t n, size_t align,
398+ void (*destructor)(void *));
399+ void AddCleanupFallback (void * elem, void (*destructor)(void *));
400+ inline void AllocateNewBlock (size_t n);
401+ inline void Init (ArenaBlock* b, size_t offset);
402+
365403 // Members are declared here to track sizeof(SerialArena) and hotness
366404 // centrally. They are (roughly) laid out in descending order of hotness.
367405
@@ -397,46 +435,8 @@ class PROTOBUF_EXPORT SerialArena {
397435 // this free list.
398436 // `cached_blocks_[i]` points to the free list for blocks of size `8+2^(i+3)`.
399437 // The array of freelists is grown when needed in `ReturnArrayMemory()`.
400- struct CachedBlock {
401- // Simple linked list.
402- CachedBlock* next;
403- };
404438 uint8_t cached_block_length_ = 0 ;
405439 CachedBlock** cached_blocks_ = nullptr ;
406-
407- // Helper getters/setters to handle relaxed operations on atomic variables.
408- ArenaBlock* head () { return head_.load (std::memory_order_relaxed); }
409- const ArenaBlock* head () const {
410- return head_.load (std::memory_order_relaxed);
411- }
412-
413- char * ptr () { return ptr_.load (std::memory_order_relaxed); }
414- const char * ptr () const { return ptr_.load (std::memory_order_relaxed); }
415- void set_ptr (char * ptr) { return ptr_.store (ptr, std::memory_order_relaxed); }
416- PROTOBUF_ALWAYS_INLINE void set_range (char * ptr, char * limit) {
417- set_ptr (ptr);
418- prefetch_ptr_ = ptr;
419- limit_ = limit;
420- prefetch_limit_ = limit;
421- }
422-
423- // Constructor is private as only New() should be used.
424- inline SerialArena (ArenaBlock* b, ThreadSafeArena& parent);
425-
426- // Constructors to handle the first SerialArena.
427- inline explicit SerialArena (ThreadSafeArena& parent);
428- inline SerialArena (FirstSerialArena, ArenaBlock* b, ThreadSafeArena& parent);
429-
430- void * AllocateAlignedFallback (size_t n);
431- void * AllocateAlignedWithCleanupFallback (size_t n, size_t align,
432- void (*destructor)(void *));
433- void AddCleanupFallback (void * elem, void (*destructor)(void *));
434- inline void AllocateNewBlock (size_t n);
435- inline void Init (ArenaBlock* b, size_t offset);
436-
437- public:
438- static constexpr size_t kBlockHeaderSize =
439- ArenaAlignDefault::Ceil (sizeof (ArenaBlock));
440440};
441441
442442inline PROTOBUF_ALWAYS_INLINE bool SerialArena::MaybeAllocateString (void *& p) {
0 commit comments