[libcxx] adds size-based __split_buffer representation to unstable ABI
#139632
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
**tl;dr** We can significantly improve the runtime performance of `std::vector` by changing its representation from three pointers to one pointer and two integers. This document explains the details of this change, along with the justifications for making it. See the [RFC] for more information. `vector` depends on `__split_buffer` for inserting elements. Changing `__split_buffer` to match `vector`'s representation simplifies the model, as it eliminates the need to convert between two different representations of a contiguous buffer in the same configuration of libc++. [RFC]: TODO
llvmbot
added
the
libc++
|
@llvm/pr-subscribers-libcxx Author: Christopher Di Bella (cjdb) Changestl;dr We can significantly improve the runtime performance of
Patch is 41.17 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/139632.diff 3 Files Affected:
diff --git a/libcxx/include/__split_buffer b/libcxx/include/__split_buffer
index 21e58f4abc6b3..9710dfec774f7 100644
--- a/libcxx/include/__split_buffer
+++ b/libcxx/include/__split_buffer
@@ -13,10 +13,12 @@
#include <__algorithm/max.h>
#include <__algorithm/move.h>
#include <__algorithm/move_backward.h>
+#include <__assert>
#include <__config>
#include <__iterator/distance.h>
#include <__iterator/iterator_traits.h>
#include <__iterator/move_iterator.h>
+#include <__memory/addressof.h>
#include <__memory/allocate_at_least.h>
#include <__memory/allocator.h>
#include <__memory/allocator_traits.h>
@@ -78,23 +80,260 @@ public:
__split_buffer,
void>;
- pointer __first_;
- pointer __begin_;
- pointer __end_;
- _LIBCPP_COMPRESSED_PAIR(pointer, __cap_, allocator_type, __alloc_);
+ struct __data {
+ pointer __first_ = nullptr;
+ pointer __begin_ = nullptr;
+#ifndef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ pointer __end_ = nullptr;
+ _LIBCPP_COMPRESSED_PAIR(pointer, __cap_ = nullptr, allocator_type, __alloc_);
+#else
+ size_type __size_ = 0;
+ _LIBCPP_COMPRESSED_PAIR(size_type, __cap_ = 0, allocator_type, __alloc_);
+#endif
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __data() = default;
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __data(const allocator_type& __alloc)
+ : __alloc_(__alloc)
+ {}
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer first() noexcept {
+ return __first_;
+ }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer first() const noexcept {
+ return __first_;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer begin() noexcept {
+ return __begin_;
+ }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer begin() const noexcept {
+ return __begin_;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer end() noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ return __begin_ + __size_;
+#else
+ return __end_;
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer end() const noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ return __begin_ + __size_;
+#else
+ return __end_;
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ return __size_;
+#else
+ return static_cast<size_type>(__end_ - __begin_);
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ return __size_ == 0;
+#else
+ return __begin_ == __end_;
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ return __cap_;
+#else
+ return static_cast<size_type>(__cap_ - __first_);
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __capacity_as_pointer() noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ return __first_ + __cap_;
+#else
+ return __cap_;
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __capacity_as_pointer() const noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ return __first_ + __cap_;
+#else
+ return __cap_;
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __update_begin(pointer __new_begin) noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ __size_ -= __new_begin - __begin_;
+#else
+ // TODO: explain why there isn't a pointer-based analogue
+#endif
+
+ __begin_ = __new_begin;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __front_spare() const noexcept {
+ return static_cast<size_type>(__begin_ - __first_);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __update_sentinel(pointer __new_end) noexcept {
+ _LIBCPP_ASSERT(__first_ <= __new_end, "__new_end cannot precede __first_");
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ __size_ += __new_end - end();
+#else
+ __end_ = __new_end;
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __update_sentinel(size_type __new_size) noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ __size_ = __new_size;
+#else
+ __end_ = __begin_ + __new_size;
+#endif
+ }
- __split_buffer(const __split_buffer&) = delete;
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __update_capacity(size_type __new_capacity) noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ __cap_ = __new_capacity;
+#else
+ __cap_ = __first_ + __new_capacity;
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __back_spare() const noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ // `__cap_ - __end_` tells us the total number of spares when in size-mode. We need to remove
+ // the __front_spare from the count.
+ return __cap_ - __size_ - __front_spare();
+#else
+ return static_cast<size_type>(__cap_ - __end_);
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ return __begin_[__size_ - 1];
+#else
+ return *(__end_ - 1);
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ return __begin_[__size_ - 1];
+#else
+ return *(__end_ - 1);
+#endif
+ }
+
+ template<class _Data2>
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __swap_without_allocator(_Data2& __other) noexcept {
+ std::swap(__first_, __other.__first_);
+ std::swap(__begin_, __other.__begin_);
+ std::swap(__cap_, __other.__cap_);
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ std::swap(__size_, __other.__size_);
+#else
+ std::swap(__end_, __other.__end_);
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void swap(__data& __other) noexcept {
+ __swap_without_allocator(__other);
+ std::__swap_allocator(__alloc_, __other.__alloc_);
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const noexcept {
+ if (__first_ == nullptr) {
+ if (__begin_ != nullptr) {
+ return false;
+ }
+
+ if (!empty()) {
+ return false;
+ }
+
+ if (capacity() != 0) {
+ return false;
+ }
+
+ return true;
+ }
+
+ if (__begin_ < __first_) {
+ return false;
+ }
+
+ if (capacity() < size()) {
+ return false;
+ }
+
+ if (end() < __begin_) {
+ return false;
+ }
+
+ return true;
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __is_full() const noexcept {
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ return __size_ == __cap_;
+#else
+ return __end_ == __cap_;
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __reset() noexcept {
+ __first_ = nullptr;
+ __begin_ = nullptr;
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ __size_ = 0;
+ __cap_ = 0;
+#else
+ __end_ = nullptr;
+ __cap_ = nullptr;
+#endif
+ }
+
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_without_alloc(__data const& __other)
+ noexcept(is_nothrow_copy_assignable<pointer>::value)
+ {
+ __first_ = __other.__first_;
+ __begin_ = __other.__begin_;
+ __cap_ = __other.__cap_;
+#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
+ __size_ = __other.__size_;
+#else
+ __end_ = __other.__end_;
+#endif
+ }
+ };
+
+ __data __data_;
+
+ __split_buffer(const __split_buffer&) = delete;
__split_buffer& operator=(const __split_buffer&) = delete;
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer()
- _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
- : __first_(nullptr), __begin_(nullptr), __end_(nullptr), __cap_(nullptr) {}
+ _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
+ : __data_{}
+ {}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(__alloc_rr& __a)
- : __first_(nullptr), __begin_(nullptr), __end_(nullptr), __cap_(nullptr), __alloc_(__a) {}
+ _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
+ : __data_(__a)
+ {}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(const __alloc_rr& __a)
- : __first_(nullptr), __begin_(nullptr), __end_(nullptr), __cap_(nullptr), __alloc_(__a) {}
+ _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
+ : __data_(__a)
+ {}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
__split_buffer(size_type __cap, size_type __start, __alloc_rr& __a);
@@ -111,36 +350,22 @@ public:
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~__split_buffer();
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __begin_; }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __begin_; }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __data_.begin(); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __data_.begin(); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __data_.end(); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __data_.end(); }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __end_; }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __end_; }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __destruct_at_end(__data_.__begin_); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const { return __data_.size(); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const { return __data_.empty(); }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __destruct_at_end(__begin_); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const { return __data_.capacity(); }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type size() const {
- return static_cast<size_type>(__end_ - __begin_);
- }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() { return *__data_.__begin_; }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const { return *__data_.__begin_; }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool empty() const { return __end_ == __begin_; }
-
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type capacity() const {
- return static_cast<size_type>(__cap_ - __first_);
- }
-
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __front_spare() const {
- return static_cast<size_type>(__begin_ - __first_);
- }
-
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI size_type __back_spare() const {
- return static_cast<size_type>(__cap_ - __end_);
- }
-
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference front() { return *__begin_; }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference front() const { return *__begin_; }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() { return *(__end_ - 1); }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const { return *(__end_ - 1); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI reference back() { return __data_.back(); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_reference back() const { return __data_.back(); }
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void shrink_to_fit() _NOEXCEPT;
@@ -149,8 +374,8 @@ public:
template <class... _Args>
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args);
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_front() { __destruct_at_begin(__begin_ + 1); }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_back() { __destruct_at_end(__end_ - 1); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_front() { __destruct_at_begin(__data_.begin() + 1); }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void pop_back() { __destruct_at_end(__data_.end() - 1); }
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n);
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __construct_at_end(size_type __n, const_reference __x);
@@ -185,66 +410,52 @@ public:
_NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__alloc_rr>);
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const;
-
private:
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__split_buffer& __c, true_type)
_NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
- __alloc_ = std::move(__c.__alloc_);
+ __data_.__alloc_ = std::move(__c.__data_.__alloc_);
}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__split_buffer&, false_type) _NOEXCEPT {}
struct _ConstructTransaction {
_LIBCPP_CONSTEXPR_SINCE_CXX20
- _LIBCPP_HIDE_FROM_ABI explicit _ConstructTransaction(pointer* __p, size_type __n) _NOEXCEPT
- : __pos_(*__p),
- __end_(*__p + __n),
- __dest_(__p) {}
+ _LIBCPP_HIDE_FROM_ABI explicit _ConstructTransaction(__split_buffer* __parent, pointer __p, size_type __n) noexcept
+ : __pos_(__p),
+ __end_(__p + __n),
+ __parent_(__parent) {}
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~_ConstructTransaction() { *__dest_ = __pos_; }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~_ConstructTransaction() {
+ __parent_->__data_.__update_sentinel(__pos_);
+ }
pointer __pos_;
const pointer __end_;
private:
- pointer* __dest_;
+ __split_buffer* __parent_;
};
};
template <class _Tp, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 bool __split_buffer<_Tp, _Allocator>::__invariants() const {
- if (__first_ == nullptr) {
- if (__begin_ != nullptr)
- return false;
- if (__end_ != nullptr)
- return false;
- if (__cap_ != nullptr)
- return false;
- } else {
- if (__begin_ < __first_)
- return false;
- if (__end_ < __begin_)
- return false;
- if (__cap_ < __end_)
- return false;
- }
- return true;
+ return __data_.__invariants();
}
-// Default constructs __n objects starting at __end_
+// Default constructs __n objects starting at `__begin_ + size()`
// throws if construction throws
// Precondition: __n > 0
// Precondition: size() + __n <= capacity()
// Postcondition: size() == size() + __n
template <class _Tp, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::__construct_at_end(size_type __n) {
- _ConstructTransaction __tx(std::addressof(this->__end_), __n);
+ _ConstructTransaction __tx(this, __data_.end(), __n);
for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_) {
- __alloc_traits::construct(__alloc_, std::__to_address(__tx.__pos_));
+ __alloc_traits::construct(__data_.__alloc_, std::__to_address(__tx.__pos_));
}
}
-// Copy constructs __n objects starting at __end_ from __x
+// Copy constructs __n objects starting at `__begin_ + size()` from __x
// throws if construction throws
// Precondition: __n > 0
// Precondition: size() + __n <= capacity()
@@ -253,30 +464,35 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::__construct_
template <class _Tp, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 void
__split_buffer<_Tp, _Allocator>::__construct_at_end(size_type __n, const_reference __x) {
- _ConstructTransaction __tx(std::addressof(this->__end_), __n);
+ _ConstructTransaction __tx(this, __data_.end(), __n);
for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_) {
- __alloc_traits::construct(__alloc_, std::__to_address(__tx.__pos_), __x);
+ __alloc_traits::construct(__data_.__alloc_, std::__to_address(__tx.__pos_), __x);
}
}
-template <class _Tp, class _Allocator>
-template <class _Iterator, class _Sentinel>
+template<class _Tp, class _Allocator>
+template<class _Iterator, class _Sentinel>
_LIBCPP_CONSTEXPR_SINCE_CXX20 void
__split_buffer<_Tp, _Allocator>::__construct_at_end_with_sentinel(_Iterator __first, _Sentinel __last) {
- __alloc_rr& __a = __alloc_;
+ __alloc_rr& __a = __data_.__alloc_;
for (; __first != __last; ++__first) {
- if (__end_ == __cap_) {
- size_type __old_cap = __cap_ - __first_;
+ if (__data_.__back_spare() == 0) {
+ size_type __old_cap = __data_.capacity();
size_type __new_cap = std::max<size_type>(2 * __old_cap, 8);
__split_buffer __buf(__new_cap, 0, __a);
- for (pointer __p = __begin_; __p != __end_; ++__p, (void)++__buf.__end_)
- __alloc_traits::construct(__buf.__alloc_, std::__to_address(__buf.__end_), std::move(*__p));
+ pointer __buf_end = __buf.__data_.end();
+ pointer __end = __data_.end();
+ for (pointer __p = __data_.__begin_; __p != __end; ++__p, (void)++__buf_end)
+ __alloc_traits::construct(__buf.__data_.__alloc_, std::__to_address(__buf_end), std::move(*__p));
+ __buf.__data_.__update_sentinel(__buf_end);
swap(__buf);
}
- __alloc_traits::construct(__a, std::__to_address(this->__end_), *__first);
- ++this->__end_;
+
+ __alloc_traits::construct(__a, std::__to_address(__data_.end()), *__first);
+ __data_.__update_sentinel(size() + 1);
}
}
+
template <class _Tp, class _Allocator>
template <class _ForwardIterator, __enable_if_t<__has_forward_iterator_category<_ForwardIterator>::value, int> >
_LIBCPP_CONSTEXPR_SINCE_CXX20 void
@@ -288,92 +504,82 @@ template <class _Tp, class _Allocator>
template <class _ForwardIterator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 void
__split_buffer<_Tp, _Allocator>::__construct_at_end_with_size(_ForwardIterator __first, size_type __n) {
- _ConstructTransaction __tx(std::addressof(this->__end_), __n);
+ _ConstructTransaction __tx(this, __data_.end(), __n);
for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_, (void)++__first) {
- __alloc_traits::construct(__alloc_, std::__to_address(__tx.__pos_), *__first);
+ __alloc_traits::construct(__data_.__alloc_, std::__to_address(__tx.__pos_), *__first);
}
}
template <class _Tp, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
__split_buffer<_Tp, _Allocator>::__destruct_at_begin(pointer __new_begin, false_type) {
- while (__begin_ != __new_begin)
- __alloc_traits::destroy(__alloc_, std::__to_address(__begin_++));
+ pointer __begin = __data_.__begin_;
+ while (__begin != __new_begin)
+ __alloc_traits::destroy(__data_.__alloc_, std::__to_address(__begin++));
+ __data_.__update_begin(__begin);
}
template <class _Tp, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
__split_buffer<_Tp, _Allocator>::__destruct_at_begin(pointer __new_begin, true_type) {
- __begin_ = __new_begin;
+ __data_.__update_begin(__new_begin);
}
template <class _Tp, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
__split_buffer<_Tp, _Allocator>::__destruct_at_end(pointer __new_last, false_type) _NOEXCEPT {
- while (__new_last != __end_)
- __alloc_traits::destroy(__alloc_, std::__to_address(--__end_));
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
-__split_buffer<_Tp, _Allocator>::__destruct_at_end(pointer __new_last, true_type) _NOEXCEPT {
- __end_ = __new_last;
+ pointer __end = __data_.end();
+ while (__new_last != __end)
+ __alloc_traits::destroy(__data_.__alloc_, std::__to_address(--__end));
+ __data_.__update_sentinel(__end);
}
template <class _Tp, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20
__split_buffer<_Tp, _Allocator>::__split_buffer(size_type __cap, size_type __start, __alloc_rr...
[truncated]
|
You can test this locally with the following command:git-clang-format --diff HEAD~1 HEAD --extensions ,h -- libcxx/include/__split_buffer libcxx/include/__vector/vector.h libcxx/include/dequeView the diff from clang-format here.diff --git a/libcxx/include/__split_buffer b/libcxx/include/__split_buffer
index 0cd1a0e36..0d8baf89e 100644
--- a/libcxx/include/__split_buffer
+++ b/libcxx/include/__split_buffer
@@ -94,22 +94,13 @@ public:
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __data() = default;
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __data(const allocator_type& __alloc)
- : __alloc_(__alloc)
- {}
+ : __alloc_(__alloc) {}
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer first() _NOEXCEPT {
- return __first_;
- }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer first() const _NOEXCEPT {
- return __first_;
- }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer first() _NOEXCEPT { return __first_; }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer first() const _NOEXCEPT { return __first_; }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer begin() _NOEXCEPT {
- return __begin_;
- }
- _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer begin() const _NOEXCEPT {
- return __begin_;
- }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer begin() _NOEXCEPT { return __begin_; }
+ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI const_pointer begin() const _NOEXCEPT { return __begin_; }
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer end() _NOEXCEPT {
#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
@@ -232,7 +223,7 @@ public:
#endif
}
- template<class _Data2>
+ template <class _Data2>
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __swap_without_allocator(_Data2& __other) _NOEXCEPT {
std::swap(__first_, __other.__first_);
std::swap(__begin_, __other.__begin_);
@@ -294,7 +285,7 @@ public:
__begin_ = nullptr;
#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
__size_ = 0;
- __cap_ = 0;
+ __cap_ = 0;
#else
__end_ = nullptr;
__cap_ = nullptr;
@@ -302,11 +293,10 @@ public:
}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_without_alloc(__data const& __other)
- _NOEXCEPT_(is_nothrow_copy_assignable<pointer>::value)
- {
+ _NOEXCEPT_(is_nothrow_copy_assignable<pointer>::value) {
__first_ = __other.__first_;
__begin_ = __other.__begin_;
- __cap_ = __other.__cap_;
+ __cap_ = __other.__cap_;
#ifdef _LIBCPP_ABI_SIZE_BASED_VECTOR
__size_ = __other.__size_;
#else
@@ -317,23 +307,20 @@ public:
__data __data_;
- __split_buffer(const __split_buffer&) = delete;
+ __split_buffer(const __split_buffer&) = delete;
__split_buffer& operator=(const __split_buffer&) = delete;
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __split_buffer()
- _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
- : __data_()
- {}
+ _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
+ : __data_() {}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(__alloc_rr& __a)
- _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
- : __data_(__a)
- {}
+ _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
+ : __data_(__a) {}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI explicit __split_buffer(const __alloc_rr& __a)
- _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
- : __data_(__a)
- {}
+ _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
+ : __data_(__a) {}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
__split_buffer(size_type __cap, size_type __start, __alloc_rr& __a);
@@ -410,6 +397,7 @@ public:
_NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<__alloc_rr>);
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __invariants() const;
+
private:
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__split_buffer& __c, true_type)
_NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value) {
@@ -421,9 +409,9 @@ private:
struct _ConstructTransaction {
_LIBCPP_CONSTEXPR_SINCE_CXX20
_LIBCPP_HIDE_FROM_ABI explicit _ConstructTransaction(__split_buffer* __parent, pointer __p, size_type __n) _NOEXCEPT
- : __pos_(__p),
- __end_(__p + __n),
- __parent_(__parent) {}
+ : __pos_(__p),
+ __end_(__p + __n),
+ __parent_(__parent) {}
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI ~_ConstructTransaction() {
__parent_->__data_.__update_sentinel(__pos_);
@@ -470,8 +458,8 @@ __split_buffer<_Tp, _Allocator>::__construct_at_end(size_type __n, const_referen
}
}
-template<class _Tp, class _Allocator>
-template<class _Iterator, class _Sentinel>
+template <class _Tp, class _Allocator>
+template <class _Iterator, class _Sentinel>
_LIBCPP_CONSTEXPR_SINCE_CXX20 void
__split_buffer<_Tp, _Allocator>::__construct_at_end_with_sentinel(_Iterator __first, _Sentinel __last) {
__alloc_rr& __a = __data_.__alloc_;
@@ -481,7 +469,7 @@ __split_buffer<_Tp, _Allocator>::__construct_at_end_with_sentinel(_Iterator __fi
size_type __new_cap = std::max<size_type>(2 * __old_cap, 8);
__split_buffer __buf(__new_cap, 0, __a);
pointer __buf_end = __buf.__data_.end();
- pointer __end = __data_.end();
+ pointer __end = __data_.end();
for (pointer __p = __data_.__begin_; __p != __end; ++__p, (void)++__buf_end)
__alloc_traits::construct(__buf.__data_.__alloc_, std::__to_address(__buf_end), std::move(*__p));
__buf.__data_.__update_sentinel(__buf_end);
@@ -537,12 +525,12 @@ __split_buffer<_Tp, _Allocator>::__destruct_at_end(pointer __new_last, false_typ
template <class _Tp, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20
__split_buffer<_Tp, _Allocator>::__split_buffer(size_type __cap, size_type __start, __alloc_rr& __a)
- : __data_(__a) {
+ : __data_(__a) {
_LIBCPP_ASSERT(__cap >= __start, "can't have a start point outside the capacity");
if (__cap > 0) {
auto __allocation = std::__allocate_at_least(__data_.__alloc_, __cap);
- __data_.__first_ = __allocation.ptr;
- __cap = __allocation.count;
+ __data_.__first_ = __allocation.ptr;
+ __cap = __allocation.count;
}
__data_.__begin_ = __data_.__first_ + __start;
@@ -576,8 +564,8 @@ __split_buffer<_Tp, _Allocator>::__split_buffer(__split_buffer&& __c, const __al
__c.__data_.__reset();
} else {
auto __allocation = std::__allocate_at_least(__data_.__alloc_, __c.size());
- __data_.__first_ = __allocation.ptr;
- __data_.__begin_ = __data_.__first_;
+ __data_.__first_ = __allocation.ptr;
+ __data_.__begin_ = __data_.__first_;
__data_.__update_sentinel(__data_.__first_);
__data_.__update_capacity(__allocation.count);
typedef move_iterator<iterator> _Ip;
@@ -655,8 +643,8 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::emplace_fron
#endif
}
-template<class _Tp, class _Allocator>
-template<class... _Args>
+template <class _Tp, class _Allocator>
+template <class... _Args>
_LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::emplace_back(_Args&&... __args) {
pointer __end = __data_.end();
if (__data_.__back_spare() == 0) {
@@ -664,7 +652,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::emplace_back
difference_type __d = __data_.__begin_ - __data_.__first_;
__d = (__d + 1) / 2;
__end = std::move(__data_.__begin_, __end, __data_.__begin_ - __d);
- __data_.__begin_ -= __d;
+ __data_.__begin_ -= __d;
__data_.__update_sentinel(__end);
} else {
size_type __c = std::max<size_type>(2 * capacity(), 1);
@@ -678,10 +666,9 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void __split_buffer<_Tp, _Allocator>::emplace_back
__data_.__update_sentinel(++__end);
}
-template<class _Tp, class _Allocator>
+template <class _Tp, class _Allocator>
_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI void
-swap(__split_buffer<_Tp, _Allocator>& __x, __split_buffer<_Tp, _Allocator>& __y) _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y)))
-{
+swap(__split_buffer<_Tp, _Allocator>& __x, __split_buffer<_Tp, _Allocator>& __y) _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
__x.swap(__y);
}
|
6 tasks
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
tl;dr We can significantly improve the runtime performance of
std::vectorby changing its representation from three pointers to one pointer and two integers. This document explains the details of this change, along with the justifications for making it. See the RFC for more information.vectordepends on__split_bufferfor inserting elements. Changing__split_bufferto matchvector's representation simplifies the model, as it eliminates the need to convert between two different representations of a contiguous buffer in the same configuration of libc++.