Skip to content

[18168355911] Improve lambda inlining when using for_each_enumerated #2704

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 15, 2025
Merged

[18168355911] Improve lambda inlining when using for_each_enumerated #2704

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
237945a
Improve lambda inlining when using `for_each_enumerated`
IvoDD Oct 13, 2025
61798e5
Apply lambda inlining for all uses of `for_each_enumerated`
IvoDD Oct 14, 2025
806b848
Improve inlining macro
IvoDD Oct 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cpp/arcticdb/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -532,6 +532,7 @@ set(arcticdb_srcs
util/decimal.cpp
util/error_code.cpp
util/global_lifetimes.cpp
util/lambda_inlining.hpp
util/memory_mapped_file.hpp
util/name_validation.cpp
util/offset_string.cpp
Expand All @@ -553,7 +554,7 @@ set(arcticdb_srcs
version/symbol_list.cpp
version/version_map_batch_methods.cpp
storage/s3/ec2_utils.cpp
)
)

add_library(arcticdb_core_object OBJECT ${arcticdb_srcs})

Expand Down
3 changes: 2 additions & 1 deletion cpp/arcticdb/arrow/arrow_handlers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <arcticdb/codec/encoding_sizes.hpp>
#include <arcticdb/codec/codec.hpp>
#include <arcticdb/util/decode_path_data.hpp>
#include <arcticdb/util/lambda_inlining.hpp>
#include <arcticdb/pipeline/column_mapping.hpp>
#include <arcticdb/column_store/string_pool.hpp>
#include <arcticdb/column_store/column_algorithms.hpp>
Expand Down Expand Up @@ -74,7 +75,7 @@ void ArrowStringHandler::convert_type(
// Benchmarks in benchmark_arrow_reads.cpp show a 20% speedup with this approach for a dense string column with few
// unique strings.
bool populate_inverted_bitset = !source_column.opt_sparse_map().has_value();
for_each_enumerated<ArcticStringColumnTag>(source_column, [&](const auto& en) {
for_each_enumerated<ArcticStringColumnTag>(source_column, [&] ARCTICDB_LAMBDA_INLINE(const auto& en) {
if (is_a_string(en.value())) {
auto [entry, is_emplaced] = unique_offsets.try_emplace(
en.value(), DictEntry{unique_offset_count, bytes, string_pool->get_const_view(en.value())}
Expand Down
3 changes: 2 additions & 1 deletion cpp/arcticdb/column_store/column_algorithms.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include <concepts>

#include <arcticdb/column_store/column.hpp>
#include <arcticdb/util/lambda_inlining.hpp>

namespace arcticdb {

Expand Down Expand Up @@ -148,7 +149,7 @@ static void transform(
util::BitSet::bulk_insert_iterator inserter(output_bitset);
arcticdb::for_each_enumerated<input_tdt>(
input_column,
[&inserter, f = std::forward<functor>(f)](auto enumerated_it) {
[&inserter, f = std::forward<functor>(f)] ARCTICDB_LAMBDA_INLINE(auto enumerated_it) {
if (f(enumerated_it.value())) {
inserter = enumerated_it.idx();
}
Expand Down
67 changes: 35 additions & 32 deletions cpp/arcticdb/processing/clause.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,42 +465,45 @@ std::vector<EntityId> AggregationClause::process(std::vector<EntityId>&& entity_
}
ssize_t previous_value_index = 0;

arcticdb::for_each_enumerated<typename col_type_info::TDT>(*col.column_, [&](auto enumerating_it) {
typename col_type_info::RawType val;
if constexpr (is_sequence_type(col_type_info::data_type)) {
auto offset = enumerating_it.value();
if (auto it = offset_to_group.find(offset); it != offset_to_group.end()) {
val = it->second;
} else {
std::optional<std::string_view> str = col.string_at_offset(offset);
if (str.has_value()) {
val = string_pool->get(*str, true).offset();
arcticdb::for_each_enumerated<typename col_type_info::TDT>(
*col.column_,
[&] ARCTICDB_LAMBDA_INLINE(auto enumerating_it) {
typename col_type_info::RawType val;
if constexpr (is_sequence_type(col_type_info::data_type)) {
auto offset = enumerating_it.value();
if (auto it = offset_to_group.find(offset); it != offset_to_group.end()) {
val = it->second;
} else {
std::optional<std::string_view> str = col.string_at_offset(offset);
if (str.has_value()) {
val = string_pool->get(*str, true).offset();
} else {
val = offset;
}
typename col_type_info::RawType val_copy(val);
offset_to_group.emplace(offset, val_copy);
}
} else {
val = offset;
val = enumerating_it.value();
}
typename col_type_info::RawType val_copy(val);
offset_to_group.emplace(offset, val_copy);
}
} else {
val = enumerating_it.value();
}

if (is_sparse) {
for (auto j = previous_value_index; j != enumerating_it.idx(); ++j) {
static constexpr size_t missing_value_group_id = 0;
*row_to_group_ptr++ = missing_value_group_id;
}
previous_value_index = enumerating_it.idx() + 1;
}
if (is_sparse) {
for (auto j = previous_value_index; j != enumerating_it.idx(); ++j) {
static constexpr size_t missing_value_group_id = 0;
*row_to_group_ptr++ = missing_value_group_id;
}
previous_value_index = enumerating_it.idx() + 1;
}

if (auto it = hash_to_group->find(val); it == hash_to_group->end()) {
*row_to_group_ptr++ = next_group_id;
auto group_id = next_group_id++;
hash_to_group->emplace(val, group_id);
} else {
*row_to_group_ptr++ = it->second;
}
});
if (auto it = hash_to_group->find(val); it == hash_to_group->end()) {
*row_to_group_ptr++ = next_group_id;
auto group_id = next_group_id++;
hash_to_group->emplace(val, group_id);
} else {
*row_to_group_ptr++ = it->second;
}
}
);

num_unique = next_group_id;
util::check(num_unique != 0, "Got zero unique values");
Expand Down
2 changes: 1 addition & 1 deletion cpp/arcticdb/processing/processing_unit.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ std::pair<std::vector<bucket_id>, std::vector<uint64_t>> get_buckets(
using TDT = typename Grouper::GrouperDescriptor;

if (col.column_->is_sparse()) {
arcticdb::for_each_enumerated<TDT>(*col.column_, [&](auto enumerating_it) {
arcticdb::for_each_enumerated<TDT>(*col.column_, [&] ARCTICDB_LAMBDA_INLINE(auto enumerating_it) {
auto opt_group = grouper.group(enumerating_it.value(), col.string_pool_);
if (ARCTICDB_LIKELY(opt_group.has_value())) {
auto bucket = bucketizer.bucket(*opt_group);
Expand Down
18 changes: 9 additions & 9 deletions cpp/arcticdb/processing/unsorted_aggregation.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ void SumAggregatorData::aggregate(
if constexpr (!is_sequence_type(col_type_info::data_type)) {
arcticdb::for_each_enumerated<typename col_type_info::TDT>(
*input_column.column_,
[&out, &groups](auto enumerating_it) {
[&out, &groups] ARCTICDB_LAMBDA_INLINE(auto enumerating_it) {
if constexpr (is_floating_point_type(col_type_info::data_type)) {
if (ARCTICDB_LIKELY(!std::isnan(enumerating_it.value()))) {
out[groups[enumerating_it.idx()]] += RawType(enumerating_it.value());
Expand Down Expand Up @@ -299,7 +299,7 @@ void aggregate_impl(
if constexpr (!is_sequence_type(col_type_info::data_type)) {
arcticdb::for_each_enumerated<typename col_type_info::TDT>(
*input_column->column_,
[&](auto row) {
[&] ARCTICDB_LAMBDA_INLINE(auto row) {
auto& group_entry = out[row_to_group[row.idx()]];
const auto& current_value = GlobalRawType(row.value());
if constexpr (std::is_floating_point_v<ColRawType>) {
Expand Down Expand Up @@ -341,7 +341,7 @@ SegmentInMemory finalize_impl(
const std::span<const RawType> group_values{
reinterpret_cast<const RawType*>(aggregated.data()), aggregated.size() / sizeof(RawType)
};
arcticdb::for_each_enumerated<typename col_type_info::TDT>(*col, [&](auto row) {
arcticdb::for_each_enumerated<typename col_type_info::TDT>(*col, [&] ARCTICDB_LAMBDA_INLINE(auto row) {
row.value() = group_values[row.idx()];
});
});
Expand Down Expand Up @@ -446,7 +446,7 @@ void MeanAggregatorData::aggregate(
}
arcticdb::for_each_enumerated<typename col_type_info::TDT>(
*input_column.column_,
[&groups, &inserter, this](auto enumerating_it) {
[&groups, &inserter, this] ARCTICDB_LAMBDA_INLINE(auto enumerating_it) {
auto& fraction = fractions_[groups[enumerating_it.idx()]];
if constexpr ((is_floating_point_type(col_type_info ::data_type))) {
if (ARCTICDB_LIKELY(!std::isnan(enumerating_it.value()))) {
Expand Down Expand Up @@ -484,7 +484,7 @@ SegmentInMemory MeanAggregatorData::finalize(const ColumnName& output_column_nam
using OutputDataTypeTag =
std::conditional_t<is_time_type(TypeTag::data_type), TypeTag, DataTypeTag<DataType::FLOAT64>>;
using OutputTypeDescriptor = typename ScalarTypeInfo<OutputDataTypeTag>::TDT;
arcticdb::for_each_enumerated<OutputTypeDescriptor>(*col, [&](auto row) {
arcticdb::for_each_enumerated<OutputTypeDescriptor>(*col, [&] ARCTICDB_LAMBDA_INLINE(auto row) {
row.value() = static_cast<typename OutputDataTypeTag::raw_type>(fractions_[row.idx()].to_double());
});
});
Expand Down Expand Up @@ -516,7 +516,7 @@ void CountAggregatorData::aggregate(
using col_type_info = ScalarTypeInfo<decltype(col_tag)>;
arcticdb::for_each_enumerated<typename col_type_info::TDT>(
*input_column.column_,
[&groups, &inserter, this](auto enumerating_it) {
[&groups, &inserter, this] ARCTICDB_LAMBDA_INLINE(auto enumerating_it) {
if constexpr (is_floating_point_type(col_type_info::data_type)) {
if (ARCTICDB_LIKELY(!std::isnan(enumerating_it.value()))) {
auto& val = aggregated_[groups[enumerating_it.idx()]];
Expand Down Expand Up @@ -548,7 +548,7 @@ SegmentInMemory CountAggregatorData::finalize(const ColumnName& output_column_na
memcpy(ptr, aggregated_.data(), sizeof(uint64_t) * unique_values);
} else {
using OutputTypeDescriptor = typename ScalarTypeInfo<DataTypeTag<DataType::UINT64>>::TDT;
arcticdb::for_each_enumerated<OutputTypeDescriptor>(*col, [&](auto row) {
arcticdb::for_each_enumerated<OutputTypeDescriptor>(*col, [&] ARCTICDB_LAMBDA_INLINE(auto row) {
row.value() = aggregated_[row.idx()];
});
}
Expand Down Expand Up @@ -629,7 +629,7 @@ SegmentInMemory FirstAggregatorData::finalize(const ColumnName& output_column_na
const std::span<const RawType> group_values{
reinterpret_cast<const RawType*>(aggregated_.data()), aggregated_.size() / sizeof(RawType)
};
arcticdb::for_each_enumerated<typename col_type_info::TDT>(*col, [&](auto row) {
arcticdb::for_each_enumerated<typename col_type_info::TDT>(*col, [&] ARCTICDB_LAMBDA_INLINE(auto row) {
row.value() = group_values[row.idx()];
});
}
Expand Down Expand Up @@ -710,7 +710,7 @@ SegmentInMemory LastAggregatorData::finalize(const ColumnName& output_column_nam
const std::span<const RawType> group_values{
reinterpret_cast<const RawType*>(aggregated_.data()), aggregated_.size() / sizeof(RawType)
};
arcticdb::for_each_enumerated<typename col_type_info::TDT>(*col, [&](auto row) {
arcticdb::for_each_enumerated<typename col_type_info::TDT>(*col, [&] ARCTICDB_LAMBDA_INLINE(auto row) {
row.value() = group_values[row.idx()];
});
}
Expand Down
33 changes: 33 additions & 0 deletions cpp/arcticdb/util/lambda_inlining.hpp
View file
Open in desktop
Copy link
Collaborator

@alexowens90 alexowens90 Oct 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we could make this less verbose by defining everything as it is here, and then two additional macros:

#define ARCTICDB_LAMBDA_INLINE(args) ARCTICDB_LAMBDA_INLINE_PRE(args) ARCTICDB_LAMBDA_INLINE_MID ARCTICDB_LAMBDA_INLINE_POST
#define ARCTICDB_LAMBDA_INLINE_NOEXCEPT(args) ARCTICDB_LAMBDA_INLINE_PRE(args) ARCTICDB_LAMBDA_INLINE_MID noexcept ARCTICDB_LAMBDA_INLINE_POST

Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#pragma once

#if defined(__clang__)
// Inlining for clang
#define ARCTICDB_LAMBDA_INLINE(...) (__VA_ARGS__) __attribute__((always_inline))
#define ARCTICDB_LAMBDA_INLINE_1(m1, ...) (__VA_ARGS__) __attribute__((always_inline)) m1
#define ARCTICDB_LAMBDA_INLINE_2(m1, m2, ...) (__VA_ARGS__) __attribute__((always_inline)) m1 m2
#elif defined(__GNUC__)
// Inlining for recent versions of gcc
#define ARCTICDB_LAMBDA_INLINE(...) [[gnu::always_inline]] (__VA_ARGS__)
#define ARCTICDB_LAMBDA_INLINE_1(m1, ...) [[gnu::always_inline]] (__VA_ARGS__) m1
#define ARCTICDB_LAMBDA_INLINE_2(m1, m2, ...) [[gnu::always_inline]] (__VA_ARGS__) m1 m2
#elif defined(_MSC_VER) && _MSC_VER >= 1927 && __cplusplus >= 202002L
// MSVC 16.7 and newer specific syntax, requires C++20 language standard to be enabled
#define ARCTICDB_LAMBDA_INLINE(...) (__VA_ARGS__) [[msvc::forceinline]]
#define ARCTICDB_LAMBDA_INLINE_1(m1, ...) (__VA_ARGS__) m1 [[msvc::forceinline]]
#define ARCTICDB_LAMBDA_INLINE_2(m1, m2, ...) (__VA_ARGS__) m1 m2 [[msvc::forceinline]]
#else
#define ARCTICDB_LAMBDA_INLINE(...) (__VA_ARGS__)
#define ARCTICDB_LAMBDA_INLINE_1(m1, ...) (__VA_ARGS__) m1
#define ARCTICDB_LAMBDA_INLINE_2(m1, m2, ...) (__VA_ARGS__) m1 m2
#endif

// Example usages of inlining a lambda looks like:
// [&] ARCTICDB_LAMBDA_INLINE(int arg1, int arg2) -> int {
// return arg+5;
// }
// [&] ARCTICDB_LAMBDA_INLINE_1(noexcept, int arg1, int arg2) -> int {
// return arg+5;
// }
// [&] ARCTICDB_LAMBDA_INLINE_2(static, noexcept, int arg1, int arg2) -> int {
// return arg+5;
// }
11 changes: 6 additions & 5 deletions cpp/arcticdb/version/version_core.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1601,7 +1601,7 @@ void copy_frame_data_to_buffer(
using src_type_info = ScalarTypeInfo<decltype(src_tag)>;
arcticdb::for_each_enumerated<typename src_type_info::TDT>(
src_column,
[typed_dst_ptr](auto enumerating_it) {
[typed_dst_ptr] ARCTICDB_LAMBDA_INLINE(auto enumerating_it) {
typed_dst_ptr[enumerating_it.idx()] =
static_cast<typename dst_type_info::RawType>(enumerating_it.value());
}
Expand Down Expand Up @@ -1629,7 +1629,7 @@ void copy_frame_data_to_buffer(
default_value
);
SourceType* typed_dst_ptr = reinterpret_cast<SourceType*>(dst_ptr);
arcticdb::for_each_enumerated<SourceTDT>(src_column, [&](const auto& row) {
arcticdb::for_each_enumerated<SourceTDT>(src_column, [&] ARCTICDB_LAMBDA_INLINE(const auto& row) {
typed_dst_ptr[row.idx()] = row.value();
});
}
Expand Down Expand Up @@ -1674,9 +1674,10 @@ void copy_frame_data_to_buffer(
src_column.opt_sparse_map(),
default_value
);
arcticdb::for_each_enumerated<typename source_type_info::TDT>(src_column, [&](const auto& row) {
typed_dst_ptr[row.idx()] = row.value();
});
arcticdb::for_each_enumerated<typename source_type_info::TDT>(
src_column,
[&] ARCTICDB_LAMBDA_INLINE(const auto& row) { typed_dst_ptr[row.idx()] = row.value(); }
);
} else {
arcticdb::for_each<typename source_type_info::TDT>(src_column, [&](const auto& value) {
*typed_dst_ptr = value;
Expand Down
Loading