Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 33 additions & 28 deletions include/boost/corosio/detail/ready_queue.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
#include <boost/corosio/detail/scheduler_op.hpp>
#include <boost/capy/continuation.hpp>

#include <bit>
#include <cstdint>

namespace boost::corosio::detail {
Expand All @@ -31,23 +30,27 @@ inline constexpr std::uintptr_t ready_cont_bit = 1;

/// Return true if a queue entry refers to a continuation (vs a scheduler_op).
inline bool
ready_is_continuation(std::uintptr_t e) noexcept
ready_is_continuation(void* e) noexcept
{
return (e & ready_cont_bit) != 0;
return (reinterpret_cast<std::uintptr_t>(e) & ready_cont_bit) != 0;
}

/// Recover the scheduler_op from an op-tagged entry.
inline scheduler_op*
ready_as_op(std::uintptr_t e) noexcept
ready_as_op(void* e) noexcept
{
return std::bit_cast<scheduler_op*>(e & ~ready_cont_bit);
// NOLINTNEXTLINE(performance-no-int-to-ptr)
return reinterpret_cast<scheduler_op*>(
reinterpret_cast<std::uintptr_t>(e) & ~ready_cont_bit);
}

/// Recover the continuation from a continuation-tagged entry.
inline capy::continuation*
ready_as_cont(std::uintptr_t e) noexcept
ready_as_cont(void* e) noexcept
{
return std::bit_cast<capy::continuation*>(e & ~ready_cont_bit);
// NOLINTNEXTLINE(performance-no-int-to-ptr)
return reinterpret_cast<capy::continuation*>(
reinterpret_cast<std::uintptr_t>(e) & ~ready_cont_bit);
}

/** A unified intrusive FIFO of scheduler_ops and continuations.
Expand All @@ -64,37 +67,36 @@ ready_as_cont(std::uintptr_t e) noexcept
*/
class ready_queue
{
std::uintptr_t head_ = 0; // tagged first entry, 0 when empty
std::uintptr_t tail_ = 0; // tagged last entry, 0 when empty
void* head_ = nullptr; // tagged first entry, nullptr when empty
void* tail_ = nullptr; // tagged last entry, nullptr when empty

// Read a node's next-link by value. A continuation's link lives in its
// void* `reserved` slot; bit_cast keeps us from forming a uintptr_t
// lvalue over that void* object (which would violate strict aliasing).
// void* `reserved` slot.
//
// GCC 12/13 false-positive: when inlining proves an entry refers to a
// continuation, -Warray-bounds still diagnoses the untaken scheduler_op
// branch against the smaller object. Fixed in GCC 14.
BOOST_COROSIO_GCC_WARNING_PUSH
BOOST_COROSIO_GCC_WARNING_DISABLE("-Warray-bounds")
static std::uintptr_t next_of(std::uintptr_t e) noexcept
static void* next_of(void* e) noexcept
{
if (ready_is_continuation(e))
return std::bit_cast<std::uintptr_t>(ready_as_cont(e)->reserved);
return ready_as_cont(e)->reserved;
return ready_as_op(e)->q_next_;
}

static void set_next(std::uintptr_t e, std::uintptr_t nxt) noexcept
static void set_next(void* e, void* nxt) noexcept
{
if (ready_is_continuation(e))
ready_as_cont(e)->reserved = std::bit_cast<void*>(nxt);
ready_as_cont(e)->reserved = nxt;
else
ready_as_op(e)->q_next_ = nxt;
}
BOOST_COROSIO_GCC_WARNING_POP

void push_entry(std::uintptr_t e) noexcept
void push_entry(void* e) noexcept
{
set_next(e, 0);
set_next(e, nullptr);
if (tail_)
set_next(tail_, e);
else
Expand All @@ -107,8 +109,8 @@ class ready_queue

ready_queue(ready_queue&& o) noexcept : head_(o.head_), tail_(o.tail_)
{
o.head_ = 0;
o.tail_ = 0;
o.head_ = nullptr;
o.tail_ = nullptr;
}

ready_queue(ready_queue const&) = delete;
Expand All @@ -118,19 +120,22 @@ class ready_queue
/// Return true if the queue holds no entries.
bool empty() const noexcept
{
return head_ == 0;
return head_ == nullptr;
}

/// Append a scheduler_op to the back of the queue.
void push(scheduler_op* op) noexcept
{
push_entry(std::bit_cast<std::uintptr_t>(op));
push_entry(reinterpret_cast<void*>(op));
}

/// Append a continuation to the back of the queue.
void push(capy::continuation& c) noexcept
{
push_entry(std::bit_cast<std::uintptr_t>(&c) | ready_cont_bit);
push_entry(
// NOLINTNEXTLINE(performance-no-int-to-ptr)
reinterpret_cast<void*>(
reinterpret_cast<std::uintptr_t>(&c) | ready_cont_bit));
}

/// Move all entries of @p other to the back in O(1); @p other is emptied.
Expand All @@ -143,19 +148,19 @@ class ready_queue
else
head_ = other.head_;
tail_ = other.tail_;
other.head_ = 0;
other.tail_ = 0;
other.head_ = nullptr;
other.tail_ = nullptr;
}

/// Remove and return the front entry as a tagged value, or 0 when empty.
std::uintptr_t pop() noexcept
/// Remove and return the front entry as a tagged value, or nullptr when empty.
void* pop() noexcept
{
auto e = head_;
if (!e)
return 0;
return nullptr;
head_ = next_of(e);
if (!head_)
tail_ = 0;
tail_ = nullptr;
return e;
}
};
Expand Down
2 changes: 1 addition & 1 deletion include/boost/corosio/detail/scheduler_op.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ class scheduler_op : public intrusive_queue<scheduler_op>::node
// the former 8-byte padding word, so size/alignment are unchanged. Used
// only while this op is in the ready queue; node.next_ is used while it
// is pending in an op_queue. An op is in one or the other, never both.
std::uintptr_t q_next_ = 0;
void* q_next_ = nullptr;
};

using op_queue = intrusive_queue<scheduler_op>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,7 @@ reactor_find_context(reactor_scheduler const* self) noexcept
@par Thread Safety
All public member functions are thread-safe.
*/
class reactor_scheduler
: public scheduler
class reactor_scheduler : public scheduler
{
public:
using context_type = reactor_scheduler_context;
Expand Down Expand Up @@ -994,7 +993,7 @@ reactor_scheduler::do_one(lock_type& lock, long timeout_us, context_type& ctx)
if (stopped_.load(std::memory_order_acquire))
return 0;

std::uintptr_t e = completed_ops_.pop();
auto const e = completed_ops_.pop();
scheduler_op* op = ready_is_continuation(e) ? nullptr : ready_as_op(e);

// Handle reactor sentinel — time to poll for I/O
Expand Down Expand Up @@ -1038,7 +1037,7 @@ reactor_scheduler::do_one(lock_type& lock, long timeout_us, context_type& ctx)
}

// Handle ready entry (op or continuation)
if (e != 0)
if (e != nullptr)
{
bool more = !completed_ops_.empty();

Expand Down
4 changes: 2 additions & 2 deletions test/unit/ready_queue.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ struct ready_queue_test
auto e2 = q.pop();
BOOST_TEST(ready_as_op(e2) == &b);

BOOST_TEST(q.pop() == 0u);
BOOST_TEST(q.pop() == nullptr);
BOOST_TEST(q.empty());
}

Expand Down Expand Up @@ -80,7 +80,7 @@ struct ready_queue_test
BOOST_TEST(q2.empty());
BOOST_TEST(ready_as_op(q1.pop()) == &a);
BOOST_TEST(ready_as_op(q1.pop()) == &b);
BOOST_TEST(q1.pop() == 0u);
BOOST_TEST(q1.pop() == nullptr);
}

void run()
Expand Down
Loading