[pull] master from ruby:master

concurrent_set.c

@@ -1,7 +1,6 @@
 #include "internal.h"
 #include "internal/gc.h"
 #include "internal/concurrent_set.h"
-#include "ruby_atomic.h"
 #include "ruby/atomic.h"
 #include "vm_sync.h"
 
@@ -109,15 +108,15 @@ static void
 concurrent_set_try_resize_without_locking(VALUE old_set_obj, VALUE *set_obj_ptr)
 {
     // Check if another thread has already resized.
-    if (RUBY_ATOMIC_VALUE_LOAD(*set_obj_ptr) != old_set_obj) {
+    if (rbimpl_atomic_value_load(set_obj_ptr, RBIMPL_ATOMIC_ACQUIRE) != old_set_obj) {
         return;
     }
 
     struct concurrent_set *old_set = RTYPEDDATA_GET_DATA(old_set_obj);
 
     // This may overcount by up to the number of threads concurrently attempting to insert
     // GC may also happen between now and the set being rebuilt
-    int expected_size = RUBY_ATOMIC_LOAD(old_set->size) - old_set->deleted_entries;
+    int expected_size = rbimpl_atomic_load(&old_set->size, RBIMPL_ATOMIC_RELAXED) - old_set->deleted_entries;
 
     struct concurrent_set_entry *old_entries = old_set->entries;
     int old_capacity = old_set->capacity;
@@ -135,13 +134,13 @@ concurrent_set_try_resize_without_locking(VALUE old_set_obj, VALUE *set_obj_ptr)
 
     for (int i = 0; i < old_capacity; i++) {
         struct concurrent_set_entry *entry = &old_entries[i];
-        VALUE key = RUBY_ATOMIC_VALUE_EXCHANGE(entry->key, CONCURRENT_SET_MOVED);
+        VALUE key = rbimpl_atomic_value_exchange(&entry->key, CONCURRENT_SET_MOVED, RBIMPL_ATOMIC_ACQUIRE);
         RUBY_ASSERT(key != CONCURRENT_SET_MOVED);
 
         if (key < CONCURRENT_SET_SPECIAL_VALUE_COUNT) continue;
         if (!RB_SPECIAL_CONST_P(key) && rb_objspace_garbage_object_p(key)) continue;
 
-        VALUE hash = RUBY_ATOMIC_VALUE_LOAD(entry->hash);
+        VALUE hash = rbimpl_atomic_value_load(&entry->hash, RBIMPL_ATOMIC_RELAXED);
         if (hash == 0) {
             // Either in-progress insert or extremely unlikely 0 hash.
             // Re-calculate the hash.
@@ -174,7 +173,7 @@ concurrent_set_try_resize_without_locking(VALUE old_set_obj, VALUE *set_obj_ptr)
         }
     }
 
-    RUBY_ATOMIC_VALUE_SET(*set_obj_ptr, new_set_obj);
+    rbimpl_atomic_value_store(set_obj_ptr, new_set_obj, RBIMPL_ATOMIC_RELEASE);
 
     RB_GC_GUARD(old_set_obj);
 }
@@ -196,7 +195,7 @@ rb_concurrent_set_find(VALUE *set_obj_ptr, VALUE key)
     VALUE hash = 0;
 
   retry:
-    set_obj = RUBY_ATOMIC_VALUE_LOAD(*set_obj_ptr);
+    set_obj = rbimpl_atomic_value_load(set_obj_ptr, RBIMPL_ATOMIC_ACQUIRE);
     RUBY_ASSERT(set_obj);
     struct concurrent_set *set = RTYPEDDATA_GET_DATA(set_obj);
 
@@ -212,7 +211,7 @@ rb_concurrent_set_find(VALUE *set_obj_ptr, VALUE key)
 
     while (true) {
         struct concurrent_set_entry *entry = &set->entries[idx];
-        VALUE curr_key = RUBY_ATOMIC_VALUE_LOAD(entry->key);
+        VALUE curr_key = rbimpl_atomic_value_load(&entry->key, RBIMPL_ATOMIC_ACQUIRE);
 
         switch (curr_key) {
           case CONCURRENT_SET_EMPTY:
@@ -225,13 +224,13 @@ rb_concurrent_set_find(VALUE *set_obj_ptr, VALUE key)
 
             goto retry;
           default: {
-            VALUE curr_hash = RUBY_ATOMIC_VALUE_LOAD(entry->hash);
+            VALUE curr_hash = rbimpl_atomic_value_load(&entry->hash, RBIMPL_ATOMIC_RELAXED);
             if (curr_hash != 0 && curr_hash != hash) break;
 
             if (UNLIKELY(!RB_SPECIAL_CONST_P(curr_key) && rb_objspace_garbage_object_p(curr_key))) {
                 // This is a weakref set, so after marking but before sweeping is complete we may find a matching garbage object.
                 // Skip it and mark it as deleted.
-                RUBY_ATOMIC_VALUE_CAS(entry->key, curr_key, CONCURRENT_SET_DELETED);
+                rbimpl_atomic_value_cas(&entry->key, curr_key, CONCURRENT_SET_DELETED, RBIMPL_ATOMIC_RELEASE, RBIMPL_ATOMIC_RELAXED);
                 break;
             }
 
@@ -259,7 +258,7 @@ rb_concurrent_set_find_or_insert(VALUE *set_obj_ptr, VALUE key, void *data)
     VALUE hash = 0;
 
   retry:
-    set_obj = RUBY_ATOMIC_VALUE_LOAD(*set_obj_ptr);
+    set_obj = rbimpl_atomic_value_load(set_obj_ptr, RBIMPL_ATOMIC_ACQUIRE);
     RUBY_ASSERT(set_obj);
     struct concurrent_set *set = RTYPEDDATA_GET_DATA(set_obj);
 
@@ -275,7 +274,7 @@ rb_concurrent_set_find_or_insert(VALUE *set_obj_ptr, VALUE key, void *data)
 
     while (true) {
         struct concurrent_set_entry *entry = &set->entries[idx];
-        VALUE curr_key = RUBY_ATOMIC_VALUE_LOAD(entry->key);
+        VALUE curr_key = rbimpl_atomic_value_load(&entry->key, RBIMPL_ATOMIC_ACQUIRE);
 
         switch (curr_key) {
           case CONCURRENT_SET_EMPTY: {
@@ -286,24 +285,24 @@ rb_concurrent_set_find_or_insert(VALUE *set_obj_ptr, VALUE key, void *data)
                 inserting = true;
             }
 
-            rb_atomic_t prev_size = RUBY_ATOMIC_FETCH_ADD(set->size, 1);
+            rb_atomic_t prev_size = rbimpl_atomic_fetch_add(&set->size, 1, RBIMPL_ATOMIC_RELAXED);
 
             if (UNLIKELY(prev_size > set->capacity / 2)) {
                 concurrent_set_try_resize(set_obj, set_obj_ptr);
 
                 goto retry;
             }
 
-            curr_key = RUBY_ATOMIC_VALUE_CAS(entry->key, CONCURRENT_SET_EMPTY, key);
+            curr_key = rbimpl_atomic_value_cas(&entry->key, CONCURRENT_SET_EMPTY, key, RBIMPL_ATOMIC_RELEASE, RBIMPL_ATOMIC_RELAXED);
             if (curr_key == CONCURRENT_SET_EMPTY) {
-                RUBY_ATOMIC_VALUE_SET(entry->hash, hash);
+                rbimpl_atomic_value_store(&entry->hash, hash, RBIMPL_ATOMIC_RELAXED);
 
                 RB_GC_GUARD(set_obj);
                 return key;
             }
             else {
                 // Entry was not inserted.
-                RUBY_ATOMIC_DEC(set->size);
+                rbimpl_atomic_sub(&set->size, 1, RBIMPL_ATOMIC_RELAXED);
 
                 // Another thread won the race, try again at the same location.
                 continue;
@@ -317,13 +316,13 @@ rb_concurrent_set_find_or_insert(VALUE *set_obj_ptr, VALUE key, void *data)
 
             goto retry;
           default: {
-            VALUE curr_hash = RUBY_ATOMIC_VALUE_LOAD(entry->hash);
+            VALUE curr_hash = rbimpl_atomic_value_load(&entry->hash, RBIMPL_ATOMIC_RELAXED);
             if (curr_hash != 0 && curr_hash != hash) break;
 
             if (UNLIKELY(!RB_SPECIAL_CONST_P(curr_key) && rb_objspace_garbage_object_p(curr_key))) {
                 // This is a weakref set, so after marking but before sweeping is complete we may find a matching garbage object.
                 // Skip it and mark it as deleted.
-                RUBY_ATOMIC_VALUE_CAS(entry->key, curr_key, CONCURRENT_SET_DELETED);
+                rbimpl_atomic_value_cas(&entry->key, curr_key, CONCURRENT_SET_DELETED, RBIMPL_ATOMIC_RELEASE, RBIMPL_ATOMIC_RELAXED);
                 break;
             }
 
@@ -363,7 +362,7 @@ rb_concurrent_set_delete_by_identity(VALUE set_obj, VALUE key)
 
     while (true) {
         struct concurrent_set_entry *entry = &set->entries[idx];
-        VALUE curr_key = RUBY_ATOMIC_VALUE_LOAD(entry->key);
+        VALUE curr_key = entry->key;
 
         switch (curr_key) {
           case CONCURRENT_SET_EMPTY:

insns.def

@@ -1598,6 +1598,7 @@ opt_succ
 (CALL_DATA cd)
 (VALUE recv)
 (VALUE val)
+// attr bool zjit_profile = true;
 {
     val = vm_opt_succ(recv);
 

zjit/bindgen/src/main.rs

@@ -73,6 +73,7 @@ fn main() {
         .allowlist_function("rb_utf8_str_new")
         .allowlist_function("rb_str_buf_append")
         .allowlist_function("rb_str_dup")
+        .allowlist_function("rb_str_getbyte")
         .allowlist_type("ruby_preserved_encindex")
         .allowlist_function("rb_class2name")
 

zjit/src/codegen.rs

@@ -363,6 +363,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio
         // If it happens we abort the compilation for now
         Insn::StringConcat { strings, state, .. } if strings.is_empty() => return Err(*state),
         Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, opnds!(strings), &function.frame_state(*state)),
+        &Insn::StringGetbyteFixnum { string, index } => gen_string_getbyte_fixnum(asm, opnd!(string), opnd!(index)),
         Insn::StringIntern { val, state } => gen_intern(asm, opnd!(val), &function.frame_state(*state)),
         Insn::ToRegexp { opt, values, state } => gen_toregexp(jit, asm, *opt, opnds!(values), &function.frame_state(*state)),
         Insn::Param { idx } => unreachable!("block.insns should not have Insn::Param({idx})"),
@@ -2109,6 +2110,11 @@ fn gen_string_concat(jit: &mut JITState, asm: &mut Assembler, strings: Vec<Opnd>
     result
 }
 
+fn gen_string_getbyte_fixnum(asm: &mut Assembler, string: Opnd, index: Opnd) -> Opnd {
+    // TODO(max): Open-code rb_str_getbyte to avoid a call
+    asm_ccall!(asm, rb_str_getbyte, string, index)
+}
+
 /// Generate a JIT entry that just increments exit_compilation_failure and exits
 fn gen_compile_error_counter(cb: &mut CodeBlock, compile_error: &CompileError) -> Result<CodePtr, CompileError> {
     let mut asm = Assembler::new();

zjit/src/cruby.rs

@@ -132,6 +132,7 @@ unsafe extern "C" {
     pub fn rb_hash_empty_p(hash: VALUE) -> VALUE;
     pub fn rb_yjit_str_concat_codepoint(str: VALUE, codepoint: VALUE);
     pub fn rb_str_setbyte(str: VALUE, index: VALUE, value: VALUE) -> VALUE;
+    pub fn rb_str_getbyte(str: VALUE, index: VALUE) -> VALUE;
     pub fn rb_vm_splat_array(flag: VALUE, ary: VALUE) -> VALUE;
     pub fn rb_vm_concat_array(ary1: VALUE, ary2st: VALUE) -> VALUE;
     pub fn rb_vm_get_special_object(reg_ep: *const VALUE, value_type: vm_special_object_type) -> VALUE;

zjit/src/cruby_methods.rs

@@ -190,6 +190,7 @@ pub fn init() -> Annotations {
     annotate!(rb_mKernel, "itself", inline_kernel_itself);
     annotate!(rb_cString, "bytesize", types::Fixnum, no_gc, leaf);
     annotate!(rb_cString, "to_s", types::StringExact);
+    annotate!(rb_cString, "getbyte", inline_string_getbyte);
     annotate!(rb_cModule, "name", types::StringExact.union(types::NilClass), no_gc, leaf, elidable);
     annotate!(rb_cModule, "===", types::BoolExact, no_gc, leaf);
     annotate!(rb_cArray, "length", types::Fixnum, no_gc, leaf, elidable);
@@ -205,6 +206,7 @@ pub fn init() -> Annotations {
     annotate!(rb_cBasicObject, "==", types::BoolExact, no_gc, leaf, elidable);
     annotate!(rb_cBasicObject, "!", types::BoolExact, no_gc, leaf, elidable);
     annotate!(rb_cBasicObject, "initialize", types::NilClass, no_gc, leaf, elidable);
+    annotate!(rb_cInteger, "succ", inline_integer_succ);
     annotate!(rb_cString, "to_s", inline_string_to_s);
     let thread_singleton = unsafe { rb_singleton_class(rb_cThread) };
     annotate!(thread_singleton, "current", types::BasicObject, no_gc, leaf);
@@ -257,3 +259,26 @@ fn inline_hash_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::Ins
     }
     None
 }
+
+fn inline_string_getbyte(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> {
+    let &[index] = args else { return None; };
+    if fun.likely_a(index, types::Fixnum, state) {
+        // String#getbyte with a Fixnum is leaf and nogc; otherwise it may run arbitrary Ruby code
+        // when converting the index to a C integer.
+        let index = fun.coerce_to(block, index, types::Fixnum, state);
+        let result = fun.push_insn(block, hir::Insn::StringGetbyteFixnum { string: recv, index });
+        return Some(result);
+    }
+    None
+}
+
+fn inline_integer_succ(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> {
+    if !args.is_empty() { return None; }
+    if fun.likely_a(recv, types::Fixnum, state) {
+        let left = fun.coerce_to(block, recv, types::Fixnum, state);
+        let right = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(VALUE::fixnum_from_usize(1)) });
+        let result = fun.push_insn(block, hir::Insn::FixnumAdd { left, right, state });
+        return Some(result);
+    }
+    None
+}