Support running Spinel as a benchmark target (#540)

Author: k0kubun

harness-spinel/harness.rb

@@ -0,0 +1,108 @@
+# A harness for Ruby implementations that compile the whole program ahead of
+# time, such as Spinel (https://github.com/matz/spinel). harness/harness.rb
+# relies on CRuby-only libraries and reflection (RubyVM, RbConfig, Fiddle,
+# Bundler, $LOAD_PATH) that a whole-program compiler refuses to compile, so
+# this file reimplements the same protocol in a portable subset of Ruby:
+#
+# * WARMUP_ITRS, MIN_BENCH_ITRS and MIN_BENCH_TIME select the iteration count.
+# * RESULT_JSON_PATH receives the JSON that run_benchmarks.rb reads.
+#
+# harness/loader.rb selects this harness automatically when RUBY_ENGINE is
+# "spinel", so `spinel -E benchmarks/fib.rb` works without any -I option. It
+# also runs on CRuby (`ruby -Iharness-spinel benchmarks/fib.rb`), which is how
+# its output is kept compatible with the default harness.
+require "json"
+
+# Warmup iterations
+WARMUP_ITRS = Integer(ENV.fetch('WARMUP_ITRS', '15'))
+
+# Minimum number of benchmarking iterations
+MIN_BENCH_ITRS = Integer(ENV.fetch('MIN_BENCH_ITRS', '10'))
+
+# Minimum benchmarking time in seconds
+MIN_BENCH_TIME = Integer(ENV.fetch('MIN_BENCH_TIME', '10'))
+
+# Do expand_path at require-time, not when returning results, before the benchmark is likely to chdir
+default_path = File.expand_path("../data/results-#{RUBY_ENGINE}-#{RUBY_ENGINE_VERSION}-#{Time.now.strftime('%F-%H%M%S')}.json", __dir__)
+YB_OUTPUT_FILE = File.expand_path(ENV.fetch("RESULT_JSON_PATH", default_path))
+
+puts RUBY_DESCRIPTION
+
+# Ractor.make_shareable is unavailable; benchmarks call this to share constants with Ractors.
+def make_shareable(obj, copy: false)
+  obj
+end
+
+def realtime
+  r0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  yield
+  Process.clock_gettime(Process::CLOCK_MONOTONIC) - r0
+end
+
+# This returns its best estimate of the Resident Set Size in bytes.
+def get_rss
+  mem_rollup_file = "/proc/#{Process.pid}/smaps_rollup"
+  if File.exist?(mem_rollup_file)
+    # Rss is always reported in kB by the Linux kernel, e.g. "Rss:   62796 kB"
+    rss_line = File.read(mem_rollup_file).lines.find { |line| line.start_with?("Rss") }
+    1024 * rss_line.split(":")[1].to_i
+  else
+    1024 * `ps -o rss= -p #{Process.pid}`.to_i
+  end
+end
+
+# Takes a block as input
+def run_benchmark(_num_itrs_hint, &block)
+  times = []
+  rss_samples = []
+  total_time = 0.0
+  num_itrs = 0
+
+  puts "itr:   time"
+  begin
+    time = realtime(&block)
+    num_itrs += 1
+
+    time_ms = (1000 * time).to_i
+    puts "%4s %6s" % ["##{num_itrs}:", "#{time_ms}ms"]
+
+    # We internally save the time in seconds to avoid loss of precision
+    times << time
+    total_time += time
+    # Sample current RSS between iterations (outside the timed block)
+    rss_samples << get_rss
+  end until num_itrs >= WARMUP_ITRS + MIN_BENCH_ITRS && total_time >= MIN_BENCH_TIME
+
+  warmup = times[0...WARMUP_ITRS]
+  bench = times[WARMUP_ITRS..-1]
+  return_results(warmup, bench, rss_samples[WARMUP_ITRS..-1])
+
+  if bench.size > 1
+    bench_ms = ((bench.sum / bench.size) * 1000.0).to_i
+    puts "Average of last #{bench.size}, non-warmup iters: #{bench_ms}ms"
+  end
+end
+
+def return_results(warmup_iterations, bench_iterations, rss_samples)
+  # Full GC before measuring RSS to lower GC variance.
+  GC.start
+
+  rss = get_rss
+  ruby_bench_results = {
+    "RUBY_DESCRIPTION" => RUBY_DESCRIPTION,
+    "warmup" => warmup_iterations,
+    "bench" => bench_iterations,
+    "rss_samples" => rss_samples,
+    "rss" => rss,
+  }
+
+  puts "RSS: %.1fMiB" % (rss / 1024.0 / 1024.0)
+
+  out_path = YB_OUTPUT_FILE
+  system('mkdir', '-p', File.dirname(out_path))
+
+  # Using default path? Print where we put it.
+  puts "Writing file #{out_path}" unless ENV["RESULT_JSON_PATH"]
+
+  File.write(out_path, JSON.generate(ruby_bench_results))
+end

harness/loader.rb

@@ -1,13 +1,17 @@
 # Use harness/harness.rb by default. You can change it with -I option.
 # i.e. ruby -Iharness benchmarks/railsbench/benchmark.rb
-retries = 0
-begin
-  require "harness"
-rescue LoadError => e
-  if retries == 0 && e.path == "harness"
-    retries += 1
-    $LOAD_PATH << File.expand_path(__dir__)
-    retry
+if RUBY_ENGINE == "spinel"
+  require_relative "../harness-spinel/harness"
+else
+  retries = 0
+  begin
+    require "harness"
+  rescue LoadError => e
+    if retries == 0 && e.path == "harness"
+      retries += 1
+      $LOAD_PATH << File.expand_path(__dir__)
+      retry
+    end
+    raise
   end
-  raise
 end

lib/argument_parser.rb

@@ -62,7 +62,7 @@ def parse(argv)
           version, *options = version.shellsplit
           executable = find_chruby_ruby(version)
           abort "Cannot find '#{version}' in chruby paths" unless executable
-          args.executables[name] = [executable, *options]
+          args.executables[name] = [*executable, *options]
         end
       end
 
@@ -208,9 +208,23 @@ def parse(argv)
 
   private
 
+  # Returns the command (an Array) to run a script with the given chruby version, or nil.
   def find_chruby_ruby(version)
     rubies_dir = ENV["RUBIES_DIR"] || "#{ENV["HOME"]}/.rubies"
-    chruby_search_paths(version, rubies_dir).find { |path| File.executable?(path) }
+    search_paths = chruby_search_paths(version, rubies_dir)
+    if (ruby = search_paths.find { |path| File.executable?(path) })
+      return [ruby]
+    end
+
+    # Spinel (https://github.com/matz/spinel) installs `bin/spinel` instead of `bin/ruby`,
+    # e.g. with `make install PREFIX=/opt/rubies/spinel`. Its -E option compiles a script
+    # and runs the binary, like `go run`, so it can take the place of a `ruby` executable.
+    spinel_paths = search_paths.map { |path| File.join(File.dirname(path), "spinel") }
+    if (spinel = spinel_paths.find { |path| File.executable?(path) })
+      return [spinel, "-E"]
+    end
+
+    nil
   end
 
   def chruby_search_paths(version, rubies_dir)

lib/benchmark_suite.rb

@@ -243,7 +243,8 @@ def compute_benchmark_env(ruby)
     env = {}
     ruby_path = IO.popen([*ruby, "-e", "print RbConfig.ruby"], err: File::NULL, &:read)
 
-    if ruby_path != RbConfig.ruby
+    # Implementations without RbConfig (e.g. Spinel) print nothing; leave PATH alone for them.
+    if !ruby_path.empty? && ruby_path != RbConfig.ruby
       env["PATH"] = "#{File.dirname(ruby_path)}#{File::PATH_SEPARATOR}#{ENV["PATH"]}"
 
       # chruby sets GEM_HOME and GEM_PATH in your shell. We have to unset it in the child
@@ -277,8 +278,9 @@ def base_cmd(ruby_description, benchmark_name)
       cmd = setarch_prefix
 
       # Pin the process to one given core to improve caching and reduce variance on CRuby
+      # and Spinel, which compiles the benchmark to a single-threaded native binary.
       # Other Rubies need to use multiple cores, e.g., for JIT threads
-      if ruby_description.start_with?('ruby ') && should_pin?(benchmark_name)
+      if (ruby_description.start_with?('ruby ') || ruby_description.start_with?('spinel ')) && should_pin?(benchmark_name)
         # Up to the last two thirds of Intel CPU cores may be slow E-Cores, so avoid using them.
         cpu = [(Etc.nprocessors / 3) - 1, 0].max
         cmd.concat(["taskset", "-c", "#{cpu}"])

test/argument_parser_test.rb

@@ -171,6 +171,20 @@ def setup_mock_ruby(path)
         end
       end
 
+      it 'runs spinel with -E when a version has bin/spinel instead of bin/ruby' do
+        Dir.mktmpdir do |tmpdir|
+          spinel_path = File.join(tmpdir, 'opt/rubies/spinel/bin/spinel')
+          setup_mock_ruby(spinel_path)
+
+          parser = ArgumentParser.new
+          parser.stub :chruby_search_paths, ->(version, rubies_dir) { [File.join(tmpdir, 'opt/rubies', version, 'bin/ruby')] } do
+            args = parser.parse(['--chruby=spinel::spinel --int-overflow=wrap'])
+
+            assert_equal [spinel_path, '-E', '--int-overflow=wrap'], args.executables['spinel']
+          end
+        end
+      end
+
       it 'aborts when ruby version not found' do
         Dir.mktmpdir do |tmpdir|
           @temp_home = tmpdir

test/benchmark_suite_test.rb

@@ -180,6 +180,32 @@
     end
   end
 
+  describe '#base_cmd' do
+    def base_cmd_for(ruby_description)
+      suite = BenchmarkSuite.new(
+        categories: ['micro'],
+        name_filters: [],
+        out_path: @out_path,
+        harness: 'harness'
+      )
+      suite.stub :linux?, true do
+        suite.stub :setarch_prefix, [] do
+          suite.send(:base_cmd, ruby_description, 'simple')
+        end
+      end
+    end
+
+    it 'pins CRuby and Spinel to one core' do
+      assert_includes base_cmd_for('ruby 4.0.7 (2026-09-15 revision 229531a6cf) +ZJIT [x86_64-linux]'), 'taskset'
+      assert_includes base_cmd_for('spinel 2026.09.12+3194 (0949a47fb) [gcc 13.3.0 (cc)]'), 'taskset'
+    end
+
+    it 'does not pin other Rubies, which use multiple cores' do
+      refute_includes base_cmd_for('jruby 10.1.1.0 (3.4.5) 2025-08-27 OpenJDK 64-Bit Server VM [x86_64-linux]'), 'taskset'
+      refute_includes base_cmd_for('truffleruby 34.0.1 [x86_64-linux]'), 'taskset'
+    end
+  end
+
   describe '#run' do
     it 'returns bench_data and bench_failures as a tuple' do
       suite = BenchmarkSuite.new(