Fix RubyTest runner reliability

HISTORY.md

@@ -1,5 +1,13 @@
 # RELEASE HISTORY
 
+## Unreleased
+
+* Fail empty runs and unmatched test file requests instead of reporting success.
+* Apply test filters at every suite level and record skipped tests and cases.
+* Record case errors, finish suite callbacks, and restore reporter output after
+  failures.
+* Fix the CLI's alternate configuration option and `Testfile` discovery.
+
 ## 0.9.0 / 2026-03-31
 
 Maintenance release. Modernized project tooling and merged CLI back in.
@@ -222,4 +230,3 @@ First release of Ruby Test.
 Changes:
 
 * It's Your Birthday!
-

README.md

@@ -63,6 +63,9 @@ test framework or its adapter, e.g.
 
     $ rubytest -r lemon -r ae test/test_*.rb
 
+Rubytest exits unsuccessfully if a requested path matches no files or if the
+selection runs no tests.
+
 Use `-h/--help` to see all available options.
 
 #### Configuration File
@@ -101,7 +104,7 @@ If you are using Rake, shelling out to `rubytest` keeps your test
 environment pristine:
 
     desc "run tests"
-    task :test
+    task :test do
       sh "rubytest"
     end
 

demo/03_runner_reliability.md

@@ -0,0 +1,122 @@
+## Runner reliability
+
+RubyTest reports an explicitly requested file that does not exist, rather than
+passing an empty run.
+
+    runner = Test::Runner.new(files: ['__rubytest_missing_file__.rb'], format: 'test')
+    cleanup_ran = false
+    runner.config.after { cleanup_ran = true }
+    missing_file = begin
+      runner.run
+      nil
+    rescue ArgumentError => error
+      error
+    end
+
+    missing_file.class.assert == ArgumentError
+    missing_file.message.include?('__rubytest_missing_file__.rb').assert == true
+    cleanup_ran.assert == true
+
+An empty suite also has an unsuccessful result.
+
+    empty_runner = Test::Runner.new(suite: [], format: 'test')
+    empty_runner.run.assert == false
+
+### Selection and skips
+
+Test descriptions can be matched at the top level and inside cases. A skipped
+test that matches the selection is still reported.
+
+    class RunnerProbe
+      def initialize(description, skip_reason = nil, &block)
+        @description = description
+        @skip_reason = skip_reason
+        @block = block
+      end
+
+      def call
+        @block.call
+      end
+
+      def skip?
+        @skip_reason
+      end
+
+      def to_s
+        @description
+      end
+    end
+
+    top = RunnerProbe.new('wanted top') { true }
+    other = RunnerProbe.new('other') { true }
+    nested = RunnerProbe.new('wanted nested') { true }
+    skipped = RunnerProbe.new('wanted skipped', 'later') { raise 'should not run' }
+
+    selected = Test::Runner.new(suite: [top, other, [nested, skipped]],
+                                match: ['wanted'], format: 'test')
+    selected.run.assert == true
+    selected.recorder[:pass].size.assert == 2
+    selected.recorder[:skip].size.assert == 1
+
+Skipping an entire case is recorded too.
+
+    skipped_case = Class.new(Array) do
+      def skip?
+        'later'
+      end
+    end.new
+    case_skip_runner = Test::Runner.new(suite: [skipped_case], format: 'test')
+    case_skip_runner.run.assert == true
+    case_skip_runner.recorder[:skip].size.assert == 1
+
+### Case errors and cleanup
+
+A case setup error is recorded, and the next test and suite cleanup still run.
+
+    broken_case = Class.new(Array) do
+      def call
+        raise 'setup failed'
+      end
+    end.new
+    following_test = RunnerProbe.new('following test') { true }
+    case_runner = Test::Runner.new(suite: [broken_case, following_test], format: 'test')
+    suite_ended = false
+    case_runner.after(:suite) { suite_ended = true }
+
+    case_runner.run.assert == false
+    case_runner.recorder[:error].size.assert == 1
+    case_runner.recorder[:pass].size.assert == 1
+    suite_ended.assert == true
+
+An assertion failure in a hash reporter records the failure and restores
+standard output.
+
+    original_stdout = $stdout
+    failing_test = RunnerProbe.new('failing test') { raise Assertion, 'expected failure' }
+    failure_runner = Test::Runner.new(suite: [failing_test], format: 'test')
+
+    failure_runner.run.assert == false
+    failure_runner.recorder[:fail].size.assert == 1
+    ($stdout.equal?(original_stdout)).assert == true
+
+Global assertionless mode treats a false return as a failure.
+
+    previous_assertionless = Test::Config.assertionless
+    begin
+      Test::Config.assertionless = true
+      false_test = RunnerProbe.new('false result') { false }
+      hard_runner = Test::Runner.new(suite: [false_test], format: 'test')
+      hard_runner.run.assert == false
+      hard_runner.recorder[:fail].size.assert == 1
+    ensure
+      Test::Config.assertionless = previous_assertionless
+    end
+
+### CLI configuration
+
+The `--config` option accepts an alternate configuration file.
+
+    require 'rubytest/cli'
+    cli = Test::CLI.new
+    cli.options.parse!(['--config', 'custom-test.rb'])
+    cli.config_file.assert == 'custom-test.rb'

lib/rubytest/cli.rb

@@ -9,7 +9,7 @@ class CLI
     # Test configuration file can be in `etc/test.rb` or `config/test.rb`, or
     # `Testfile` or '.test` with optional `.rb` extension, in that order of
     # precedence. To use a different file there is the -c/--config option.
-    GLOB_CONFIG = '{etc/test.rb,config/test.rb,testfile.rb,testfile,.test.rb,.test}'
+    GLOB_CONFIG = '{etc/test.rb,config/test.rb,Testfile.rb,Testfile,testfile.rb,testfile,.test.rb,.test}'
 
     # Convenience method for invoking the CLI.
     #
@@ -95,7 +95,7 @@ def options
           conf.requires.concat makelist(file)
         end
         opt.on '-c', '--config FILE', "use alternate config file" do |file|
-          conf.config_files << file
+          conf.config_file = file
         end
         opt.on '-V' , '--verbose', 'provide extra detail in reports' do
           conf.verbose = true
@@ -141,6 +141,10 @@ def config_file
       @config_file
     end
 
+    def config_file=(file)
+      @config_file = file
+    end
+
     def profile
       @profile
     end

lib/rubytest/config.rb

@@ -51,7 +51,7 @@ def self.assertionless
 
     #
     def self.assertionless=(boolean)
-      @assertionaless = !!boolean
+      @assertionless = !!boolean
     end
 
     # Find and cache project root directory.

lib/rubytest/format/abstract.rb

@@ -44,11 +44,11 @@ def begin_test(test)
       end
 
       #
-      def skip_case(test_case)
+      def skip_case(test_case, reason=nil)
       end
 
       #
-      def skip_test(test)
+      def skip_test(test, reason=nil)
       end
 
       #

lib/rubytest/format/abstract_hash.rb

@@ -62,10 +62,11 @@ def begin_test(test)
     #
     # @return [Hash]
     #
-    def skip_test(test)
+    def skip_test(test, reason=nil)
       h = {}
       h['type'  ] = 'test'
       h['status'] = 'omit'
+      h['reason'] = reason if reason
 
       merge_subtype      h, test
       merge_setup        h, test
@@ -169,7 +170,8 @@ def todo(test, exception)
     #
     def end_test(test)
       super(test)
-      $stdout, $stderr = @stdout, @stderr
+    ensure
+      $stdout, $stderr = @stdout, @stderr if @stdout && @stderr
     end
 
     #
@@ -206,7 +208,7 @@ def end_suite(suite)
     #
     def merge_priority(hash, test, exception)
       level = exception.priority
-      h['priority'] = level.to_i
+      hash['priority'] = level.to_i
     end
 
     #
@@ -233,8 +235,8 @@ def merge_comparison(hash, test, exception)
 
     # Add source location information to hash.
     def merge_source(hash, test)
-      if test.respond_to?('source_location')
-        file, line = source_location
+      if test.respond_to?(:source_location)
+        file, line = test.source_location
         hash['file'   ] = file
         hash['line'   ] = line
         hash['source' ] = code(file, line).to_str
@@ -280,8 +282,8 @@ def merge_coverage(hash, test)
 
     #
     def merge_output(hash)
-      hash['stdout'] = $stdout.string
-      hash['stderr'] = $stderr.string
+      hash['stdout'] = $stdout.respond_to?(:string) ? $stdout.string : ''
+      hash['stderr'] = $stderr.respond_to?(:string) ? $stderr.string : ''
     end
 
     #

lib/rubytest/format/dotprogress.rb

@@ -35,7 +35,7 @@ def end_suite(suite)
       puts
 
       if runner.verbose?
-        unless record[:omit].empty?
+        unless record[:skip].empty?
           puts "SKIPPED\n\n"
           record[:skip].each do |test, reason|
             puts "    #{test}".ansi(:bold)
@@ -80,7 +80,11 @@ def end_suite(suite)
         end
       end
 
-      puts tally
+      if total.zero?
+        puts 'No tests were run.'
+      else
+        puts tally
+      end
     end
 
   end

lib/rubytest/recorder.rb

@@ -18,6 +18,10 @@ def skip_test(test, reason)
       self[:skip] << [test, reason]
     end
 
+    def skip_case(test_case, reason)
+      self[:skip] << [test_case, reason]
+    end
+
     # Add `test` to pass set.
     def pass(test)
       self[:pass] << test
@@ -39,9 +43,11 @@ def todo(test, exception)
     #  self[:omit] << [test, exception]
     #end
 
-    # Returns true if their are no test errors or failures.
+    # Returns true if tests were recorded without errors or failures.
     def success?
-      self[:error].size + self[:fail].size > 0 ? false : true
+      return false unless self[:error].empty? && self[:fail].empty?
+
+      [:pass, :todo, :skip].any?{ |status| !self[status].empty? }
     end
 
     # Ignore any other signals.