From 32315e8423d4d146782df75280a76c555b625b37 Mon Sep 17 00:00:00 2001
From: Thomas Sawyer <transfire@gmail.com>
Date: Mon, 28 Sep 2026 12:26:10 -0400
Subject: [PATCH] Fix RubyTest runner reliability :fix:

---
 HISTORY.md                           |   9 +-
 README.md                            |   5 +-
 demo/03_runner_reliability.md        | 122 +++++++++++++++++++++++
 lib/rubytest/cli.rb                  |   8 +-
 lib/rubytest/config.rb               |   2 +-
 lib/rubytest/format/abstract.rb      |   4 +-
 lib/rubytest/format/abstract_hash.rb |  16 +--
 lib/rubytest/format/dotprogress.rb   |   8 +-
 lib/rubytest/recorder.rb             |  10 +-
 lib/rubytest/runner.rb               | 139 +++++++++++++++------------
 10 files changed, 245 insertions(+), 78 deletions(-)
 create mode 100644 demo/03_runner_reliability.md

diff --git a/HISTORY.md b/HISTORY.md
index 6e104fd..4afc7ee 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,13 @@
 # RELEASE HISTORY
 
+## Unreleased
+
+* Fail empty runs and unmatched test file requests instead of reporting success.
+* Apply test filters at every suite level and record skipped tests and cases.
+* Record case errors, finish suite callbacks, and restore reporter output after
+  failures.
+* Fix the CLI's alternate configuration option and `Testfile` discovery.
+
 ## 0.9.0 / 2026-03-31
 
 Maintenance release. Modernized project tooling and merged CLI back in.
@@ -222,4 +230,3 @@ First release of Ruby Test.
 Changes:
 
 * It's Your Birthday!
-
diff --git a/README.md b/README.md
index 60f1a35..5c0415a 100644
--- a/README.md
+++ b/README.md
@@ -63,6 +63,9 @@ test framework or its adapter, e.g.
 
     $ rubytest -r lemon -r ae test/test_*.rb
 
+Rubytest exits unsuccessfully if a requested path matches no files or if the
+selection runs no tests.
+
 Use `-h/--help` to see all available options.
 
 #### Configuration File
@@ -101,7 +104,7 @@ If you are using Rake, shelling out to `rubytest` keeps your test
 environment pristine:
 
     desc "run tests"
-    task :test
+    task :test do
       sh "rubytest"
     end
 
diff --git a/demo/03_runner_reliability.md b/demo/03_runner_reliability.md
new file mode 100644
index 0000000..1f90f4d
--- /dev/null
+++ b/demo/03_runner_reliability.md
@@ -0,0 +1,122 @@
+## Runner reliability
+
+RubyTest reports an explicitly requested file that does not exist, rather than
+passing an empty run.
+
+    runner = Test::Runner.new(files: ['__rubytest_missing_file__.rb'], format: 'test')
+    cleanup_ran = false
+    runner.config.after { cleanup_ran = true }
+    missing_file = begin
+      runner.run
+      nil
+    rescue ArgumentError => error
+      error
+    end
+
+    missing_file.class.assert == ArgumentError
+    missing_file.message.include?('__rubytest_missing_file__.rb').assert == true
+    cleanup_ran.assert == true
+
+An empty suite also has an unsuccessful result.
+
+    empty_runner = Test::Runner.new(suite: [], format: 'test')
+    empty_runner.run.assert == false
+
+### Selection and skips
+
+Test descriptions can be matched at the top level and inside cases. A skipped
+test that matches the selection is still reported.
+
+    class RunnerProbe
+      def initialize(description, skip_reason = nil, &block)
+        @description = description
+        @skip_reason = skip_reason
+        @block = block
+      end
+
+      def call
+        @block.call
+      end
+
+      def skip?
+        @skip_reason
+      end
+
+      def to_s
+        @description
+      end
+    end
+
+    top = RunnerProbe.new('wanted top') { true }
+    other = RunnerProbe.new('other') { true }
+    nested = RunnerProbe.new('wanted nested') { true }
+    skipped = RunnerProbe.new('wanted skipped', 'later') { raise 'should not run' }
+
+    selected = Test::Runner.new(suite: [top, other, [nested, skipped]],
+                                match: ['wanted'], format: 'test')
+    selected.run.assert == true
+    selected.recorder[:pass].size.assert == 2
+    selected.recorder[:skip].size.assert == 1
+
+Skipping an entire case is recorded too.
+
+    skipped_case = Class.new(Array) do
+      def skip?
+        'later'
+      end
+    end.new
+    case_skip_runner = Test::Runner.new(suite: [skipped_case], format: 'test')
+    case_skip_runner.run.assert == true
+    case_skip_runner.recorder[:skip].size.assert == 1
+
+### Case errors and cleanup
+
+A case setup error is recorded, and the next test and suite cleanup still run.
+
+    broken_case = Class.new(Array) do
+      def call
+        raise 'setup failed'
+      end
+    end.new
+    following_test = RunnerProbe.new('following test') { true }
+    case_runner = Test::Runner.new(suite: [broken_case, following_test], format: 'test')
+    suite_ended = false
+    case_runner.after(:suite) { suite_ended = true }
+
+    case_runner.run.assert == false
+    case_runner.recorder[:error].size.assert == 1
+    case_runner.recorder[:pass].size.assert == 1
+    suite_ended.assert == true
+
+An assertion failure in a hash reporter records the failure and restores
+standard output.
+
+    original_stdout = $stdout
+    failing_test = RunnerProbe.new('failing test') { raise Assertion, 'expected failure' }
+    failure_runner = Test::Runner.new(suite: [failing_test], format: 'test')
+
+    failure_runner.run.assert == false
+    failure_runner.recorder[:fail].size.assert == 1
+    ($stdout.equal?(original_stdout)).assert == true
+
+Global assertionless mode treats a false return as a failure.
+
+    previous_assertionless = Test::Config.assertionless
+    begin
+      Test::Config.assertionless = true
+      false_test = RunnerProbe.new('false result') { false }
+      hard_runner = Test::Runner.new(suite: [false_test], format: 'test')
+      hard_runner.run.assert == false
+      hard_runner.recorder[:fail].size.assert == 1
+    ensure
+      Test::Config.assertionless = previous_assertionless
+    end
+
+### CLI configuration
+
+The `--config` option accepts an alternate configuration file.
+
+    require 'rubytest/cli'
+    cli = Test::CLI.new
+    cli.options.parse!(['--config', 'custom-test.rb'])
+    cli.config_file.assert == 'custom-test.rb'
diff --git a/lib/rubytest/cli.rb b/lib/rubytest/cli.rb
index fe70efd..4b4147b 100644
--- a/lib/rubytest/cli.rb
+++ b/lib/rubytest/cli.rb
@@ -9,7 +9,7 @@ class CLI
     # Test configuration file can be in `etc/test.rb` or `config/test.rb`, or
     # `Testfile` or '.test` with optional `.rb` extension, in that order of
     # precedence. To use a different file there is the -c/--config option.
-    GLOB_CONFIG = '{etc/test.rb,config/test.rb,testfile.rb,testfile,.test.rb,.test}'
+    GLOB_CONFIG = '{etc/test.rb,config/test.rb,Testfile.rb,Testfile,testfile.rb,testfile,.test.rb,.test}'
 
     # Convenience method for invoking the CLI.
     #
@@ -95,7 +95,7 @@ def options
           conf.requires.concat makelist(file)
         end
         opt.on '-c', '--config FILE', "use alternate config file" do |file|
-          conf.config_files << file
+          conf.config_file = file
         end
         opt.on '-V' , '--verbose', 'provide extra detail in reports' do
           conf.verbose = true
@@ -141,6 +141,10 @@ def config_file
       @config_file
     end
 
+    def config_file=(file)
+      @config_file = file
+    end
+
     def profile
       @profile
     end
diff --git a/lib/rubytest/config.rb b/lib/rubytest/config.rb
index e516264..a1340a5 100644
--- a/lib/rubytest/config.rb
+++ b/lib/rubytest/config.rb
@@ -51,7 +51,7 @@ def self.assertionless
 
     #
     def self.assertionless=(boolean)
-      @assertionaless = !!boolean
+      @assertionless = !!boolean
     end
 
     # Find and cache project root directory.
diff --git a/lib/rubytest/format/abstract.rb b/lib/rubytest/format/abstract.rb
index 9d84b88..8963b2d 100644
--- a/lib/rubytest/format/abstract.rb
+++ b/lib/rubytest/format/abstract.rb
@@ -44,11 +44,11 @@ def begin_test(test)
       end
 
       #
-      def skip_case(test_case)
+      def skip_case(test_case, reason=nil)
       end
 
       #
-      def skip_test(test)
+      def skip_test(test, reason=nil)
       end
 
       #
diff --git a/lib/rubytest/format/abstract_hash.rb b/lib/rubytest/format/abstract_hash.rb
index 5e60678..e4f4fef 100644
--- a/lib/rubytest/format/abstract_hash.rb
+++ b/lib/rubytest/format/abstract_hash.rb
@@ -62,10 +62,11 @@ def begin_test(test)
     #
     # @return [Hash]
     #
-    def skip_test(test)
+    def skip_test(test, reason=nil)
       h = {}
       h['type'  ] = 'test'
       h['status'] = 'omit'
+      h['reason'] = reason if reason
 
       merge_subtype      h, test
       merge_setup        h, test
@@ -169,7 +170,8 @@ def todo(test, exception)
     #
     def end_test(test)
       super(test)
-      $stdout, $stderr = @stdout, @stderr
+    ensure
+      $stdout, $stderr = @stdout, @stderr if @stdout && @stderr
     end
 
     #
@@ -206,7 +208,7 @@ def end_suite(suite)
     #
     def merge_priority(hash, test, exception)
       level = exception.priority
-      h['priority'] = level.to_i
+      hash['priority'] = level.to_i
     end
 
     #
@@ -233,8 +235,8 @@ def merge_comparison(hash, test, exception)
 
     # Add source location information to hash.
     def merge_source(hash, test)
-      if test.respond_to?('source_location')
-        file, line = source_location
+      if test.respond_to?(:source_location)
+        file, line = test.source_location
         hash['file'   ] = file
         hash['line'   ] = line
         hash['source' ] = code(file, line).to_str
@@ -280,8 +282,8 @@ def merge_coverage(hash, test)
 
     #
     def merge_output(hash)
-      hash['stdout'] = $stdout.string
-      hash['stderr'] = $stderr.string
+      hash['stdout'] = $stdout.respond_to?(:string) ? $stdout.string : ''
+      hash['stderr'] = $stderr.respond_to?(:string) ? $stderr.string : ''
     end
 
     #
diff --git a/lib/rubytest/format/dotprogress.rb b/lib/rubytest/format/dotprogress.rb
index 4bef5bb..23af477 100644
--- a/lib/rubytest/format/dotprogress.rb
+++ b/lib/rubytest/format/dotprogress.rb
@@ -35,7 +35,7 @@ def end_suite(suite)
       puts
 
       if runner.verbose?
-        unless record[:omit].empty?
+        unless record[:skip].empty?
           puts "SKIPPED\n\n"
           record[:skip].each do |test, reason|
             puts "    #{test}".ansi(:bold)
@@ -80,7 +80,11 @@ def end_suite(suite)
         end
       end
 
-      puts tally
+      if total.zero?
+        puts 'No tests were run.'
+      else
+        puts tally
+      end
     end
 
   end
diff --git a/lib/rubytest/recorder.rb b/lib/rubytest/recorder.rb
index e6c7925..28ec69f 100644
--- a/lib/rubytest/recorder.rb
+++ b/lib/rubytest/recorder.rb
@@ -18,6 +18,10 @@ def skip_test(test, reason)
       self[:skip] << [test, reason]
     end
 
+    def skip_case(test_case, reason)
+      self[:skip] << [test_case, reason]
+    end
+
     # Add `test` to pass set.
     def pass(test)
       self[:pass] << test
@@ -39,9 +43,11 @@ def todo(test, exception)
     #  self[:omit] << [test, exception]
     #end
 
-    # Returns true if their are no test errors or failures.
+    # Returns true if tests were recorded without errors or failures.
     def success?
-      self[:error].size + self[:fail].size > 0 ? false : true
+      return false unless self[:error].empty? && self[:fail].empty?
+
+      [:pass, :todo, :skip].any?{ |status| !self[status].empty? }
     end
 
     # Ignore any other signals.
diff --git a/lib/rubytest/runner.rb b/lib/rubytest/runner.rb
index d7ea0b4..a94d47c 100644
--- a/lib/rubytest/runner.rb
+++ b/lib/rubytest/runner.rb
@@ -139,20 +139,27 @@ def run
         # applied and before test files are required.
         config.before.call if config.before
 
-        test_files.each do |test_file|
-          require test_file
-        end
-
-        @reporter  = reporter_load(format)
-        @recorder  = Recorder.new
+        begin
+          test_files.each do |test_file|
+            require test_file
+          end
 
-        @observers = [advice, @recorder, @reporter]
+          @reporter  = reporter_load(format)
+          @recorder  = Recorder.new
 
-        observers.each{ |o| o.begin_suite(suite) }
-        run_thru(suite)
-        observers.each{ |o| o.end_suite(suite) }
+          @observers = [advice, @recorder, @reporter]
 
-        config.after.call if config.after
+          started = false
+          begin
+            observers.each{ |o| o.begin_suite(suite) }
+            started = true
+            run_thru(suite)
+          ensure
+            observers.each{ |o| o.end_suite(suite) } if started
+          end
+        ensure
+          config.after.call if config.after
+        end
       end
 
       recorder.success?
@@ -174,13 +181,13 @@ def ignore_callers
 
     #
     def run_thru(list)
-      list.each do |t|
+      select(list).each do |t|
         if t.respond_to?(:each)
           run_case(t)
         elsif t.respond_to?(:call)
           run_test(t)
         else
-          #run_note(t) ?
+          raise TypeError, "not a test or test case: #{t.inspect}"
         end
       end
     end
@@ -194,15 +201,21 @@ def run_case(tcase)
 
       observers.each{ |o| o.begin_case(tcase) }
 
-      if tcase.respond_to?(:call)
-        tcase.call do
-          run_thru( select(tcase) )
+      begin
+        if tcase.respond_to?(:call)
+          tcase.call do
+            run_thru(tcase)
+          end
+        else
+          run_thru(tcase)
         end
-      else
-        run_thru( select(tcase) )
+      rescue *OPEN_ERRORS
+        raise
+      rescue Exception => exception
+        observers.each{ |o| o.error(tcase, exception) }
+      ensure
+        observers.each{ |o| o.end_case(tcase) }
       end
-
-      observers.each{ |o| o.end_case(tcase) }
     end
 
     # Run a test.
@@ -217,28 +230,26 @@ def run_test(test)
 
       observers.each{ |o| o.begin_test(test) }
       begin
-        success = test.call
-        if config.hard? && !success  # TODO: separate run_test method to speed things up?
-          raise Assertion, "failure of #{test}"
+        exception = nil
+        begin
+          success = test.call
+          raise Assertion, "failure of #{test}" if config.hard? && !success
+        rescue *OPEN_ERRORS
+          raise
+        rescue NotImplementedError => exception
+          result = :todo
+        rescue Exception => exception
+          result = exception.assertion? ? :fail : :error
         else
-          observers.each{ |o| o.pass(test) }
+          result = :pass
         end
-      rescue *OPEN_ERRORS => exception
-        raise exception
-      rescue NotImplementedError => exception
-        #if exception.assertion?  # TODO: May require assertion? for todo in future
-          observers.each{ |o| o.todo(test, exception) }
-        #else
-        #  observers.each{ |o| o.error(test, exception) }
-        #end
-      rescue Exception => exception
-        if exception.assertion?
-          observers.each{ |o| o.fail(test, exception) }
-        else
-          observers.each{ |o| o.error(test, exception) }
+
+        observers.each do |o|
+          exception ? o.public_send(result, test, exception) : o.pass(test)
         end
+      ensure
+        observers.each{ |o| o.end_test(test) }
       end
-      observers.each{ |o| o.end_test(test) }
     end
 
     # TODO: Make sure this filtering code is correct for the complex 
@@ -249,29 +260,36 @@ def run_test(test)
     #
     # @return [Array] selected test cases
     def select(cases)
+      return cases if cases.respond_to?(:ordered?) && cases.ordered?
+      return cases if config.match.empty? && config.units.empty? && config.tags.empty?
+
       selected = []
-      if cases.respond_to?(:ordered?) && cases.ordered?
-        cases.each do |tc|
+      cases.each do |tc|
+        unless tc.respond_to?(:each) || tc.respond_to?(:call)
+          raise TypeError, "not a test or test case: #{tc.inspect}"
+        end
+
+        # Keep cases so their descendants can be filtered. The case itself
+        # may have a different description, unit, or tags from its tests.
+        if tc.respond_to?(:each)
           selected << tc
+          next
         end
-      else
-        cases.each do |tc|
-          next if tc.respond_to?(:skip?) && tc.skip?
-          next if !config.match.empty? && !config.match.any?{ |m| m =~ tc.to_s }
 
-          if !config.units.empty?
-            next unless tc.respond_to?(:unit)
-            next unless config.units.find{ |u| tc.unit.start_with?(u) }
-          end
+        next if !config.match.empty? && !config.match.any?{ |m| tc.to_s.include?(m) }
 
-          if !config.tags.empty?
-            next unless tc.respond_to?(:tags)
-            tc_tags = [tc.tags].flatten.map{ |t| t.to_s }
-            next if (config.tags & tc_tags).empty?
-          end
+        if !config.units.empty?
+          next unless tc.respond_to?(:unit)
+          next unless config.units.any?{ |u| tc.unit.to_s.start_with?(u) }
+        end
 
-          selected << tc
+        if !config.tags.empty?
+          next unless tc.respond_to?(:tags)
+          tc_tags = [tc.tags].flatten.map{ |t| t.to_s }
+          next if (config.tags & tc_tags).empty?
         end
+
+        selected << tc
       end
       selected
     end
@@ -320,12 +338,13 @@ def reporter_list
     #
     # @return [Array<String>]
     def resolve_test_files
-      list = config.files.flatten
-      list = list.map{ |f| Dir[f] }.flatten
-      list = list.map{ |f| File.directory?(f) ? Dir[File.join(f, '**/*.rb')] : f }
-      list = list.flatten.uniq
-      list = list.map{ |f| File.expand_path(f) } 
-      list
+      config.files.flatten.flat_map do |pattern|
+        files = Dir[pattern].flat_map do |file|
+          File.directory?(file) ? Dir[File.join(file, '**/*.rb')] : [file]
+        end
+        raise ArgumentError, "no test files match #{pattern.inspect}" if files.empty?
+        files
+      end.uniq.map{ |file| File.expand_path(file) }
     end
 
     # Change to directory and run block.