Route results through Recorder and bundle core reporters

HISTORY.md

@@ -2,6 +2,14 @@
 
 ## Unreleased
 
+* Bring summary, outline, and progress reporters into Rubytest and add a TAP 13
+  reporter that counts completed results, including skipped and broken cases.
+* Remove CLI shortcuts for the unbundled TAP-Y and TAP-J formats.
+* Remove the old Recorder status callbacks and status-indexed result table.
+* Preserve test and teardown exceptions together in a result, and turn outcome
+  hook failures into recorded errors.
+* Route structured test results through the recorder to reporters, while
+  retaining Advice hooks and older reporter callbacks.
 * Fail empty runs and unmatched test file requests instead of reporting success.
 * Apply test filters at every suite level and record skipped tests and cases.
 * Record case errors, finish suite callbacks, and restore reporter output after

README.md

@@ -39,6 +39,25 @@ configured to do so.
 That is the crux of Rubytest specification. Rubytest supports some
 additional features that can make its usage even more convenient.
 
+### Results and reporting
+
+The runner turns each completed test into a `Test::Result` with its status,
+exception or skip reason, elapsed time, and captured output when requested.
+If both the test and a hook raise, the result retains both exceptions.
+Skipped and broken cases also produce results. `Test::Recorder` keeps these
+results, calculates the run summary, and forwards reporting events to the
+selected reporter. Advice hooks remain with the runner, where they can affect
+test execution.
+
+Reporters implement `record(result)` for completed outcomes and
+`finish(summary)` for the final totals. Suite and case boundary callbacks are
+available for formats that show nesting.
+
+The built-in formats are `dotprogress` (default), `summary` (one status line per
+test), `outline` (nested cases), `progress` (running count and timing), and
+`tap` (TAP version 13). Select one with `rubytest -f FORMAT`. Formats load only
+when selected; TAP captures test output and emits it as diagnostics.
+
 
 ## Installation
 

demo/03_runner_reliability.md

@@ -55,8 +55,8 @@ test that matches the selection is still reported.
     selected = Test::Runner.new(suite: [top, other, [nested, skipped]],
                                 match: ['wanted'], format: 'test')
     selected.run.assert == true
-    selected.recorder[:pass].size.assert == 2
-    selected.recorder[:skip].size.assert == 1
+    selected.recorder.summary.counts[:pass].assert == 2
+    selected.recorder.summary.counts[:skip].assert == 1
 
 Skipping an entire case is recorded too.
 
@@ -67,7 +67,7 @@ Skipping an entire case is recorded too.
     end.new
     case_skip_runner = Test::Runner.new(suite: [skipped_case], format: 'test')
     case_skip_runner.run.assert == true
-    case_skip_runner.recorder[:skip].size.assert == 1
+    case_skip_runner.recorder.summary.counts[:skip].assert == 1
 
 ### Case errors and cleanup
 
@@ -84,8 +84,8 @@ A case setup error is recorded, and the next test and suite cleanup still run.
     case_runner.after(:suite) { suite_ended = true }
 
     case_runner.run.assert == false
-    case_runner.recorder[:error].size.assert == 1
-    case_runner.recorder[:pass].size.assert == 1
+    case_runner.recorder.summary.counts[:error].assert == 1
+    case_runner.recorder.summary.counts[:pass].assert == 1
     suite_ended.assert == true
 
 An assertion failure in a hash reporter records the failure and restores
@@ -96,7 +96,7 @@ standard output.
     failure_runner = Test::Runner.new(suite: [failing_test], format: 'test')
 
     failure_runner.run.assert == false
-    failure_runner.recorder[:fail].size.assert == 1
+    failure_runner.recorder.summary.counts[:fail].assert == 1
     ($stdout.equal?(original_stdout)).assert == true
 
 Global assertionless mode treats a false return as a failure.
@@ -107,7 +107,7 @@ Global assertionless mode treats a false return as a failure.
       false_test = RunnerProbe.new('false result') { false }
       hard_runner = Test::Runner.new(suite: [false_test], format: 'test')
       hard_runner.run.assert == false
-      hard_runner.recorder[:fail].size.assert == 1
+      hard_runner.recorder.summary.counts[:fail].assert == 1
     ensure
       Test::Config.assertionless = previous_assertionless
     end

demo/04_result_pipeline.md

@@ -0,0 +1,160 @@
+## Result pipeline
+
+The runner sends completed outcomes to the recorder. The recorder keeps them
+and forwards the same result objects, along with case boundaries, to a reporter.
+
+    events = []
+    reporter = Object.new
+    reporter.define_singleton_method(:begin_suite) { |suite| events << :begin_suite }
+    reporter.define_singleton_method(:begin_case) { |tcase| events << :begin_case }
+    reporter.define_singleton_method(:begin_test) { |test| events << :begin_test }
+    reporter.define_singleton_method(:record) { |result| events << result }
+    reporter.define_singleton_method(:end_test) { |test| events << :end_test }
+    reporter.define_singleton_method(:end_case) { |tcase| events << :end_case }
+    reporter.define_singleton_method(:finish) { |summary| events << summary }
+
+    runner_type = Class.new(Test::Runner) do
+      define_method(:reporter_load) { |_format| reporter }
+    end
+
+    passing = -> { true }
+    failing = -> { raise Assertion, 'no' }
+    broken = -> { raise 'unexpected' }
+    pending = -> { raise NotImplementedError, 'later' }
+    skipped = -> { raise 'must not run' }
+    skipped.define_singleton_method(:skip?) { 'later' }
+
+    skipped_case = [-> { raise 'must not run' }]
+    skipped_case.define_singleton_method(:skip?) { 'case later' }
+    broken_case = Class.new(Array) do
+      def call
+        raise 'case setup failed'
+      end
+    end.new
+
+    runner = runner_type.new(suite: [[passing, failing], broken, pending, skipped,
+                                     skipped_case, broken_case], format: 'test')
+    runner.run.assert == false
+
+    statuses = runner.recorder.results.map(&:status)
+    statuses.assert == [:pass, :fail, :error, :todo, :skip, :skip, :error]
+    runner.recorder.results.map(&:kind).assert ==
+      [:test, :test, :test, :test, :test, :case, :case]
+    runner.recorder.results.find { |result| result.status == :fail }.test.assert.equal? failing
+    runner.recorder.summary.counts[:skip].assert == 2
+    events.grep(Test::Result).map(&:object_id).assert ==
+      runner.recorder.results.map(&:object_id)
+    events.count(:begin_case).assert == 2
+    events.count(:end_case).assert == 2
+    summary = events.last
+    summary.class.assert == Test::RunSummary
+    summary.counts[:error].assert == 2
+    summary.success?.assert == false
+
+An after hook can turn an otherwise passing test into a failed result before
+that result is recorded.
+
+    hook_reporter = Object.new
+    hook_reporter.define_singleton_method(:record) { |result| }
+    hook_reporter.define_singleton_method(:finish) { |summary| }
+    hook_runner_type = Class.new(Test::Runner) do
+      define_method(:reporter_load) { |_format| hook_reporter }
+    end
+    hook_runner = hook_runner_type.new(suite: [-> { true }], format: 'test')
+    hook_runner.after(:test) { raise Assertion, 'verification failed' }
+    hook_runner.run.assert == false
+    hook_runner.recorder.results.first.status.assert == :fail
+    hook_runner.recorder.results.first.exception.message.include?('verification failed').assert == true
+
+If both a test and teardown fail, the error takes precedence while the result
+keeps both exceptions. An outcome hook that raises also becomes a result.
+
+    double_runner = hook_runner_type.new(suite: [-> { raise Assertion, 'body failed' }],
+                                         format: 'test')
+    double_runner.after(:test) { raise 'cleanup failed' }
+    double_runner.run.assert == false
+    double_result = double_runner.recorder.results.first
+    double_result.status.assert == :error
+    double_result.exception.message.assert == 'cleanup failed'
+    double_result.exceptions.first.message.include?('body failed').assert == true
+    double_result.exceptions.last.message.assert == 'cleanup failed'
+
+    outcome_runner = hook_runner_type.new(suite: [-> { true }], format: 'test')
+    outcome_runner.upon(:pass) { raise 'outcome hook failed' }
+    outcome_runner.run.assert == false
+    outcome_runner.recorder.results.first.status.assert == :error
+    outcome_runner.recorder.results.first.exception.message.assert == 'outcome hook failed'
+
+A failing before hook is recorded as an error; the test body is not called,
+and the after hook still runs.
+
+    calls = 0
+    cleanup_ran = false
+    before_runner = hook_runner_type.new(suite: [-> { calls += 1 }], format: 'test')
+    before_runner.before(:test) { raise 'setup failed' }
+    before_runner.after(:test) { cleanup_ran = true }
+    before_runner.run.assert == false
+    before_runner.recorder.results.first.status.assert == :error
+    calls.assert == 0
+    cleanup_ran.assert == true
+
+A case after hook error is recorded after its child test and case reporting
+still ends.
+
+    case_end_events = []
+    case_end_reporter = Object.new
+    case_end_reporter.define_singleton_method(:record) { |result| case_end_events << result.status }
+    case_end_reporter.define_singleton_method(:end_case) { |tcase| case_end_events << :end_case }
+    case_end_reporter.define_singleton_method(:finish) { |summary| }
+    case_end_runner_type = Class.new(Test::Runner) do
+      define_method(:reporter_load) { |_format| case_end_reporter }
+    end
+    case_end_runner = case_end_runner_type.new(suite: [[-> { true }]], format: 'test')
+    case_end_runner.after(:case) { raise 'case teardown failed' }
+    case_end_runner.run.assert == false
+    case_end_events.assert == [:pass, :error, :end_case]
+
+    case_end_events.clear
+    case_body_calls = 0
+    case_cleanup_ran = false
+    case_start_runner = case_end_runner_type.new(suite: [[-> { case_body_calls += 1 }]], format: 'test')
+    case_start_runner.before(:case) { raise 'case setup failed' }
+    case_start_runner.after(:case) { case_cleanup_ran = true }
+    case_start_runner.run.assert == false
+    case_end_events.assert == [:error, :end_case]
+    case_body_calls.assert == 0
+    case_cleanup_ran.assert == true
+
+The hash reporter receives captured output in its result, and the runner
+restores the process streams after execution.
+
+    require 'rubytest/format/test'
+    hash_reporter_type = Class.new(Test::Reporters::Test) do
+      attr_reader :rows
+
+      def initialize(runner)
+        super
+        @rows = []
+      end
+
+      def record(result)
+        @rows << super
+      end
+    end
+    hash_runner_type = Class.new(Test::Runner) do
+      define_method(:reporter_load) { |_format| hash_reporter_type.new(self) }
+    end
+    original_stdout, original_stderr = $stdout, $stderr
+    output_runner = hash_runner_type.new(suite: [-> { print 'hello'; warn 'oops' }], format: 'test')
+    output_runner.upon(:pass) { print ' from hook' }
+    output_runner.run.assert == true
+    row = output_runner.reporter.rows.first
+    row['stdout'].assert == 'hello from hook'
+    row['stderr'].assert == "oops\n"
+    ($stdout.equal?(original_stdout)).assert == true
+    ($stderr.equal?(original_stderr)).assert == true
+
+    case_output_runner = hash_runner_type.new(suite: [skipped_case], format: 'test')
+    case_output_runner.run.assert == true
+    case_output_runner.reporter.rows.first['type'].assert == 'case'
+    case_output_runner.reporter.rows.first['status'].assert == 'omit'

demo/05_builtin_reporters.md

@@ -0,0 +1,75 @@
+## Built-in reporters
+
+The human-readable formats consume the same results, including pending tests
+and skipped cases. They can be selected without installing another gem.
+
+    require 'stringio'
+
+    probe_type = Class.new do
+      def initialize(label, reason = nil, &action)
+        @label, @reason, @action = label, reason, action
+      end
+
+      def call
+        @action.call
+      end
+
+      def skip?
+        @reason
+      end
+
+      def to_s
+        @label
+      end
+    end
+
+    passing = probe_type.new('passing') { true }
+    failing = probe_type.new('failing') { raise Assertion, 'wrong value' }
+    pending = probe_type.new('pending') { raise NotImplementedError, 'later' }
+    skipped = probe_type.new('skipped', 'not available') { raise 'must not run' }
+    skipped_case = Class.new(Array) do
+      def skip?; 'case unavailable'; end
+      def to_s; 'skipped case'; end
+    end.new([passing])
+    broken_case = Class.new(Array) do
+      def call; raise 'case setup failed'; end
+      def to_s; 'broken case'; end
+    end.new([passing])
+    suite = [[passing, failing], pending, skipped, skipped_case, broken_case]
+
+    render = lambda do |format, tests = suite|
+      require "rubytest/format/#{format}"
+      output = StringIO.new
+      reporter_type = Test::Reporters.const_get(format.capitalize)
+      runner_type = Class.new(Test::Runner) do
+        define_method(:reporter_load) { |_name| reporter_type.new(self, output: output) }
+      end
+      runner = runner_type.new(suite: tests, format: format)
+      [runner.run, output.string, runner.recorder.summary]
+    end
+
+    %w[summary outline progress].each do |format|
+      success, output, report = render.call(format)
+      success.assert == false
+      report.total.assert == 6
+      output.include?('failing').assert == true
+      output.include?('skipped case').assert == true
+      output.include?('6').assert == true
+    end
+
+TAP emits one test point for every result. Test output is captured and appears
+as diagnostics, so it cannot corrupt the TAP stream. The final plan uses the
+number actually recorded, including the broken and skipped cases.
+
+    output_test = probe_type.new('output') { print "ok 999 - injected\n"; true }
+    tap_suite = [[output_test, failing], pending, skipped, skipped_case, broken_case]
+    success, output, report = render.call('tap', tap_suite)
+    success.assert == false
+    lines = output.lines.map(&:chomp)
+    lines.first.assert == 'TAP version 13'
+    lines.last.assert == '1..6'
+    lines.grep(/\A(?:ok|not ok) \d+ - /).size.assert == report.total
+    lines.grep(/# SKIP /).size.assert == 2
+    lines.grep(/# TODO /).size.assert == 1
+    lines.grep(/\A# stdout: ok 999 - injected/).size.assert == 1
+    lines.grep(/\Anot ok \d+ - broken case/).size.assert == 1

lib/rubytest.rb

@@ -12,6 +12,7 @@ module Test
 require_relative 'rubytest/core_ext'
 require_relative 'rubytest/code_snippet'
 require_relative 'rubytest/config'
+require_relative 'rubytest/result'
 require_relative 'rubytest/recorder'
 require_relative 'rubytest/advice'
 require_relative 'rubytest/runner'

lib/rubytest/cli.rb

@@ -62,12 +62,6 @@ def options
         opt.on '-f', '--format NAME', 'report format' do |name|
           conf.format = name
         end
-        opt.on '-y', '--tapy', 'shortcut for -f tapy' do
-          conf.format = 'tapy'
-        end
-        opt.on '-j', '--tapj', 'shortcut for -f tapj' do
-          conf.format = 'tapj'
-        end
 
         opt.on '-t', '--tag TAG', 'select tests by tag' do |tag|
           conf.tags.concat makelist(tag)