Fix #49: Mark entire run as todo_pass if those are the only issues

Also update POD to make these rules clear

Changes

@@ -1,9 +1,11 @@
 Revision history for Perl module TestRail::API
 
-0.027 2015-06-?? TEODESIAN
+0.027 2015-06-12 TEODESIAN
     - If no tests are run (environment fail), set test status to 'retest' on non case-per-ok tests
     - Fix issue where having a ~/.testrailrc would prevent make test from allowing install by cpan client
     - Add Plan Summarizer function
+    - Always append the full raw TAP to all results
+    - If the only oddities in a test are passing TODO tests, mark entire run as TODO PASS.
 
 0.026 2015-06-06 TEODESIAN
     - Add --no-match option to testrail-tests to find orphan tests in a tree

lib/Test/LWP/UserAgent/TestRailMock.pm

@@ -1153,7 +1153,7 @@ $VAR4 = bless( {
                  'content-type' => 'application/json; charset=utf-8',
                  'server' => 'Apache/2.4.7 (Ubuntu)'
                }, 'HTTP::Headers' );
-$VAR5 = '[{"id":15,"case_id":8,"status_id":3,"assignedto_id":null,"run_id":22,"title":"STORAGE TANKS SEARED","type_id":6,"priority_id":4,"estimate":null,"estimate_forecast":null,"refs":null,"milestone_id":null,"custom_preconds":null,"custom_steps":null,"custom_expected":null},{"id":15,"case_id":8,"status_id":3,"assignedto_id":null,"run_id":22,"title":"NOT SO SEARED AFTER ARR"},{"id":15,"case_id":8,"status_id":3,"assignedto_id":null,"run_id":22,"title":"skipall.test"},{"id":16,"case_id":9,"status_id":3,"assignedto_id":null,"run_id":22,"title":"notests.test"},{"id":17,"case_id":10,"status_id":1,"assignedto_id":null,"run_id":22,"title":"pass.test"}]';
+$VAR5 = '[{"id":15,"case_id":8,"status_id":3,"assignedto_id":null,"run_id":22,"title":"STORAGE TANKS SEARED","type_id":6,"priority_id":4,"estimate":null,"estimate_forecast":null,"refs":null,"milestone_id":null,"custom_preconds":null,"custom_steps":null,"custom_expected":null},{"id":15,"case_id":8,"status_id":3,"assignedto_id":null,"run_id":22,"title":"NOT SO SEARED AFTER ARR"},{"id":15,"case_id":8,"status_id":3,"assignedto_id":null,"run_id":22,"title":"skipall.test"},{"id":16,"case_id":9,"status_id":3,"assignedto_id":null,"run_id":22,"title":"notests.test"},{"id":17,"case_id":10,"status_id":1,"assignedto_id":null,"run_id":22,"title":"pass.test"},{"id":18,"case_id":10,"status_id":1,"assignedto_id":null,"run_id":22,"title":"todo_pass.test"}]';
 $mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5));
 
 }
@@ -1320,6 +1320,34 @@ $mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4,
 
 }
 
+{
+
+$VAR1 = 'index.php?/api/v2/add_result/18';
+$VAR2 = '200';
+$VAR3 = 'OK';
+$VAR4 = bless( {
+                 'connection' => 'close',
+                 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.5',
+                 'client-response-num' => 1,
+                 'date' => 'Tue, 23 Dec 2014 20:02:11 GMT',
+                 'client-peer' => '192.168.122.217:80',
+                 'content-length' => '174',
+                 '::std_case' => {
+                                   'client-date' => 'Client-Date',
+                                   'x-powered-by' => 'X-Powered-By',
+                                   'client-response-num' => 'Client-Response-Num',
+                                   'client-peer' => 'Client-Peer'
+                                 },
+                 'client-date' => 'Tue, 23 Dec 2014 20:02:11 GMT',
+                 'content-type' => 'application/json; charset=utf-8',
+                 'server' => 'Apache/2.4.7 (Ubuntu)'
+               }, 'HTTP::Headers' );
+$VAR5 = '{"id":10,"test_id":18,"status_id":8,"created_by":1,"created_on":1419364931,"assignedto_id":null,"comment":"REAPER FORCES INBOUND","version":null,"elapsed":null,"defects":null}';
+$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5));
+
+}
+
+
 {
 
 $VAR1 = 'index.php?/api/v2/add_result/10';

lib/Test/Rail/Parser.pm

@@ -87,6 +87,17 @@ This module also attempts to calculate the elapsed time to run each test if it i
 
 The constructor will terminate if the statuses 'pass', 'fail', 'retest', 'skip', 'todo_pass', and 'todo_fail' are not registered as result internal names in your TestRail install.
 
+If you are not in case_per_ok mode, the global status of the case will be set according to the following rules:
+
+    1. If there are no issues whatsoever besides TODO failing tests & skips, mark as PASS
+    2. If there are any non-skipped or TODOed fails OR a bad plan (extra/missing tests), mark as FAIL
+    3. If there are only SKIPs (e.g. plan => skip_all), mark as SKIP
+    4. If the only issues with the test are TODO tests that pass, mark as TODO PASS (to denote these TODOs for removal).
+    5. If no tests are run at all, mark as 'retest'.  This is making the assumption that such failures are due to test environment being setup improperly;
+       which can be remediated and retested.
+
+Step results will always be whatever status is relevant to the particular step.
+
 =cut
 
 sub new {
@@ -457,9 +468,10 @@ sub EOFCallback {
     my $test_name  = basename($self->{'file'});
 
     my $status = $self->{'tr_opts'}->{'ok'}->{'id'};
-    $status = $self->{'tr_opts'}->{'not_ok'}->{'id'} if $self->has_problems();
-    $status = $self->{'tr_opts'}->{'retest'}->{'id'} if !$self->tests_run(); #No tests were run, env fail
-    $status = $self->{'tr_opts'}->{'skip'}->{'id'} if $self->skip_all();
+    $status = $self->{'tr_opts'}->{'not_ok'}->{'id'}    if $self->has_problems();
+    $status = $self->{'tr_opts'}->{'retest'}->{'id'}    if !$self->tests_run(); #No tests were run, env fail
+    $status = $self->{'tr_opts'}->{'todo_pass'}->{'id'} if $self->todo_passed() && !$self->failed(); #If no fails, but a TODO pass, mark as TODO PASS
+    $status = $self->{'tr_opts'}->{'skip'}->{'id'}      if $self->skip_all(); #Skip all, whee
 
     #Optional args
     my $notes          = $self->{'tr_opts'}->{'test_notes'};

t/Test-Rail-Parser.t

@@ -7,7 +7,7 @@ use Scalar::Util qw{reftype};
 use TestRail::API;
 use Test::LWP::UserAgent::TestRailMock;
 use Test::Rail::Parser;
-use Test::More 'tests' => 58;
+use Test::More 'tests' => 62;
 use Test::Fatal qw{exception};
 
 #Same song and dance as in TestRail-API.t
@@ -128,7 +128,7 @@ isa_ok($tap,"Test::Rail::Parser");
 if (!$res) {
     $tap->run();
     is($tap->{'errors'},0,"No errors encountered uploading case results");
-    is($tap->{'global_status'},5, "Test global result is FAIL when one subtest fails");
+    is($tap->{'global_status'},5, "Test global result is FAIL when one subtest fails even if there are TODO passes");
 }
 
 #Default mode
@@ -496,4 +496,30 @@ if (!$res) {
     is($tap->{'errors'},0,"No errors encountered uploading case results");
     is($tap->{'global_status'},1, "Test global result is PASS on ok test");
 }
+
+undef $tap;
+$res = exception {
+    $tap = Test::Rail::Parser->new({
+        'source'              => 't/todo_pass.test',
+        'apiurl'              => $apiurl,
+        'user'                => $login,
+        'pass'                => $pw,
+        'debug'               => $debug,
+        'browser'             => $browser,
+        'run'                 => 'BogoRun',
+        'project'             => 'TestProject',
+        'merge'               => 1,
+        'spawn'               => 9,
+    });
+};
+is($res,undef,"TR Parser doesn't explode on instantiation");
+isa_ok($tap,"Test::Rail::Parser");
+
+if (!$res) {
+    $tap->run();
+    is($tap->{'errors'},0,"No errors encountered uploading case results");
+    is($tap->{'global_status'},8, "Test global result is TODO PASS on todo pass test");
+}
+
+
 0;

t/faker.test

@@ -3,9 +3,19 @@
 use strict;
 use warnings;
 
-use Test::More 'tests' => 2;
+use Test::More 'tests' => 4;
 
 is('expected','expected','Expected result OK');
 is('expected','unexpected','Unexpected result not OK');
 
+TODO: {
+    local $TODO = 'Making sure this test is still marked fail, not todo pass';
+    pass('whoop deee');
+}
+
+SKIP: {
+    skip('cuz I can',1);
+    fail('har har');
+}
+
 0;

t/todo_pass.test

@@ -0,0 +1,7 @@
+use Test::More 'tests' => 3;
+pass("yay!");
+TODO: {
+    local $TODO = 'Ez duz it';
+    ok(1,'ruh roh');
+    ok(0,'diddley');
+}