Fix #42 - Set cases with no tests run (env fail) to retest

Changes

@@ -1,5 +1,8 @@
 Revision history for Perl module TestRail::API
 
+0.027 2015-06-?? TEODESIAN
+    - If no tests are run (environment fail), set test status to 'retest' on non case-per-ok tests
+
 0.026 2015-06-06 TEODESIAN
     - Add --no-match option to testrail-tests to find orphan tests in a tree
     - Upload full raw results to TestRail when not in step_results or case_per_ok mode

dist.ini

@@ -1,6 +1,6 @@
 name = TestRail-API
 main_module = lib/TestRail/API.pm
-version = 0.026
+version = 0.027
 author = George S. Baugh <teodesian@cpan.org>
 license = Perl_5
 copyright_holder = George S. Baugh

lib/Test/LWP/UserAgent/TestRailMock.pm

@@ -1153,7 +1153,7 @@ $VAR4 = bless( {
                  'content-type' => 'application/json; charset=utf-8',
                  'server' => 'Apache/2.4.7 (Ubuntu)'
                }, 'HTTP::Headers' );
-$VAR5 = '[{"id":15,"case_id":8,"status_id":3,"assignedto_id":null,"run_id":22,"title":"STORAGE TANKS SEARED","type_id":6,"priority_id":4,"estimate":null,"estimate_forecast":null,"refs":null,"milestone_id":null,"custom_preconds":null,"custom_steps":null,"custom_expected":null},{"id":15,"case_id":8,"status_id":3,"assignedto_id":null,"run_id":22,"title":"NOT SO SEARED AFTER ARR"},{"id":15,"case_id":8,"status_id":3,"assignedto_id":null,"run_id":22,"title":"skipall.test"} ]';
+$VAR5 = '[{"id":15,"case_id":8,"status_id":3,"assignedto_id":null,"run_id":22,"title":"STORAGE TANKS SEARED","type_id":6,"priority_id":4,"estimate":null,"estimate_forecast":null,"refs":null,"milestone_id":null,"custom_preconds":null,"custom_steps":null,"custom_expected":null},{"id":15,"case_id":8,"status_id":3,"assignedto_id":null,"run_id":22,"title":"NOT SO SEARED AFTER ARR"},{"id":15,"case_id":8,"status_id":3,"assignedto_id":null,"run_id":22,"title":"skipall.test"},{"id":16,"case_id":9,"status_id":3,"assignedto_id":null,"run_id":22,"title":"notests.test"},{"id":17,"case_id":10,"status_id":1,"assignedto_id":null,"run_id":22,"title":"pass.test"}]';
 $mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5));
 
 }
@@ -1268,6 +1268,60 @@ $mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4,
 
 {
 
+$VAR1 = 'index.php?/api/v2/add_result/16';
+$VAR2 = '200';
+$VAR3 = 'OK';
+$VAR4 = bless( {
+                 'connection' => 'close',
+                 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.5',
+                 'client-response-num' => 1,
+                 'date' => 'Tue, 23 Dec 2014 20:02:11 GMT',
+                 'client-peer' => '192.168.122.217:80',
+                 'content-length' => '174',
+                 '::std_case' => {
+                                   'client-date' => 'Client-Date',
+                                   'x-powered-by' => 'X-Powered-By',
+                                   'client-response-num' => 'Client-Response-Num',
+                                   'client-peer' => 'Client-Peer'
+                                 },
+                 'client-date' => 'Tue, 23 Dec 2014 20:02:11 GMT',
+                 'content-type' => 'application/json; charset=utf-8',
+                 'server' => 'Apache/2.4.7 (Ubuntu)'
+               }, 'HTTP::Headers' );
+$VAR5 = '{"id":9,"test_id":16,"status_id":2,"created_by":1,"created_on":1419364931,"assignedto_id":null,"comment":"REAPER FORCES INBOUND","version":null,"elapsed":null,"defects":null}';
+$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5));
+
+}
+
+{
+
+$VAR1 = 'index.php?/api/v2/add_result/17';
+$VAR2 = '200';
+$VAR3 = 'OK';
+$VAR4 = bless( {
+                 'connection' => 'close',
+                 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.5',
+                 'client-response-num' => 1,
+                 'date' => 'Tue, 23 Dec 2014 20:02:11 GMT',
+                 'client-peer' => '192.168.122.217:80',
+                 'content-length' => '174',
+                 '::std_case' => {
+                                   'client-date' => 'Client-Date',
+                                   'x-powered-by' => 'X-Powered-By',
+                                   'client-response-num' => 'Client-Response-Num',
+                                   'client-peer' => 'Client-Peer'
+                                 },
+                 'client-date' => 'Tue, 23 Dec 2014 20:02:11 GMT',
+                 'content-type' => 'application/json; charset=utf-8',
+                 'server' => 'Apache/2.4.7 (Ubuntu)'
+               }, 'HTTP::Headers' );
+$VAR5 = '{"id":10,"test_id":17,"status_id":1,"created_by":1,"created_on":1419364931,"assignedto_id":null,"comment":"REAPER FORCES INBOUND","version":null,"elapsed":null,"defects":null}';
+$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5));
+
+}
+
+{
+
 $VAR1 = 'index.php?/api/v2/add_result/10';
 $VAR2 = '200';
 $VAR3 = 'OK';

lib/Test/Rail/Parser.pm

@@ -85,6 +85,8 @@ In both this mode and step_results, the file name of the test is expected to cor
 
 This module also attempts to calculate the elapsed time to run each test if it is run by a prove plugin rather than on raw TAP.
 
+The constructor will terminate if the statuses 'pass', 'fail', 'retest', 'skip', 'todo_pass', and 'todo_fail' are not registered as result internal names in your TestRail install.
+
 =cut
 
 sub new {
@@ -144,21 +146,24 @@ sub new {
 
     #Discover possible test statuses
     $tropts->{'statuses'} = $tr->getPossibleTestStatuses();
-    my @ok = grep {$_->{'name'} eq 'passed'} @{$tropts->{'statuses'}};
+    my @ok     = grep {$_->{'name'} eq 'passed'} @{$tropts->{'statuses'}};
     my @not_ok = grep {$_->{'name'} eq 'failed'} @{$tropts->{'statuses'}};
-    my @skip = grep {$_->{'name'} eq 'skip'} @{$tropts->{'statuses'}};
-    my @todof = grep {$_->{'name'} eq 'todo_fail'} @{$tropts->{'statuses'}};
-    my @todop = grep {$_->{'name'} eq 'todo_pass'} @{$tropts->{'statuses'}};
+    my @skip   = grep {$_->{'name'} eq 'skip'} @{$tropts->{'statuses'}};
+    my @todof  = grep {$_->{'name'} eq 'todo_fail'} @{$tropts->{'statuses'}};
+    my @todop  = grep {$_->{'name'} eq 'todo_pass'} @{$tropts->{'statuses'}};
+    my @retest = grep {$_->{'name'} eq 'retest'} @{$tropts->{'statuses'}};
     confess("No status with internal name 'passed' in TestRail!") unless scalar(@ok);
     confess("No status with internal name 'failed' in TestRail!") unless scalar(@not_ok);
     confess("No status with internal name 'skip' in TestRail!") unless scalar(@skip);
     confess("No status with internal name 'todo_fail' in TestRail!") unless scalar(@todof);
     confess("No status with internal name 'todo_pass' in TestRail!") unless scalar(@todop);
-    $tropts->{'ok'} = $ok[0];
-    $tropts->{'not_ok'} = $not_ok[0];
-    $tropts->{'skip'} = $skip[0];
+    confess("No status with internal name 'retest' in TestRail!") unless scalar(@retest);
+    $tropts->{'ok'}        = $ok[0];
+    $tropts->{'not_ok'}    = $not_ok[0];
+    $tropts->{'skip'}      = $skip[0];
     $tropts->{'todo_fail'} = $todof[0];
     $tropts->{'todo_pass'} = $todop[0];
+    $tropts->{'retest'}    = $retest[0];
 
     #Grab run
     my $run_id = $tropts->{'run_id'};
@@ -453,6 +458,7 @@ sub EOFCallback {
 
     my $status = $self->{'tr_opts'}->{'ok'}->{'id'};
     $status = $self->{'tr_opts'}->{'not_ok'}->{'id'} if $self->has_problems();
+    $status = $self->{'tr_opts'}->{'retest'}->{'id'} if !$self->tests_run(); #No tests were run, env fail
     $status = $self->{'tr_opts'}->{'skip'}->{'id'} if $self->skip_all();
 
     #Optional args
@@ -464,6 +470,7 @@ sub EOFCallback {
 
     print "# Setting results...\n";
     my $cres = _set_result($run_id,$test_name,$status,$notes,$options,$custom_options);
+    $self->{'global_status'} = $status;
 
     undef $self->{'tr_opts'} unless $self->{'tr_opts'}->{'debug'};
 
@@ -477,13 +484,14 @@ sub _set_result {
 
     print "# Test elapsed: ".$options->{'elapsed'}."\n" if $options->{'elapsed'};
 
-    print "# Attempting to find case by title '".$test_name."'...\n";
+    print "# Attempting to find case by title '".$test_name." in run $run_id'...\n";
     $tc = $self->{'tr_opts'}->{'testrail'}->getTestByName($run_id,$test_name);
     if (!defined($tc) || (reftype($tc) || 'undef') ne 'HASH') {
         cluck("ERROR: Could not find test case: $tc");
         $self->{'errors'}++;
         return 0;
     }
+
     my $xid = $tc ? $tc->{'id'} : '???';
 
     my $cres;

t/Test-Rail-Parser.t

@@ -7,7 +7,7 @@ use Scalar::Util qw{reftype};
 use TestRail::API;
 use Test::LWP::UserAgent::TestRailMock;
 use Test::Rail::Parser;
-use Test::More 'tests' => 48;
+use Test::More 'tests' => 58;
 use Test::Fatal qw{exception};
 
 #Same song and dance as in TestRail-API.t
@@ -128,6 +128,7 @@ isa_ok($tap,"Test::Rail::Parser");
 if (!$res) {
     $tap->run();
     is($tap->{'errors'},0,"No errors encountered uploading case results");
+    is($tap->{'global_status'},5, "Test global result is FAIL when one subtest fails");
 }
 
 #Default mode
@@ -233,6 +234,7 @@ isa_ok($tap,"Test::Rail::Parser");
 if (!$res) {
     $tap->run();
     is($tap->{'errors'},0,"No errors encountered uploading case results");
+    is($tap->{'global_status'},6, "Test global result is SKIP on skip all");
 }
 
 #Ok, let's test the plan, config, and spawn bits.
@@ -447,4 +449,51 @@ $res = exception {
 };
 isnt($res,undef,"TR Parser explodes on instantiation with invalid section");
 
+undef $tap;
+$res = exception {
+    $tap = Test::Rail::Parser->new({
+        'source'              => 't/notests.test',
+        'apiurl'              => $apiurl,
+        'user'                => $login,
+        'pass'                => $pw,
+        'debug'               => $debug,
+        'browser'             => $browser,
+        'run'                 => 'BogoRun',
+        'project'             => 'TestProject',
+        'merge'               => 1,
+        'spawn'               => 9,
+    });
+};
+is($res,undef,"TR Parser doesn't explode on instantiation");
+isa_ok($tap,"Test::Rail::Parser");
+
+if (!$res) {
+    $tap->run();
+    is($tap->{'errors'},0,"No errors encountered uploading case results");
+    is($tap->{'global_status'},4, "Test global result is RETEST on env fail");
+}
+
+undef $tap;
+$res = exception {
+    $tap = Test::Rail::Parser->new({
+        'source'              => 't/pass.test',
+        'apiurl'              => $apiurl,
+        'user'                => $login,
+        'pass'                => $pw,
+        'debug'               => $debug,
+        'browser'             => $browser,
+        'run'                 => 'BogoRun',
+        'project'             => 'TestProject',
+        'merge'               => 1,
+        'spawn'               => 9,
+    });
+};
+is($res,undef,"TR Parser doesn't explode on instantiation");
+isa_ok($tap,"Test::Rail::Parser");
+
+if (!$res) {
+    $tap->run();
+    is($tap->{'errors'},0,"No errors encountered uploading case results");
+    is($tap->{'global_status'},1, "Test global result is PASS on ok test");
+}
 0;

t/notests.test

@@ -0,0 +1,5 @@
+use strict;
+use warnings;
+use Test::More 'tests' => 2;
+note "whee";
+die();

t/pass.test

@@ -0,0 +1,2 @@
+use Test::More 'tests' => 1;
+pass("yay!");

t/skipall.test

@@ -1 +1,2 @@
-use Test::More 'skip_all' => 'cause I can';
+use Test::More 'skip_all' => 'cause I can', 'tests' => 1;
+pass('whee');