Fix #52: Add --autoclose feature

Changes

@@ -7,6 +7,8 @@ Revision history for Perl module TestRail::API
     - Fix issue where specifying sections past the first defined in a project would fail to restrict spawning to said sections
     - Add Plan Summarizer function
     - Always append the full raw TAP to all results
+    - Add closePlan and closeRun functions to TestRail::API
+    - Add option to binaries, plugin to close plan/run if no untested/retest exist at end of TAP parse
 
 0.026 2015-06-06 TEODESIAN
     - Add --no-match option to testrail-tests to find orphan tests in a tree

bin/testrail-report

@@ -77,7 +77,8 @@ overall result of the test will be used as the pass/fail for the test.
 
 =head3 RESULT OPTIONS
 
-    --version : String describing the version of the system under test.
+    --version   : String describing the version of the system under test.
+    --autoclose : If there are no more tests in 'untested' or 'retest' status for the specified run/plan, close it.
 
 =head2 PROVE PLUGIN:
 
@@ -203,6 +204,8 @@ PARAMETERS:
   [RESULT OPTIONS]
 
     --version : String describing the version of the system under test.
+    --autoclose : If there are no more tests in 'untested' or 'retest'
+                  status for the specified run/plan, close it.
 
 PROVE PLUGIN:
 
@@ -227,7 +230,7 @@ TESTING OPTIONS:
 
 #Main loop------------
 
-my ($help,$apiurl,$user,$password,$project,$run,$case_per_ok,$step_results,$mock,$configs,$plan,$version,$spawn,$sections);
+my ($help,$apiurl,$user,$password,$project,$run,$case_per_ok,$step_results,$mock,$configs,$plan,$version,$spawn,$sections,$autoclose);
 
 #parse switches
 GetOptions(
@@ -244,6 +247,7 @@ GetOptions(
     'version=s'      => \$version,
     'spawn=i'        => \$spawn,
     'section=s@'     => \$sections,
+    'autoclose'      => \$autoclose,
     'help'           => \$help
 );
 
@@ -364,6 +368,7 @@ foreach my $phil (@files) {
         'result_options' => $result_options,
         'spawn'        => $spawn,
         'sections'     => $sections,
+        'autoclose'    => $autoclose,
         'merge'        => 1
     });
     $tap->run();

dist.ini

@@ -109,6 +109,9 @@ stopwords = getPlansPaginated
 stopwords = getRunsPaginated
 stopwords = getPossibleTestStatuses
 stopwords = userInput
+stopwords = autoclose
+stopwords = closeRun
+stopwords = closePlan
 
 [PkgVersion]
 [AutoPrereqs]

lib/App/Prove/Plugin/TestRail.pm

@@ -36,6 +36,7 @@ If \$HOME/.testrailrc exists, it will be parsed for any of these values in a new
     step_results=sr_sys_name
     spawn=123
     sections=section1:section2:section3: ... :sectionN
+    autoclose=0
 
 Note that passing configurations as filters for runs inside of plans are separated by colons.
 Values passed in via query string will override values in \$HOME/.testrailrc.
@@ -81,18 +82,19 @@ sub load {
     $app->merge(1);
 
     #XXX I can't figure out for the life of me any other way to pass this data. #YOLO
-    $ENV{'TESTRAIL_APIURL'}   = $params->{apiurl};
-    $ENV{'TESTRAIL_USER'}     = $params->{user};
-    $ENV{'TESTRAIL_PASS'}     = $params->{password};
-    $ENV{'TESTRAIL_PROJ'}     = $params->{project};
-    $ENV{'TESTRAIL_RUN'}      = $params->{run};
-    $ENV{'TESTRAIL_PLAN'}     = $params->{plan};
-    $ENV{'TESTRAIL_CONFIGS'}  = $params->{configs};
-    $ENV{'TESTRAIL_VERSION'}  = $params->{version};
-    $ENV{'TESTRAIL_CASEOK'}   = $params->{case_per_ok};
-    $ENV{'TESTRAIL_STEPS'}    = $params->{step_results};
-    $ENV{'TESTRAIL_SPAWN'}    = $params->{spawn};
-    $ENV{'TESTRAIL_SECTIONS'} = $params->{sections};
+    $ENV{'TESTRAIL_APIURL'}    = $params->{apiurl};
+    $ENV{'TESTRAIL_USER'}      = $params->{user};
+    $ENV{'TESTRAIL_PASS'}      = $params->{password};
+    $ENV{'TESTRAIL_PROJ'}      = $params->{project};
+    $ENV{'TESTRAIL_RUN'}       = $params->{run};
+    $ENV{'TESTRAIL_PLAN'}      = $params->{plan};
+    $ENV{'TESTRAIL_CONFIGS'}   = $params->{configs};
+    $ENV{'TESTRAIL_VERSION'}   = $params->{version};
+    $ENV{'TESTRAIL_CASEOK'}    = $params->{case_per_ok};
+    $ENV{'TESTRAIL_STEPS'}     = $params->{step_results};
+    $ENV{'TESTRAIL_SPAWN'}     = $params->{spawn};
+    $ENV{'TESTRAIL_SECTIONS'}  = $params->{sections};
+    $ENV{'TESTRAIL_AUTOCLOSE'} = $params->{autoclose};
     return $class;
 }
 

lib/Test/Rail/Harness.pm

@@ -49,14 +49,17 @@ sub make_parser {
     $args->{'project'} = $ENV{'TESTRAIL_PROJ'};
     $args->{'run'}     = $ENV{'TESTRAIL_RUN'};
     $args->{'plan'}    = $ENV{'TESTRAIL_PLAN'};
+    
     @configs = split(/:/,$ENV{'TESTRAIL_CONFIGS'}) if $ENV{'TESTRAIL_CONFIGS'};
-    $args->{'configs'} = \@configs if scalar(@configs);
+    $args->{'configs'}        = \@configs if scalar(@configs);
     $args->{'result_options'} = {'version' => $ENV{'TESTRAIL_VERSION'}} if $ENV{'TESTRAIL_VERSION'};
     $args->{'case_per_ok'}    = $ENV{'TESTRAIL_CASEOK'};
     $args->{'step_results'}   = $ENV{'TESTRAIL_STEPS'};
     $args->{'spawn'}          = $ENV{'TESTRAIL_SPAWN'};
+    
     @sections = split(/:/,$ENV{'TESTRAIL_SECTIONS'}) if $ENV{'TESTRAIL_SECTIONS'};
-    $args->{'sections'} = \@sections if scalar(@sections);
+    $args->{'sections'}  = \@sections if scalar(@sections);
+    $args->{'autoclose'} = $ENV{'TESTRAIL_AUTOCLOSE'};
 
     #for Testability of plugin
     if ($ENV{'TESTRAIL_MOCKED'}) {

lib/Test/Rail/Parser.pm

@@ -76,6 +76,8 @@ Get the TAP Parser ready to talk to TestRail, and register a bunch of callbacks
 
 =item B<sections> - ARRAYREF (optional): Restrict a spawned run to cases in these particular sections.
 
+=item B<autoclose> - BOOLEAN (optional): If no cases in the run/plan are marked 'untested' or 'retest', go ahead and close the run.  Default false.
+
 =back
 
 =back
@@ -128,6 +130,7 @@ sub new {
         'configs'      => delete $opts->{'configs'} // [],
         'spawn'        => delete $opts->{'spawn'},
         'sections'     => delete $opts->{'sections'},
+        'autoclose'    => delete $opts->{'autoclose'},
         #Stubs for extension by subclassers
         'result_options'        => delete $opts->{'result_options'},
         'result_custom_options' => delete $opts->{'result_custom_options'}
@@ -253,7 +256,6 @@ sub new {
         confess("Could not spawn run with requested parameters!") if !$tropts->{'run_id'};
         print "# Success!\n"
     }
-
     confess("No run ID provided, and no run with specified name exists in provided project/plan!") if !$tropts->{'run_id'};
 
     $self = $class->SUPER::new($opts);
@@ -454,6 +456,7 @@ sub EOFCallback {
 
     if ($self->{'tr_opts'}->{'case_per_ok'}) {
         print "# Nothing left to do.\n";
+        $self->_test_closure();
         undef $self->{'tr_opts'} unless $self->{'tr_opts'}->{'debug'};
         return 1;
     }
@@ -483,6 +486,7 @@ sub EOFCallback {
 
     print "# Setting results...\n";
     my $cres = _set_result($run_id,$test_name,$status,$notes,$options,$custom_options);
+    $self->_test_closure();
     $self->{'global_status'} = $status;
 
     undef $self->{'tr_opts'} unless $self->{'tr_opts'}->{'debug'};
@@ -550,6 +554,28 @@ sub _compute_elapsed {
     return $datestr;
 }
 
+sub _test_closure {
+    my ($self) = @_;
+    return unless $self->{'tr_opts'}->{'autoclose'};
+    my $is_plan = $self->{'tr_opts'}->{'plan'} ? 1 : 0;
+    my $id      = $self->{'tr_opts'}->{'plan'} ? $self->{'tr_opts'}->{'plan'}->{'id'} : $self->{'tr_opts'}->{'run'};
+  
+    if ($is_plan) {
+        my $plan_summary = $self->{'tr_opts'}->{'testrail'}->getPlanSummary($id);
+
+        return if ( $plan_summary->{'totals'}->{'untested'} + $plan_summary->{'totals'}->{'retest'} );
+        print "# No more outstanding cases detected.  Closing Plan.\n";
+        $self->{'plan_closed'} = 1;
+        return $self->{'tr_opts'}->{'testrail'}->closePlan($id);
+    }
+
+    my ($run_summary) = $self->{'tr_opts'}->{'testrail'}->getRunSummary($id);
+    return if ( $run_summary->{'run_status'}->{'untested'} + $run_summary->{'run_status'}->{'retest'} );
+    print "# No more outstanding cases detected.  Closing Run.\n";
+    $self->{'run_closed'} = 1;
+    return $self->{'tr_opts'}->{'testrail'}->closeRun($self->{'tr_opts'}->{'run_id'});
+}
+
 1;
 
 __END__

lib/TestRail/API.pm

@@ -1217,6 +1217,29 @@ sub getRunByID {
     return $self->_doRequest("index.php?/api/v2/get_run/$run_id");
 }
 
+=head2 B<closeRun (run_id)>
+
+Close the specified run.
+
+=over 4
+
+=item INTEGER C<RUN ID> - ID of desired run.
+
+=back
+
+Returns run definition HASHREF on success, false on failure.
+
+    $tr->closeRun(90210);
+
+=cut
+
+sub closeRun {
+    my ($self,$run_id) = @_;
+    confess("Object methods must be called by an instance") unless ref($self);
+    confess("Run ID must be integer") unless $self->_checkInteger($run_id);
+    return $self->_doRequest("index.php?/api/v2/close_run/$run_id",'POST');
+}
+
 =head2 B<getRunSummary(runs)>
 
 Returns array of hashrefs describing the # of tests in the run(s) with the available statuses.
@@ -1236,6 +1259,8 @@ Returns ARRAY of run HASHREFs with the added key 'run_status' holding a hashref
 
 sub getRunSummary {
     my ($self,@runs) = @_;
+    confess("Object methods must be called by an instance") unless ref($self);
+    confess("All Plans passed must be HASHREFs") unless scalar( grep {(reftype($_) || 'undef') eq 'HASH'} @runs ) == scalar(@runs);
 
     #Translate custom statuses
     my $statuses = $self->getPossibleTestStatuses();
@@ -1548,6 +1573,8 @@ sub getPlanByID {
 =head2 B<getPlanSummary(plan_ID)>
 
 Returns hashref describing the various pass, fail, etc. percentages for tests in the plan.
+The 'totals' key has total cases in each status ('status' => count)
+The 'percentages' key has the same, but as a percentage of the total.
 
 =over 4
 
@@ -1590,7 +1617,7 @@ sub getPlanSummary {
 
 =head2 B<createRunInPlan (plan_id,suite_id,name,description,milestone_id,assigned_to_id,config_ids,case_ids)>
 
-Create a run.
+Create a run in a plan.
 
 =over 4
 
@@ -1646,6 +1673,29 @@ sub createRunInPlan {
     return $result;
 }
 
+=head2 B<closePlan (plan_id)>
+
+Close the specified plan.
+
+=over 4
+
+=item INTEGER C<PLAN ID> - ID of desired plan.
+
+=back
+
+Returns plan definition HASHREF on success, false on failure.
+
+    $tr->closePlan(75020);
+
+=cut
+
+sub closePlan {
+    my ($self,$plan_id) = @_;
+    confess("Object methods must be called by an instance") unless ref($self);
+    confess("Plan ID must be integer") unless $self->_checkInteger($plan_id);
+    return $self->_doRequest("index.php?/api/v2/close_plan/$plan_id",'POST');
+}
+
 =head1 MILESTONE METHODS
 
 =head2 B<createMilestone (project_id,name,description,due_on)>

t/App-Prove-Plugin-Testrail.t

@@ -3,7 +3,7 @@
 use strict;
 use warnings;
 
-use Test::More 'tests' => 5;
+use Test::More 'tests' => 6;
 use Test::Fatal;
 use App::Prove;
 use App::Prove::Plugin::TestRail;
@@ -39,3 +39,8 @@ $prove = App::Prove->new();
 $prove->process_args("-PTestRail=apiurl=http://some.testlink.install/,user=someUser,password=somePassword,project=TestProject,run=bogoRun,version=0.014,case_per_ok=1,spawn=9,sections=fake.test:CARBON LIQUEFACTION",'t/fake.test');
 
 is (exception {$prove->run()},undef,"Running TR parser can discriminate by sections correctly");
+
+$prove = App::Prove->new();
+$prove->process_args("-PTestRail=apiurl=http://some.testlink.install/,user=someUser,password=somePassword,project=TestProject,plan=FinalPlan,run=FinalRun,configs=testConfig,version=0.014,case_per_ok=1,autoclose=1",'t/fake.test');
+
+is (exception {$prove->run()},undef,"Running TR parser with autoclose works correctly");

t/Test-Rail-Parser.t

@@ -7,7 +7,7 @@ use Scalar::Util qw{reftype};
 use TestRail::API;
 use Test::LWP::UserAgent::TestRailMock;
 use Test::Rail::Parser;
-use Test::More 'tests' => 62;
+use Test::More 'tests' => 78;
 use Test::Fatal qw{exception};
 
 #Same song and dance as in TestRail-API.t
@@ -521,5 +521,111 @@ if (!$res) {
     is($tap->{'global_status'},8, "Test global result is TODO PASS on todo pass test");
 }
 
+#Check autoclose functionality against Run with all tests in run status.
+undef $tap;
+$res = exception {
+    $tap = Test::Rail::Parser->new({
+        'source'              => 't/skip.test',
+        'apiurl'              => $apiurl,
+        'user'                => $login,
+        'pass'                => $pw,
+        'debug'               => $debug,
+        'browser'             => $browser,
+        'run'                 => 'FinalRun',
+        'project'             => 'TestProject',
+        'merge'               => 1,
+        'autoclose'           => 1,
+        'spawn'               => 9,
+    });
+};
+is($res,undef,"TR Parser doesn't explode on instantiation");
+isa_ok($tap,"Test::Rail::Parser");
+
+if (!$res) {
+    $tap->run();
+    is($tap->{'errors'},0,"No errors encountered uploading case results");
+    is($tap->{'run_closed'},1, "Run closed by parser when all tests done");
+}
+
+#Check autoclose functionality against Run with not all tests in run status.
+undef $tap;
+$res = exception {
+    $tap = Test::Rail::Parser->new({
+        'source'              => 't/todo_pass.test',
+        'apiurl'              => $apiurl,
+        'user'                => $login,
+        'pass'                => $pw,
+        'debug'               => $debug,
+        'browser'             => $browser,
+        'run'                 => 'BogoRun',
+        'project'             => 'TestProject',
+        'merge'               => 1,
+        'autoclose'           => 1,
+        'spawn'               => 9,
+    });
+};
+is($res,undef,"TR Parser doesn't explode on instantiation");
+isa_ok($tap,"Test::Rail::Parser");
+
+if (!$res) {
+    $tap->run();
+    is($tap->{'errors'},0,"No errors encountered uploading case results");
+    is($tap->{'run_closed'},undef, "Run not closed by parser when results are outstanding");
+}
+
+#Check that autoclose works against plan wiht all tests in run status
+undef $tap;
+$res = exception {
+    $tap = Test::Rail::Parser->new({
+        'source'              => 't/fake.test',
+        'apiurl'              => $apiurl,
+        'user'                => $login,
+        'pass'                => $pw,
+        'debug'               => $debug,
+        'browser'             => $browser,
+        'run'                 => 'FinalRun',
+        'plan'                => 'FinalPlan',
+        'project'             => 'TestProject',
+        'configs'             => ['testConfig'],
+        'merge'               => 1,
+        'autoclose'           => 1,
+        'case_per_ok'         => 1
+    });
+};
+is($res,undef,"TR Parser doesn't explode on instantiation");
+isa_ok($tap,"Test::Rail::Parser");
+
+if (!$res) {
+    $tap->run();
+    is($tap->{'errors'},0,"No errors encountered uploading case results");
+    is($tap->{'plan_closed'},1, "Plan closed by parser when all tests done");
+}
+
+#Check that autoclose works against plan wiht all tests not in run status
+undef $tap;
+$res = exception {
+    $tap = Test::Rail::Parser->new({
+        'source'              => 't/fake.test',
+        'apiurl'              => $apiurl,
+        'user'                => $login,
+        'pass'                => $pw,
+        'debug'               => $debug,
+        'browser'             => $browser,
+        'run'                 => 'BogoRun',
+        'plan'                => 'BogoPlan',
+        'project'             => 'TestProject',
+        'spawn'               => 9,
+        'merge'               => 1,
+        'autoclose'           => 1,
+        'case_per_ok'         => 1
+    });
+};
+is($res,undef,"TR Parser doesn't explode on instantiation");
+isa_ok($tap,"Test::Rail::Parser");
+
+if (!$res) {
+    $tap->run();
+    is($tap->{'errors'},0,"No errors encountered uploading case results");
+    is($tap->{'plan_closed'},undef, "Plan not closed by parser when results are outstanding");
+}
 
-0;

t/TestRail-API-mockOnly.t

@@ -3,7 +3,7 @@ use warnings;
 
 #Test things we can only mock, because the API doesn't support them.
 
-use Test::More 'tests' => 9;
+use Test::More 'tests' => 13;
 use TestRail::API;
 use Test::LWP::UserAgent::TestRailMock;
 use Scalar::Util qw{reftype};
@@ -34,3 +34,9 @@ $projResType = $tr->getTestResultFieldByName('moo_results');
 is($projResType,0,"Bad name returns no result field");
 $projResType = $tr->getTestResultFieldByName('step_results',66669);
 is($projResType,-3,"Bad project returns no result field");
+
+# I can't delete closed plans, so...test closePlan et cetera
+is(reftype($tr->closeRun(666)),'HASH',"Can close run that exists");
+is($tr->closeRun(90210),-404,"Can't close run that doesn't exist");
+is(reftype($tr->closePlan(23)),'HASH',"Can close plan that exists");
+is($tr->closePlan(75020),-404,"Can't close plan that doesn't exist");

t/arg_types.t

@@ -2,7 +2,7 @@ use strict;
 use warnings;
 
 use TestRail::API;
-use Test::More 'tests' => 139;
+use Test::More 'tests' => 141;
 use Test::Fatal;
 use Class::Inspector;
 use Test::LWP::UserAgent;
@@ -74,6 +74,8 @@ isnt( exception {$tr->createRunInPlan() },undef,'createRunInPlan returns error w
 isnt( exception {$tr->translateConfigNamesToIds()}, undef,'translateConfigNamesToIds returns error when no arguments are passed');
 isnt( exception {$tr->userNamesToIds()}, undef,'userNamesToIds returns error when no arguments are passed');
 isnt( exception {$tr->statusNamesToIds()}, undef,'statusNamesToIds returns error when no arguments are passed');
+isnt( exception {$tr->getRunSummary()}, undef,'getRunSummary returns error when no arguments are passed');
+isnt( exception {$tr->getPlanSummary()}, undef,'getPlanSummary returns error when no arguments are passed');
 
 #1-arg functions
 is(exception {$tr->deleteCase(1)},            undef,'deleteCase returns no error when int arg passed');

t/fake.tap

@@ -0,0 +1,19 @@
+t/fake.test ..
+1..2
+ok 1 - STORAGE TANKS SEARED
+# whee
+not ok 2 - NOT SO SEARED AFTER ARR
+
+#   Failed test 'NOT SO SEARED AFTER ARR'
+#   at t/fake.test line 10.
+# Looks like you failed 1 test of 2.
+Dubious, test returned 1 (wstat 256, 0x100)
+Failed 1/2 subtests
+
+Test Summary Report
+-------------------
+t/fake.test (Wstat: 256 Tests: 2 Failed: 1)
+  Failed test:  2
+  Non-zero exit status: 1
+Files=1, Tests=2,  2 wallclock secs ( 0.02 usr  0.00 sys +  0.01 cusr  0.00 csys =  0.03 CPU)
+Result: FAIL

t/testrail-report.t

@@ -1,7 +1,7 @@
 use strict;
 use warnings;
 
-use Test::More 'tests' => 14;
+use Test::More 'tests' => 16;
 
 my @args = ($^X,qw{bin/testrail-report --apiurl http://testrail.local --user "test@fake.fake" --password "fake" --project "CRUSH ALL HUMANS" --run "SEND T-1000 INFILTRATION UNITS BACK IN TIME" --mock t/test_multiple_files.tap});
 my $out = `@args`;
@@ -43,6 +43,12 @@ is($? >> 8, 0, "Exit code OK reported with spawn");
 $matches = () = $out =~ m/with specified sections/ig;
 is($matches,1,"Attempts to spawn work");
 
+#Test that the autoclose option works
+@args = ($^X,qw{bin/testrail-report --apiurl http://testrail.local --user "test@fake.fake" --password "fake" --project "TestProject" --run "FinalRun" --plan "FinalPlan" --config "testConfig" --case-ok --autoclose --mock t/fake.tap});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK when doing autoclose");
+like($out,qr/closing plan/i,"Run closure reported to user");
+
 #Test that help works
 @args = ($^X,qw{bin/testrail-report --help});
 $out = `@args`;

t/testrail-runs.t

@@ -8,7 +8,7 @@ my @args = ($^X,qw{bin/testrail-runs --apiurl http://testrail.local --user "test
 my $out = `@args`;
 is($? >> 8, 0, "Exit code OK looking for runs with passes");
 chomp $out;
-like($out,qr/^TestingSuite\nOtherOtherSuite$/,"Gets run correctly looking for passes");
+like($out,qr/^TestingSuite\nOtherOtherSuite\nFinalRun$/,"Gets run correctly looking for passes");
 
 #check status filters
 @args = ($^X,qw{bin/testrail-runs --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j "TestProject" --mock --status passed});