|
|
@@ -62,9 +62,7 @@ Get the TAP Parser ready to talk to TestRail, and register a bunch of callbacks
|
|
|
|
|
|
=item B<project_id> - INTEGER (optional): ID of project containing your desired run. Required if project not passed.
|
|
|
|
|
|
-=item B<step_results> - STRING (optional): 'internal name' of the 'step_results' type field available for your project. Mutually exclusive with case_per_ok
|
|
|
-
|
|
|
-=item B<case_per_ok> - BOOLEAN (optional): Consider test files to correspond to section names, and test steps (OKs) to correspond to tests in TestRail. Mutually exclusive with step_results.
|
|
|
+=item B<step_results> - STRING (optional): 'internal name' of the 'step_results' type field available for your project.
|
|
|
|
|
|
=item B<result_options> - HASHREF (optional): Extra options to set with your result. See L<TestRail::API>'s createTestResults function for more information.
|
|
|
|
|
|
@@ -84,14 +82,13 @@ Get the TAP Parser ready to talk to TestRail, and register a bunch of callbacks
|
|
|
|
|
|
=back
|
|
|
|
|
|
-It is worth noting that if neither step_results or case_per_ok is passed, that the test will be passed if it has no problems of any sort, failed otherwise.
|
|
|
In both this mode and step_results, the file name of the test is expected to correspond to the test name in TestRail.
|
|
|
|
|
|
This module also attempts to calculate the elapsed time to run each test if it is run by a prove plugin rather than on raw TAP.
|
|
|
|
|
|
The constructor will terminate if the statuses 'pass', 'fail', 'retest', 'skip', 'todo_pass', and 'todo_fail' are not registered as result internal names in your TestRail install.
|
|
|
|
|
|
-If you are not in case_per_ok mode, the global status of the case will be set according to the following rules:
|
|
|
+The global status of the case will be set according to the following rules:
|
|
|
|
|
|
1. If there are no issues whatsoever besides TODO failing tests & skips, mark as PASS
|
|
|
2. If there are any non-skipped or TODOed fails OR a bad plan (extra/missing tests), mark as FAIL
|
|
|
@@ -146,7 +143,6 @@ sub new {
|
|
|
'project' => delete $opts->{'project'},
|
|
|
'project_id' => delete $opts->{'project_id'},
|
|
|
'step_results' => delete $opts->{'step_results'},
|
|
|
- 'case_per_ok' => delete $opts->{'case_per_ok'},
|
|
|
'plan' => delete $opts->{'plan'},
|
|
|
'configs' => delete $opts->{'configs'} // [],
|
|
|
'testsuite_id' => delete $opts->{'testsuite_id'},
|
|
|
@@ -160,7 +156,6 @@ sub new {
|
|
|
};
|
|
|
|
|
|
confess("plan passed, but no run passed!") if !$tropts->{'run'} && $tropts->{'plan'};
|
|
|
- confess("case_per_ok and step_results options are mutually exclusive") if ($tropts->{'case_per_ok'} && $tropts->{'step_results'});
|
|
|
|
|
|
#Allow natural confessing from constructor
|
|
|
my $tr = TestRail::API->new($tropts->{'apiurl'},$tropts->{'user'},$tropts->{'pass'},$tropts->{'encoding'},$tropts->{'debug'});
|
|
|
@@ -342,6 +337,7 @@ sub unknownCallback {
|
|
|
#try to pick out the filename if we are running this on TAP in files, where App::Prove is uninvolved
|
|
|
my $file = TestRail::Utils::getFilenameFromTapLine($line);
|
|
|
$self->{'file'} = $file if $file;
|
|
|
+ return;
|
|
|
}
|
|
|
|
|
|
=head2 commentCallback
|
|
|
@@ -361,17 +357,13 @@ sub commentCallback {
|
|
|
if ($line =~ m/^#TESTDESC:\s*/) {
|
|
|
$self->{'tr_opts'}->{'test_desc'} = $line;
|
|
|
$self->{'tr_opts'}->{'test_desc'} =~ s/^#TESTDESC:\s*//g;
|
|
|
- return;
|
|
|
}
|
|
|
-
|
|
|
- #keep all comments before a test that aren't these special directives to save in NOTES field of reportTCResult
|
|
|
- $self->{'tr_opts'}->{'test_notes'} .= "$line\n";
|
|
|
+ return;
|
|
|
}
|
|
|
|
|
|
=head2 testCallback
|
|
|
|
|
|
If we are using step_results, append it to the step results array for use at EOF.
|
|
|
-If we are using case_per_ok, update TestRail per case.
|
|
|
Otherwise, do nothing.
|
|
|
|
|
|
=cut
|
|
|
@@ -388,33 +380,19 @@ sub testCallback {
|
|
|
$self->{'lasttime'} = $tm;
|
|
|
}
|
|
|
|
|
|
- #Default assumption is that case name is step text (case_per_ok), unless...
|
|
|
my $line = $test->as_string;
|
|
|
my $tline = $line;
|
|
|
$tline = "[".strftime("%H:%M:%S %b %e %Y",localtime($self->{'lasttime'}))." ($self->{elapse_display})] $line" if $self->{'track_time'};
|
|
|
$self->{'raw_output'} .= "$tline\n";
|
|
|
|
|
|
#Don't do anything if we don't want to map TR case => ok or use step-by-step results
|
|
|
- if ( !($self->{'tr_opts'}->{'step_results'} || $self->{'tr_opts'}->{'case_per_ok'}) ) {
|
|
|
- print "# Neither step_results or case_per_ok set. No action to be taken, except on a whole test basis.\n" if $self->{'tr_opts'}->{'debug'};
|
|
|
+ if ( !$self->{'tr_opts'}->{'step_results'} ) {
|
|
|
+ print "# step_results not set. No action to be taken, except on a whole test basis.\n" if $self->{'tr_opts'}->{'debug'};
|
|
|
return 1;
|
|
|
}
|
|
|
- if ($self->{'tr_opts'}->{'step_results'} && $self->{'tr_opts'}->{'case_per_ok'}) {
|
|
|
- cluck("ERROR: step_options and case_per_ok options are mutually exclusive!");
|
|
|
- $self->{'errors'}++;
|
|
|
- return 0;
|
|
|
- }
|
|
|
-
|
|
|
- #Can't upload unplanned tests when it's case_per_ok
|
|
|
- if ($test->is_unplanned() && $self->{'tr_opts'}->{'case_per_ok'}) {
|
|
|
- cluck("ERROR: Unplanned test detected. Will not attempt to upload results.");
|
|
|
- $self->{'errors'}++;
|
|
|
- return 0;
|
|
|
- }
|
|
|
|
|
|
$line =~ s/^(ok|not ok)\s[0-9]*\s-\s//g;
|
|
|
my $test_name = $line;
|
|
|
- my $run_id = $self->{'tr_opts'}->{'run_id'};
|
|
|
|
|
|
print "# Assuming test name is '$test_name'...\n" if $self->{'tr_opts'}->{'debug'} && !$self->{'tr_opts'}->{'step_results'};
|
|
|
|
|
|
@@ -456,36 +434,19 @@ sub testCallback {
|
|
|
#If this is a TODO, set the reason in the notes
|
|
|
$self->{'tr_opts'}->{'test_notes'} .= "\nTODO reason: $todo_reason\n" if $todo_reason;
|
|
|
|
|
|
- #Setup step options and exit if that's the mode we be rollin'
|
|
|
- if ($self->{'tr_opts'}->{'step_results'}) {
|
|
|
- my $sr_sys_name = $self->{'tr_opts'}->{'step_results'}->{'name'};
|
|
|
- $self->{'tr_opts'}->{'result_custom_options'} = {} if !defined $self->{'tr_opts'}->{'result_custom_options'};
|
|
|
- $self->{'tr_opts'}->{'result_custom_options'}->{$sr_sys_name} = [] if !defined $self->{'tr_opts'}->{'result_custom_options'}->{$sr_sys_name};
|
|
|
- #TimeStamp every particular step
|
|
|
-
|
|
|
- $line = "[".strftime("%H:%M:%S %b %e %Y",localtime($self->{'lasttime'}))." ($self->{elapse_display})] $line" if $self->{'track_time'};
|
|
|
- #XXX Obviously getting the 'expected' and 'actual' from the tap DIAGs would be ideal
|
|
|
- push(
|
|
|
- @{$self->{'tr_opts'}->{'result_custom_options'}->{$sr_sys_name}},
|
|
|
- TestRail::API::buildStepResults($line,"OK",$status_name,$status)
|
|
|
- );
|
|
|
- print "# Appended step results.\n" if $self->{'tr_opts'}->{'debug'};
|
|
|
- return 1;
|
|
|
- }
|
|
|
-
|
|
|
- #Optional args
|
|
|
- my $notes = $self->{'tr_opts'}->{'test_notes'};
|
|
|
- my $options = $self->{'tr_opts'}->{'result_options'};
|
|
|
- my $custom_options = $self->{'tr_opts'}->{'result_custom_options'};
|
|
|
-
|
|
|
- $self->_set_result($run_id,$test_name,$status,$notes,$options,$custom_options);
|
|
|
- #Re-start the shot clock
|
|
|
- $self->{'starttime'} = time();
|
|
|
-
|
|
|
- #Blank out test description in anticipation of next test
|
|
|
- # also blank out notes
|
|
|
- $self->{'tr_opts'}->{'test_notes'} = undef;
|
|
|
- $self->{'tr_opts'}->{'test_desc'} = undef;
|
|
|
+ my $sr_sys_name = $self->{'tr_opts'}->{'step_results'}->{'name'};
|
|
|
+ $self->{'tr_opts'}->{'result_custom_options'} = {} if !defined $self->{'tr_opts'}->{'result_custom_options'};
|
|
|
+ $self->{'tr_opts'}->{'result_custom_options'}->{$sr_sys_name} = [] if !defined $self->{'tr_opts'}->{'result_custom_options'}->{$sr_sys_name};
|
|
|
+ #TimeStamp every particular step
|
|
|
+
|
|
|
+ $line = "[".strftime("%H:%M:%S %b %e %Y",localtime($self->{'lasttime'}))." ($self->{elapse_display})] $line" if $self->{'track_time'};
|
|
|
+ #XXX Obviously getting the 'expected' and 'actual' from the tap DIAGs would be ideal
|
|
|
+ push(
|
|
|
+ @{$self->{'tr_opts'}->{'result_custom_options'}->{$sr_sys_name}},
|
|
|
+ TestRail::API::buildStepResults($line,"OK",$status_name,$status)
|
|
|
+ );
|
|
|
+ print "# Appended step results.\n" if $self->{'tr_opts'}->{'debug'};
|
|
|
+ return 1;
|
|
|
}
|
|
|
|
|
|
=head2 bailoutCallback
|
|
|
@@ -516,7 +477,6 @@ sub bailoutCallback {
|
|
|
=head2 EOFCallback
|
|
|
|
|
|
If we are running in step_results mode, send over all the step results to TestRail.
|
|
|
-If we are running in case_per_ok mode, do nothing.
|
|
|
Otherwise, upload the overall results of the test to TestRail.
|
|
|
|
|
|
=cut
|
|
|
@@ -529,13 +489,6 @@ sub EOFCallback {
|
|
|
$self->{'tr_opts'}->{'result_options'}->{'elapsed'} = _compute_elapsed($self->{'starttime'},time());
|
|
|
}
|
|
|
|
|
|
- if ($self->{'tr_opts'}->{'case_per_ok'}) {
|
|
|
- print "# Nothing left to do.\n";
|
|
|
- $self->_test_closure();
|
|
|
- undef $self->{'tr_opts'} unless $self->{'tr_opts'}->{'debug'};
|
|
|
- return 1;
|
|
|
- }
|
|
|
-
|
|
|
#Fail if the file is not set
|
|
|
if (!defined($self->{'file'})) {
|
|
|
cluck("ERROR: Cannot detect filename, will not be able to find a Test Case with that name");
|
|
|
@@ -572,8 +525,7 @@ sub EOFCallback {
|
|
|
}
|
|
|
|
|
|
#Optional args
|
|
|
- my $notes = $self->{'tr_opts'}->{'test_notes'};
|
|
|
- $notes = $self->{'raw_output'};
|
|
|
+ my $notes = $self->{'raw_output'};
|
|
|
my $options = $self->{'tr_opts'}->{'result_options'};
|
|
|
my $custom_options = $self->{'tr_opts'}->{'result_custom_options'};
|
|
|
|
