Fix #44: add new binary testrail-results

Changes

@@ -2,6 +2,7 @@ Revision history for Perl module TestRail::API
 
 0.037 2016-04-?? TEODESIAN
     - Fix incorrect POD for TestRail::API::createRunInPlan
+    - Add testrail-results binary and TestRail::Utils::Find::getResults.
 
 0.036 2016-04-25 TEODESIAN
     - Fix using wrong perl during testsuite when running binaries

bin/testrail-results

@@ -0,0 +1,246 @@
+#!/usr/bin/env perl
+# ABSTRACT: List results for specified test(s).
+# PODNAME: TestRail::Bin::Results
+
+=head1 SYNOPSIS
+
+  testrail-results [OPTIONS] test1 test2 ...
+
+  require `which testrail-results`;
+  TestRail::Bin::Results::run('args' => \@args);
+
+=head1 DESCRIPTION
+
+testrail-results - List results for specified test(s).
+
+Searches across multiple runs (and projects) for results for broad-based metrics; especially useful for diagnosing unreliable tests, and establishing defect density for certain features.
+
+Can be used as the modulino TestRail::Bin::Tests.
+Has a single 'run' function which accepts a hash with the 'args' parameter being the array of arguments.
+
+=head1 WARNING
+
+Searching across all projects can take a very long time for highly active TestRail Installations.
+However, cross project metrics are also very useful.
+
+As such, the results from prior searches in json mode may be provided, and the runs previously analyzed therein will not be investigated again.
+
+It is up to the caller to integrate this data into their analysis as may be appropriate.
+
+=head1 PARAMETERS:
+
+=head2 MANDATORY PARAMETERS
+
+=over 4
+
+--apiurl   : full URL to get to TestRail index document
+
+--password : Your TestRail Password, or a valid API key (TestRail 4.2 and above).
+
+--user     : Your TestRail User Name.
+
+=back
+
+All mandatory options not passed with the above switches, or in your ~/.testrailrc will be prompted for.
+
+=head2 SEMI-OPTIONAL PARAMETERS
+
+=over 4
+
+-e --encoding : Character encoding of arguments.  Defaults to UTF-8. See L<Encode::Supported> for supported encodings.
+
+=back
+
+=head2 OPTIONAL PARAMETERS
+
+=over 4
+
+-j --project : Restrict search to provided project name.  May be passed multiple times.
+
+-r --run     : Restrict search to runs with the provided name.  May be passed multiple times.
+
+-p --plan    : Restrict search to plans with the provided name. May be passed multiple times.
+
+-g --grep    : Restrict results printed to those matching the provided pattern. Great for looking for specific failure conditions.
+
+-c --cachefile : Load the provided file as a place to pick up your search from.
+
+--json       : Print results as a JSON serialization.
+
+=back
+
+=head1 CONFIGURATION FILE
+
+In your \$HOME, (or the current directory, if your system has no concept of a home directory) put a file called .testrailrc with key=value syntax separated by newlines.
+Valid Keys are the same as documented by L<App::Prove::Plugin::TestRail>.
+All options specified thereby are overridden by passing the command-line switches above.
+
+=head1 MISCELLANEOUS OPTIONS:
+
+=over 4
+
+--help : show this output
+
+=back
+
+=cut
+
+package TestRail::Bin::Results;
+
+use strict;
+use warnings;
+use utf8;
+
+use TestRail::API;
+use TestRail::Utils;
+use TestRail::Utils::Find;
+
+use Getopt::Long qw{GetOptionsFromArray};
+use File::HomeDir qw{my_home};
+use JSON::MaybeXS ();
+use Statistics::Descriptive;
+
+if (!caller()) {
+    my ($out,$code) = run('args' => \@ARGV);
+    print "$out\n";
+    exit $code;
+}
+
+sub run {
+    my %params = @_;
+    my $opts ={};
+
+    #Parse config file if we are missing api url/key or user
+    my $homedir = my_home() || '.';
+    if (-e $homedir . '/.testrailrc') {
+        $opts = TestRail::Utils::parseConfig($homedir);
+    }
+
+    GetOptionsFromArray($params{'args'},
+        'apiurl=s'        => \$opts->{'apiurl'},
+        'password=s'      => \$opts->{'password'},
+        'user=s'          => \$opts->{'user'},
+        'j|project=s@'    => \$opts->{'projects'},
+        'p|plan=s@'       => \$opts->{'plans'},
+        'r|run=s@'        => \$opts->{'runs'},
+        'e|encoding=s'    => \$opts->{'encoding'},
+        'g|grep=s'        => \$opts->{'pattern'},
+        'c|cachefile=s'   => \$opts->{'cachefile'},
+        'json'            => \$opts->{'json'},
+        'h|help'          => \$opts->{'help'},
+    );
+
+    if ($opts->{help}) { return ('',TestRail::Utils::help()); }
+
+    die("No tests passed") unless scalar(@{$params{'args'}});
+    die("Prior search file passed does not exist") if $opts->{'cachefile'} && !( -e $opts->{'cachefile'});
+
+    $opts->{'browser'} = $params{'browser'};
+
+    TestRail::Utils::interrogateUser($opts,qw{apiurl user password});
+
+    my $tr = TestRail::Utils::getHandle($opts);
+    my $prior_search;
+    my $prior_runs = [];
+    if ($opts->{'cachefile'}) {
+        my $raw_text = '';
+        open(my $fh, '<', $opts->{'cachefile'}) or die "Could not open $opts->{cachefile}";
+        while (<$fh>) {
+            $raw_text .= $_;
+        }
+        close($fh);
+        $prior_search = JSON::MaybeXS::decode_json($raw_text);
+        foreach my $key (keys(%$prior_search)) {
+            push(@$prior_runs,@{$prior_search->{$key}->{'seen_runs'}});
+        }
+    }
+
+    my $res = TestRail::Utils::Find::getResults($tr,$opts,$prior_runs,@{$params{'args'}});
+
+    my $statuses = $tr->getPossibleTestStatuses();
+    my %status_map;
+    @status_map{map {$_->{'id'}} @$statuses} = map {$_->{'label'}} @$statuses;
+
+    my ($out,$out_json) = ('',{});
+    foreach my $case (keys(%$res)) {
+        $out .= "#############################\n";
+        my $num_runs = 0;
+        my $casetotals = {};
+        my $total_elapsed = 0;
+        my $avg_elapsed = 0;
+        my $median_runtime = 0;
+        my $elapsetotals = [];
+        my $seen_runs    = [];
+
+        foreach my $casedef (@{$res->{$case}}) {
+            push(@$seen_runs, $casedef->{run_id});
+            $num_runs++;
+            #$out .= "Found case '$case' in run $casedef->{run_id}\n";
+            foreach my $result (@{$casedef->{results}}) {
+                $casetotals->{$result->{status_id}}++;
+                push(@$elapsetotals,_elapsed2secs($result->{'elapsed'}));
+            }
+        }
+
+        my $pattern_output = '';
+        $out_json->{$case}->{search_string} = $opts->{'pattern'};
+        $pattern_output = " using search string '$opts->{pattern}'" if $opts->{'pattern'};
+
+        $out .= "$case was present in $num_runs runs$pattern_output.\n";
+        $out_json->{$case}->{'num_runs'}  = $num_runs;
+        $out_json->{$case}->{'seen_runs'} = $seen_runs;
+
+        #Collect time statistics
+        my $timestats = Statistics::Descriptive::Full->new();
+        $timestats->add_data(@$elapsetotals);
+
+        $out_json->{$case}->{total_elapsed}   = $timestats->sum() || 0;
+        $out .= "Total time spent running this test: $out_json->{$case}->{total_elapsed} seconds\n";
+        $out_json->{$case}->{median_elapsed}  = $timestats->median() || 0;
+        $out .= "Median time spent running this test: $out_json->{$case}->{median_elapsed} seconds\n";
+        $out_json->{$case}->{average_elapsed} = $timestats->mean() || 0;
+        $out .= "Mean time spent running this test: $out_json->{$case}->{average_elapsed} seconds\n";
+        $out_json->{$case}->{stdev_elapsed}   = $timestats->standard_deviation() || 0;
+        $out .= "Standard deviations of runtime in test: $out_json->{$case}->{stdev_elapsed}\n";
+        $out_json->{$case}->{max_elapsed}     = $timestats->max() || 0;
+        $out .= "Maximum time spent running this test: $out_json->{$case}->{max_elapsed} seconds\n";
+        $out_json->{$case}->{min_elapsed}     = $timestats->min() || 0;
+        $out .= "Minimum time spent running this test: $out_json->{$case}->{min_elapsed} seconds\n";
+        $out_json->{$case}->{times_executed}  = $timestats->count() || 0;
+        $out .= "Num times this test has been executed: $out_json->{$case}->{times_executed}\n";
+
+        foreach my $status (keys(%$casetotals)) {
+            $out .= "$status_map{$status}: $casetotals->{$status}\n";
+            $out_json->{$case}->{$status_map{$status}} = $casetotals->{$status};
+        }
+    }
+
+    if ($opts->{'json'}) {
+        my $coder = JSON::MaybeXS->new;
+        return ($coder->encode($out_json),0);
+    }
+
+    $out .= "#############################";
+    return ($out,0);
+}
+
+sub _elapsed2secs {
+    my $stamp = shift;
+    return 0 if !$stamp;
+    my ($seconds)         = $stamp =~ m/(\d*)s/;
+    my ($seconds_minutes) = $stamp =~ m/(\d*)m/;
+    my ($seconds_hours)   = $stamp =~ m/(\d*)h/;
+    return ($seconds || 0) + ($seconds_minutes ? $seconds_minutes * 60 : 0) + ($seconds_hours ? $seconds_hours * 3600 : 0);
+}
+
+1;
+
+__END__
+
+L<TestRail::API>
+
+L<File::HomeDir> for the finding of .testrailrc
+
+=head1 SPECIAL THANKS
+
+Thanks to cPanel Inc, for graciously funding the creation of this distribution.

dist.ini

@@ -1,6 +1,6 @@
 name = TestRail-API
 main_module = lib/TestRail/API.pm
-version = 0.036
+version = 0.037
 author = George S. Baugh <teodesian@cpan.org>
 license = Perl_5
 copyright_holder = George S. Baugh
@@ -126,6 +126,8 @@ stopwords = getPlanSummary
 stopwords = getRunSummary
 stopwords = judgements
 stopwords = bailoutCallback
+stopwords = findResults
+stopwords = cachefile
 
 [PkgVersion]
 [AutoPrereqs]

lib/TestRail/API.pm

@@ -1988,13 +1988,17 @@ Gets all possible statuses a test can be set to.
 
 Returns ARRAYREF of status definition HASHREFs.
 
+Caches the result for the lifetime of the TestRail::API object.
+
 =cut
 
 sub getPossibleTestStatuses {
     state $check = compile(Object);
     my ($self) = $check->(@_);
+    return $self->{'status_cache'} if $self->{'status_cache'};
 
-    return $self->_doRequest('index.php?/api/v2/get_statuses');
+    $self->{'status_cache'} = $self->_doRequest('index.php?/api/v2/get_statuses');
+    return $self->{'status_cache'};
 }
 
 =head2 statusNamesToIds(names)

lib/TestRail/Utils/Find.pm

@@ -314,6 +314,63 @@ sub findCases {
     return $ret;
 }
 
+=head2 getResults(options, $prior_runs, @cases)
+
+Get results for tests by name, filtered by the provided options, and skipping any runs found in the provided ARRAYREF of run IDs.
+
+Probably should have called this findResults, but we all prefer to get results right?
+
+=cut
+
+sub getResults {
+    my ($tr,$opts,$prior_runs,@cases) = @_;
+    my $res = {};
+    my $projects = $tr->getProjects();
+
+    #TODO obey status filtering
+    #TODO obey result notes text grepping
+    foreach my $project (@$projects) {
+        next if $opts->{projects} && !( grep { $_ eq $project->{'name'} } @{$opts->{'projects'}} );
+        my $runs = $tr->getRuns($project->{'id'});
+
+        #Translate plan names to ids
+        my $plans = $tr->getPlans($project->{'id'}) || [];
+        $opts->{'runs'} //= [];
+        my $plan_filters = [];
+        foreach my $plan (@$plans) {
+            $plan = $tr->getPlanByID($plan->{'id'});
+            my $plan_runs = $tr->getChildRuns($plan);
+            push(@$runs,@$plan_runs) if $plan_runs;
+        }
+
+        if ($opts->{'plans'}) {
+            @$plan_filters = map { $_->{'id'} } grep { my $p = $_; grep { $p->{'name'} eq $_} @{$opts->{'plans'}} } @$plans;
+        }
+
+        foreach my $run (@$runs) {
+            next if scalar(@{$opts->{runs}}) && !( grep { $_ eq $run->{'name'} } @{$opts->{'runs'}} );
+            next if scalar(@$plan_filters) && !( grep { $run->{'plan_id'} ? $_ eq $run->{'plan_id'} : undef } @$plan_filters );
+            next if grep { $run->{id} eq $_ } @$prior_runs;
+            foreach my $case (@cases) {
+                my $c = $tr->getTestByName($run->{'id'},basename($case));
+                next unless ref $c eq 'HASH';
+
+                $res->{$case} //= [];
+                $c->{results} = $tr->getTestResults($c->{'id'},$tr->{'global_limit'},0);
+
+                #Filter by provided pattern, if any
+                if ($opts->{'pattern'}) {
+                    my $pattern = $opts->{pattern};
+                    @{$c->{results}} = grep { my $comment = $_->{comment} || ''; $comment =~ m/$pattern/i } @{$c->{results}};
+                }
+
+                push(@{$res->{$case}}, $c) if scalar(@{$c->{results}}); #Make sure they weren't filtered out
+            }
+        }
+    }
+    return $res;
+}
+
 1;
 
 __END__

t/TestRail-Utils-Find.t

@@ -231,4 +231,6 @@ like(exception {TestRail::Utils::Find::findCases($opts,@$cases)},qr/Directory pa
 $opts->{'directory'} = 'bogoDir/';
 like(exception {TestRail::Utils::Find::findCases($opts,@$cases)},qr/No such directory/i,"Bad directory being passed results in error");
 
+#XXX Deliberately omitting the tests for getResults.  It's adequately covered (for now) by testrail-results unit test
+
 #Test synchronize

t/testrail-results.t

@@ -0,0 +1,123 @@
+use strict;
+use warnings;
+
+use FindBin;
+
+use lib $FindBin::Bin.'/../bin';
+require 'testrail-results';
+
+use lib $FindBin::Bin.'/lib';
+use Test::LWP::UserAgent::TestRailMock;
+
+use Test::More 'tests' => 22;
+use Capture::Tiny qw{capture_merged};
+
+no warnings qw{redefine once};
+*TestRail::API::getTests = sub {
+    my ($self,$run_id) = @_;
+    return [
+        {
+            'id' => 666,
+            'title' => 'fake.test',
+            'run_id' => $run_id
+        }
+    ];
+};
+
+*TestRail::API::getTestResults = sub {
+    return [
+        {
+            'elapsed' => '1s',
+            'status_id'  => 5
+        },
+        {
+            'elapsed' => '2s',
+            'status_id' => 4,
+            'comment'   => 'zippy'
+        }
+    ];
+};
+
+*TestRail::API::getPlanByID = sub {
+    return {
+        'id' => 40000,
+        'name' => 'mah dubz plan',
+        'entries' => [{
+            'runs' => [
+                {
+                    'name' => 'planrun',
+                    'id'   => '999',
+                    'plan_id' => 40000
+                }
+            ]
+        }]
+    };
+};
+
+use warnings;
+
+#check doing things over all projects/plans/runs
+my @args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake t/fake.test };
+my ($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
+is($code, 0, "Exit code OK looking for results of fake.test");
+like($out,qr/fake\.test was present in 514 runs/,"Gets correct # of runs with test inside it");
+
+#check project filters
+@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --project TestProject t/fake.test };
+($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
+is($code, 0, "Exit code OK looking for results of fake.test");
+like($out,qr/fake\.test was present in 10 runs/,"Gets correct # of runs with test inside it when filtering by project name");
+
+#check plan filters
+@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --plan };
+push(@args,'mah dubz plan', 't/fake.test');
+($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
+is($code, 0, "Exit code OK looking for results of fake.test");
+like($out,qr/fake\.test was present in 257 runs/,"Gets correct # of runs with test inside it when filtering by plan name");
+
+#check run filters
+@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --run FinalRun t/fake.test};
+($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
+is($code, 0, "Exit code OK looking for results of fake.test");
+like($out,qr/fake\.test was present in 1 runs/,"Gets correct # of runs with test inside it when filtering by run name");
+
+#check pattern filters
+@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --grep zippy t/fake.test};
+($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
+is($code, 0, "Exit code OK looking for results of fake.test");
+like($out,qr/Retest: 514/,"Gets correct # & status of runs with test inside it when grepping");
+unlike($out,qr/Failed: 514/,"Gets correct # & status of runs with test inside it when grepping");
+
+@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --json t/fake.test };
+($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
+is($code, 0, "Exit code OK looking for results of fake.test in json mode");
+like($out,qr/num_runs/,"Gets # of runs with test inside it in json mode");
+
+#For making the test data to test the caching
+#open(my $fh, '>', "t/data/faketest_cache.json");
+#print $fh $out;
+#close($fh);
+
+#Check caching
+@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --json --cachefile t/data/faketest_cache.json t/fake.test };
+($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
+is($code, 0, "Exit code OK looking for results of fake.test in json mode");
+chomp $out;
+is($out,"{}","Caching mode works");
+
+#Check time parser
+is(TestRail::Bin::Results::_elapsed2secs('1s'),1,"elapsed2secs works : seconds");
+is(TestRail::Bin::Results::_elapsed2secs('1m'),60,"elapsed2secs works : minutes");
+is(TestRail::Bin::Results::_elapsed2secs('1h'),3600,"elapsed2secs works : hours");
+is(TestRail::Bin::Results::_elapsed2secs('1s1m1h'),3661,"elapsed2secs works :smh");
+
+#Check help output
+@args = qw{--help};
+$0 = $FindBin::Bin.'/../bin/testrail-runs';
+($out,(undef,$code)) = capture_merged {TestRail::Bin::Results::run('args' => \@args)};
+is($code, 0, "Exit code OK asking for help");
+like($out,qr/encoding of arguments/i,"Help output OK");
+
+#Make sure that the binary itself processes args correctly
+$out = `$^X $0 --help`;
+like($out,qr/encoding of arguments/i,"Appears we can run binary successfully");