Fix #118: implement a --fast mode in testrail-results

Also add in --perfile and multi-cachefile for further optimization

bin/testrail-results

@@ -67,10 +67,14 @@ All mandatory options not passed with the above switches, or in your ~/.testrail
 
 -d --defect  : Restrict results printed to those related to the provided defect(s).  May be passed multiple times.
 
--c --cachefile : Load the provided file as a place to pick up your search from.
+-c --cachefile : Load the provided file as a place to pick up your search from.  May be passed multiple times.
+
+-f --fast : Reduce the number of required HTTP requests at the cost of historical data (previous statuses in runs).  Speeds up gathering a great deal when checking many tests.
 
 --json       : Print results as a JSON serialization.
 
+--perfile : Output JSON summary data per file to the provided directory when in json mode
+
 =back
 
 =head1 CONFIGURATION FILE
@@ -104,6 +108,7 @@ use File::HomeDir qw{my_home};
 use JSON::MaybeXS ();
 use Statistics::Descriptive;
 use List::MoreUtils qw{uniq};
+use File::Basename qw{basename};
 
 if (!caller()) {
     my ($out,$code) = run('args' => \@ARGV);
@@ -132,15 +137,16 @@ sub run {
         'g|grep=s'        => \$opts->{'pattern'},
         'd|defect=s@'     => \$opts->{'defects'},
         'v|version=s@'    => \$opts->{'versions'},
-        'c|cachefile=s'   => \$opts->{'cachefile'},
+        'c|cachefile=s@'  => \$opts->{'cachefile'},
+        'f|fast'          => \$opts->{'fast'},
         'json'            => \$opts->{'json'},
+        'perfile=s'       => \$opts->{'perfile'},
         'h|help'          => \$opts->{'help'},
     );
 
     if ($opts->{help}) { return ('',TestRail::Utils::help()); }
 
     die("No tests passed") unless scalar(@{$params{'args'}});
-    die("Prior search file passed does not exist") if $opts->{'cachefile'} && !( -e $opts->{'cachefile'});
 
     $opts->{'browser'} = $params{'browser'};
 
@@ -151,21 +157,27 @@ sub run {
     my $prior_runs = [];
     my $prior_plans = [];
     if ($opts->{'cachefile'}) {
-        my $raw_text = '';
-        open(my $fh, '<', $opts->{'cachefile'}) or die "Could not open $opts->{cachefile}";
-        while (<$fh>) {
-            $raw_text .= $_;
-        }
-        close($fh);
-        $prior_search = JSON::MaybeXS::decode_json($raw_text);
-        foreach my $key (keys(%$prior_search)) {
-            push(@$prior_runs,@{$prior_search->{$key}->{'seen_runs'}});
-            push(@$prior_plans,@{$prior_search->{$key}->{'seen_plans'}});
+        foreach my $cf (@{$opts->{cachefile}}) {
+            die("Prior search file '$cf' passed does not exist") if $cf && !( -e $cf);
+            my $raw_text = '';
+            open(my $fh, '<', $cf) or die "Could not open $cf";
+            while (<$fh>) {
+                $raw_text .= $_;
+            }
+            close($fh);
+            $prior_search = JSON::MaybeXS::decode_json($raw_text);
+            foreach my $key (keys(%$prior_search)) {
+                push(@$prior_runs, @{$prior_search->{$key}->{'seen_runs'}});
+                push(@$prior_plans,@{$prior_search->{$key}->{'seen_plans'}});
+            }
+            @$prior_plans = uniq(@$prior_plans);
+            @$prior_runs  = uniq(@$prior_runs);
+            $opts->{'plan_ids'} = $prior_plans;
+            $opts->{'run_ids'}  = $prior_runs;
         }
-        $opts->{'plan_ids'} = $prior_plans;
     }
 
-    my ($res,$seen_plans) = TestRail::Utils::Find::getResults($tr,$opts,$prior_runs,@{$params{'args'}});
+    my ($res,$seen_plans,$seen_runs) = TestRail::Utils::Find::getResults($tr,$opts,@{$params{'args'}});
 
     #Make sure subsequent runs keep ignoring the prior plans
     push(@$seen_plans,@$prior_plans);
@@ -185,10 +197,8 @@ sub run {
         my $avg_elapsed = 0;
         my $median_runtime = 0;
         my $elapsetotals = [];
-        my $seen_runs    = $prior_runs;
 
         foreach my $casedef (@{$res->{$case}}) {
-            push(@$seen_runs, $casedef->{run_id});
             $num_runs++;
             #$out .= "Found case '$case' in run $casedef->{run_id}\n";
             foreach my $result (@{$casedef->{results}}) {
@@ -198,11 +208,11 @@ sub run {
                     $versions_by_status->{$result->{status_id}} //= [];
                     push(@{$versions_by_status->{$result->{status_id}}},$result->{version}) if $result->{version};
                 }
-                push(@$defects, $result->{defects}) if $result->{defects};
+                push(@$defects, $result->{defects})    if $result->{defects} && ref($result->{defects}) ne 'ARRAY';
+                push(@$defects, @{$result->{defects}}) if $result->{defects} && ref($result->{defects}) eq 'ARRAY';
                 push(@$elapsetotals,_elapsed2secs($result->{'elapsed'}));
             }
         }
-        @$seen_runs = uniq(@$seen_runs);
         @$defects   = uniq(@$defects);
         foreach my $st (keys(%$versions_by_status)) {
             @{$versions_by_status->{$st}} = uniq(@{$versions_by_status->{$st}});
@@ -253,7 +263,17 @@ sub run {
 
     if ($opts->{'json'}) {
         my $coder = JSON::MaybeXS->new;
-        return ($coder->encode($out_json),0);
+        return ($coder->encode($out_json),0) unless $opts->{'perfile'};
+
+        die("no such directory $opts->{perfile}") unless -d $opts->{perfile};
+
+        foreach my $test (keys(%$out_json)) {
+            my $tn = basename($test);
+            open( my $fh, '>', "$opts->{perfile}/$tn.json") or die "could not open $opts->{perfile}/$tn.json";
+            print $fh $coder->encode({ $test => $out_json->{$test} });
+            close $fh;
+        }
+        return('',0);
     }
 
     $out .= "#############################";

dist.ini

@@ -131,6 +131,7 @@ stopwords = findResults
 stopwords = cachefile
 stopwords = getChildSections
 stopwords = POSTs
+stopwords = perfile
 
 [PkgVersion]
 [AutoPrereqs]

lib/TestRail/API.pm

@@ -1405,7 +1405,7 @@ sub getRunResults {
     return $results;
 }
 
-=head2 B<getRunResultsPaginated(run_id,limit,offset)
+=head2 B<getRunResultsPaginated(run_id,limit,offset)>
 
 =cut
 

lib/TestRail/Utils/Find.pm

@@ -8,7 +8,7 @@ use warnings;
 
 use Carp qw{confess cluck};
 use Scalar::Util qw{blessed};
-use List::Util qw{any};
+use List::Util qw{any first};
 use List::MoreUtils qw{uniq};
 
 use File::Find;
@@ -323,7 +323,7 @@ sub findCases {
     return $ret;
 }
 
-=head2 getResults(options, $prior_runs, @cases)
+=head2 getResults(options, @cases)
 
 Get results for tests by name, filtered by the provided options, and skipping any runs found in the provided ARRAYREF of run IDs.
 
@@ -337,43 +337,58 @@ Valid Options:
 
 =item B<plans> - ARRAYREF of plan names to check.
 
-=item B<plan_ids> - ARRAYREF of plan IDs to check.
-
 =item B<runs> - ARRAYREF of runs names to check.
 
+=item B<plan_ids> - ARRAYREF of plan IDs to NOT check.
+
+=item B<run_ids> - ARRAYREF of run IDs to NOT check.
+
 =item B<pattern> - Pattern to filter case results on.
 
 =item B<defects> - ARRAYREF of defects of which at least one must be present in a result.
 
+=item B<fast> - Whether to get only the latest result from the test in your run(s).  This can significantly speed up operations when gathering metrics for large numbers of tests.
+
 =back
 
 =cut
 
 sub getResults {
-    my ($tr,$opts,$prior_runs,@cases) = @_;
+    my ($tr,$opts,@cases) = @_;
     my $res = {};
     my $projects = $tr->getProjects();
 
-
-    my $prior_plans = [];
+    my (@seenRunIds,@seenPlanIds);
 
     #TODO obey status filtering
     #TODO obey result notes text grepping
     foreach my $project (@$projects) {
         next if $opts->{projects} && !( grep { $_ eq $project->{'name'} } @{$opts->{'projects'}} );
         my $runs = $tr->getRuns($project->{'id'});
+        push(@seenRunIds, map { $_->{id} } @$runs);
 
         #Translate plan names to ids
         my $plans = $tr->getPlans($project->{'id'}) || [];
+        push(@seenPlanIds, map { $_->{id} } @$plans);
 
         #Filter out plans which do not match our filters to prevent a call to getPlanByID
         if ($opts->{'plans'}) {
             @$plans = grep { my $p = $_; any { $p->{'name'} eq $_ } @{$opts->{'plans'}} } @$plans;
         }
 
+        #Filter out runs which do not match our filters
+        if ($opts->{'runs'}) {
+            @$runs = grep { my $r = $_; any { $r->{'name'} eq $_ } @{$opts->{'runs'}} } @$runs;
+        }
+
         #Filter out prior plans
         if ($opts->{'plan_ids'}) {
-            @$plans = grep { my $p = $_; any { $p->{'id'} eq $_ } @{$opts->{'plan_ids'}} } @$plans;
+            @$plans = grep { my $p = $_; !any { $p->{'id'} eq $_ } @{$opts->{'plan_ids'}} } @$plans;
+        }
+
+        #Filter out prior runs
+        if ($opts->{'run_ids'}) {
+            @$runs = grep { my $r = $_; !any { $r->{'id'} eq $_ } @{$opts->{'run_ids'}} } @$runs;
         }
 
         $opts->{'runs'} //= [];
@@ -382,9 +397,33 @@ sub getResults {
             my $plan_runs = $tr->getChildRuns($plan);
             push(@$runs,@$plan_runs) if $plan_runs;
         }
+
         foreach my $run (@$runs) {
             next if scalar(@{$opts->{runs}}) && !( grep { $_ eq $run->{'name'} } @{$opts->{'runs'}} );
-            next if grep { $run->{id} eq $_ } @$prior_runs;
+
+            if ($opts->{fast}) {
+                my @csz = @cases;
+                @csz = grep { ref($_) eq 'HASH' } map {
+                    my $cname = basename($_);
+                    my $c = $tr->getTestByName($run->{id},$cname);
+                    $c->{name} = $cname if $c;
+                    $c
+                } @csz;
+                next unless scalar(@csz);
+
+                my $results = $tr->getRunResults($run->{id});
+                foreach my $c (@csz) {
+                    $res->{$c->{name}} //= [];
+                    my $cres = first { $c->{id} == $_->{test_id} } @$results;
+                    next unless $cres;
+
+                    $c->{results} = [$cres];
+                    $c = _filterResults($opts,$c);
+
+                    push(@{$res->{$c->{name}}}, $c) if scalar(@{$c->{results}});
+                }
+                next;
+            }
 
             foreach my $case (@cases) {
                 my $c = $tr->getTestByName($run->{'id'},basename($case));
@@ -392,43 +431,45 @@ sub getResults {
 
                 $res->{$case} //= [];
                 $c->{results} = $tr->getTestResults($c->{'id'},$tr->{'global_limit'},0);
-
-                #Filter by provided pattern, if any
-                if ($opts->{'pattern'}) {
-                    my $pattern = $opts->{pattern};
-                    @{$c->{results}} = grep { my $comment = $_->{comment} || ''; $comment =~ m/$pattern/i } @{$c->{results}};
-                }
-
-                #Filter by the provided case IDs, if any
-                if (ref($opts->{'defects'}) eq 'ARRAY' && scalar(@{$opts->{defects}})) {
-                    @{$c->{results}} = grep {
-                        my $defects = $_->{defects};
-                        any {
-                            my $df_case = $_;
-                            any { $df_case eq $_ } @{$opts->{defects}};
-                        } @$defects
-                    } @{$c->{results}};
-                }
-
-                #Filter by the provided versions, if any
-                if (ref($opts->{'versions'}) eq 'ARRAY' && scalar(@{$opts->{versions}})) {
-                    @{$c->{results}} = grep {
-                        my $version = $_->{version};
-                        any { $version eq $_ } @{$opts->{versions}};
-                    } @{$c->{results}};
-                }
-
+                $c = _filterResults($opts,$c);
 
                 push(@{$res->{$case}}, $c) if scalar(@{$c->{results}}); #Make sure they weren't filtered out
             }
         }
+    }
 
-        push(@$prior_plans, map {$_->{'id'}} @$plans);
+    return ($res,\@seenPlanIds,\@seenRunIds);
+}
+
+sub _filterResults {
+    my ($opts,$c) = @_;
+
+    #Filter by provided pattern, if any
+    if ($opts->{'pattern'}) {
+        my $pattern = $opts->{pattern};
+        @{$c->{results}} = grep { my $comment = $_->{comment} || ''; $comment =~ m/$pattern/i } @{$c->{results}};
     }
 
-    @$prior_plans = uniq(@$prior_plans);
+    #Filter by the provided case IDs, if any
+    if (ref($opts->{'defects'}) eq 'ARRAY' && scalar(@{$opts->{defects}})) {
+        @{$c->{results}} = grep {
+            my $defects = $_->{defects};
+            any {
+                my $df_case = $_;
+                any { $df_case eq $_ } @{$opts->{defects}};
+            } @$defects
+        } @{$c->{results}};
+    }
+
+    #Filter by the provided versions, if any
+    if (ref($opts->{'versions'}) eq 'ARRAY' && scalar(@{$opts->{versions}})) {
+        @{$c->{results}} = grep {
+            my $version = $_->{version};
+            any { $version eq $_ } @{$opts->{versions}};
+        } @{$c->{results}};
+    }
 
-    return ($res,$prior_plans);
+    return $c;
 }
 
 1;

t/TestRail-API.t

@@ -219,8 +219,8 @@ is($summary->{'run_status'}->{'Passed'},int(!$is_mock),"Gets # of passed cases c
 is($summary->{'run_status'}->{'Untested'},int($is_mock),"Gets # of untested cases correctly");
 
 #Get run results
-my $results = $tr->getRunResults($new_run->{'id'});
-is(scalar(@$results),3,"Correct # of results returned by getRunResults");
+my $run_results = $tr->getRunResults($new_run->{'id'});
+is(scalar(@$run_results),3,"Correct # of results returned by getRunResults");
 
 #Test configuration methods
 my $configs = $tr->getConfigurations($new_project->{'id'});

t/TestRail-Utils-Lock.t

@@ -54,9 +54,14 @@ is($ret,0,"Verify that no tests are locked, as they either are of the wrong type
 $tr->{'browser'} = Test::LWP::UserAgent::TestRailMock::lockMockStep4();
 
 #Simulate lock collision
+$tr->{tests_cache} = {};
 my ($lockStatusID) = $tr->statusNamesToIds('locked');
 my ($project,$plan,$run) = TestRail::Utils::getRunInformation($tr,$opts);
-capture { $ret = TestRail::Utils::Lock::lockTest($tr->getTestByName($run->{'id'},'lockme.test'),$lockStatusID,'race.bannon',$tr) };
+capture {
+    $ret = TestRail::Utils::Lock::lockTest(
+        $tr->getTestByName($run->{'id'},'lockme.test'),$lockStatusID,'race.bannon',$tr
+    )
+};
 is($ret ,0,"False returned when race condition is simulated");
 $tr->{'browser'} = Test::LWP::UserAgent::TestRailMock::lockMockStep5();
 

t/testrail-results.t

@@ -9,9 +9,11 @@ require 'testrail-results';
 use lib $FindBin::Bin.'/lib';
 use Test::LWP::UserAgent::TestRailMock;
 
-use Test::More 'tests' => 26;
+use Test::More 'tests' => 31;
 use Capture::Tiny qw{capture_merged};
 use List::MoreUtils qw{uniq};
+use Test::Fatal;
+use File::Temp qw{tempdir};
 
 no warnings qw{redefine once};
 *TestRail::API::getTests = sub {
@@ -61,7 +63,7 @@ use warnings;
 my @args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake t/fake.test };
 my ($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
 is($code, 0, "Exit code OK looking for results of fake.test");
-like($out,qr/fake\.test was present in 515 runs/,"Gets correct # of runs with test inside it");
+like($out,qr/fake\.test was present in 509 runs/,"Gets correct # of runs with test inside it");
 
 #check project filters
 @args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --project TestProject t/fake.test };
@@ -74,7 +76,7 @@ like($out,qr/fake\.test was present in 10 runs/,"Gets correct # of runs with tes
 push(@args,'mah dubz plan', 't/fake.test');
 ($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
 is($code, 0, "Exit code OK looking for results of fake.test");
-like($out,qr/fake\.test was present in 259 runs/,"Gets correct # of runs with test inside it when filtering by plan name");
+like($out,qr/fake\.test was present in 253 runs/,"Gets correct # of runs with test inside it when filtering by plan name");
 
 #check run filters
 @args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --run FinalRun t/fake.test};
@@ -86,7 +88,7 @@ like($out,qr/fake\.test was present in 1 runs/,"Gets correct # of runs with test
 @args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --grep zippy t/fake.test};
 ($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
 is($code, 0, "Exit code OK looking for results of fake.test");
-like($out,qr/Retest: 515/,"Gets correct # & status of runs with test inside it when grepping");
+like($out,qr/Retest: 509/,"Gets correct # & status of runs with test inside it when grepping");
 unlike($out,qr/Failed: 515/,"Gets correct # & status of runs with test inside it when grepping");
 
 @args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --json t/fake.test };
@@ -94,6 +96,28 @@ unlike($out,qr/Failed: 515/,"Gets correct # & status of runs with test inside it
 is($code, 0, "Exit code OK looking for results of fake.test in json mode");
 like($out,qr/num_runs/,"Gets # of runs with test inside it in json mode");
 
+#For making the test data to test the caching
+#open(my $fh, '>', "t/data/faketest_cache.json");
+#print $fh $out;
+#close($fh);
+
+#Check caching
+@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --json --cachefile t/data/faketest_cache.json t/fake.test };
+($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
+is($code, 0, "Exit code OK looking for results of fake.test in json mode");
+chomp $out;
+is($out,"{}","Caching mode works");
+
+#check pertest mode
+@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --json --perfile bogus.dir t/fake.test };
+like( exception { TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args) }, qr/no such dir/i, "Bad pertest dir throws");
+my $dir = tempdir( CLEANUP => 1);
+@args = (qw{--apiurl http://testrail.local --user test@fake.fake --password fake --json --perfile}, $dir,  't/fake.test');
+($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
+is($code, 0, "Exit code OK looking for results of fake.test in json mode");
+chomp $out;
+ok(-f "$dir/fake.test.json","pertest write works");
+
 #check defect & version filters
 {
     no warnings qw{redefine once};
@@ -111,17 +135,22 @@ like($out,qr/num_runs/,"Gets # of runs with test inside it in json mode");
 
 }
 
-#For making the test data to test the caching
-#open(my $fh, '>', "t/data/faketest_cache.json");
-#print $fh $out;
-#close($fh);
+#check fast mode
+{
+    no warnings qw{redefine once};
+    local *TestRail::API::getTestByName = sub { return { 'id' => '666', 'run_id' => 123, 'elapsed' => 1 } };
+    local *TestRail::API::getRunResults = sub {
+        return [
+            { 'test_id' => 666, 'status_id' => 2, 'defects' => ['YOLO-666'], version => 666},
+            { 'test_id' => 666, 'defects' => undef, 'status_id' => 1, 'version' => 333}
+        ];
+    };
+    @args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --defect YOLO-666 --fast t/skip.test };
+    ($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
+    is($code, 0, "Exit code OK looking for defects of skip.test -- fastmode");
+    like($out,qr/YOLO-666/,"Gets # of runs with defects inside it -- fastmode");
 
-#Check caching
-@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --json --cachefile t/data/faketest_cache.json t/fake.test };
-($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args);
-is($code, 0, "Exit code OK looking for results of fake.test in json mode");
-chomp $out;
-is($out,"{}","Caching mode works");
+}
 
 #Check time parser
 is(TestRail::Bin::Results::_elapsed2secs('1s'),1,"elapsed2secs works : seconds");