TestRail-Perl/bin/testrail-results

#!/usr/bin/env perl
# ABSTRACT: List results for specified test(s).
# PODNAME: TestRail::Bin::Results

=head1 SYNOPSIS

  testrail-results [OPTIONS] test1 test2 ...

  require `which testrail-results`;
  TestRail::Bin::Results::run('args' => \@args);

=head1 DESCRIPTION

testrail-results - List results for specified test(s).

Searches across multiple runs (and projects) for results for broad-based metrics; especially useful for diagnosing unreliable tests, and establishing defect density for certain features.

Can be used as the modulino TestRail::Bin::Tests.
Has a single 'run' function which accepts a hash with the 'args' parameter being the array of arguments.

=head1 WARNING

Searching across all projects can take a very long time for highly active TestRail Installations.
However, cross project metrics are also very useful.

As such, the results from prior searches in json mode may be provided, and the runs previously analyzed therein will not be investigated again.

It is up to the caller to integrate this data into their analysis as may be appropriate.

=head1 PARAMETERS:

=head2 MANDATORY PARAMETERS

=over 4

--apiurl   : full URL to get to TestRail index document

--password : Your TestRail Password, or a valid API key (TestRail 4.2 and above).

--user     : Your TestRail User Name.

=back

All mandatory options not passed with the above switches, or in your ~/.testrailrc will be prompted for.

=head2 SEMI-OPTIONAL PARAMETERS

=over 4

-e --encoding : Character encoding of arguments.  Defaults to UTF-8. See L<Encode::Supported> for supported encodings.

=back

=head2 OPTIONAL PARAMETERS

=over 4

-j --project : Restrict search to provided project name.  May be passed multiple times.

-r --run     : Restrict search to runs with the provided name.  May be passed multiple times.

-p --plan    : Restrict search to plans with the provided name. May be passed multiple times.

-g --grep    : Restrict results printed to those matching the provided pattern. Great for looking for specific failure conditions.

-v --version : Restrict results printed to those tested on the provided version(s).  May be passed multiple times.

-d --defect  : Restrict results printed to those related to the provided defect(s).  May be passed multiple times.

-c --cachefile : Load the provided file as a place to pick up your search from.

--json       : Print results as a JSON serialization.

=back

=head1 CONFIGURATION FILE

In your \$HOME, (or the current directory, if your system has no concept of a home directory) put a file called .testrailrc with key=value syntax separated by newlines.
Valid Keys are the same as documented by L<App::Prove::Plugin::TestRail>.
All options specified thereby are overridden by passing the command-line switches above.

=head1 MISCELLANEOUS OPTIONS:

=over 4

--help : show this output

=back

=cut

package TestRail::Bin::Results;

use strict;
use warnings;
use utf8;

use TestRail::API;
use TestRail::Utils;
use TestRail::Utils::Find;

use Getopt::Long qw{GetOptionsFromArray};
use File::HomeDir qw{my_home};
use JSON::MaybeXS ();
use Statistics::Descriptive;
use List::MoreUtils qw{uniq};

if (!caller()) {
    my ($out,$code) = run('args' => \@ARGV);
    print "$out\n";
    exit $code;
}

sub run {
    my %params = @_;
    my $opts ={};

    #Parse config file if we are missing api url/key or user
    my $homedir = my_home() || '.';
    if (-e $homedir . '/.testrailrc') {
        $opts = TestRail::Utils::parseConfig($homedir);
    }

    GetOptionsFromArray($params{'args'},
        'apiurl=s'        => \$opts->{'apiurl'},
        'password=s'      => \$opts->{'password'},
        'user=s'          => \$opts->{'user'},
        'j|project=s@'    => \$opts->{'projects'},
        'p|plan=s@'       => \$opts->{'plans'},
        'r|run=s@'        => \$opts->{'runs'},
        'e|encoding=s'    => \$opts->{'encoding'},
        'g|grep=s'        => \$opts->{'pattern'},
        'd|defect=s@'     => \$opts->{'defects'},
        'v|version=s@'    => \$opts->{'versions'},
        'c|cachefile=s'   => \$opts->{'cachefile'},
        'json'            => \$opts->{'json'},
        'h|help'          => \$opts->{'help'},
    );

    if ($opts->{help}) { return ('',TestRail::Utils::help()); }

    die("No tests passed") unless scalar(@{$params{'args'}});
    die("Prior search file passed does not exist") if $opts->{'cachefile'} && !( -e $opts->{'cachefile'});

    $opts->{'browser'} = $params{'browser'};

    TestRail::Utils::interrogateUser($opts,qw{apiurl user password});

    my $tr = TestRail::Utils::getHandle($opts);
    my $prior_search;
    my $prior_runs = [];
    my $prior_plans = [];
    if ($opts->{'cachefile'}) {
        my $raw_text = '';
        open(my $fh, '<', $opts->{'cachefile'}) or die "Could not open $opts->{cachefile}";
        while (<$fh>) {
            $raw_text .= $_;
        }
        close($fh);
        $prior_search = JSON::MaybeXS::decode_json($raw_text);
        foreach my $key (keys(%$prior_search)) {
            push(@$prior_runs,@{$prior_search->{$key}->{'seen_runs'}});
            push(@$prior_plans,@{$prior_search->{$key}->{'seen_plans'}});
        }
        $opts->{'plan_ids'} = $prior_plans;
    }

    my ($res,$seen_plans) = TestRail::Utils::Find::getResults($tr,$opts,$prior_runs,@{$params{'args'}});

    #Make sure subsequent runs keep ignoring the prior plans
    push(@$seen_plans,@$prior_plans);

    my $statuses = $tr->getPossibleTestStatuses();
    my %status_map;
    @status_map{map {$_->{'id'}} @$statuses} = map {$_->{'label'}} @$statuses;

    my ($out,$out_json) = ('',{});
    foreach my $case (keys(%$res)) {
        $out .= "#############################\n";
        my $num_runs = 0;
        my $casetotals = {};
        my $versions_by_status = {};
        my $defects = [];
        my $total_elapsed = 0;
        my $avg_elapsed = 0;
        my $median_runtime = 0;
        my $elapsetotals = [];
        my $seen_runs    = $prior_runs;

        foreach my $casedef (@{$res->{$case}}) {
            push(@$seen_runs, $casedef->{run_id});
            $num_runs++;
            #$out .= "Found case '$case' in run $casedef->{run_id}\n";
            foreach my $result (@{$casedef->{results}}) {
                $casetotals->{$result->{status_id}}++;
                $versions_by_status->{$result->{status_id}} //= [];
                push(@{$versions_by_status->{$result->{status_id}}},$result->{version}) if $result->{version};
                push(@$defects, $result->{defects}) if $result->{defects};
                push(@$elapsetotals,_elapsed2secs($result->{'elapsed'}));
            }
        }
        @$seen_runs = uniq(@$seen_runs);
        @$defects   = uniq(@$defects);
        foreach my $st (keys(%$versions_by_status)) {
            @{$versions_by_status->{$st}} = uniq(@{$versions_by_status->{$st}});
        }

        my $pattern_output = '';
        $out_json->{$case}->{search_string} = $opts->{'pattern'};
        $pattern_output = " using search string '$opts->{pattern}'" if $opts->{'pattern'};

        $out .= "$case was present in $num_runs runs$pattern_output.\n";
        $out_json->{$case}->{'num_runs'}   = $num_runs;
        $out_json->{$case}->{'seen_runs'}  = $seen_runs;
        $out_json->{$case}->{'seen_plans'} = $seen_plans;

        #Collect time statistics
        my $timestats = Statistics::Descriptive::Full->new();
        $timestats->add_data(@$elapsetotals);

        $out_json->{$case}->{total_elapsed}   = $timestats->sum() || 0;
        $out .= "Total time spent running this test: $out_json->{$case}->{total_elapsed} seconds\n";
        $out_json->{$case}->{median_elapsed}  = $timestats->median() || 0;
        $out .= "Median time spent running this test: $out_json->{$case}->{median_elapsed} seconds\n";
        $out_json->{$case}->{average_elapsed} = $timestats->mean() || 0;
        $out .= "Mean time spent running this test: $out_json->{$case}->{average_elapsed} seconds\n";
        $out_json->{$case}->{stdev_elapsed}   = $timestats->standard_deviation() || 0;
        $out .= "Standard deviations of runtime in test: $out_json->{$case}->{stdev_elapsed}\n";
        $out_json->{$case}->{max_elapsed}     = $timestats->max() || 0;
        $out .= "Maximum time spent running this test: $out_json->{$case}->{max_elapsed} seconds\n";
        $out_json->{$case}->{min_elapsed}     = $timestats->min() || 0;
        $out .= "Minimum time spent running this test: $out_json->{$case}->{min_elapsed} seconds\n";
        $out_json->{$case}->{times_executed}  = $timestats->count() || 0;
        $out .= "Num times this test has been executed: $out_json->{$case}->{times_executed}\n";

        foreach my $status (keys(%$casetotals)) {
            $out .= "$status_map{$status}: $casetotals->{$status}\n";
            $out_json->{$case}->{$status_map{$status}} = $casetotals->{$status};
        }

        foreach my $status (keys(%$versions_by_status)) {
            $out .= "Versions $status_map{$status} in:\n".join(',',@{$versions_by_status->{$status}})."\n";
        }
        $out_json->{$case}->{versions_by_status} = $versions_by_status;

        $out .= "\nDefects related to case:\n".join(',',@$defects)."\n" if @$defects;
        $out_json->{$case}->{defects} = $defects;

    }

    if ($opts->{'json'}) {
        my $coder = JSON::MaybeXS->new;
        return ($coder->encode($out_json),0);
    }

    $out .= "#############################";
    return ($out,0);
}

sub _elapsed2secs {
    my $stamp = shift;
    return 0 if !$stamp;
    my ($seconds)         = $stamp =~ m/(\d*)s/;
    my ($seconds_minutes) = $stamp =~ m/(\d*)m/;
    my ($seconds_hours)   = $stamp =~ m/(\d*)h/;
    return ($seconds || 0) + ($seconds_minutes ? $seconds_minutes * 60 : 0) + ($seconds_hours ? $seconds_hours * 3600 : 0);
}

1;

__END__

L<TestRail::API>

L<File::HomeDir> for the finding of .testrailrc

=head1 SPECIAL THANKS

Thanks to cPanel Inc, for graciously funding the creation of this distribution.