Re-instate integration tests of binaries

bin/testrail-bulk-mark-results

@@ -103,7 +103,8 @@ GetOptions(
     'c|config=s@'     => \$opts->{'configs'},
     'a|assignedto=s@' => \$opts->{'users'},
     'e|encoding=s'    => \$opts->{'encoding'},
-    'h|help'          => \$opts->{'help'}
+    'h|help'          => \$opts->{'help'},
+    'mock'            => \$opts->{'mock'}
 );
 
 if ($opts->{help}) { TestRail::Utils::help(); }
@@ -114,7 +115,8 @@ my $reason = $ARGV[1];
 die("No status to set provided.") unless $status;
 TestRail::Utils::interrogateUser($opts,qw{apiurl user password project run});
 
-my $tr = TestRail::API->new($opts->{apiurl},$opts->{user},$opts->{password},$opts->{'encoding'},$opts->{'debug'});
+my $tr = TestRail::Utils::getHandle($opts);
+
 $opts->{'set_status_to'} = $status;
 $opts->{'reason'} = $reason;
 my $results = TestRail::Utils::Results::bulkMarkResults($opts,$tr);

bin/testrail-lock

@@ -115,7 +115,8 @@ GetOptions(
     'n|no-recurse'    => \$opts->{'no-recurse'},
     't|case-type=s@'  => \$opts->{'case-types'},
     'e|encoding=s'    => \$opts->{'encoding'},
-    'h|help'          => \$opts->{'help'}
+    'h|help'          => \$opts->{'help'},
+    'mock'            => \$opts->{'mock'}
 );
 
 if ($opts->{help}) { TestRail::Utils::help(); }
@@ -123,7 +124,8 @@ $opts->{'hostname'} = hostname;
 
 TestRail::Utils::interrogateUser($opts,qw{apiurl user password project run lockname});
 
-my $tr = TestRail::API->new($opts->{apiurl},$opts->{user},$opts->{password},$opts->{'encoding'});
+my $tr = TestRail::Utils::getHandle($opts);
+
 my $ret = TestRail::Utils::pickAndLockTest($opts,$tr);
 
 exit 255 if !$ret;

bin/testrail-report

@@ -131,7 +131,8 @@ GetOptions(
     'section=s@'     => \$opts{sections},
     'autoclose'      => \$opts{autoclose},
     'e|encoding=s'   => \$opts{encoding},
-    'help'           => \$opts{help}
+    'help'           => \$opts{help},
+    'mock'           => \$opts{mock}
 );
 
 if ($opts{help}) { TestRail::Utils::help(); }
@@ -154,6 +155,12 @@ TestRail::Utils::interrogateUser(\%opts,qw{apiurl user password project run});
 
 $opts{result_options} = {'version' => $opts{version}} if $opts{version};
 
+if ($opts{'mock'}) {
+    require Test::LWP::UserAgent::TestRailMock;
+    $opts{'browser'} = $Test::LWP::UserAgent::TestRailMock::mockObject;
+    $opts{'debug'} = 0;
+}
+
 my $tap;
 foreach my $phil (@files) {
     $tap = Test::Rail::Parser->new({
@@ -177,7 +184,6 @@ foreach my $phil (@files) {
         'merge'          => 1
     });
     $tap->run();
-
 }
 
 print "Done.\n";

bin/testrail-runs

@@ -93,14 +93,15 @@ GetOptions(
     'e|encoding=s' => \$opts->{'encoding'},
     'l|lifo'       => \$opts->{'lifo'},
     'm|milesort'   => \$opts->{'milesort'},
-    'h|help'       => \$opts->{'help'}
+    'h|help'       => \$opts->{'help'},
+    'mock'         => \$opts->{'mock'}
 );
 
 if ($opts->{help}) { TestRail::Utils::help(); }
 
 TestRail::Utils::interrogateUser($opts,qw{apiurl user password project});
 
-my $tr = TestRail::API->new($opts->{apiurl},$opts->{user},$opts->{password},$opts->{'encoding'},$opts->{'debug'});
+my $tr = TestRail::Utils::getHandle($opts);
 
 my $runs = TestRail::Utils::Find::findRuns($opts,$tr);
 

bin/testrail-tests

@@ -111,14 +111,15 @@ GetOptions(
     'n|no-recurse'    => \$opts->{'no-recurse'},
     'e|encoding=s'    => \$opts->{'encoding'},
     'extension=s'     => \$opts->{'extension'},
-    'h|help'          => \$opts->{'help'}
+    'h|help'          => \$opts->{'help'},
+    'mock'            => \$opts->{'mock'}
 );
 
 if ($opts->{help}) { TestRail::Utils::help(); }
 
 TestRail::Utils::interrogateUser($opts,qw{apiurl user password project run});
 
-my $tr = TestRail::API->new($opts->{apiurl},$opts->{user},$opts->{password},$opts->{'encoding'},$opts->{'debug'});
+my $tr = TestRail::Utils::getHandle($opts);
 
 my ($cases) = TestRail::Utils::Find::getTests($opts,$tr);
 die "No cases in TestRail!\n" unless $cases;

lib/TestRail/Utils.pm

@@ -210,6 +210,30 @@ sub getRunInformation {
     return ($project, $plan, $run, $milestone);
 }
 
+=head2 getHandle(opts)
+
+Convenience method for binaries and testing.
+Returns a new TestRail::API when passed an options hash such as is built by most of the binaries,
+or returned by parseConfig.
+
+Has a special 'mock' hash key that can only be used by those testing this distribution.
+
+=cut
+
+sub getHandle {
+    my $opts = shift;
+
+    $opts->{'debug'} = 1 if ($opts->{'mock'});
+    my $tr = TestRail::API->new($opts->{apiurl},$opts->{user},$opts->{password},$opts->{'encoding'},$opts->{'debug'});
+    if ($opts->{'mock'}) {
+        use lib 't/lib'; #Unit tests will always run from the main dir during make test
+        require Test::LWP::UserAgent::TestRailMock;
+        $opts->{'browser'} = $Test::LWP::UserAgent::TestRailMock::mockObject;
+        $opts->{'debug'} = 0;
+    }
+    return $tr;
+}
+
 1;
 
 __END__

t/testrail-bulk-mark-results.t

@@ -1,9 +1,18 @@
 use strict;
 use warnings;
 
-use Test::More "tests" => 2;
+use Test::More "tests" => 4;
 
 my @args = ($^X,qw{bin/testrail-bulk-mark-results --help});
 my $out = `@args`;
 is($? >> 8, 0, "Exit code OK asking for help");
 like($out,qr/encoding of arguments/i,"Help output OK");
+
+#check plan mode
+@args = ($^X,qw{bin/testrail-bulk-mark-results --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j "CRUSH ALL HUMANS" -r "SEND T-1000 INFILTRATION UNITS BACK IN TIME" --mock blocked "Build was bad."});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK running against normal run");
+chomp $out;
+like($out,qr/set the status of 1 cases to blocked/,"Sets test correctly in single run mode");
+
+#TODO more thorough testing

t/testrail-report.t

@@ -1,11 +1,59 @@
 use strict;
 use warnings;
 
-use Test::More 'tests' => 2;
+use Test::More 'tests' => 16;
 
-#Test that help works
-my @args = ($^X,qw{bin/testrail-report --help});
+my @args = ($^X,qw{bin/testrail-report --apiurl http://testrail.local --user "test@fake.fake" --password "fake" --project "CRUSH ALL HUMANS" --run "SEND T-1000 INFILTRATION UNITS BACK IN TIME" --mock t/test_multiple_files.tap});
 my $out = `@args`;
+is($? >> 8, 0, "Exit code OK reported with multiple files");
+my $matches = () = $out =~ m/Reporting result of case/ig;
+is($matches,2,"Attempts to upload multiple times");
+
+@args = ($^X,qw{bin/testrail-report --apiurl http://testrail.local --user "test@fake.fake" --password "fake" --project "CRUSH ALL HUMANS" --run "SEND T-1000 INFILTRATION UNITS BACK IN TIME" --case-ok --mock t/test_multiple_files.tap});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK reported with multiple files (case-ok mode)");
+$matches = () = $out =~ m/Reporting result of case/ig;
+is($matches,4,"Attempts to upload multiple times (case-ok mode)");
+
+#Test version, case-ok
+@args = ($^X,qw{bin/testrail-report --apiurl http://testrail.local --user "test@fake.fake" --password "fake" --project "TestProject" --run "TestingSuite" --case-ok --version '1.0.14' --mock t/test_subtest.tap});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK reported with subtests (case-ok mode)");
+$matches = () = $out =~ m/Reporting result of case/ig;
+is($matches,2,"Attempts to upload do not do subtests (case-ok mode)");
+
+#Test plans/configs
+@args = ($^X,qw{bin/testrail-report --apiurl http://testrail.local --user "test@fake.fake" --password "fake" --project "TestProject" --run "Executing the great plan" --plan "GosPlan" --config "testConfig"  --case-ok --mock t/test_subtest.tap});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK reported with plans");
+$matches = () = $out =~ m/Reporting result of case.*OK/ig;
+is($matches,2,"Attempts to to plans work");
+
+#Test that spawn works
+@args = ($^X,qw{bin/testrail-report --apiurl http://testrail.local --user "test@fake.fake" --password "fake" --project "TestProject" --run "TestingSuite2" --spawn 9 --case-ok --mock t/test_subtest.tap});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK reported with spawn");
+$matches = () = $out =~ m/Reporting result of case.*OK/ig;
+is($matches,2,"Attempts to spawn work");
+
+#Test that spawn works w/sections
+@args = ($^X,qw{bin/testrail-report --apiurl http://testrail.local --user "test@fake.fake" --password "fake" --project "TestProject" --run "TestingSuite2" --spawn 9 --case-ok --section "CARBON LIQUEFACTION" --mock t/test_subtest.tap});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK reported with spawn");
+$matches = () = $out =~ m/with specified sections/ig;
+is($matches,1,"Attempts to spawn work");
+
+#Test that the autoclose option works
+@args = ($^X,qw{bin/testrail-report --apiurl http://testrail.local --user "test@fake.fake" --password "fake" --project "TestProject" --run "FinalRun" --plan "FinalPlan" --config "testConfig" --case-ok --autoclose --mock t/fake.tap});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK when doing autoclose");
+like($out,qr/closing plan/i,"Run closure reported to user");
+
+#Test that help works
+@args = ($^X,qw{bin/testrail-report --help});
+$out = `@args`;
 is($? >> 8, 0, "Exit code OK reported with help");
-my $matches = () = $out =~ m/encoding of arguments/ig;
+$matches = () = $out =~ m/encoding of arguments/ig;
 is($matches,1,"Help output OK");
+
+

t/testrail-runs.t

@@ -1,10 +1,52 @@
 use strict;
 use warnings;
 
-use Test::More 'tests' => 2;
+use Test::More 'tests' => 14;
 
-#help options
-my @args = ($^X,qw{bin/testrail-runs --help});
+#check status filters
+my @args = ($^X,qw{bin/testrail-runs --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j "TestProject" --mock});
 my $out = `@args`;
+is($? >> 8, 0, "Exit code OK looking for runs with passes");
+chomp $out;
+like($out,qr/^OtherOtherSuite\nTestingSuite\nFinalRun\nlockRun$/,"Gets run correctly looking for passes");
+
+#check LIFO sort
+@args = ($^X,qw{bin/testrail-runs --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j "TestProject" --lifo --mock});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK looking for runs with passes");
+chomp $out;
+like($out,qr/^lockRun\nTestingSuite\nFinalRun\nOtherOtherSuite$/,"LIFO sort works");
+
+#check milesort
+@args = ($^X,qw{bin/testrail-runs --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j "TestProject" --milesort --mock});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK looking for runs with passes");
+chomp $out;
+like($out,qr/^TestingSuite\nFinalRun\nlockRun\nOtherOtherSuite$/,"milesort works");
+
+
+#check status filters
+@args = ($^X,qw{bin/testrail-runs --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j "TestProject" --mock --status passed});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK looking for runs with passes, which should fail to return results");
+chomp $out;
+is($out,'',"Gets no runs correctly looking for passes");
+
+@args = ($^X,qw{bin/testrail-runs --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j "CRUSH ALL HUMANS" --mock --status passed});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK looking for runs with passes");
+chomp $out;
+like($out,qr/SEND T-1000 INFILTRATION UNITS BACK IN TIME$/,"Gets run correctly looking for passes");
+
+#TODO check configs for real next time
+@args = ($^X,qw{bin/testrail-runs --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j "TestProject" --mock --config testConfig});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK looking for runs with passes");
+chomp $out;
+is($out,'',"Gets no run correctly when filtering by unassigned config");
+
+#help options
+@args = ($^X,qw{bin/testrail-runs --help});
+$out = `@args`;
 is($? >> 8, 0, "Exit code OK looking for help");
 like($out,qr/encoding of arguments/i,"Help output OK");

t/testrail-tests.t

@@ -1,9 +1,96 @@
 use strict;
 use warnings;
 
-use Test::More "tests" => 2;
+use Test::More "tests" => 30;
 
-my @args = ($^X,qw{bin/testrail-tests --help});
+#check plan mode
+my @args = ($^X,qw{bin/testrail-tests --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -p "GosPlan" -r "Executing the great plan" -m t --config testConfig --mock --no-recurse});
 my $out = `@args`;
+is($? >> 8, 0, "Exit code OK running plan mode, no recurse");
+chomp $out;
+like($out,qr/skipall\.test$/,"Gets test correctly in plan mode, no recurse");
+
+#check no-match
+@args = ($^X,qw{bin/testrail-tests --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -p "GosPlan" -r "Executing the great plan" --no-match t --config testConfig --mock});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK running plan mode, no match");
+chomp $out;
+unlike($out,qr/skipall\.test/,"Omits test correctly in plan mode, recurse, no-match");
+unlike($out,qr/NOT SO SEARED AFTER ARR/,"Omits non-file test correctly in plan mode, recurse, no-match");
+like($out,qr/faker\.test/,"Omits non-file test correctly in plan mode, recurse, no-match");
+
+#check no-match, no recurse
+@args = ($^X,qw{bin/testrail-tests --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -p "GosPlan" -r "Executing the great plan" --no-match t --config testConfig --mock --no-recurse});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK running plan mode, no match, no recurse");
+chomp $out;
+unlike($out,qr/skipall\.test/,"Omits test correctly in plan mode, no recurse, no-match");
+unlike($out,qr/NOT SO SEARED AFTER ARR/,"Omits non-file test correctly in plan mode, no recurse, no-match");
+like($out,qr/faker\.test/,"Omits non-file test correctly in plan mode, no recurse, no-match");
+
+
+@args = ($^X,qw{bin/testrail-tests --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -p "GosPlan" -r "Executing the great plan" --config testConfig -m t --mock});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK running plan mode, recurse");
+chomp $out;
+like($out,qr/skipall\.test$/,"Gets test correctly in plan mode, recurse");
+
+#check non plan mode
+@args = ($^X,qw{bin/testrail-tests --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -r "TestingSuite" -m t --mock --no-recurse});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK running no plan mode, no recurse");
+chomp $out;
+like($out,qr/skipall\.test$/,"Gets test correctly in no plan mode, no recurse");
+
+@args = ($^X,qw{bin/testrail-tests --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -r "TestingSuite" -m t --mock});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK running no plan mode, recurse");
+chomp $out;
+like($out,qr/skipall\.test$/,"Gets test correctly in no plan mode, recurse");
+
+#Negative case, filtering by config
+@args = ($^X,qw{bin/testrail-tests --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -p "GosPlan" -r "Executing the great plan" -m t --mock --config testPlatform1});
+$out = `@args`;
+isnt($? >> 8, 0, "Exit code not OK when passing invalid configs for plan");
+
+#check assignedto filters
+@args = ($^X,qw{bin/testrail-tests --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -p "GosPlan" -r "Executing the great plan" --mock --config "testConfig" --assignedto teodesian});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK when filtering by assignment");
+like($out,qr/skipall\.test$/,"Gets test correctly when filtering by assignment");
+
+@args = ($^X,qw{bin/testrail-tests --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -p "GosPlan" -r "Executing the great plan" --mock --config "testConfig" --assignedto billy});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK when filtering by assignment");
+chomp $out;
+is($out,"","Gets no tests correctly when filtering by wrong assignment");
+
+#check status filters
+@args = ($^X,qw{bin/testrail-tests --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -p "GosPlan" -r "Executing the great plan" -m t --mock --config "testConfig" --status "passed"});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK when filtering by status");
+like($out,qr/skipall\.test$/,"Gets test correctly when filtering by status");
+
+@args = ($^X,qw{bin/testrail-tests --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -p "GosPlan" -r "Executing the great plan" --mock --config "testConfig" --status "failed"});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK when filtering by status");
+chomp $out;
+is($out,"","Gets no tests correctly when filtering by wrong status");
+
+#Verify no-match returns non path
+@args = ($^X,qw{bin/testrail-tests --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -r "TestingSuite" --mock});
+$out = `@args`;
+is($? >> 8, 0, "Exit code OK running no plan mode, no recurse");
+chomp $out;
+like($out,qr/\nskipall\.test$/,"Gets test correctly in no plan mode, no recurse");
+
+#Verify no-match returns non path
+@args = ($^X,qw{bin/testrail-tests --help});
+$out = `@args`;
 is($? >> 8, 0, "Exit code OK asking for help");
 like($out,qr/encoding of arguments/i,"Help output OK");
+
+#Verify no-match and match are mutually exclusive
+@args = ($^X,qw{bin/testrail-tests --no-match t/ --match t/qa --apiurl http://testrail.local --user "test@fake.fake" --password "fake" -j TestProject -r "TestingSuite" --mock});
+$out = `@args`;
+isnt($? >> 8, 0, "Exit code not OK asking for mutually exclusive match options");