Make planemo test a little less error prone.

 - Insist that Galaxy produce json and xunit test data. Previously planemo would try to work through without these files and just issue warnings - but the state could be inconsistent.
 - Some initial unit tests for planemo test - mocking out the actual running of Galaxy (so we can simulate problems generating these files).

Author: jmchilton

planemo/galaxy/test/actions.py

@@ -4,7 +4,7 @@
 import click
 
 from . import structures as test_structures
-from planemo.io import info, warn, shell_join
+from planemo.io import error, info, warn, shell_join
 from planemo.galaxy.run import (
     run_galaxy_command,
     setup_venv,
@@ -14,12 +14,12 @@
 
 from galaxy.tools.deps.commands import shell
 
-XUNIT_UPGRADE_MESSAGE = ("This version of Galaxy does not support xUnit - "
-                         "please update to newest development brach.")
-NO_XUNIT_MESSAGE = ("Cannot locate xUnit report option for tests - update "
-                    "Galaxy for more detailed breakdown.")
-NO_JSON_MESSAGE = ("Cannot locate json report option for tests - update "
-                   "Galaxy for more detailed breakdown.")
+NO_XUNIT_REPORT_MESSAGE = ("Cannot locate xUnit report [%s] for tests - "
+                           "required to build planemo report and summarize "
+                           "tests.")
+NO_JSON_REPORT_MESSAGE = ("Cannot locate JSON report [%s] for tests - "
+                          "required to build planemo report and summarize "
+                          "tests.")
 NO_TESTS_MESSAGE = "No tests were executed - see Galaxy output for details."
 ALL_TESTS_PASSED_MESSAGE = "All %d test(s) executed passed."
 PROBLEM_COUNT_MESSAGE = ("There were problems with %d test(s) - out of %d "
@@ -29,7 +29,7 @@
 GENERIC_TESTS_PASSED_MESSAGE = "No failing tests encountered."
 
 
-def run_in_config(ctx, config, **kwds):
+def run_in_config(ctx, config, run=run_galaxy_command, **kwds):
     """Run Galaxy tests with the run_tests.sh command.
 
     The specified `config` object describes the context for tool
@@ -42,7 +42,7 @@ def run_in_config(ctx, config, **kwds):
     if job_output_files is None:
         job_output_files = os.path.join(config_directory, "jobfiles")
 
-    xunit_supported, xunit_report_file = _xunit_state(kwds, config)
+    xunit_report_file = _xunit_state(kwds, config)
     structured_report_file = _structured_report_file(kwds, config)
 
     info("Testing using galaxy_root %s", config.galaxy_root)
@@ -75,7 +75,7 @@ def run_in_config(ctx, config, **kwds):
         test_cmd,
     )
     action = "Testing tools"
-    return_code = run_galaxy_command(
+    return_code = run(
         ctx,
         cmd,
         config.env,
@@ -85,9 +85,15 @@ def run_in_config(ctx, config, **kwds):
         update_cp_args = (job_output_files, config.test_data_dir)
         shell('cp -r "%s"/* "%s"' % update_cp_args)
 
-    if xunit_report_file and (not os.path.exists(xunit_report_file)):
-        warn(NO_XUNIT_MESSAGE)
-        xunit_report_file = None
+    if not os.path.exists(xunit_report_file):
+        message = NO_XUNIT_REPORT_MESSAGE % xunit_report_file
+        error(message)
+        raise Exception(message)
+
+    if not os.path.exists(structured_report_file):
+        message = NO_JSON_REPORT_MESSAGE % structured_report_file
+        error(message)
+        raise Exception(message)
 
     test_results = test_structures.GalaxyTestResults(
         structured_report_file,
@@ -243,33 +249,31 @@ def _print_command_line(structured_data, test_id):
 
 
 def _xunit_state(kwds, config):
-    xunit_supported = True
-    if shell("grep -q xunit '%s'/run_tests.sh" % config.galaxy_root):
-        xunit_supported = False
+    # This has been supported in Galaxy for well over a year, just going to assume
+    # it from here on out.
+    # xunit_supported = True
+    # if shell("grep -q xunit '%s'/run_tests.sh" % config.galaxy_root):
+    #    xunit_supported = False
 
     xunit_report_file = kwds.get("test_output_xunit", None)
-    if xunit_report_file is None and xunit_supported:
+    if xunit_report_file is None:
         xunit_report_file = os.path.join(config.config_directory, "xunit.xml")
-    elif xunit_report_file is not None and not xunit_supported:
-        warn(XUNIT_UPGRADE_MESSAGE)
-        xunit_report_file = None
 
-    return xunit_supported, xunit_report_file
+    return xunit_report_file
 
 
 def _structured_report_file(kwds, config):
-    structured_data_supported = True
-    if shell("grep -q structured_data '%s'/run_tests.sh" % config.galaxy_root):
-        structured_data_supported = False
+    # This has been supported in Galaxy for well over a year, just going to assume
+    # it from here on out.
+    # structured_data_supported = True
+    # if shell("grep -q structured_data '%s'/run_tests.sh" % config.galaxy_root):
+    #    structured_data_supported = False
 
     structured_report_file = None
     structured_report_file = kwds.get("test_output_json", None)
-    if structured_report_file is None and structured_data_supported:
+    if structured_report_file is None:
         conf_dir = config.config_directory
         structured_report_file = os.path.join(conf_dir, "structured_data.json")
-    elif structured_report_file is not None and not structured_data_supported:
-        warn(NO_JSON_MESSAGE)
-        structured_report_file = None
 
     return structured_report_file
 

tests/data/tt_success.html

@@ -0,0 +1 @@
+{"tests": [{"data": {"status": "success", "inputs": {"input1": {"src": "hda", "id": "2891970512fa2d5a"}}, "job": {"inputs": {"input1": {"src": "hda", "id": "2891970512fa2d5a", "uuid": "3b1f7b73-d96b-4ca8-be81-3aa7a24bac13"}}, "update_time": "2016-05-11T00:55:26.723036", "tool_id": "cat", "outputs": {"out_file1": {"src": "hda", "id": "5729865256bc2525", "uuid": "c7bf3ebb-1bed-45ce-b473-aefba37773ec"}}, "stdout": "", "exit_code": 0, "state": "ok", "create_time": "2016-05-11T00:55:23.253017", "params": {"chromInfo": "\"/home/john/workspace/galaxy/tool-data/shared/ucsc/chrom/hg17.len\"", "dbkey": "\"hg17\"", "queries": "[]"}, "stderr": "", "model_class": "Job", "id": "5729865256bc2525"}}, "id": "functional.test_toolbox.TestForTool_cat.test_tool_000000", "has_data": true}], "version": "0.1", "exit_code": 0, "summary": {"num_skips": 0, "num_errors": 0, "num_failures": 0, "num_tests": 1}}

tests/data/tt_success.json

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8"?><testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0"><testcase classname="functional.test_toolbox.TestForTool_cat" name="test_tool_000000" time="12.805"></testcase></testsuite>

tests/data/tt_success.xml

@@ -1,23 +1,85 @@
-""" Tests for the galaxy_test module :) """
+"""Tests for the `planeo.galaxy.test` module."""
 
 import os
+import shutil
 
-from .test_utils import TEST_DATA_DIR
+from .test_utils import (
+    test_context,
+    TempDirectoryTestCase,
+    TEST_DATA_DIR,
+)
 
 from planemo.galaxy.test import structures
 from planemo.galaxy.test.actions import passed
+from planemo.galaxy.test.actions import run_in_config
+
 
 nose_1_3_report = os.path.join(TEST_DATA_DIR, "xunit_nose_1_3.xml")
 nose_0_11_report = os.path.join(TEST_DATA_DIR, "xunit_nose_0_11.xml")
 
 xunit_report_with_failure = os.path.join(TEST_DATA_DIR, "xunit_failure.xml")
 
 
+class RunInConfigTestCase(TempDirectoryTestCase):
+    """Test cases for ``run_in_config``."""
+
+    def setUp(self):
+        """Setup mock keywords, context, and Galaxy config for tests."""
+        super(RunInConfigTestCase, self).setUp()
+        td = self.temp_directory
+        self.ctx = test_context()
+        self.config = _MockConfig(td)
+        self.kwds = {
+            "test_output": os.path.join(td, "tests.html"),
+            "test_output_json": os.path.join(td, "tests.json"),
+            "test_output_xunit": os.path.join(td, "tests.xml"),
+        }
+
+    def test_normal_execution(self):
+        """Test normal operation of run_in_config."""
+        def mock_galaxy_run(ctx_, command, env, action):
+            assert ctx_ is self.ctx
+            assert env["test_key"] == "test_value"
+            self._copy_good_artifacts(["xml", "html", "json"])
+            return 0
+
+        assert self._do_run(mock_galaxy_run) == 0
+
+    def test_failed_to_produce_xunit(self):
+        """Test an exception is thrown if not XUnit report is produced."""
+        def mock_galaxy_run(ctx_, command, env, action):
+            self._copy_good_artifacts(["json", "html"])
+            return 0
+
+        with self.assertRaises(Exception):
+            self._do_run(mock_galaxy_run)
+
+    def test_failed_to_produce_json(self):
+        """Test an exception is thrown if not XUnit report is produced."""
+        def mock_galaxy_run(ctx_, command, env, action):
+            self._copy_good_artifacts(["xml", "html"])
+            return 0
+
+        with self.assertRaises(Exception):
+            self._do_run(mock_galaxy_run)
+
+    def _copy_good_artifacts(self, extensions):
+        for extension in extensions:
+            source = os.path.join(TEST_DATA_DIR, "tt_success.%s" % extension)
+            destination = os.path.join(self.temp_directory, "tests.%s" % extension)
+            shutil.copy(source, destination)
+
+    def _do_run(self, mock_run_function):
+        return run_in_config(self.ctx, self.config, run=mock_run_function, **self.kwds)
+
+
 def get_test_id_new():
+    """Test ID parsing on newer nose dependency."""
     _get_test_id(nose_1_3_report)
 
 
 def get_test_id_old():
+    """Test ID parsing on older nose dependency."""
     _get_test_id(nose_0_11_report)
 
 
@@ -33,10 +95,19 @@ def _get_test_id(path):
 
 
 def test_passed():
+    """Test :func:`passed`."""
     xml_tree = structures.parse_xunit_report(xunit_report_with_failure)
     root = xml_tree.getroot()
     good_testcase_el = structures.find_cases(root)[0]
     assert passed(good_testcase_el)
 
     bad_testcase_el = structures.find_cases(root)[1]
     assert not passed(bad_testcase_el)
+
+
+class _MockConfig(object):
+
+    def __init__(self, temp_directory):
+        self.config_directory = temp_directory
+        self.env = {"test_key": "test_value"}
+        self.galaxy_root = os.path.join(self.config_directory, "galaxy-dev")