Add output diff rendering (#18)

Author: niosus

.gitignore

@@ -27,3 +27,6 @@ build/
 dist/
 /result.md
 /results.md
+
+.DS_Store
+.vscode/

Pipfile

@@ -9,8 +9,6 @@ schema = "*"
 cpplint = "*"
 datetime = "*"
 black = "*"
+typing-extensions = "*"
 
 [dev-packages]
-
-[requires]
-python_version = "3.8"

Pipfile.lock

@@ -10,18 +10,21 @@
 TABLE_TEMPLATE = "| {hw_name} | {task_name} | {test_name} | {result_sign} |\n"
 TABLE_SEPARATOR = "|---|---|---|:---:|\n"
 
+ENTRY_TEMPLATE = """
+**`{name}`**
+```{syntax}
+{content}
+```
+"""
+
+STATUS_CODE_TEMPLATE = """
+**`Status code`** {code}
+"""
+
 ERROR_TEMPLATE = """
 <details><summary><b>{hw_name} | {task_name} | {test_name}</b></summary>
 
-**`stderr`**
-```apiblueprint
-{stderr}
-```
-
-**`stdout`**
-```
-{stdout}
-```
+{entries}
 
 --------
 
@@ -120,10 +123,30 @@ def _add_error(
         if expired:
             self._errors += EXPIRED_TEMPLATE.format(hw_name=hw_name)
             return
+        entries = STATUS_CODE_TEMPLATE.format(code=test_result.status)
+        if test_result.output_mismatch:
+            if test_result.output_mismatch.input:
+                entries += ENTRY_TEMPLATE.format(
+                    name="Input",
+                    syntax="",
+                    content=test_result.output_mismatch.input,
+                )
+            entries += ENTRY_TEMPLATE.format(
+                name="Output mismatch",
+                syntax="diff",
+                content=test_result.output_mismatch.diff(),
+            )
+        if test_result.stderr:
+            entries += ENTRY_TEMPLATE.format(
+                name="stderr", syntax="css", content=test_result.stderr
+            )
+        if test_result.stdout:
+            entries += ENTRY_TEMPLATE.format(
+                name="stdout", syntax="", content=test_result.stdout
+            )
         self._errors += ERROR_TEMPLATE.format(
             hw_name=hw_name,
             task_name=task_name,
             test_name=test_name,
-            stderr=test_result.stderr,
-            stdout=test_result.stdout,
+            entries=entries,
         )

homework_checker/core/md_writer.py

@@ -13,11 +13,6 @@
 
 log = logging.getLogger("GHC")
 
-
-OUTPUT_MISMATCH_MESSAGE = """Given input: '{input}'
-Your output '{actual}'
-Expected output: '{expected}'"""
-
 BUILD_SUCCESS_TAG = "Build succeeded"
 STYLE_ERROR_TAG = "Style errors"
 
@@ -289,14 +284,24 @@ def _run_test(self: CppTask, test_node: dict, executable_folder: Path):
         our_output, error = tools.convert_to(self._output_type, run_result.stdout)
         if not our_output:
             # Conversion has failed.
-            run_result.stderr = error
-            return run_result
+            return tools.CmdResult(
+                status=tools.CmdResult.FAILURE,
+                stdout=run_result.stdout,
+                stderr=error,
+            )
         expected_output, error = tools.convert_to(
             self._output_type, test_node[Tags.EXPECTED_OUTPUT_TAG]
         )
         if our_output != expected_output:
-            run_result.stderr = OUTPUT_MISMATCH_MESSAGE.format(
-                actual=our_output, input=input_str, expected=expected_output
+            return tools.CmdResult(
+                status=tools.CmdResult.FAILURE,
+                stdout=run_result.stdout,
+                stderr=run_result.stderr,
+                output_mismatch=tools.OutputMismatch(
+                    input=input_str,
+                    expected_output=expected_output,
+                    actual_output=our_output,
+                ),
             )
         return run_result
 
@@ -342,13 +347,23 @@ def _run_test(
         our_output, error = tools.convert_to(self._output_type, run_result.stdout)
         if not our_output:
             # Conversion has failed.
-            run_result.stderr = error
-            return run_result
+            return tools.CmdResult(
+                status=tools.CmdResult.FAILURE,
+                stdout=run_result.stdout,
+                stderr=error,
+            )
         expected_output, error = tools.convert_to(
             self._output_type, test_node[Tags.EXPECTED_OUTPUT_TAG]
         )
         if our_output != expected_output:
-            run_result.stderr = OUTPUT_MISMATCH_MESSAGE.format(
-                actual=our_output, input=input_str, expected=expected_output
+            return tools.CmdResult(
+                status=tools.CmdResult.FAILURE,
+                stdout=run_result.stdout,
+                stderr=run_result.stderr,
+                output_mismatch=tools.OutputMismatch(
+                    input=input_str,
+                    expected_output=expected_output,
+                    actual_output=our_output,
+                ),
             )
         return run_result

homework_checker/core/tasks.py

@@ -52,7 +52,11 @@ homeworks:
               Another line
               test_me.sh
           - name: Test wrong output
-            expected_output: Different output that doesn't match generated one
+            expected_output: |
+              Hello World!
+              Expected non-matching line
+
+              test_me.sh
       - name: Test input piping
         language: cpp
         folder: task_5

homework_checker/core/tests/data/homework/example_job.yml

@@ -13,6 +13,7 @@
 import datetime
 import signal
 import shutil
+import difflib
 import hashlib
 
 from .schema_tags import OutputTags
@@ -138,32 +139,77 @@ def parse_git_url(git_url: str) -> Tuple[Optional[str], Optional[str], Optional[
     return domain, user, project
 
 
+class OutputMismatch:
+    def __init__(self, input: str, expected_output: str, actual_output: str) -> None:
+        """Initialize the output mismatch class."""
+        self._input = input
+        self._expected_output = expected_output
+        self._actual_output = actual_output
+
+    @property
+    def input(self: OutputMismatch) -> str:
+        """Get input."""
+        return self._input
+
+    @property
+    def expected_output(self: OutputMismatch) -> str:
+        """Get expected output."""
+        return self._expected_output
+
+    @property
+    def actual_output(self: OutputMismatch) -> str:
+        """Get actual output."""
+        return self._actual_output
+
+    def diff(self: OutputMismatch) -> str:
+        actual = str(self._actual_output)
+        expected = str(self._expected_output)
+        diff = difflib.unified_diff(
+            actual.split("\n"),
+            expected.split("\n"),
+            fromfile="Actual output",
+            tofile="Expected output",
+        )
+        diff_str = ""
+        for line in diff:
+            diff_str += line + "\n"
+        return diff_str
+
+    def __repr__(self: OutputMismatch) -> str:
+        """Representation of the output mismatch object."""
+        return "input: {}, expected: {}, actual: {}".format(
+            self._input, self._expected_output, self._actual_output
+        )
+
+
 class CmdResult:
     """A small container for command result."""
 
     SUCCESS = 0
     FAILURE = 13
+    TIMEOUT = 42
 
     def __init__(
-        self: CmdResult, returncode: int = None, stdout: str = None, stderr: str = None
+        self: CmdResult,
+        status: int,
+        stdout: str = None,
+        stderr: str = None,
+        output_mismatch: OutputMismatch = None,
     ):
         """Initialize either stdout of stderr."""
-        self._returncode = returncode
+        self._status = status
         self._stdout = stdout
         self._stderr = stderr
+        self._output_mismatch = output_mismatch
 
     def succeeded(self: CmdResult) -> bool:
         """Check if the command succeeded."""
-        if self.returncode is not None:
-            return self.returncode == CmdResult.SUCCESS
-        if self.stderr:
-            return False
-        return True
+        return self._status == CmdResult.SUCCESS
 
     @property
-    def returncode(self: CmdResult) -> Optional[int]:
-        """Get returncode."""
-        return self._returncode
+    def status(self: CmdResult) -> int:
+        """Get status."""
+        return self._status
 
     @property
     def stdout(self: CmdResult) -> Optional[str]:
@@ -175,24 +221,26 @@ def stderr(self: CmdResult) -> Optional[str]:
         """Get stderr."""
         return self._stderr
 
-    @stderr.setter
-    def stderr(self, value: str):
-        self._returncode = None  # We can't rely on returncode anymore
-        self._stderr = value
+    @property
+    def output_mismatch(self: CmdResult) -> Optional[OutputMismatch]:
+        """Get output_mismatch."""
+        return self._output_mismatch
 
     @staticmethod
     def success() -> CmdResult:
         """Return a cmd result that is a success."""
-        return CmdResult(stdout="Success!")
+        return CmdResult(status=CmdResult.SUCCESS)
 
     def __repr__(self: CmdResult) -> str:
-        """Representatin of command result."""
-        stdout = self.stdout
-        if not stdout:
-            stdout = ""
-        if self.stderr:
-            return "stdout: {}, stderr: {}".format(stdout.strip(), self.stderr.strip())
-        return stdout.strip()
+        """Representation of command result."""
+        repr = "status: {} ".format(self._status)
+        if self._stdout:
+            repr += "stdout: {} ".format(self._stdout)
+        if self._stderr:
+            repr += "stderr: {} ".format(self._stderr)
+        if self._output_mismatch:
+            repr += "output_mismatch: {}".format(self._output_mismatch)
+        return repr.strip()
 
 
 def run_command(
@@ -228,21 +276,21 @@ def run_command(
             timeout=timeout,
         )
         return CmdResult(
-            returncode=process.returncode,
+            status=process.returncode,
             stdout=process.stdout.decode("utf-8"),
             stderr=process.stderr.decode("utf-8"),
         )
     except subprocess.CalledProcessError as error:
         output_text = error.output.decode("utf-8")
-        log.error("command '%s' finished with code: %s", error.cmd, error.returncode)
+        log.error("command '%s' finished with code: %s", error.cmd, error.status)
         log.debug("command output: \n%s", output_text)
-        return CmdResult(returncode=error.returncode, stderr=output_text)
+        return CmdResult(status=error.status, stderr=output_text)
     except subprocess.TimeoutExpired as error:
         output_text = "Timeout: command '{}' ran longer than {} seconds".format(
             error.cmd.strip(), error.timeout
         )
         log.error(output_text)
-        return CmdResult(returncode=1, stderr=output_text)
+        return CmdResult(status=CmdResult.TIMEOUT, stderr=output_text)
 
 
 def __run_subprocess(
@@ -281,11 +329,11 @@ def __run_subprocess(
             raise TimeoutExpired(
                 process.args, timeout, output=stdout, stderr=stderr
             ) from timeout_error
-        retcode = process.poll()
-        if retcode is None:
-            retcode = 1
-        if check and retcode:
+        return_code = process.poll()
+        if return_code is None:
+            return_code = 1
+        if check and return_code:
             raise CalledProcessError(
-                retcode, process.args, output=stdout, stderr=stderr
+                return_code, process.args, output=stdout, stderr=stderr
             )
-    return CompletedProcess(process.args, retcode, stdout, stderr)
+    return CompletedProcess(process.args, return_code, stdout, stderr)

homework_checker/core/tools.py

@@ -6,7 +6,7 @@
 from setuptools import find_packages
 from setuptools.command.install import install
 
-VERSION_STRING = "1.1.0"
+VERSION_STRING = "1.2.0"
 
 PACKAGE_NAME = "homework_checker"