Better output ordering (#10)

* Better output ordering

* Format with black

* Fix the tests

* Remove clutter in output

Author: niosus

homework_checker/checker.py

@@ -20,6 +20,7 @@ class Checker:
     """Check homework."""
 
     TESTS_TAG = "tests"
+    NUMBERING_TAG = "____"
 
     def __init__(self: "Checker", job_file_path: Path):
         """Initialize the checker from file."""
@@ -41,22 +42,22 @@ def check_homework(self: "Checker", homework_node: dict) -> HomeworkResultDict:
         deadline_datetime = datetime.strptime(deadline_str, tools.DATE_PATTERN)
         if datetime.now() > deadline_datetime:
             results[tools.EXPIRED_TAG] = {}
-        for task_node in homework_node[Tags.TASKS_TAG]:
+        for idx, task_node in enumerate(homework_node[Tags.TASKS_TAG]):
             task = Task.from_yaml_node(
                 task_node=task_node,
                 student_hw_folder=current_folder,
                 job_file=self._job_file_path,
             )
             if not task:
                 continue
-            results[task.name] = task.check()
+            results[tools.add_number_to_name(idx, task.name)] = task.check()
         return results
 
     def check_all_homeworks(self: "Checker") -> Dict[str, HomeworkResultDict]:
         """Run over all Tasks in all homeworks."""
         results: Dict[str, HomeworkResultDict] = {}
-        for homework_node in self._base_node[Tags.HOMEWORKS_TAG]:
-            hw_name = homework_node[Tags.NAME_TAG]
+        for idx, homework_node in enumerate(self._base_node[Tags.HOMEWORKS_TAG]):
+            hw_name = tools.add_number_to_name(idx, homework_node[Tags.NAME_TAG])
             current_homework_results = self.check_homework(homework_node)
             if current_homework_results:
                 results[hw_name] = current_homework_results

homework_checker/md_writer.py

@@ -4,35 +4,48 @@
 
 from .tools import EXPIRED_TAG
 from .tools import CmdResult
+from .tools import remove_number_from_name
 from .checker import HomeworkResultDict
 
 TABLE_TEMPLATE = "| {hw_name} | {task_name} | {test_name} | {result_sign} |\n"
 TABLE_SEPARATOR = "|---|---|---|:---:|\n"
 
-ERROR_TEMPLATE = """### `[{hw_name}][{task_name}][{test_name}]:`
+ERROR_TEMPLATE = """
+<details><summary><b>{hw_name} | {task_name} | {test_name}</b></summary>
 
-*stderr*:
+**`stderr`**
 ```apiblueprint
 {stderr}
 ```
-*stdout*:
+
+**`stdout`**
 ```
 {stdout}
 ```
+
 --------
+
+</details>
+
 """
 
 EXPIRED_TEMPLATE = """
 
-### `[{hw_name}][Past Deadline][Errors Hidden]`
+<details><summary><b>{hw_name}</b></summary>
+
+The deadline for this homework is exceeded. Errors hidden.
+
+-------
+ 
+</details>
 
 """
 
 SEPARATOR = "--------\n"
-FINISHING_NOTE = "With 💙 from homework bot 🤖\n"
+FINISHING_NOTE = "With ❤️ from Homework Bot 🤖\n"
 
-SUCCESS_TAG = "✔"
-FAILED_TAG = "✘"
+SUCCESS_TAG = "✅"
+FAILED_TAG = "❌"
 
 
 class MdWriter:
@@ -52,18 +65,21 @@ def __init__(self: "MdWriter"):
     def update(self: "MdWriter", hw_results: Dict[str, HomeworkResultDict]):
         """Update the table of completion."""
         for hw_name, hw_dict in sorted(hw_results.items()):
+            hw_name = remove_number_from_name(hw_name)
             need_hw_name = True
             expired = False
             if EXPIRED_TAG in hw_dict:
                 expired = True
             for task_name, ex_dict in sorted(hw_dict.items()):
+                task_name = remove_number_from_name(task_name)
                 if task_name == EXPIRED_TAG:
                     # Maybe there is a better way to handle this, but I don't
                     # want to dig into this right now. We have added the
                     # EXPIRED_TAG to this dict and need to ignore it here.
                     continue
                 need_task_name = True
                 for test_name, test_result in sorted(ex_dict.items()):
+                    test_name = remove_number_from_name(test_name)
                     result_sign = SUCCESS_TAG if test_result.succeeded() else FAILED_TAG
                     extended_hw_name = (
                         hw_name + " `[PAST DEADLINE]`" if expired else hw_name
@@ -83,7 +99,7 @@ def write_md_file(self: "MdWriter", md_file_path: Path):
         md_file_content = "# Test results\n"
         md_file_content += self._md_table
         if self._errors:
-            md_file_content += "\n## Encountered errors\n"
+            md_file_content += "\n# Encountered errors\n"
             md_file_content += self._errors
         md_file_content += SEPARATOR
         md_file_content += FINISHING_NOTE

homework_checker/tasks.py

@@ -18,8 +18,8 @@
 Your output '{actual}'
 Expected output: '{expected}'"""
 
-BUILD_SUCCESS_TAG = "0. Build succeeded"
-STYLE_ERROR_TAG = "0. Style errors"
+BUILD_SUCCESS_TAG = "Build succeeded"
+STYLE_ERROR_TAG = "Style errors"
 
 TOTAL_ERRORS_FOUND_TAG = "Total errors found"
 
@@ -80,6 +80,12 @@ def __init__(
             self._test_nodes = task_node[Tags.TESTS_TAG]
         else:
             self._test_nodes = []  # Sometimes we don't have tests.
+        self.__test_counter = 0
+
+    def __with_number_prefix(self: Task, test_name: str) -> str:
+        """Get the test name with number."""
+        self.__test_counter += 1
+        return tools.add_number_to_name(self.__test_counter, test_name)
 
     def check(self: Task) -> Task.ResultDictType:
         """Iterate over the tests and check them."""
@@ -94,7 +100,9 @@ def run_all_tests(
             if Tags.INJECT_FOLDER_TAG not in test_node:
                 # There is no need to rebuild the code. We can just run our tests.
                 test_result = self._run_test(test_node, executable_folder)
-                results[test_node[Tags.NAME_TAG]] = test_result
+                results[
+                    self.__with_number_prefix(test_node[Tags.NAME_TAG])
+                ] = test_result
                 return results
             # There are folders to inject, so we will have to rebuild with the newly
             # injected folders. We do it in a new temp folder.
@@ -111,14 +119,16 @@ def run_all_tests(
                 test_result = self._run_test(
                     test_node=test_node, executable_folder=build_folder
                 )
-                results[test_node[Tags.NAME_TAG]] = test_result
+                results[
+                    self.__with_number_prefix(test_node[Tags.NAME_TAG])
+                ] = test_result
             return results
 
         with tools.TempDirCopy(self._student_task_folder) as code_folder:
             # Build the source if this is needed.
             build_result, build_folder = self._build_if_needed(code_folder)
             if build_result:
-                results[BUILD_SUCCESS_TAG] = build_result
+                results[self.__with_number_prefix(BUILD_SUCCESS_TAG)] = build_result
                 if not build_result.succeeded():
                     # The build has failed, so no further testing needed.
                     return results

homework_checker/tests/test_checker.py

@@ -5,6 +5,8 @@
 import unittest
 
 from pathlib import Path
+from typing import Dict
+
 from homework_checker.checker import Checker
 from homework_checker.md_writer import MdWriter
 from homework_checker import tools
@@ -13,6 +15,16 @@
 class TestChecker(unittest.TestCase):
     """Test the checker."""
 
+    @staticmethod
+    def __sanitize_results(results: Dict[str, Dict[str, tools.CmdResult]]):
+        sanitized_results = {}
+        for key, value in results.items():
+            sanitized_value = {}
+            for inner_key, inner_value in value.items():
+                sanitized_value[tools.remove_number_from_name(inner_key)] = inner_value
+            sanitized_results[tools.remove_number_from_name(key)] = sanitized_value
+        return sanitized_results
+
     def test_everything(self: TestChecker):
         """Check all homeworks and Tasks."""
         path_to_job = (
@@ -25,6 +37,7 @@ def test_everything(self: TestChecker):
         )
         checker = Checker(path_to_job)
         results = checker.check_all_homeworks()
+        results = TestChecker.__sanitize_results(results)
         self.assertEqual(len(results), 3)
         self.assertNotIn("Non existing homework", results)
 

homework_checker/tests/test_task.py

@@ -50,6 +50,14 @@ def __get_homework_name_and_task(
             self.fail("Task cannot be None")
         return homework_node[Tags.NAME_TAG], task
 
+    @staticmethod
+    def __sanitize_results(results: dict) -> dict:
+        """Sanitize the outputs of the tasks."""
+        sanitized_results = {}
+        for key, value in results.items():
+            sanitized_results[tools.remove_number_from_name(key)] = value
+        return sanitized_results
+
     def test_check_simple_cpp_io_task(self: "TestTask"):
         """Check that we can build and run cpp code and get some output."""
 
@@ -59,6 +67,7 @@ def test_check_simple_cpp_io_task(self: "TestTask"):
         self.assertEqual(homework_name, "Sample homework")
         self.assertEqual(task.name, "Simple cpp tasks")
         results = task.check()
+        results = TestTask.__sanitize_results(results)
         expected_number_of_build_outputs = 1
         expected_number_of_test_outputs = 3
         self.assertEqual(
@@ -81,6 +90,7 @@ def test_check_build_failure_task(self: "TestTask"):
         self.assertEqual(homework_name, "Sample homework")
         self.assertEqual(task.name, "Build failure task")
         results = task.check()
+        results = TestTask.__sanitize_results(results)
         expected_number_of_build_outputs = 1
         expected_number_of_test_outputs = 0
         self.assertEqual(
@@ -99,6 +109,7 @@ def test_check_cmake_arithmetics_task(self: "TestTask"):
         self.assertEqual(homework_name, "Sample homework")
         self.assertEqual(task.name, "CMake build arithmetics task")
         results = task.check()
+        results = TestTask.__sanitize_results(results)
         expected_number_of_build_outputs = 1
         expected_number_of_test_outputs = 2
         expected_number_of_code_style_outputs = 0
@@ -122,6 +133,7 @@ def test_check_bash_task(self: "TestTask"):
         self.assertEqual(homework_name, "Sample homework")
         self.assertEqual(task.name, "Bash task")
         results = task.check()
+        results = TestTask.__sanitize_results(results)
         expected_number_of_build_outputs = 0
         expected_number_of_test_outputs = 2
         expected_number_of_code_style_outputs = 0
@@ -132,6 +144,7 @@ def test_check_bash_task(self: "TestTask"):
             + expected_number_of_code_style_outputs,
             "Wrong results: {}".format(results),
         )
+        print(results)
         self.assertTrue(results["Test output"].succeeded())
         self.assertFalse(results["Test wrong output"].succeeded())
 
@@ -144,6 +157,7 @@ def test_check_wrong_output_type(self: "TestTask"):
         self.assertEqual(homework_name, "Homework where things go wrong")
         self.assertEqual(task.name, "Return number task")
         results = task.check()
+        results = TestTask.__sanitize_results(results)
         expected_number_of_build_outputs = 1
         expected_number_of_test_outputs = 1
         expected_number_of_code_style_outputs = 0
@@ -170,6 +184,7 @@ def test_timing_out_task(self: "TestTask"):
         self.assertEqual(homework_name, "Homework where things go wrong")
         self.assertEqual(task.name, "While loop task")
         results = task.check()
+        results = TestTask.__sanitize_results(results)
         expected_number_of_build_outputs = 1
         expected_number_of_test_outputs = 1
         expected_number_of_code_style_outputs = 0
@@ -198,6 +213,7 @@ def test_google_tests_task(self: "TestTask"):
         self.assertEqual(homework_name, "Homework with injections")
         self.assertEqual(task.name, "Google Tests")
         results = task.check()
+        results = TestTask.__sanitize_results(results)
         expected_number_of_build_outputs = 1
         expected_number_of_test_outputs = 3
         expected_number_of_code_style_outputs = 0
@@ -222,6 +238,7 @@ def test_bash_task_with_injections(self: "TestTask"):
         self.assertEqual(homework_name, "Homework with injections")
         self.assertEqual(task.name, "Bash with many folders")
         results = task.check()
+        results = TestTask.__sanitize_results(results)
         expected_number_of_build_outputs = 0
         expected_number_of_test_outputs = 1
         expected_number_of_code_style_outputs = 0

homework_checker/tools.py

@@ -22,6 +22,7 @@
 MAX_DATE_STR = datetime.datetime.max.strftime(DATE_PATTERN)
 
 EXPIRED_TAG = "expired"
+NUMBER_SPLIT_TAG = "____"
 
 log = logging.getLogger("GHC")
 
@@ -31,6 +32,18 @@ def get_unique_str(seed: str) -> str:
     return hashlib.md5(seed.encode("utf-8")).hexdigest()
 
 
+def add_number_to_name(number: int, name: str) -> str:
+    """Add a number before a string."""
+    return str(number) + NUMBER_SPLIT_TAG + name
+
+
+def remove_number_from_name(name: str) -> str:
+    """Add a number before a string."""
+    if NUMBER_SPLIT_TAG not in name:
+        return name
+    return name.split(NUMBER_SPLIT_TAG)[1]
+
+
 class TempDirCopy:
     """docstring for TempDirCopy"""