test: Add --exclude-example flag and use to skip basic_determinitic with PCT

MLD_CONFIG_NO_RANDOMIZED_API and MLD_CONFIG_KEYGEN_PCT are incompatible and
hence the config variation test that enables PCT currently errors in the
basic_deterministic example (with the appopriate error message that the options
are incompatible).

This commit ports the --exclude-example option for the test script and uses
it in CI for the PCT test.

We also consolidate various small differences to the mlkem-native test script
(whitespace changes, etc.).

Signed-off-by: Matthias J. Kannwischer <matthias@kannwischer.eu>

Author: mkannwischer

.github/actions/config-variations/action.yml

@@ -30,6 +30,7 @@ runs:
         acvp: true
         opt: ${{ inputs.opt }}
         examples: true
+        extra_args: "--exclude-example basic_deterministic"
     - name: "PCT enabled + broken"
       if: ${{ inputs.tests == 'all' || contains(inputs.tests, 'pct-enabled-broken') }}
       shell: bash

.github/actions/functest/action.yml

@@ -51,6 +51,9 @@ inputs:
   stack:
     description: Determine whether to run stack analysis or not
     default: "false"
+  extra_args:
+    description: Additional arguments to pass to the tests script
+    default: ""
 runs:
   using: composite
   steps:
@@ -98,7 +101,7 @@ runs:
         shell: ${{ env.SHELL }}
         run: |
           make clean
-          ./scripts/tests all --exec-wrapper="${{ inputs.exec_wrapper }}" --cross-prefix="${{ inputs.cross_prefix }}" --cflags="${{ inputs.cflags }}" --ldflags="${{ inputs.ldflags }}" --opt=${{ inputs.opt }} --${{ env.FUNC }} --${{ env.KAT }} --${{ env.ACVP }} --${{ env.EXAMPLES }} --${{ env.STACK }} -v
+          ./scripts/tests all --exec-wrapper="${{ inputs.exec_wrapper }}" --cross-prefix="${{ inputs.cross_prefix }}" --cflags="${{ inputs.cflags }}" --ldflags="${{ inputs.ldflags }}" --opt=${{ inputs.opt }} --${{ env.FUNC }} --${{ env.KAT }} --${{ env.ACVP }} --${{ env.EXAMPLES }} --${{ env.STACK }} -v ${{ inputs.extra_args }}
       - name: Post ${{ env.MODE }} Tests
         shell: ${{ env.SHELL }}
         if: success() || failure()

.github/actions/multi-functest/action.yml

@@ -48,6 +48,9 @@ inputs:
   stack:
     description: Determine whether to run stack analysis or not
     default: "false"
+  extra_args:
+    description: Additional arguments to pass to the tests script
+    default: ""
 runs:
   using: composite
   steps:
@@ -68,6 +71,7 @@ runs:
           acvp: ${{ inputs.acvp }}
           examples: ${{ inputs.examples }}
           stack: ${{ inputs.stack }}
+          extra_args: ${{ inputs.extra_args }}
       - name: Cross x86_64 Tests
         if: ${{ (inputs.compile_mode == 'all' || inputs.compile_mode == 'cross-x86_64') && (success() || failure()) }}
         uses: ./.github/actions/functest
@@ -87,6 +91,7 @@ runs:
           acvp: ${{ inputs.acvp }}
           examples: ${{ inputs.examples }}
           stack: ${{ inputs.stack }}
+          extra_args: ${{ inputs.extra_args }}
       - name: Cross aarch64 Tests
         if: ${{ (inputs.compile_mode == 'all' || inputs.compile_mode == 'cross-aarch64') && (success() || failure()) }}
         uses: ./.github/actions/functest
@@ -106,6 +111,7 @@ runs:
           acvp: ${{ inputs.acvp }}
           examples: ${{ inputs.examples }}
           stack: ${{ inputs.stack }}
+          extra_args: ${{ inputs.extra_args }}
       - name: Cross ppc64le Tests
         if: ${{ (inputs.compile_mode == 'all' || inputs.compile_mode == 'cross-ppc64le') && (success() || failure()) }}
         uses: ./.github/actions/functest
@@ -125,6 +131,7 @@ runs:
           acvp: ${{ inputs.acvp }}
           examples: ${{ inputs.examples }}
           stack: ${{ inputs.stack }}
+          extra_args: ${{ inputs.extra_args }}
       - name: Cross aarch64_be Tests
         if: ${{ (inputs.compile_mode == 'all' || inputs.compile_mode == 'cross-aarch64_be') && (success() || failure()) }}
         uses: ./.github/actions/functest
@@ -144,6 +151,7 @@ runs:
           acvp: ${{ inputs.acvp }}
           examples: ${{ inputs.examples }}
           stack: ${{ inputs.stack }}
+          extra_args: ${{ inputs.extra_args }}
       - name: Cross riscv64 Tests
         if: ${{ (inputs.compile_mode == 'all' || inputs.compile_mode == 'cross-riscv64') && (success() || failure()) }}
         uses: ./.github/actions/functest
@@ -163,3 +171,4 @@ runs:
           acvp: ${{ inputs.acvp }}
           examples: ${{ inputs.examples }}
           stack: ${{ inputs.stack }}
+          extra_args: ${{ inputs.extra_args }}

scripts/tests

@@ -11,12 +11,12 @@ See the command line interface for more information."""
 import platform
 import argparse
 import os
+import re
 import sys
 import time
 import logging
 import subprocess
 import json
-import re
 
 from enum import Enum
 from functools import reduce
@@ -582,6 +582,12 @@ class Tests:
             l = TEST_TYPES.examples()
         else:
             l = list(map(TEST_TYPES.from_string, self.args.l))
+
+        # Filter out excluded examples
+        if hasattr(self.args, "exclude_example") and self.args.exclude_example:
+            excluded = [TEST_TYPES.from_string(ex) for ex in self.args.exclude_example]
+            l = [e for e in l if e not in excluded]
+
         for e in l:
             self._compile_schemes(e, None)
             self._run_scheme(e, None, None)
@@ -648,6 +654,8 @@ class Tests:
         self.check_fail()
 
     def stack(self):
+        """Stack usage analysis"""
+
         def _stack(opt):
             self._compile_schemes(TEST_TYPES.STACK, opt)
             if self.args.run:
@@ -769,6 +777,7 @@ class Tests:
                         )
                         exit(1)
                     continue
+
                 end = time.time()
                 dur = int(end - start)
                 if p.returncode != 0:
@@ -944,6 +953,25 @@ def cli():
         dest="examples",
         help="Do not run examples",
     )
+    all_parser.add_argument(
+        "--exclude-example",
+        help="Exclude specific examples from running (can be used multiple times)",
+        choices=[
+            "bring_your_own_fips202",
+            # "bring_your_own_fips202_static",
+            # "custom_backend",
+            "basic",
+            "basic_deterministic",
+            "monolithic_build",
+            # "monolithic_build_native",
+            "monolithic_build_multilevel",
+            # "monolithic_build_multilevel_native",
+            # "multilevel_build",
+            # "multilevel_build_native",
+        ],
+        action="append",
+        default=[],
+    )
 
     stack_group = all_parser.add_mutually_exclusive_group()
     stack_group.add_argument(
@@ -964,7 +992,6 @@ def cli():
     acvp_parser = cmd_subparsers.add_parser(
         "acvp", help="Run ACVP client", parents=[common_parser]
     )
-
     acvp_parser.add_argument(
         "--version",
         default="v1.1.0.40",
@@ -980,9 +1007,9 @@ def cli():
         "-l",
         help="Explicitly list the examples to run; can be called multiple times",
         choices=[
+            "bring_your_own_fips202",
             "basic",
             "basic_deterministic",
-            "bring_your_own_fips202",
             "monolithic_build",
             "monolithic_build_multilevel",
         ],
@@ -1124,6 +1151,8 @@ def cli():
     if not hasattr(args, "l"):
         args.l = None
 
+    os.chdir(os.path.join(os.path.dirname(__file__), ".."))
+
     if args.cmd == "all":
         Tests(args).all()
     elif args.cmd == "examples":