fix: cancel other evaluation runs on fatal WCS error

Currently when there is a fatal error, e.g. in the executor (especially
custom ones), WCS is aborting and "An error occurred" is printed, but
the other kicked-off evaluations still run. This commit fixes this.

Author: devversion

runner/orchestration/generate.ts

@@ -58,10 +58,11 @@ export async function generateCodeAndAssess(options: AssessmentConfig): Promise<
   // Ensure cleanup logic runs when the evaluation is aborted.
   options.abortSignal?.addEventListener('abort', cleanup);
 
-  const ratingLlm = await getRunnerByName('genkit');
-
   await assertValidModelName(options.model, env.executor);
 
+  const ratingLlm = await getRunnerByName('genkit');
+  const allTasksAbortCtrl = new AbortController();
+
   try {
     const promptsToProcess = (
       await getCandidateExecutablePrompts(env, options.localMode, options.promptFilter)
@@ -129,7 +130,11 @@ export async function generateCodeAndAssess(options: AssessmentConfig): Promise<
                   env,
                   ratingLlm,
                   rootPromptDef,
-                  combineAbortSignals(timeoutAbortSignal, options.abortSignal),
+                  combineAbortSignals(
+                    allTasksAbortCtrl.signal,
+                    timeoutAbortSignal,
+                    options.abortSignal,
+                  ),
                   workerConcurrencyQueue,
                   progress,
                 ),
@@ -183,7 +188,7 @@ export async function generateCodeAndAssess(options: AssessmentConfig): Promise<
     const details = {
       summary: await prepareSummary(
         ratingLlm,
-        new AbortController().signal, // Note: AI summarization is currently not abortable.
+        allTasksAbortCtrl.signal,
         options.model,
         env,
         results,
@@ -211,6 +216,10 @@ export async function generateCodeAndAssess(options: AssessmentConfig): Promise<
       results,
       details,
     } satisfies RunInfo;
+  } catch (e) {
+    // Ensure all other running evaluations are cancelled.
+    allTasksAbortCtrl.abort();
+    throw e;
   } finally {
     await cleanup();