fix(evaluate): postprocess the concurrent outputs for o1

terryyz · terryyz · commit ba18aaf6c92e · 2024-12-21T02:14:47.000+08:00
diff --git a/bigcodebench/provider/openai.py b/bigcodebench/provider/openai.py
@@ -57,7 +57,7 @@ def _codegen_batch_via_concurrency(self, messages: List[str], num_samples: int)
         batches = concurrent_call(
             num_samples, self._codegen_api_batch, messages, num_samples=1
         )
-        return [b[0] for b in batches]
+        return [[element for sublist in item for element in sublist] for item in zip(*batches)]
 
     def is_direct_completion(self) -> bool:
         return False

Original file line number	Diff line number	Diff line change
`@@ -57,7 +57,7 @@ def _codegen_batch_via_concurrency(self, messages: List[str], num_samples: int)`
`57`	`57`	`batches = concurrent_call(`
`58`	`58`	`num_samples, self._codegen_api_batch, messages, num_samples=1`
`59`	`59`	`)`
`60`		`- return [b[0] for b in batches]`
	`60`	`+ return [[element for sublist in item for element in sublist] for item in zip(*batches)]`
`61`	`61`
`62`	`62`	`def is_direct_completion(self) -> bool:`
`63`	`63`	`return False`