diff --git a/examples/alphaevolve_math_problems/matmul/evaluator.py b/examples/alphaevolve_math_problems/matmul/evaluator.py index e7144d3ae4..5869c9866d 100644 --- a/examples/alphaevolve_math_problems/matmul/evaluator.py +++ b/examples/alphaevolve_math_problems/matmul/evaluator.py @@ -20,6 +20,12 @@ BENCHMARK = 32 +# Fixed target tensor dimensions for this example: = <2, 4, 5>. +# The evaluator must check the dimensions a candidate reports against these: +# the reported values are produced inside the EVOLVE-BLOCK and must not be +# trusted (see https://github.com/algorithmicsuperintelligence/openevolve/issues/514). +TARGET_N, TARGET_M, TARGET_P = 2, 4, 5 + def verify_tensor_decomposition( decomposition: tuple[np.ndarray, np.ndarray, np.ndarray], n: int, m: int, p: int, rank: int @@ -30,8 +36,10 @@ def verify_tensor_decomposition( if not all(isinstance(arr, np.ndarray) for arr in decomposition) or not decomposition: raise ValueError("Decomposition must be a tuple of NumPy arrays.") if any(arr.size == 0 for arr in decomposition): - print("Warning: One or more decomposition arrays are empty. Verification skipped.") - return + # An empty factor matrix can never reproduce the target tensor, so this + # is a failed verification -- not something to silently skip. Skipping + # would let a candidate score without solving the task (see #514). + raise ValueError("Verification failed: one or more decomposition arrays are empty.") # Check that each factor matrix has the correct shape. factor_matrix_1, factor_matrix_2, factor_matrix_3 = decomposition @@ -78,7 +86,9 @@ def evaluate(program_path: str): sys.path.insert(0, program_dir) program = __import__(module_name) start_time = time.time() - decomposition, n, m, p, loss, rank = program.run() + # NOTE: the reported n, m, p and rank come from the EVOLVE-BLOCK and + # are deliberately not trusted below (see #514). + decomposition, n, m, p, loss, _reported_rank = program.run() end_time = time.time() eval_time = end_time - start_time except Exception as err: @@ -87,6 +97,22 @@ def evaluate(program_path: str): if program_dir in sys.path: sys.path.remove(program_dir) + # The candidate must solve the fixed <2, 4, 5> problem: reject any + # reported dimensions that do not match the target. + if (n, m, p) != (TARGET_N, TARGET_M, TARGET_P): + raise ValueError( + f"Candidate reported dimensions (n, m, p) = {(n, m, p)}; " + f"expected {(TARGET_N, TARGET_M, TARGET_P)}." + ) + + # Derive the rank from the actual factor matrices instead of trusting + # the candidate-reported value: a candidate could otherwise claim an + # arbitrarily low rank and score above the state of the art. + try: + rank = int(decomposition[0].shape[1]) + except (AttributeError, IndexError, TypeError) as exc: + raise ValueError(f"Could not determine rank from the decomposition: {exc}") + verify_tensor_decomposition(decomposition, n, m, p, rank) success_threshold = 1e-6 diff --git a/examples/function_minimization/evaluator.py b/examples/function_minimization/evaluator.py index f16318125b..93d68940a0 100644 --- a/examples/function_minimization/evaluator.py +++ b/examples/function_minimization/evaluator.py @@ -42,6 +42,18 @@ def safe_float(value): return 0.0 +def _objective_function(x, y): + """The objective being minimized. + + Always recomputed by the evaluator from the returned point: a + candidate-reported value comes from the EVOLVE-BLOCK and must not be + trusted for scoring -- a candidate could otherwise claim an arbitrary + score without finding a good point. + See https://github.com/algorithmicsuperintelligence/openevolve/issues/514 + """ + return np.sin(x) * np.cos(y) + np.sin(x * y) + (x**2 + y**2) / 20 + + def evaluate(program_path): """ Evaluate the program by running it multiple times and checking how close @@ -104,12 +116,15 @@ def evaluate(program_path): # Handle different result formats if isinstance(result, tuple): if len(result) == 3: - x, y, value = result + # Never trust the candidate-reported value: it is produced + # inside the EVOLVE-BLOCK, so it is recomputed from (x, y) + # below (see _objective_function). + x, y, _reported_value = result elif len(result) == 2: # Assume it's (x, y) and calculate value x, y = result # Calculate the function value since it wasn't returned - value = np.sin(x) * np.cos(y) + np.sin(x * y) + (x**2 + y**2) / 20 + value = _objective_function(x, y) print(f"Trial {trial}: Got 2 values, calculated function value: {value}") else: print( @@ -127,7 +142,9 @@ def evaluate(program_path): # Ensure all values are float x = safe_float(x) y = safe_float(y) - value = safe_float(value) + # Always (re)compute the objective from (x, y) instead of trusting + # a candidate-reported value (see _objective_function). + value = _objective_function(x, y) # Check if the result is valid (not NaN or infinite) if ( @@ -296,12 +313,15 @@ def evaluate_stage1(program_path): # Handle different result formats if isinstance(result, tuple): if len(result) == 3: - x, y, value = result + # Never trust the candidate-reported value: it is produced + # inside the EVOLVE-BLOCK, so it is recomputed from (x, y) + # below (see _objective_function). + x, y, _reported_value = result elif len(result) == 2: # Assume it's (x, y) and calculate value x, y = result # Calculate the function value since it wasn't returned - value = np.sin(x) * np.cos(y) + np.sin(x * y) + (x**2 + y**2) / 20 + value = _objective_function(x, y) print(f"Stage 1: Got 2 values, calculated function value: {value}") else: print( @@ -343,7 +363,9 @@ def evaluate_stage1(program_path): # Ensure all values are float x = safe_float(x) y = safe_float(y) - value = safe_float(value) + # Always (re)compute the objective from (x, y) instead of trusting + # a candidate-reported value (see _objective_function). + value = _objective_function(x, y) # Check if the result is valid if ( diff --git a/tests/test_example_evaluator_integrity.py b/tests/test_example_evaluator_integrity.py new file mode 100644 index 0000000000..45781b4bb2 --- /dev/null +++ b/tests/test_example_evaluator_integrity.py @@ -0,0 +1,167 @@ +""" +Regression tests for example evaluator integrity. + +Example evaluators must never trust values the candidate program reports about +itself: those values are produced inside the EVOLVE-BLOCK, i.e. under the LLM's +control during evolution, so a candidate could otherwise claim an arbitrary +score without solving the task. + +See https://github.com/algorithmicsuperintelligence/openevolve/issues/514 +""" + +import importlib.util +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +EXAMPLES_DIR = Path(__file__).resolve().parent.parent / "examples" +MATMUL_EVALUATOR = EXAMPLES_DIR / "alphaevolve_math_problems" / "matmul" / "evaluator.py" +FUNCMIN_EVALUATOR = EXAMPLES_DIR / "function_minimization" / "evaluator.py" + + +def load_evaluator(name, path): + """Load an example evaluator module directly from its file path.""" + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class TempProgramTestCase(unittest.TestCase): + """Writes candidate programs to temp files and evaluates them.""" + + def write_program(self, module_name, source): + self._tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(self._tmpdir.cleanup) + # The matmul evaluator imports the program via __import__, which caches + # in sys.modules: make sure a re-run never sees a stale module. + self.addCleanup(sys.modules.pop, module_name, None) + path = Path(self._tmpdir.name) / f"{module_name}.py" + path.write_text(textwrap.dedent(source)) + return str(path) + + +# An exact rank-40 decomposition of the <2, 4, 5> matmul tensor, built from +# indicator outer products: T[a, b, c] = sum_r A[a, r] * B[b, r] * C[c, r]. +_RANK_40_PROGRAM = """ + import numpy as np + + N, M, P = 2, 4, 5 + + def _decomposition(): + n_cols = N * M * P + A = np.zeros((N * M, n_cols)) + B = np.zeros((M * P, n_cols)) + C = np.zeros((N * P, n_cols)) + r = 0 + for i in range(N): + for j in range(M): + for k in range(P): + A[i * M + j, r] = 1.0 + B[j * P + k, r] = 1.0 + C[k * N + i, r] = 1.0 + r += 1 + return A, B, C + + def run(): + # Lie about the rank: the true rank is 40, claim 2. + return _decomposition(), N, M, P, 0.0, 2 +""" + + +class TestMatmulEvaluatorIntegrity(TempProgramTestCase): + @classmethod + def setUpClass(cls): + cls.evaluator = load_evaluator("matmul_evaluator", MATMUL_EVALUATOR) + + def test_empty_decomposition_scores_zero(self): + """The issue's exact repro: empty arrays + claimed rank must score 0, + not skip verification and score 32 / claimed_rank.""" + program = self.write_program( + "cheat_empty_matmul", + """ + import numpy as np + + def run(): + return (np.zeros(0), np.zeros(0), np.zeros(0)), 2, 4, 5, 0.0, 2 + """, + ) + result = self.evaluator.evaluate(program) + self.assertEqual(result["combined_score"], 0.0) + self.assertIn("error", result) + + def test_reported_rank_is_ignored(self): + """The score must reflect the actual decomposition (rank 40 -> 32/40), + not the candidate-claimed rank (which would give 32/2 = 16).""" + program = self.write_program("cheat_rank_matmul", _RANK_40_PROGRAM) + result = self.evaluator.evaluate(program) + self.assertNotIn("error", result) + self.assertEqual(result["rank"], 40) + self.assertAlmostEqual(result["combined_score"], 32 / 40) + + def test_reported_dimensions_are_checked(self): + """A valid decomposition for <2, 4, 5> must not score when the + candidate reports dimensions of a different problem.""" + program = self.write_program( + "cheat_dims_matmul", + _RANK_40_PROGRAM.replace( + "return _decomposition(), N, M, P, 0.0, 2", + "return _decomposition(), 3, 4, 5, 0.0, 40", + ), + ) + result = self.evaluator.evaluate(program) + self.assertEqual(result["combined_score"], 0.0) + self.assertIn("error", result) + + +class TestFunctionMinimizationEvaluatorIntegrity(TempProgramTestCase): + @classmethod + def setUpClass(cls): + cls.evaluator = load_evaluator("funcmin_evaluator", FUNCMIN_EVALUATOR) + + def _metrics(self, result): + return result.metrics if hasattr(result, "metrics") else result + + def test_reported_value_is_recomputed(self): + """Returning (x, y, claimed_value) must score exactly like returning + (x, y): the claimed value is recomputed from the point.""" + honest = self.write_program( + "honest_funcmin", + """ + def run_search(): + return (0.5, 0.5) + """, + ) + cheat = self.write_program( + "cheat_funcmin", + """ + def run_search(): + # Claim the global-minimum value at a non-minimum point. + return (0.5, 0.5, -1.519) + """, + ) + honest_metrics = self._metrics(self.evaluator.evaluate(honest)) + cheat_metrics = self._metrics(self.evaluator.evaluate(cheat)) + self.assertEqual(cheat_metrics["combined_score"], honest_metrics["combined_score"]) + # The claimed global-minimum value must not leak into the score: + # f(0.5, 0.5) != -1.519, so value_score cannot be 1.0. + self.assertLess(cheat_metrics["value_score"], 1.0) + + def test_stage1_recomputes_reported_value(self): + """The cascade stage-1 evaluator must not trust the reported value + either.""" + cheat = self.write_program( + "cheat_funcmin_stage1", + """ + def run_search(): + return (0.5, 0.5, -1.519) + """, + ) + metrics = self._metrics(self.evaluator.evaluate_stage1(cheat)) + self.assertLess(metrics["value_score"], 1.0) + + +if __name__ == "__main__": + unittest.main()