expression-parsing: read sum(expr) without limits as an unevaluated sum

expression_parsing/CHANGELOG.md

@@ -53,6 +53,7 @@ The first public release. Further tidy-up after the 0.2.0 extraction: previewing
 
 ### Fixed
 
+- With `elementary_functions`, `sum(expr)` without limits is an unevaluated sum, `UNEVALUATED_SUM(expr)` (exported; an undefined function `sum`), which `sympy_to_latex` prints as `\sum{\left(expr \right)}`. It became `summation(expr)`, which SymPy rejects without limits, so every response to such a question failed to parse. Two such sums are equal when their summands are; nothing is factored out, as the index is unknown. `sum(f, (k, a, b))` is still summed. An applied `sum(...)` is a sum even where the task defines `sum` as a symbol code or alias (`ALWAYS_FUNCTIONS` in `symbol_tables`); on its own, `sum` is still the task's symbol. compareExpressions had read it as that symbol, by accident of its substitution order.
 - (since 0.3.0rc1) `parse_latex` leaves division between numbers unevaluated, as the lambda-feedback fork did: `\frac{10}{2}`, `10/2` and `10 \div 2` give `10/2`, and `\frac{3}{6}` gives `3/6`. In rc1 latex2sympy2_extended divided two integers as a `Rational` (`5` and `1/2`), so a "written as" criterion couldn't tell `\frac{3}{6}` from `\frac{1}{2}`. `2\frac{1}{2}` is the product `2*1/2` again (rc1 read it as the mixed number `5/2`).
 - `parse_expression` leaves every SymPy function unevaluated, not only the ones SymPy's own `evaluate=False` covers (trigonometric, `exp`, `log`, ...). `factorial`, `binomial` and the number-theory functions computed while parsing: `10000000!` took minutes, and `100000!` gave an integer too long to print. As a result `5!` now parses as `factorial(5)`, not `120`, like `sin(0)` stays `sin(0)`; with `simplify` they are still computed. `parse_latex` likewise leaves `\binom` unevaluated (`\binom{5}{2}` gives `binomial(5, 2)`, not `10`): latex2sympy2_extended computed it, and `\binom{100000000}{50000000}` didn't finish. Operator classes called by name (`Pow`, `Add`, `Eq`, ...) are left unevaluated too: `Pow(9,Pow(9,9))` never finished, in every mode.
 - **One LaTeX differential broke every later LaTeX parse in the process.** latex2sympy2 (and the lambda-feedback fork) overwrote a module-level variable when parsing `\mathrm{d}x`, after which every `parse_latex` call failed with "argument of type 'Symbol' is not iterable"; in a warm evaluation-function container, one learner's answer broke LaTeX for every later request. latex2sympy2_extended keeps no state between calls. (Upstream issue OrangeX4/latex2sympy#4, closed unfixed.)

expression_parsing/src/lambdafeedback/expression_parsing/__init__.py

@@ -56,7 +56,7 @@
     substitutions_sort_key,
 )
 from .symbol_tables import UNICODE_OPERATORS
-from .sympy_parsing import SympyParsingConfig, parse_expression, parse_expressions, sympy_symbols
+from .sympy_parsing import UNEVALUATED_SUM, SympyParsingConfig, parse_expression, parse_expressions, sympy_symbols
 
 # Library convention: callers decide where log records go (``logging.basicConfig(level=logging.DEBUG)`` shows them).
 logging.getLogger(__name__).addHandler(logging.NullHandler())
@@ -72,6 +72,7 @@
     "MAX_NUMBER_DIGITS",
     "MAX_NUMBER_EXPONENT",
     "MAX_SIMPLIFY_POWER",
+    "UNEVALUATED_SUM",
     "UNICODE_OPERATORS",
     "AbsoluteValueNotationError",
     "BracketNotationError",

expression_parsing/src/lambdafeedback/expression_parsing/latex.py

@@ -5,7 +5,7 @@
 import logging
 from collections.abc import Iterator, Mapping
 from string import ascii_letters
-from typing import Any, Literal, TypeAlias
+from typing import Any, Literal, TypeAlias, override
 
 from latex2sympy2_extended import NormalizationConfig, normalize_latex
 from latex2sympy2_extended.antlr_parser import PSLexer
@@ -22,6 +22,7 @@
 from .params import SymbolSpec
 from .preprocessing import expand_plus_minus, split_equation
 from .symbol_tables import MINUS_PLUS, PLUS_MINUS
+from .sympy_parsing import UNEVALUATED_SUM
 
 # Symbol aliases are parsed with ``Symbol`` as the only name: an alias that
 # needs anything more (a number, a function call) falls back to a plain name.
@@ -52,7 +53,15 @@ def _replace_command(latex: str, command: str, replacement: str) -> str:
 
 
 class _LatexPrinter(LatexPrinter):
-    """SymPy's LaTeX printer, but printing logarithms with a base as ``\\log_{b}``."""
+    """SymPy's LaTeX printer, but printing logarithms with a base as ``\\log_{b}``, and limitless sums as ``\\sum``."""
+
+    @override
+    def _print_Function(self, expr: Any, exp: Any = None) -> str:
+        if expr.func is not UNEVALUATED_SUM:
+            return str(super()._print_Function(expr, exp))
+        # The summand stays bracketed: in 2 y \sum x, the y could be read as summed too.
+        tex = rf"\sum{{\left({self._print(expr.args[0])} \right)}}"
+        return rf"\left({tex}\right)^{{{exp}}}" if exp is not None else tex
 
     def _print_log(self, expr: Any, exp: Any = None) -> str:
         if len(expr.args) > 1:

expression_parsing/src/lambdafeedback/expression_parsing/substitution.py

@@ -7,6 +7,7 @@
 from typing import TYPE_CHECKING
 
 from .symbol_tables import (
+    ALWAYS_FUNCTIONS,
     ELEMENTARY_FUNCTIONS,
     EMBEDDABLE_FUNCTION_NAMES,
     GREEK_SYMBOLS,
@@ -196,10 +197,11 @@ def substitute_input_symbols(expr: str, params: ExpressionParams) -> str:
     if params.elementary_functions:
         for table, original in ((ELEMENTARY_FUNCTIONS, _function_original), (GREEK_SYMBOLS, str)):
             for name, aliases in table:
-                if name in symbol_aliases:
+                taken = set() if name in ALWAYS_FUNCTIONS else symbol_aliases
+                if name in taken:
                     continue  # the task uses this name for one of its own symbols
                 spellings = [name] if name in expr else []
-                spellings += [alias for alias in aliases if alias in expr and alias not in symbol_aliases]
+                spellings += [alias for alias in aliases if alias in expr and alias not in taken]
                 substitutions += [(original(spelling), " " + name) for spelling in spellings]
 
     for code, spec in params.symbols.items():

expression_parsing/src/lambdafeedback/expression_parsing/symbol_tables.py

@@ -60,6 +60,13 @@ def _with_upper_case(entries: list[tuple[str, list[str]]]) -> tuple[SymbolAliase
 including on their own, they are read as letters.
 """
 
+ALWAYS_FUNCTIONS = frozenset({"summation"})
+"""Function names whose spellings stay that function where applied, even if a task uses one as a symbol alias.
+
+A task may name a symbol ``sum`` just to show it as ``\\sum``; ``sum(...)``
+is still a sum, while ``sum`` on its own is the task's symbol.
+"""
+
 GREEK_SYMBOLS: tuple[SymbolAliases, ...] = tuple(
     SymbolAliases(name, tuple(aliases))
     for name, aliases in [

expression_parsing/src/lambdafeedback/expression_parsing/sympy_parsing.py

@@ -14,7 +14,7 @@
 from typing import Any
 
 import sympy
-from sympy import Basic, Chi, E, Equality, Function, I, Lambda, Set, Symbol, Tuple, beta, gamma, zeta
+from sympy import Basic, Chi, E, Equality, Function, I, Lambda, Set, Symbol, Tuple, beta, gamma, summation, zeta
 from sympy.parsing.sympy_parser import (
     _token_splittable,
     auto_number,
@@ -95,13 +95,27 @@ def sympy_symbols(symbols: Iterable[str]) -> dict[str, Symbol]:
     return {name: Symbol(name) for name in symbols}
 
 
+UNEVALUATED_SUM = Function("sum")
+"""A sum written without an index or limits (``sum(x^n)``): an undefined function of its summand.
+
+Two such sums are equal when their summands are; nothing is pulled out of
+one (``2*sum(x)`` is not ``sum(2*x)``), since which symbol is the index is unknown.
+"""
+
+
+def _summation(*args: Any) -> Any:
+    """``summation``, or :data:`UNEVALUATED_SUM` when called with just a summand."""
+    return UNEVALUATED_SUM(*args) if len(args) == 1 else summation(*args)
+
+
 def _base_symbol_dict(params: ExpressionParams) -> dict[str, Any]:
     """Names SymPy would otherwise read as special objects, unless enabled by the parameters."""
     special: dict[str, Any] = {"beta": beta, "gamma": gamma, "zeta": zeta, "Lambda": Lambda, "Chi": Chi}
     if not params.special_functions:
         special = {name: Symbol(name) for name in special}
     return {
         **special,
+        "summation": _summation,
         "I": I if params.complex_numbers else Symbol("I"),
         "N": Symbol("N"),
         "O": Symbol("O"),

expression_parsing/tests/test_regressions.py

@@ -10,6 +10,7 @@
 from sympy import Abs, Eq, Function, Integral, Rational, Symbol, sqrt, srepr
 
 from lambdafeedback.expression_parsing import (
+    UNEVALUATED_SUM,
     AbsoluteValueNotationError,
     BracketNotationError,
     ExpressionParams,
@@ -743,6 +744,62 @@ def test_shorter_input_still_parses(self):
         assert len(parse_expression("+".join(["x"] * 300), parsing_params()).args) == 300  # left unevaluated
 
 
+class TestSumWithoutLimits:
+    """With elementary_functions, ``sum(x^n)`` became ``summation(x^n)``, which SymPy rejects without limits."""
+
+    @staticmethod
+    def parse(expr, **params):
+        params = ExpressionParams.from_dict({"elementary_functions": True, **params})
+        return parse_expression(preprocess_expression(expr, params), SympyParsingConfig.from_params(params))
+
+    @pytest.mark.parametrize(
+        "params",
+        [
+            {},
+            {"symbols": {"sum": {"latex": r"\sum", "aliases": []}}},
+            {"symbols": {"S": {"latex": r"\sum", "aliases": ["sum"]}}},
+            {"input_symbols": [["sum", []]]},
+        ],
+    )
+    @pytest.mark.parametrize("simplify", [False, True])
+    def test_is_an_unevaluated_sum(self, params, simplify):
+        x, n = Symbol("x"), Symbol("n")
+        assert self.parse("sum(x^n)", simplify=simplify, **params) == UNEVALUATED_SUM(x**n)
+
+    def test_a_task_alias_still_names_the_symbol_where_not_applied(self):
+        params = {"symbols": {"S": {"latex": r"\sum", "aliases": ["sum"]}}}
+        x, n = Symbol("x"), Symbol("n")
+        assert self.parse("sum(x^n) + sum*n", simplify=True, **params) == UNEVALUATED_SUM(x**n) + Symbol("S") * n
+
+    def test_equal_when_the_summands_are(self):
+        assert self.parse("sum(x+x)", simplify=True) == self.parse("sum(2*x)", simplify=True)
+        assert self.parse("2*sum(x)", simplify=True) != self.parse("sum(2*x)", simplify=True)
+
+    @pytest.mark.parametrize(
+        ("expr", "expected"),
+        [("sum(2*k - 1, (k, 1, n))", Symbol("n") ** 2), ("summation(x, (x, 1, 3))", 6)],
+    )
+    def test_with_limits_is_still_summed(self, expr, expected):
+        assert self.parse(expr) == expected
+
+    @pytest.mark.parametrize(
+        ("expr", "latex"),
+        [
+            ("sum(x^n)", r"\sum{\left(x^{n} \right)}"),
+            ("2*y*sum(x+1)", r"2 y \sum{\left(x + 1 \right)}"),
+            ("sum(x)^2", r"\left(\sum{\left(x \right)}\right)^{2}"),
+            ("sin(x)^2", r"\sin^{2}{\left(x \right)}"),
+        ],
+    )
+    def test_latex_is_a_sum(self, expr, latex):
+        assert sympy_to_latex(self.parse(expr, simplify=True), {}) == latex
+
+    def test_reported_answer_parses(self):
+        params = {"symbols": {"sum": {"latex": r"\sum_{k=0}^{\infty}", "aliases": []}}}
+        for expr in ("1/2 + (2/π) * sum (1/(2k+1)) * sin((2k+1) * x)", "1/2 + (2/ pi)sum( sin((2k+1)x)/(2k+1))"):
+            assert self.parse(expr, simplify=True, **params).has(UNEVALUATED_SUM)
+
+
 def test_parsing_logs_its_decisions_at_debug_only(caplog):
     import logging