Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions criteria/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,10 @@ Behaviour differences to be aware of when adopting:
- **Tags in `to_dict()`.** lf_toolkit only includes `tags` with `include_test_data=True`; `EvaluationResult.serialise()` always did.
- **Stripping.** Feedback text is stripped when resolved (it used to be stripped when serialised), so the stored texts are already trimmed.

### Added

- `build_quantity_criteria_parser(reserved_expressions, token_list=, productions=)` parses criteria on physical quantities (`response matches answer`, `dimension(response) matches dimension(answer)`, and `=`, `<=`, `>=`, `<`, `>` between quantities). Its defaults are `quantity_grammar.QUANTITY_TOKENS` / `QUANTITY_PRODUCTIONS`. It was `generate_criteria_parser` in compareExpressions' physical-quantity context, and its parse trees are unchanged. It takes the same arguments as `build_criteria_parser`, so either can fill the same slot. `reserved_expressions` isn't used yet: only `response` and `answer` scan as quantities.

### Changed

- DEBUG logging through module loggers (`lambdafeedback.criteria.*`), with a `NullHandler` on the package logger; nothing is logged at INFO or above. See the repository README. `generate_feedback` logs each evaluation run, the criteria it reported and replacements followed.
Expand Down
4 changes: 4 additions & 0 deletions criteria/src/lambdafeedback/criteria/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

- :func:`build_criteria_parser` parses criteria such as
``response = answer where a = 2``.
- :func:`build_quantity_criteria_parser` parses criteria on physical
quantities, such as ``response matches answer``.
- :class:`CriteriaGraph` links evaluations to the criteria they establish,
runs them (:meth:`CriteriaGraph.generate_feedback`), turns the criteria
reached into feedback on a result object such as
Expand All @@ -26,6 +28,7 @@
ReachedCriteria,
ResultLike,
)
from .quantity_grammar import build_quantity_criteria_parser
from .tree import CriteriaTree

# Library convention: callers decide where log records go (``logging.basicConfig(level=logging.DEBUG)`` shows them).
Expand Down Expand Up @@ -53,4 +56,5 @@
"ResultLike",
"__version__",
"build_criteria_parser",
"build_quantity_criteria_parser",
]
78 changes: 78 additions & 0 deletions criteria/src/lambdafeedback/criteria/quantity_grammar.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
"""Grammar and parser for criteria on physical quantities, such as ``response matches answer``.

Criteria compare two quantities with ``matches``, ``=``, ``<=``, ``>=``,
``<`` or ``>``, or compare their dimensions with
``dimension(QUANTITY) matches dimension(QUANTITY)``. ``response`` and
``answer`` scan as ``QUANTITY``; other text is ``INPUT``.
"""

from __future__ import annotations

from collections.abc import Mapping, Sequence
from typing import Any

from lambdafeedback.slr_parsing import (
Action,
CatchAll,
Lexeme,
SLRParser,
TokenPattern,
TokenSpec,
create_node,
infix,
operate,
)

from .grammar import END_SYMBOL, NULL_SYMBOL, START_SYMBOL


def _spaced(form: str) -> Lexeme:
"""``form``, with any spaces around it."""
return Lexeme((form,), space_before=True, space_after=True)


_OPERATORS = ("matches", "dimension", "=", "<=", ">=", "<", ">")

QUANTITY_TOKENS: tuple[TokenSpec, ...] = (
TokenPattern(START_SYMBOL, START_SYMBOL),
TokenPattern(END_SYMBOL, END_SYMBOL),
TokenPattern(NULL_SYMBOL, NULL_SYMBOL),
TokenPattern(_spaced("BOOL"), "BOOL"),
TokenPattern(_spaced("QUANTITY"), "QUANTITY"),
TokenPattern(_spaced("DIMENSION"), "DIMENSION"),
TokenPattern(Lexeme(("(",), space_after=True), "START_DELIMITER"),
TokenPattern(Lexeme((")",), space_before=True), "END_DELIMITER"),
TokenPattern("response", "QUANTITY"),
TokenPattern("answer", "QUANTITY"),
CatchAll("INPUT", "INPUT"),
# Operators are labelled with the spaces around them (e.g. `` matches ``), as in compareExpressions.
*(TokenPattern(_spaced(operator), f" {operator} ") for operator in _OPERATORS),
)

QUANTITY_PRODUCTIONS: tuple[tuple[str, str, Action], ...] = (
("START", "BOOL", create_node),
("BOOL", "QUANTITY matches QUANTITY", infix),
("BOOL", "DIMENSION matches DIMENSION", infix),
("BOOL", "QUANTITY=QUANTITY", infix),
("BOOL", "QUANTITY<=QUANTITY", infix),
("BOOL", "QUANTITY>=QUANTITY", infix),
("BOOL", "QUANTITY<QUANTITY", infix),
("BOOL", "QUANTITY>QUANTITY", infix),
("QUANTITY", "INPUT", create_node),
("DIMENSION", "dimension(QUANTITY)", operate(1)),
)


def build_quantity_criteria_parser(
reserved_expressions: Mapping[str, Mapping[str, Any]],
token_list: Sequence[TokenSpec] = QUANTITY_TOKENS,
productions: Sequence[tuple[str, str, Action | None]] = QUANTITY_PRODUCTIONS,
) -> SLRParser:
"""A parser for criteria on physical quantities.

Takes the same arguments as :func:`.build_criteria_parser`, so either can
be used where a criteria parser is built from the reserved expressions.
``reserved_expressions`` isn't used yet: only ``response`` and ``answer``
scan as ``QUANTITY``.
"""
return SLRParser(token_list, productions, START_SYMBOL, END_SYMBOL, NULL_SYMBOL)
36 changes: 35 additions & 1 deletion criteria/tests/test_criteria.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@

import pytest

from lambdafeedback.criteria import CriteriaGraph, CriteriaGraphError, build_criteria_parser
from lambdafeedback.criteria import (
CriteriaGraph,
CriteriaGraphError,
build_criteria_parser,
build_quantity_criteria_parser,
)
from lambdafeedback.slr_parsing import append_last, create_node, infix, join, proceed

RESERVED = {"learner": {"response": None}, "task": {"answer": None}}
Expand Down Expand Up @@ -46,6 +51,35 @@ def test_where_clause_parses(self):
assert node.label == "WHERE"


def _parse_quantity(criterion):
parser = build_quantity_criteria_parser(RESERVED)
return parser.parse(parser.scan(criterion))[0]


class TestQuantityCriteriaParser:
@pytest.mark.parametrize("operator", ["matches", "=", "<=", ">=", "<", ">"])
def test_relation_between_quantities(self, operator):
node = _parse_quantity(f"response {operator} answer")
assert node.label == f" {operator} "
assert [(child.label, child.content_string()) for child in node.children] == [
("QUANTITY", "response"),
("QUANTITY", "answer"),
]

def test_dimensions_match(self):
node = _parse_quantity("dimension(response) matches dimension(answer)")
assert node.label == " matches "
assert [child.label for child in node.children] == [" dimension ", " dimension "]
assert [child.children[0].content_string() for child in node.children] == ["response", "answer"]

def test_other_text_is_a_quantity(self):
node = _parse_quantity("response matches 9.81 m/s^2")
assert [(child.label, child.content_string()) for child in node.children] == [
("QUANTITY", "response"),
("INPUT", "9.81 m/s^2"),
]


class TestCriteriaGraph:
def test_build_and_serialise(self):
g = CriteriaGraph("root")
Expand Down
Loading