Fix string operators like startswith not using an index because of the unicode regex flag #965

docs/changelog.rst

@@ -8,6 +8,7 @@ Development
 ===========
 - (Fill this out as you fix issues and develop your features).
 - Fix partial ``Document.reload()`` leaving custom ``db_field`` values marked as changed.
+- Fix ``__startswith``, ``__exact`` and the other string query operators not using indexes: they no longer send the ``u`` regex option that Python adds to every compiled ``str`` pattern #965
 
 Changes in 1.0.0
 ================

mongoengine/fields.py

@@ -12,7 +12,7 @@
 
 import gridfs
 import pymongo
-from bson import Binary, DBRef, ObjectId
+from bson import Binary, DBRef, ObjectId, Regex
 from bson.decimal128 import Decimal128, create_decimal128_context
 from pymongo import ReturnDocument
 
@@ -182,11 +182,13 @@ def prepare_query_value(self, op, value):
                 regex = value
 
             if op == "regex":
-                value = re.compile(regex, flags)
+                re.compile(regex, flags)  # raise re.error early on an invalid pattern
             else:
                 # escape unsafe characters which could lead to a re.error
-                value = re.escape(value)
-                value = re.compile(regex % value, flags)
+                regex = regex % re.escape(value)
+            # Not re.compile: a str pattern always gets re.UNICODE, sent as the `u` option,
+            # which stops MongoDB from using an index for a prefix regex (#965).
+            value = Regex(regex, flags)
         return super().prepare_query_value(op, value)
 
 

tests/fields/test_string_field.py

@@ -41,3 +41,28 @@ class Person(Document):
 
         person = Person(name="a friendl name", userid="7a757668sqjdkqlsdkq")
         person.validate()
+
+    def test_string_operators_use_an_index(self):
+        class Person(Document):
+            name = StringField()
+            meta = {"indexes": ["name"]}
+
+        Person.drop_collection()
+        Person.ensure_indexes()
+        Person.objects.insert([Person(name=f"person{i}") for i in range(100)])
+
+        stats = Person.objects(name__startswith="person7").explain()["executionStats"]
+
+        assert stats["nReturned"] == 11
+        assert stats["totalKeysExamined"] <= 12
+
+    def test_string_operators_keep_case_insensitive_flag(self):
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+        Person(name="Guido").save()
+
+        assert Person.objects(name__istartswith="gui").count() == 1
+        assert Person.objects(name__startswith="gui").count() == 0
+        assert Person.objects(name__not__istartswith="gui").count() == 0