diff --git a/apps/pages/middleware.py b/apps/pages/middleware.py index 39553e36f..9bd200a8d 100644 --- a/apps/pages/middleware.py +++ b/apps/pages/middleware.py @@ -5,6 +5,7 @@ from django import http from django.conf import settings +from sitetree.sitetreeapp import get_sitetree from apps.pages.models import Page from apps.pages.views import PageView @@ -48,6 +49,10 @@ def __call__(self, request): new_url = f"{scheme}://{request.get_host()}{new_path}" return http.HttpResponsePermanentRedirect(new_url) if page is not None: + # The 404 response was already rendered for this request, and sitetree keys its + # per-request state on the request object, so it would keep resolving variables + # against that finished 404 context. Reset it so the page renders with its own. + get_sitetree().init(context=None) response = PageView.as_view()(request, path=full_path) if hasattr(response, "render"): response.render() diff --git a/apps/pages/tests/test_views.py b/apps/pages/tests/test_views.py index ef5b0e654..5b6ac2c41 100644 --- a/apps/pages/tests/test_views.py +++ b/apps/pages/tests/test_views.py @@ -1,6 +1,7 @@ from django.contrib.redirects.models import Redirect from django.contrib.sites.models import Site +from apps.pages.models import Page from apps.pages.tests.base import BasePageTests @@ -23,6 +24,15 @@ def test_with_query_string(self): r = self.client.get("/one/?foo") self.assertEqual(r.context["page"], self.p1) + def test_page_shadowed_by_view_returning_404(self): + """A page renders when a matched view 404s first, though the 404 page was already rendered.""" + page = Page.objects.create( + title="Minutes", path="psf/records/board/minutes/2014-01-03", content="Whatever", is_published=True + ) + r = self.client.get("/psf/records/board/minutes/2014-01-03/") + self.assertEqual(r.status_code, 200) + self.assertEqual(r.context["page"], page) + def test_redirect(self): """ Check that redirects still have priority over pages. diff --git a/gunicorn.conf b/gunicorn.conf index aba67137d..6d768a932 100644 --- a/gunicorn.conf +++ b/gunicorn.conf @@ -14,5 +14,7 @@ loglevel = 'info' accesslog = '-' access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' +control_socket_disable = True + def when_ready(server): open('/tmp/app-initialized', 'w').close() diff --git a/pydotorg/search.py b/pydotorg/search.py new file mode 100644 index 000000000..251e47eb8 --- /dev/null +++ b/pydotorg/search.py @@ -0,0 +1,40 @@ +"""Site search result highlighter built on Haystack.""" + +from haystack.utils.highlighting import Highlighter as HaystackHighlighter + + +class Highlighter(HaystackHighlighter): + """Haystack highlighter that finds the densest window in linear rather than quadratic time. + + Haystack compares every match offset with every later one, so a query of common + letters against a long page keeps a worker busy past its timeout. + """ + + def find_window(self, highlight_locations): + """Return the earliest ``max_length`` window containing the most matches.""" + words_found = sorted(offset for offsets in highlight_locations.values() for offset in offsets) + + if not words_found: + return 0, self.max_length + + if len(words_found) == 1: + return words_found[0], words_found[0] + self.max_length + + best_start, best_end = 0, self.max_length + if words_found[0] > self.max_length: + best_start, best_end = words_found[0], words_found[0] + self.max_length + + # Slide the window end forward as the start advances; a window only wins + # if it holds at least two matches and strictly more than any earlier one. + highest_density = 1 + end = 0 + for count, start in enumerate(words_found[:-1]): + end = max(end, count + 1) + while end < len(words_found) and words_found[end] - start < self.max_length: + end += 1 + density = end - count + if density > highest_density: + best_start, best_end = start, start + self.max_length + highest_density = density + + return best_start, best_end diff --git a/pydotorg/settings/base.py b/pydotorg/settings/base.py index 5f51d931e..611579caf 100644 --- a/pydotorg/settings/base.py +++ b/pydotorg/settings/base.py @@ -259,6 +259,9 @@ HONEYPOT_FIELD_NAME = "email_body_text" HONEYPOT_VALUE = "write your message" +### Haystack +HAYSTACK_CUSTOM_HIGHLIGHTER = "pydotorg.search.Highlighter" + ### Blog Feed URL PYTHON_BLOG_FEED_URL = "https://blog.python.org/feeds/posts/default?alt=rss" PYTHON_BLOG_URL = "https://blog.python.org" diff --git a/pydotorg/tests/test_search.py b/pydotorg/tests/test_search.py new file mode 100644 index 000000000..a69718bc3 --- /dev/null +++ b/pydotorg/tests/test_search.py @@ -0,0 +1,25 @@ +import random + +from django.template import Context, Template +from django.test import SimpleTestCase +from haystack.utils.highlighting import Highlighter as HaystackHighlighter + +from pydotorg.search import Highlighter + + +class HighlighterTests(SimpleTestCase): + def test_find_window_matches_haystack(self): + rng = random.Random(1234) # noqa: S311 - not used for security, just test data + for max_length in [5, 20, 200]: + for _ in range(300): + locations = {word: sorted(rng.sample(range(500), rng.randint(0, 15))) for word in ["py", "thon", "x"]} + with self.subTest(max_length=max_length, locations=locations): + self.assertEqual( + Highlighter("q", max_length=max_length).find_window(locations), + HaystackHighlighter("q", max_length=max_length).find_window(locations), + ) + + def test_highlight_many_matches(self): + template = Template("{% load highlight %}{% highlight text with query max_length 500 %}") + rendered = template.render(Context({"text": "asdfghjkl " * 20000, "query": "A S D F G H J K L"})) + self.assertTrue(rendered.startswith('a'))