Speed up copyright candidate line selection

CHANGELOG.rst

@@ -26,6 +26,11 @@ Next release
   ``ValueError: did not find any 'pre_scan' hooks``.
   https://github.com/aboutcode-org/scancode-toolkit/issues/5363
 
+- Speed up copyright detection: skip markup stripping on lines without markup,
+  run gibberish detection only on lines with a year or a copyright marker and
+  stop normalizing lines twice in gibberish detection. Results are unchanged.
+  https://github.com/aboutcode-org/scancode-toolkit/issues/5374
+
 v33.0.0rc1 - 2026-05-14
 ------------------------
 

src/cluecode/copyrights.py

@@ -4343,19 +4343,20 @@ def is_candidate(prepared_line):
 
         return False
 
+    # check for years and markers first: gibberish detection is much slower
+    lowered_line = prepared_line.lower()
+    if not (
+        copyrights_hint.years(prepared_line)
+        or any(marker in lowered_line for marker in copyrights_hint.statement_markers)
+    ):
+        return False
+
     if gibberish_detector.detect_gibberish(prepared_line):
         if TRACE:
             logger_debug(f'is_candidate: gibberish_detector.detect_gibberish:\n{prepared_line!r}')
         return False
 
-    if copyrights_hint.years(prepared_line):
-        return True
-    else:
-        pass
-    prepared_line = prepared_line.lower()
-    for marker in copyrights_hint.statement_markers:
-        if marker in prepared_line:
-            return True
+    return True
 
 
 def is_inside_statement(

src/textcode/gibberish.py

@@ -103,7 +103,7 @@ def train(self, bigfile=big_file_path, goodfile=good_file_path,
         self.persist_model()
 
     def detect_gibberish(self, text):
-        text = ''.join(self.normalize(text))
+        # avg_transition_prob() already normalizes text through ngram()
         return self.avg_transition_prob(text, self.mat) < self.thresh
 
     def percent_gibberish(self, text):

src/textcode/markup.py

@@ -305,9 +305,27 @@ def strip_known_markup_from_text(text):
     """
     Str_ip markup tags from ``text`` using a list of tags
     """
+    if not may_have_known_markup(text):
+        # nothing in text could be dropped by keep_tag: only normalize spaces
+        return " ".join(text.split())
     return get_demarkuped_text(text, splitter=split_on_tags, keeper=keep_tag)
 
 
+def may_have_known_markup(text):
+    """
+    Return True if ``text`` may contain a token that ``keep_tag`` would drop.
+    Return False only when splitting ``text`` with ``split_on_tags`` cannot
+    yield any such token: there is no "<" or ">", no skipped attribute and
+    no tag without angle brackets.
+    """
+    if "<" in text or ">" in text:
+        return True
+    if any(attribute in text for attribute in SKIP_ATTRIBUTES):
+        return True
+    tlow = text.lower()
+    return any(tag in tlow for tag in BARE_TAGS)
+
+
 ALL_TAGS = frozenset(
     [
         "<a",
@@ -1928,6 +1946,9 @@ def strip_known_markup_from_text(text):
     ]
 )
 
+# tags that can match a token without any "<" or ">"
+BARE_TAGS = tuple(sorted(t for t in ALL_TAGS if "<" not in t and ">" not in t))
+
 
 SKIP_ATTRIBUTES = (
     "href=",

tests/cluecode/test_copyrights_basic.py

@@ -249,6 +249,16 @@ def test_is_candidate_should_not_select_line_with_junk_hex(self):
         line = prepare_text_line(line)
         assert not copyrights.is_candidate(line)
 
+    def test_is_candidate_should_not_select_gibberish_line_with_year_and_marker(self):
+        line = 'Copyright 2012 kjhgqwkzxqjvbn qwxzjkvq'
+        line = prepare_text_line(line)
+        assert not copyrights.is_candidate(line)
+
+    def test_is_candidate_should_not_select_line_without_year_or_marker(self):
+        line = 'static void printBlockName(raw_ostream &OS, const BasicBlock *BB);'
+        line = prepare_text_line(line)
+        assert not copyrights.is_candidate(line)
+
     def test_is_candidate_should_select_line_with_iso_date_year(self):
         line = 'sig 3 ccd6f801 2006-11-15 nathan mittler <nathan.mittler@gmail.com>'
         line = prepare_text_line(line)

tests/textcode/test_markup.py

@@ -287,6 +287,41 @@ def test_strip_markup_text_tagset(test_text, expected_demarkup):
     assert markup.strip_known_markup_from_text(test_text) == expected_demarkup
 
 
+no_markup_tests = [
+    "Copyright (c) 2003-2024 The LLVM project.",
+    "  some   text\twith  spaces  ",
+    "Contact MAILTO:foo@example.com for details",
+    "",
+]
+
+maybe_markup_tests = [
+    "Copyright <b>2024</b> Foo",
+    "a > b",
+    "@end copying",
+    "see mailto:foo@example.com",
+    'xmlns="http://www.w3.org/1999/xhtml" by Foo',
+    "HTTP://WWW.W3.ORG/2000/SVG Copyright Foo",
+]
+
+
+@pytest.mark.parametrize("test_text", no_markup_tests)
+def test_may_have_known_markup_is_false_for_plain_text(test_text):
+    assert not markup.may_have_known_markup(test_text)
+
+
+@pytest.mark.parametrize("test_text", maybe_markup_tests)
+def test_may_have_known_markup_is_true_for_markup_or_skipped_tokens(test_text):
+    assert markup.may_have_known_markup(test_text)
+
+
+@pytest.mark.parametrize("test_text", no_markup_tests + maybe_markup_tests)
+def test_strip_known_markup_from_text_fast_path_matches_full_split(test_text):
+    expected = markup.get_demarkuped_text(
+        test_text, splitter=markup.split_on_tags, keeper=markup.keep_tag
+    )
+    assert markup.strip_known_markup_from_text(test_text) == expected
+
+
 split_tests = [
     (
         """<a href="https://licenses.nuget.org/Apache-2.0">Apache-2.0</a>""",