From cb255027ba5e9b7830b77a0454e6274459b543ce Mon Sep 17 00:00:00 2001 From: Nachiket Date: Sat, 24 Oct 2026 21:52:35 +0530 Subject: [PATCH] Speed up copyright candidate line selection Copyright detection got noticeably slower on large source trees, with most of the extra time spent in gibberish detection and markup stripping that run on nearly every line. - Check for years and copyright markers before running gibberish detection in is_candidate(). A line without them is never a candidate, so gibberish detection now only runs on the few lines that could be. - Skip the tag splitting in strip_known_markup_from_text() when a line has no "<" or ">", no skipped attribute and no bare tag, since keep_tag() would keep every token and only spaces get normalized. - Do not normalize text twice in detect_gibberish(): ngram() already normalizes it. Detections are unchanged: the scan output for the llvm/lib tree from llvm-project 18.1.8 is byte-identical before and after, and copyright detection on it is about 19% faster. Reference: https://github.com/aboutcode-org/scancode-toolkit/issues/5374 Signed-off-by: Nachiket --- CHANGELOG.rst | 5 ++++ src/cluecode/copyrights.py | 17 ++++++------ src/textcode/gibberish.py | 2 +- src/textcode/markup.py | 21 +++++++++++++++ tests/cluecode/test_copyrights_basic.py | 10 +++++++ tests/textcode/test_markup.py | 35 +++++++++++++++++++++++++ 6 files changed, 81 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index abc3ae41ad2..030bd081168 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -26,6 +26,11 @@ Next release ``ValueError: did not find any 'pre_scan' hooks``. https://github.com/aboutcode-org/scancode-toolkit/issues/5363 +- Speed up copyright detection: skip markup stripping on lines without markup, + run gibberish detection only on lines with a year or a copyright marker and + stop normalizing lines twice in gibberish detection. Results are unchanged. + https://github.com/aboutcode-org/scancode-toolkit/issues/5374 + v33.0.0rc1 - 2026-05-14 ------------------------ diff --git a/src/cluecode/copyrights.py b/src/cluecode/copyrights.py index 29876084168..8b6c1b62b96 100644 --- a/src/cluecode/copyrights.py +++ b/src/cluecode/copyrights.py @@ -4343,19 +4343,20 @@ def is_candidate(prepared_line): return False + # check for years and markers first: gibberish detection is much slower + lowered_line = prepared_line.lower() + if not ( + copyrights_hint.years(prepared_line) + or any(marker in lowered_line for marker in copyrights_hint.statement_markers) + ): + return False + if gibberish_detector.detect_gibberish(prepared_line): if TRACE: logger_debug(f'is_candidate: gibberish_detector.detect_gibberish:\n{prepared_line!r}') return False - if copyrights_hint.years(prepared_line): - return True - else: - pass - prepared_line = prepared_line.lower() - for marker in copyrights_hint.statement_markers: - if marker in prepared_line: - return True + return True def is_inside_statement( diff --git a/src/textcode/gibberish.py b/src/textcode/gibberish.py index 0af96532c15..c966cac172c 100644 --- a/src/textcode/gibberish.py +++ b/src/textcode/gibberish.py @@ -103,7 +103,7 @@ def train(self, bigfile=big_file_path, goodfile=good_file_path, self.persist_model() def detect_gibberish(self, text): - text = ''.join(self.normalize(text)) + # avg_transition_prob() already normalizes text through ngram() return self.avg_transition_prob(text, self.mat) < self.thresh def percent_gibberish(self, text): diff --git a/src/textcode/markup.py b/src/textcode/markup.py index 6d28ccb3c94..8eb9d0e4e75 100644 --- a/src/textcode/markup.py +++ b/src/textcode/markup.py @@ -305,9 +305,27 @@ def strip_known_markup_from_text(text): """ Str_ip markup tags from ``text`` using a list of tags """ + if not may_have_known_markup(text): + # nothing in text could be dropped by keep_tag: only normalize spaces + return " ".join(text.split()) return get_demarkuped_text(text, splitter=split_on_tags, keeper=keep_tag) +def may_have_known_markup(text): + """ + Return True if ``text`` may contain a token that ``keep_tag`` would drop. + Return False only when splitting ``text`` with ``split_on_tags`` cannot + yield any such token: there is no "<" or ">", no skipped attribute and + no tag without angle brackets. + """ + if "<" in text or ">" in text: + return True + if any(attribute in text for attribute in SKIP_ATTRIBUTES): + return True + tlow = text.lower() + return any(tag in tlow for tag in BARE_TAGS) + + ALL_TAGS = frozenset( [ "" +BARE_TAGS = tuple(sorted(t for t in ALL_TAGS if "<" not in t and ">" not in t)) + SKIP_ATTRIBUTES = ( "href=", diff --git a/tests/cluecode/test_copyrights_basic.py b/tests/cluecode/test_copyrights_basic.py index 1fbafcf5488..dd2fe436296 100644 --- a/tests/cluecode/test_copyrights_basic.py +++ b/tests/cluecode/test_copyrights_basic.py @@ -249,6 +249,16 @@ def test_is_candidate_should_not_select_line_with_junk_hex(self): line = prepare_text_line(line) assert not copyrights.is_candidate(line) + def test_is_candidate_should_not_select_gibberish_line_with_year_and_marker(self): + line = 'Copyright 2012 kjhgqwkzxqjvbn qwxzjkvq' + line = prepare_text_line(line) + assert not copyrights.is_candidate(line) + + def test_is_candidate_should_not_select_line_without_year_or_marker(self): + line = 'static void printBlockName(raw_ostream &OS, const BasicBlock *BB);' + line = prepare_text_line(line) + assert not copyrights.is_candidate(line) + def test_is_candidate_should_select_line_with_iso_date_year(self): line = 'sig 3 ccd6f801 2006-11-15 nathan mittler ' line = prepare_text_line(line) diff --git a/tests/textcode/test_markup.py b/tests/textcode/test_markup.py index 28c512c01ec..a25a3cc00d8 100644 --- a/tests/textcode/test_markup.py +++ b/tests/textcode/test_markup.py @@ -287,6 +287,41 @@ def test_strip_markup_text_tagset(test_text, expected_demarkup): assert markup.strip_known_markup_from_text(test_text) == expected_demarkup +no_markup_tests = [ + "Copyright (c) 2003-2024 The LLVM project.", + " some text\twith spaces ", + "Contact MAILTO:foo@example.com for details", + "", +] + +maybe_markup_tests = [ + "Copyright 2024 Foo", + "a > b", + "@end copying", + "see mailto:foo@example.com", + 'xmlns="http://www.w3.org/1999/xhtml" by Foo', + "HTTP://WWW.W3.ORG/2000/SVG Copyright Foo", +] + + +@pytest.mark.parametrize("test_text", no_markup_tests) +def test_may_have_known_markup_is_false_for_plain_text(test_text): + assert not markup.may_have_known_markup(test_text) + + +@pytest.mark.parametrize("test_text", maybe_markup_tests) +def test_may_have_known_markup_is_true_for_markup_or_skipped_tokens(test_text): + assert markup.may_have_known_markup(test_text) + + +@pytest.mark.parametrize("test_text", no_markup_tests + maybe_markup_tests) +def test_strip_known_markup_from_text_fast_path_matches_full_split(test_text): + expected = markup.get_demarkuped_text( + test_text, splitter=markup.split_on_tags, keeper=markup.keep_tag + ) + assert markup.strip_known_markup_from_text(test_text) == expected + + split_tests = [ ( """Apache-2.0""",