diff --git a/CHANGELOG.rst b/CHANGELOG.rst index abc3ae41ad2..030bd081168 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -26,6 +26,11 @@ Next release ``ValueError: did not find any 'pre_scan' hooks``. https://github.com/aboutcode-org/scancode-toolkit/issues/5363 +- Speed up copyright detection: skip markup stripping on lines without markup, + run gibberish detection only on lines with a year or a copyright marker and + stop normalizing lines twice in gibberish detection. Results are unchanged. + https://github.com/aboutcode-org/scancode-toolkit/issues/5374 + v33.0.0rc1 - 2026-05-14 ------------------------ diff --git a/src/cluecode/copyrights.py b/src/cluecode/copyrights.py index 29876084168..8b6c1b62b96 100644 --- a/src/cluecode/copyrights.py +++ b/src/cluecode/copyrights.py @@ -4343,19 +4343,20 @@ def is_candidate(prepared_line): return False + # check for years and markers first: gibberish detection is much slower + lowered_line = prepared_line.lower() + if not ( + copyrights_hint.years(prepared_line) + or any(marker in lowered_line for marker in copyrights_hint.statement_markers) + ): + return False + if gibberish_detector.detect_gibberish(prepared_line): if TRACE: logger_debug(f'is_candidate: gibberish_detector.detect_gibberish:\n{prepared_line!r}') return False - if copyrights_hint.years(prepared_line): - return True - else: - pass - prepared_line = prepared_line.lower() - for marker in copyrights_hint.statement_markers: - if marker in prepared_line: - return True + return True def is_inside_statement( diff --git a/src/textcode/gibberish.py b/src/textcode/gibberish.py index 0af96532c15..c966cac172c 100644 --- a/src/textcode/gibberish.py +++ b/src/textcode/gibberish.py @@ -103,7 +103,7 @@ def train(self, bigfile=big_file_path, goodfile=good_file_path, self.persist_model() def detect_gibberish(self, text): - text = ''.join(self.normalize(text)) + # avg_transition_prob() already normalizes text through ngram() return self.avg_transition_prob(text, self.mat) < self.thresh def percent_gibberish(self, text): diff --git a/src/textcode/markup.py b/src/textcode/markup.py index 6d28ccb3c94..8eb9d0e4e75 100644 --- a/src/textcode/markup.py +++ b/src/textcode/markup.py @@ -305,9 +305,27 @@ def strip_known_markup_from_text(text): """ Str_ip markup tags from ``text`` using a list of tags """ + if not may_have_known_markup(text): + # nothing in text could be dropped by keep_tag: only normalize spaces + return " ".join(text.split()) return get_demarkuped_text(text, splitter=split_on_tags, keeper=keep_tag) +def may_have_known_markup(text): + """ + Return True if ``text`` may contain a token that ``keep_tag`` would drop. + Return False only when splitting ``text`` with ``split_on_tags`` cannot + yield any such token: there is no "<" or ">", no skipped attribute and + no tag without angle brackets. + """ + if "<" in text or ">" in text: + return True + if any(attribute in text for attribute in SKIP_ATTRIBUTES): + return True + tlow = text.lower() + return any(tag in tlow for tag in BARE_TAGS) + + ALL_TAGS = frozenset( [ "" +BARE_TAGS = tuple(sorted(t for t in ALL_TAGS if "<" not in t and ">" not in t)) + SKIP_ATTRIBUTES = ( "href=", diff --git a/tests/cluecode/test_copyrights_basic.py b/tests/cluecode/test_copyrights_basic.py index 1fbafcf5488..dd2fe436296 100644 --- a/tests/cluecode/test_copyrights_basic.py +++ b/tests/cluecode/test_copyrights_basic.py @@ -249,6 +249,16 @@ def test_is_candidate_should_not_select_line_with_junk_hex(self): line = prepare_text_line(line) assert not copyrights.is_candidate(line) + def test_is_candidate_should_not_select_gibberish_line_with_year_and_marker(self): + line = 'Copyright 2012 kjhgqwkzxqjvbn qwxzjkvq' + line = prepare_text_line(line) + assert not copyrights.is_candidate(line) + + def test_is_candidate_should_not_select_line_without_year_or_marker(self): + line = 'static void printBlockName(raw_ostream &OS, const BasicBlock *BB);' + line = prepare_text_line(line) + assert not copyrights.is_candidate(line) + def test_is_candidate_should_select_line_with_iso_date_year(self): line = 'sig 3 ccd6f801 2006-11-15 nathan mittler ' line = prepare_text_line(line) diff --git a/tests/textcode/test_markup.py b/tests/textcode/test_markup.py index 28c512c01ec..a25a3cc00d8 100644 --- a/tests/textcode/test_markup.py +++ b/tests/textcode/test_markup.py @@ -287,6 +287,41 @@ def test_strip_markup_text_tagset(test_text, expected_demarkup): assert markup.strip_known_markup_from_text(test_text) == expected_demarkup +no_markup_tests = [ + "Copyright (c) 2003-2024 The LLVM project.", + " some text\twith spaces ", + "Contact MAILTO:foo@example.com for details", + "", +] + +maybe_markup_tests = [ + "Copyright 2024 Foo", + "a > b", + "@end copying", + "see mailto:foo@example.com", + 'xmlns="http://www.w3.org/1999/xhtml" by Foo', + "HTTP://WWW.W3.ORG/2000/SVG Copyright Foo", +] + + +@pytest.mark.parametrize("test_text", no_markup_tests) +def test_may_have_known_markup_is_false_for_plain_text(test_text): + assert not markup.may_have_known_markup(test_text) + + +@pytest.mark.parametrize("test_text", maybe_markup_tests) +def test_may_have_known_markup_is_true_for_markup_or_skipped_tokens(test_text): + assert markup.may_have_known_markup(test_text) + + +@pytest.mark.parametrize("test_text", no_markup_tests + maybe_markup_tests) +def test_strip_known_markup_from_text_fast_path_matches_full_split(test_text): + expected = markup.get_demarkuped_text( + test_text, splitter=markup.split_on_tags, keeper=markup.keep_tag + ) + assert markup.strip_known_markup_from_text(test_text) == expected + + split_tests = [ ( """Apache-2.0""",