diff --git a/AUTHORS.rst b/AUTHORS.rst index 9a8224dc7da..4b8f48fcb1e 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -36,6 +36,7 @@ The following organizations or individuals have contributed to ScanCode: - Hanif Ali @hanif-ali - Horie Issei @is2ei - James Ward @jamesward +- Jatin Pandey @satiricalguru - Jelmer Vernooij @jelmer - Jens Keim @pepper-jk - Kunal Chhabra @kunalchhabra37 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index abc3ae41ad2..bc0ec8ca2af 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,9 @@ Changelog Next release -------------- +- Preserve Unicode characters in extracted URLs and remove surrounding typographic quotation marks. + https://github.com/aboutcode-org/scancode-toolkit/issues/4475 + - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 diff --git a/src/cluecode/finder.py b/src/cluecode/finder.py index 4364b494551..f5aae92712a 100644 --- a/src/cluecode/finder.py +++ b/src/cluecode/finder.py @@ -47,13 +47,14 @@ def logger_debug(*args): """ -def find(location, patterns): +def find(location, patterns, *, ascii_matches=True): """ Yield match and matched lines for patterns found in file at location as a tuple of (key, found text, text line). `patterns` is a list of tuples (key, compiled regex). Note: the location can be a list of lines for testing convenience. + Set `ascii_matches` to False to keep the original matched characters. """ if TRACE: from pprint import pformat @@ -67,7 +68,7 @@ def find(location, patterns): if TRACE: logger_debug('find: yielding match: key=%(key)r, ' 'match=%(match)r,\n line=%(line)r' % locals()) - yield key, toascii(match), line, line_number + yield key, toascii(match) if ascii_matches else match, line, line_number def unique_filter(matches): @@ -233,7 +234,7 @@ def find_urls(location, unique=True): `location` can be a list of strings for testing. """ patterns = [('urls', urls_regex(),)] - matches = find(location, patterns) + matches = find(location, patterns, ascii_matches=False) if TRACE: matches = list(matches) for m in matches: @@ -291,6 +292,27 @@ def verbatim_crlf_url_cleaner(matches): yield key, url, line, line_number +URL_TEXT_TRANSLATION = str.maketrans({ + '“': '"', '”': '"', '«': '"', '»': '"', + '‘': "'", '’': "'", + '\u200b': None, # zero-width space inserted by formatted text + '\u2060': None, # word joiner inserted by formatted text +}) + +URL_TRAILING_PUNCTUATION = string.punctuation + '¬†、。！？' + +# A bare origin followed by CJK text could be a real Unicode path. Recover +# bare origins only when a second explicit link and sentence punctuation +# make it clear that the match has swallowed a pair of links in prose. +HTTP_ORIGIN = r'https?://[a-z0-9.-]+(?::[0-9]+)?/' +CJK_TEXT_CHAR = r'[\u3040-\u30ff\u3400-\u9fff]' +LINKS_IN_CJK_PROSE = re.compile( + rf'({HTTP_ORIGIN}){CJK_TEXT_CHAR}+、{CJK_TEXT_CHAR}+' + rf'({HTTP_ORIGIN}){CJK_TEXT_CHAR}+。', + re.IGNORECASE, +) + + def end_of_url_cleaner(matches): """ Given an iterable of URL matches, return an iterable where junk characters @@ -298,11 +320,18 @@ def end_of_url_cleaner(matches): This is not entirely correct, but works practically. """ for key, url, line, line_number in matches: + url = url.translate(URL_TEXT_TRANSLATION) + joined_links = LINKS_IN_CJK_PROSE.fullmatch(url) + if joined_links: + for origin in joined_links.groups(): + yield key, origin, line, line_number + continue + if not url.endswith('/'): url = url.replace(u'<', u'<') url = url.replace(u'>', u'>') url = url.replace(u'&', u'&') - url = url.rstrip(string.punctuation) + url = url.rstrip(URL_TRAILING_PUNCTUATION) url = url.split(u'\\')[0] url = url.split(u'<')[0] url = url.split(u'>')[0] diff --git a/src/licensedcode/data/licenses/cc-by-sa-2.1-jp.LICENSE b/src/licensedcode/data/licenses/cc-by-sa-2.1-jp.LICENSE index 682c92efd11..0fe5947017c 100644 --- a/src/licensedcode/data/licenses/cc-by-sa-2.1-jp.LICENSE +++ b/src/licensedcode/data/licenses/cc-by-sa-2.1-jp.LICENSE @@ -9,7 +9,8 @@ spdx_license_key: CC-BY-SA-2.1-JP other_urls: - https://creativecommons.org/licenses/by-sa/2.1/jp/legalcode ignorable_urls: - - https://creativecommons.org/http:/www.creativecommons.jp/ + - http://www.creativecommons.jp/ + - https://creativecommons.org/ --- アトリビューション—シェアアライク 2.1 diff --git a/src/licensedcode/data/rules/cc-by-nc-sa-4.0_zh.RULE b/src/licensedcode/data/rules/cc-by-nc-sa-4.0_zh.RULE index 9d89b26ebf3..8658f37d49c 100644 --- a/src/licensedcode/data/rules/cc-by-nc-sa-4.0_zh.RULE +++ b/src/licensedcode/data/rules/cc-by-nc-sa-4.0_zh.RULE @@ -2,7 +2,7 @@ license_expression: cc-by-nc-sa-4.0 is_license_notice: yes ignorable_urls: - - http://creativecommons.org/licenses/by-nc-sa/4.0 + - http://creativecommons.org/licenses/by-nc-sa/4.0/ - https://github.com/kesenhoo/android-training-course-in-chinese --- diff --git a/src/licensedcode/data/rules/gpl-2.0-plus_617.RULE b/src/licensedcode/data/rules/gpl-2.0-plus_617.RULE index 1b284dd19eb..dfa41266d3a 100644 --- a/src/licensedcode/data/rules/gpl-2.0-plus_617.RULE +++ b/src/licensedcode/data/rules/gpl-2.0-plus_617.RULE @@ -3,7 +3,7 @@ license_expression: gpl-2.0-plus is_license_notice: yes relevance: 100 ignorable_urls: - - https://spdx.org/licenses/GPL2.0orlater.html + - https://spdx.org/licenses/GPL%E2%80%912.0%E2%80%91or%E2%80%91later.html --- This code is subject to the {{GNU General Public License}}. diff --git a/tests/cluecode/test_finder.py b/tests/cluecode/test_finder.py index 8625bba4fac..4a3ded38714 100644 --- a/tests/cluecode/test_finder.py +++ b/tests/cluecode/test_finder.py @@ -751,6 +751,51 @@ def test_find_urls_does_not_crash_on_mojibake_bytes(self): result = find_urls_tester(lines) assert result == expected + def test_find_urls_with_special_unicode_characters(self): + # Reproduces issue #4475 + lines = [ + 'https://sk.wikipedia.org/wiki/Administratívne_členenie_Slovenska', + 'https://ru.wikipedia.org/wiki/Федеральные_округа_Российской_Федерации', + ] + expected = [ + 'https://sk.wikipedia.org/wiki/Administrat%C3%ADvne_%C4%8Dlenenie_Slovenska', + 'https://ru.wikipedia.org/wiki/%D0%A4%D0%B5%D0%B4%D0%B5%D1%80%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5_%D0%BE%D0%BA%D1%80%D1%83%D0%B3%D0%B0_%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D0%B9%D1%81%D0%BA%D0%BE%D0%B9_%D0%A4%D0%B5%D0%B4%D0%B5%D1%80%D0%B0%D1%86%D0%B8%D0%B8', + ] + result = find_urls_tester(lines) + assert result == expected + + def test_find_urls_with_unicode_and_typographic_quotes(self): + lines = [ + '“https://fr.wikipedia.org/wiki/café”.', + '«https://ru.wikipedia.org/wiki/путь»', + ] + assert find_urls_tester(lines) == [ + 'https://fr.wikipedia.org/wiki/caf%C3%A9', + 'https://ru.wikipedia.org/wiki/%D0%BF%D1%83%D1%82%D1%8C', + ] + + def test_find_urls_in_unspaced_japanese_prose(self): + lines = [ + '参考は https://creativecommons.org/から、' + '連絡先はhttp://www.creativecommons.jp/から確認できます。', + ] + assert find_urls_tester(lines) == [ + 'https://creativecommons.org/', + 'http://www.creativecommons.jp/', + ] + + def test_find_urls_strips_trailing_unicode_punctuation(self): + lines = [ + 'http://www.mozilla.org/MPL/.¬† Software distributed under the license.', + 'https://ja.wikipedia.org/wiki/東京・日本。', + 'https://ja.wikipedia.org/からくり', + ] + assert find_urls_tester(lines) == [ + 'http://www.mozilla.org/MPL', + 'https://ja.wikipedia.org/wiki/%E6%9D%B1%E4%BA%AC%E3%83%BB%E6%97%A5%E6%9C%AC', + 'https://ja.wikipedia.org/%E3%81%8B%E3%82%89%E3%81%8F%E3%82%8A', + ] + def test_find_in_go_does_not_crash_with_unicode_error(self): test_file = self.get_test_loc('finder/url/verify.go') patterns = [('urls', urls_regex(),)]