Preserve Unicode URL characters and clean typographic quotes

AUTHORS.rst

@@ -36,6 +36,7 @@ The following organizations or individuals have contributed to ScanCode:
 - Hanif Ali @hanif-ali
 - Horie Issei @is2ei
 - James Ward @jamesward
+- Jatin Pandey @satiricalguru
 - Jelmer Vernooij @jelmer
 - Jens Keim @pepper-jk
 - Kunal Chhabra @kunalchhabra37

CHANGELOG.rst

@@ -4,6 +4,9 @@ Changelog
 Next release
 --------------
 
+- Preserve Unicode characters in extracted URLs and remove surrounding typographic quotation marks.
+  https://github.com/aboutcode-org/scancode-toolkit/issues/4475
+
 - Fix the optional ``licenses`` extra dependency typo to install
   ``licensedcode-data``.
   https://github.com/aboutcode-org/scancode-toolkit/pull/5056

src/cluecode/finder.py

@@ -47,13 +47,14 @@ def logger_debug(*args):
 """
 
 
-def find(location, patterns):
+def find(location, patterns, *, ascii_matches=True):
     """
     Yield match and matched lines for patterns found in file at location as a
     tuple of (key, found text, text line). `patterns` is a list of tuples (key,
     compiled regex).
 
     Note: the location can be a list of lines for testing convenience.
+    Set `ascii_matches` to False to keep the original matched characters.
     """
     if TRACE:
         from pprint import pformat
@@ -67,7 +68,7 @@ def find(location, patterns):
                 if TRACE:
                     logger_debug('find: yielding match: key=%(key)r, '
                           'match=%(match)r,\n    line=%(line)r' % locals())
-                yield key, toascii(match), line, line_number
+                yield key, toascii(match) if ascii_matches else match, line, line_number
 
 
 def unique_filter(matches):
@@ -233,7 +234,7 @@ def find_urls(location, unique=True):
     `location` can be a list of strings for testing.
     """
     patterns = [('urls', urls_regex(),)]
-    matches = find(location, patterns)
+    matches = find(location, patterns, ascii_matches=False)
     if TRACE:
         matches = list(matches)
         for m in matches:
@@ -291,18 +292,46 @@ def verbatim_crlf_url_cleaner(matches):
         yield key, url, line, line_number
 
 
+URL_TEXT_TRANSLATION = str.maketrans({
+    '“': '"', '”': '"', '«': '"', '»': '"',
+    '‘': "'", '’': "'",
+    '\u200b': None,  # zero-width space inserted by formatted text
+    '\u2060': None,  # word joiner inserted by formatted text
+})
+
+URL_TRAILING_PUNCTUATION = string.punctuation + '¬†、。！？'
+
+# A bare origin followed by CJK text could be a real Unicode path. Recover
+# bare origins only when a second explicit link and sentence punctuation
+# make it clear that the match has swallowed a pair of links in prose.
+HTTP_ORIGIN = r'https?://[a-z0-9.-]+(?::[0-9]+)?/'
+CJK_TEXT_CHAR = r'[\u3040-\u30ff\u3400-\u9fff]'
+LINKS_IN_CJK_PROSE = re.compile(
+    rf'({HTTP_ORIGIN}){CJK_TEXT_CHAR}+、{CJK_TEXT_CHAR}+'
+    rf'({HTTP_ORIGIN}){CJK_TEXT_CHAR}+。',
+    re.IGNORECASE,
+)
+
+
 def end_of_url_cleaner(matches):
     """
     Given an iterable of URL matches, return an iterable where junk characters
     commonly found at the end of a URL are removed.
     This is not entirely correct, but works practically.
     """
     for key, url, line, line_number in matches:
+        url = url.translate(URL_TEXT_TRANSLATION)
+        joined_links = LINKS_IN_CJK_PROSE.fullmatch(url)
+        if joined_links:
+            for origin in joined_links.groups():
+                yield key, origin, line, line_number
+            continue
+
         if not url.endswith('/'):
             url = url.replace(u'&lt;', u'<')
             url = url.replace(u'&gt;', u'>')
             url = url.replace(u'&amp;', u'&')
-            url = url.rstrip(string.punctuation)
+            url = url.rstrip(URL_TRAILING_PUNCTUATION)
             url = url.split(u'\\')[0]
             url = url.split(u'<')[0]
             url = url.split(u'>')[0]

src/licensedcode/data/licenses/cc-by-sa-2.1-jp.LICENSE

@@ -9,7 +9,8 @@ spdx_license_key: CC-BY-SA-2.1-JP
 other_urls:
     - https://creativecommons.org/licenses/by-sa/2.1/jp/legalcode
 ignorable_urls:
-    - https://creativecommons.org/http:/www.creativecommons.jp/
+    - http://www.creativecommons.jp/
+    - https://creativecommons.org/
 ---
 
 アトリビューション—シェアアライク 2.1

src/licensedcode/data/rules/cc-by-nc-sa-4.0_zh.RULE

@@ -2,7 +2,7 @@
 license_expression: cc-by-nc-sa-4.0
 is_license_notice: yes
 ignorable_urls:
-    - http://creativecommons.org/licenses/by-nc-sa/4.0
+    - http://creativecommons.org/licenses/by-nc-sa/4.0/
     - https://github.com/kesenhoo/android-training-course-in-chinese
 ---
 

src/licensedcode/data/rules/gpl-2.0-plus_617.RULE

@@ -3,7 +3,7 @@ license_expression: gpl-2.0-plus
 is_license_notice: yes
 relevance: 100
 ignorable_urls:
-    - https://spdx.org/licenses/GPL2.0orlater.html
+    - https://spdx.org/licenses/GPL%E2%80%912.0%E2%80%91or%E2%80%91later.html
 ---
 
 This code is subject to the {{GNU General Public License}}.

tests/cluecode/test_finder.py

@@ -751,6 +751,51 @@ def test_find_urls_does_not_crash_on_mojibake_bytes(self):
         result = find_urls_tester(lines)
         assert result == expected
 
+    def test_find_urls_with_special_unicode_characters(self):
+        # Reproduces issue #4475
+        lines = [
+            'https://sk.wikipedia.org/wiki/Administratívne_členenie_Slovenska',
+            'https://ru.wikipedia.org/wiki/Федеральные_округа_Российской_Федерации',
+        ]
+        expected = [
+            'https://sk.wikipedia.org/wiki/Administrat%C3%ADvne_%C4%8Dlenenie_Slovenska',
+            'https://ru.wikipedia.org/wiki/%D0%A4%D0%B5%D0%B4%D0%B5%D1%80%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5_%D0%BE%D0%BA%D1%80%D1%83%D0%B3%D0%B0_%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D0%B9%D1%81%D0%BA%D0%BE%D0%B9_%D0%A4%D0%B5%D0%B4%D0%B5%D1%80%D0%B0%D1%86%D0%B8%D0%B8',
+        ]
+        result = find_urls_tester(lines)
+        assert result == expected
+
+    def test_find_urls_with_unicode_and_typographic_quotes(self):
+        lines = [
+            '“https://fr.wikipedia.org/wiki/café”.',
+            '«https://ru.wikipedia.org/wiki/путь»',
+        ]
+        assert find_urls_tester(lines) == [
+            'https://fr.wikipedia.org/wiki/caf%C3%A9',
+            'https://ru.wikipedia.org/wiki/%D0%BF%D1%83%D1%82%D1%8C',
+        ]
+
+    def test_find_urls_in_unspaced_japanese_prose(self):
+        lines = [
+            '参考は https://creativecommons.org/から、'
+            '連絡先はhttp://www.creativecommons.jp/から確認できます。',
+        ]
+        assert find_urls_tester(lines) == [
+            'https://creativecommons.org/',
+            'http://www.creativecommons.jp/',
+        ]
+
+    def test_find_urls_strips_trailing_unicode_punctuation(self):
+        lines = [
+            'http://www.mozilla.org/MPL/.  Software distributed under the license.',
+            'https://ja.wikipedia.org/wiki/東京・日本。',
+            'https://ja.wikipedia.org/からくり',
+        ]
+        assert find_urls_tester(lines) == [
+            'http://www.mozilla.org/MPL',
+            'https://ja.wikipedia.org/wiki/%E6%9D%B1%E4%BA%AC%E3%83%BB%E6%97%A5%E6%9C%AC',
+            'https://ja.wikipedia.org/%E3%81%8B%E3%82%89%E3%81%8F%E3%82%8A',
+        ]
+
     def test_find_in_go_does_not_crash_with_unicode_error(self):
         test_file = self.get_test_loc('finder/url/verify.go')
         patterns = [('urls', urls_regex(),)]