Skip to content

Add PURL_TYPES and enforce validation of PURL types #188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kdeldycke wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add PURL_TYPES and enforce validation of PURL types #188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,14 @@ Changelog
0.18.0 (unlreleased)
--------------------

- Add ``PURL_TYPES`` constant sourced from the ``purl-types-index.json`` registry.
Enforce validation of PackageURL types.
Remove ``sourceforge`` and ``gitlab`` support from ``url2purl`` and ``purl2url``.
Remove ``rubygems`` route alias from ``purl2url`` (the registered type is ``gem``).
https://github.com/package-url/packageurl-python/issues/181
https://github.com/package-url/packageurl-python/issues/155
https://github.com/package-url/packageurl-python/pull/188

- Add support for Python 3.14

- Drop support for Python 3.8
Expand Down
56 changes: 52 additions & 4 deletions src/packageurl/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,49 @@
"""


PURL_TYPES: set[str] = {
"alpm",
"apk",
"bitbucket",
"bitnami",
"cargo",
"cocoapods",
"composer",
"conan",
"conda",
"cpan",
"cran",
"deb",
"docker",
"gem",
"generic",
"github",
"golang",
"hackage",
"hex",
"huggingface",
"luarocks",
"maven",
"mlflow",
"npm",
"nuget",
"oci",
"pub",
"pypi",
"qpkg",
"rpm",
"swid",
"swift",
}
"""Registered pURL types.

Sourced from the `purl-types-index.json
<https://github.com/package-url/purl-spec/blob/main/purl-types-index.json>`_
registry, which is automatically regenerated whenever a new pURL type is
registered.
"""


class ValidationSeverity(str, Enum):
ERROR = "error"
WARNING = "warning"
Expand Down Expand Up @@ -124,8 +167,15 @@ def normalize_type(type: AnyStr | None, encode: bool | None = True) -> str | Non

type_str = type if isinstance(type, str) else type.decode("utf-8")
quoter = get_quoter(encode)
type_str = quoter(type_str)
return type_str.strip().lower() or None
type_str = quoter(type_str).strip().lower()
if not type_str:
return None
if type_str not in PURL_TYPES:
raise ValueError(
f"Invalid purl type: {type_str!r}. "
f"Must be one of: {', '.join(sorted(PURL_TYPES))}."
)
return type_str


def normalize_namespace(
Expand All @@ -140,7 +190,6 @@ def normalize_namespace(
"bitbucket",
"github",
"pypi",
"gitlab",
"composer",
"luarocks",
"qpkg",
Expand Down Expand Up @@ -195,7 +244,6 @@ def normalize_name(
"bitbucket",
"github",
"pypi",
"gitlab",
"composer",
"luarocks",
"oci",
Expand Down
21 changes: 3 additions & 18 deletions src/packageurl/contrib/purl2url.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ def get_repo_download_url_by_package_type(
download_url_by_type = {
"github": f"https://github.com/{namespace}/{name}/archive/{version}.{archive_extension}",
"bitbucket": f"https://bitbucket.org/{namespace}/{name}/get/{version}.{archive_extension}",
"gitlab": f"https://gitlab.com/{namespace}/{name}/-/archive/{version}/{name}-{version}.{archive_extension}",
}
return download_url_by_type.get(type)

Expand Down Expand Up @@ -158,21 +157,7 @@ def build_github_repo_url(purl):
return repo_url


@repo_router.route("pkg:gitlab/.*")
def build_gitlab_repo_url(purl):
"""
Return a gitlab repo URL from the `purl` string.
"""
purl_data = PackageURL.from_string(purl)

namespace = purl_data.namespace
name = purl_data.name

if name and namespace:
return f"https://gitlab.com/{namespace}/{name}"


@repo_router.route("pkg:(gem|rubygems)/.*")
@repo_router.route("pkg:gem/.*")
def build_rubygems_repo_url(purl):
"""
Return a rubygems repo URL from the `purl` string.
Expand Down Expand Up @@ -351,7 +336,7 @@ def build_cargo_download_url(purl):
return f"https://crates.io/api/v1/crates/{name}/{version}/download"


@download_router.route("pkg:(gem|rubygems)/.*")
@download_router.route("pkg:gem/.*")
def build_rubygems_download_url(purl):
"""
Return a rubygems download URL from the `purl` string.
Expand Down Expand Up @@ -435,7 +420,7 @@ def build_nuget_download_url(purl):
return f"https://www.nuget.org/api/v2/package/{name}/{version}"


@download_router.route("pkg:gitlab/.*", "pkg:bitbucket/.*", "pkg:github/.*")
@download_router.route("pkg:bitbucket/.*", "pkg:github/.*")
def build_repo_download_url(purl):
"""
Return a gitlab download URL from the `purl` string.
Expand Down
137 changes: 0 additions & 137 deletions src/packageurl/contrib/url2purl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,75 +372,6 @@ def build_composer_purl(uri):
register_pattern("nuget", nuget_api_pattern)


# https://sourceforge.net/projects/turbovnc/files/3.1/turbovnc-3.1.tar.gz/download
# https://sourceforge.net/projects/scribus/files/scribus/1.6.0/scribus-1.6.0.tar.gz/download
# https://sourceforge.net/projects/ventoy/files/v1.0.96/Ventoy%201.0.96%20release%20source%20code.tar.gz/download
# https://sourceforge.net/projects/geoserver/files/GeoServer/2.23.4/geoserver-2.23.4-war.zip/download
sourceforge_download_pattern = (
r"^https?://.*sourceforge.net/projects/"
r"(?P<name>.+)/"
r"files/"
r"(?i:(?P=name)/)?" # optional case-insensitive name segment repeated
r"v?(?P<version>[0-9\.]+)/" # version restricted to digits and dots
r"(?i:(?P=name)).*(?P=version).*" # case-insensitive matching for {name}-{version}
r"(/download)$" # ending with "/download"
)

register_pattern("sourceforge", sourceforge_download_pattern)


# https://sourceforge.net/projects/spacesniffer/files/spacesniffer_1_3_0_2.zip/download
sourceforge_download_pattern_bis = (
r"^https?://.*sourceforge.net/projects/"
r"(?P<name>.+)/"
r"files/"
r"(?i:(?P=name))_*(?P<version>[0-9_]+).*"
r"(/download)$" # ending with "/download"
)

register_pattern("sourceforge", sourceforge_download_pattern_bis)


@purl_router.route("https?://.*sourceforge.net/project/.*")
def build_sourceforge_purl(uri):
# We use a more general route pattern instead of using `sourceforge_pattern`
# below by itself because we want to capture all sourceforge download URLs,
# even the ones that do not fit `sourceforge_pattern`. This helps prevent
# url2purl from attempting to create a generic PackageURL from a sourceforge
# URL that we can't handle.

# http://master.dl.sourceforge.net/project/libpng/zlib/1.2.3/zlib-1.2.3.tar.bz2
sourceforge_pattern = (
r"^https?://.*sourceforge.net/projects?/"
r"(?P<namespace>([^/]+))/" # do not allow more "/" segments
r"(OldFiles/)?"
r"(?P<name>.+)/"
r"(?P<version>[v0-9\.]+)/" # version restricted to digits and dots
r"(?P=name).*(?P=version).*" # {name}-{version} repeated in the filename
r"[^/]$" # not ending with "/"
)

sourceforge_purl = purl_from_pattern("sourceforge", sourceforge_pattern, uri)

if not sourceforge_purl:
# Get the project name from `uri` and use that as the Package name
# http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip
split_uri = uri.split("/project/")

# http://master.dl.sourceforge.net, aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip
if len(split_uri) >= 2:
# aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip
remaining_uri_path = split_uri[1]
# aloyscore, aloyscore, 0.1a1%2520stable, 0.1a1_stable_AloysCore.zip
remaining_uri_path_segments = remaining_uri_path.split("/")
if remaining_uri_path_segments:
project_name = remaining_uri_path_segments[0] # aloyscore
sourceforge_purl = PackageURL(
type="sourceforge", name=project_name, qualifiers={"download_url": uri}
)
return sourceforge_purl


# https://crates.io/api/v1/crates/rand/0.7.2/download
cargo_pattern = r"^https?://crates.io/api/v1/crates/(?P<name>.+)/(?P<version>.+)(\/download)$"

Expand Down Expand Up @@ -667,74 +598,6 @@ def build_bitbucket_purl(url):
)


@purl_router.route("https?://gitlab\\.com/(?!.*/archive/).*")
def build_gitlab_purl(url):
"""
Return a PackageURL object from Gitlab `url`.
For example:
https://gitlab.com/TG1999/firebase/-/tree/1a122122/views
https://gitlab.com/TG1999/firebase/-/tree
https://gitlab.com/TG1999/firebase/-/master
https://gitlab.com/tg1999/Firebase/-/tree/master
https://gitlab.com/tg1999/Firebase/-/commit/bf04e5f289885cf2f20a92b387bcc6df33e30809
"""
# https://gitlab.com/<ns>/<name>/-/commit/<sha>
commit_pattern = (
r"https?://gitlab.com/"
r"(?P<namespace>[^/]+)/(?P<name>[^/]+)/-/commit/"
r"(?P<version>[0-9a-fA-F]{7,64})/?$"
)

commit_matche = re.search(commit_pattern, url)
if commit_matche:
return PackageURL(
type="gitlab",
namespace=commit_matche.group("namespace"),
name=commit_matche.group("name"),
version=commit_matche.group("version"),
qualifiers={},
subpath="",
)

segments = get_path_segments(url)

if not len(segments) >= 2:
return
namespace = segments[0]
name = segments[1]
version = None
subpath = None

# https://gitlab.com/TG1999/firebase/master
if (len(segments) >= 3) and segments[2] != "-" and segments[2] != "tree":
version = segments[2]
subpath = "/".join(segments[3:])

# https://gitlab.com/TG1999/firebase/-/tree/master
if len(segments) >= 5 and (segments[2] == "-" and segments[3] == "tree"):
version = segments[4]
subpath = "/".join(segments[5:])

return PackageURL(
type="gitlab",
namespace=namespace,
name=name,
version=version,
subpath=subpath,
)


# https://gitlab.com/hoppr/hoppr/-/archive/v1.11.1-dev.2/hoppr-v1.11.1-dev.2.tar.gz
gitlab_archive_pattern = (
r"^https?://gitlab.com/"
r"(?P<namespace>.+)/(?P<name>.+)/-/archive/(?P<version>.+)/"
r"(?P=name)-(?P=version).*"
r"[^/]$"
)

register_pattern("gitlab", gitlab_archive_pattern)


# https://hackage.haskell.org/package/cli-extras-0.2.0.0/cli-extras-0.2.0.0.tar.gz
hackage_download_pattern = (
r"^https?://hackage.haskell.org/package/"
Expand Down
37 changes: 0 additions & 37 deletions tests/contrib/data/url2purl.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,31 +155,6 @@
"https://www.nuget.org/api/v2/package/MvvmLightLibs/4.1.23": "pkg:nuget/MvvmLightLibs@4.1.23",
"https://www.nuget.org/api/v2/package/Twilio/3.4.1": "pkg:nuget/Twilio@3.4.1",
"https://api.nuget.org/v3-flatcontainer/newtonsoft.json/10.0.1/newtonsoft.json.10.0.1.nupkg": "pkg:nuget/newtonsoft.json@10.0.1",
"http://master.dl.sourceforge.net/project/zznotes/zznotes/1.1.2/zznotes-1.1.2.tar.gz": "pkg:sourceforge/zznotes/zznotes@1.1.2",
"http://master.dl.sourceforge.net/project/zapping/zvbi/0.2.35/zvbi-0.2.35.tar.bz2": "pkg:sourceforge/zapping/zvbi@0.2.35",
"http://master.dl.sourceforge.net/project/libpng/zlib/1.2.3/zlib-1.2.3.tar.bz2": "pkg:sourceforge/libpng/zlib@1.2.3",
"http://master.dl.sourceforge.net/project/xmlstar/xmlstarlet/1.0.0/xmlstarlet-1.0.0-1.src.rpm": "pkg:sourceforge/xmlstar/xmlstarlet@1.0.0",
"http://master.dl.sourceforge.net/project/wxmozilla/wxMozilla/0.5.5/wxMozilla-0.5.5.exe": "pkg:sourceforge/wxmozilla/wxMozilla@0.5.5",
"http://iweb.dl.sourceforge.net/project/sblim/sblim-cim-client2/2.2.5/sblim-cim-client2-2.2.5-src.zip": "pkg:sourceforge/sblim/sblim-cim-client2@2.2.5",
"http://master.dl.sourceforge.net/project/zinnia/zinnia-win32/0.06/zinnia-win32-0.06.zip": "pkg:sourceforge/zinnia/zinnia-win32@0.06",
"http://iweb.dl.sourceforge.net/project/findbugs/findbugs/1.3.4/findbugs-1.3.4.tar.gz/": "pkg:sourceforge/findbugs?download_url=http://iweb.dl.sourceforge.net/project/findbugs/findbugs/1.3.4/findbugs-1.3.4.tar.gz/",
"http://master.dl.sourceforge.net/project/arestc/net/sf/arestc/arestc/0.1.4/arestc-0.1.4-javadoc.jar": "pkg:sourceforge/arestc?download_url=http://master.dl.sourceforge.net/project/arestc/net/sf/arestc/arestc/0.1.4/arestc-0.1.4-javadoc.jar",
"http://master.dl.sourceforge.net/project/intraperson/OldFiles/intraperson/0.28/intraperson-0.28.tar.gz": "pkg:sourceforge/intraperson/intraperson@0.28",
"http://master.dl.sourceforge.net/project/pwiki/pwiki/0.1.2/0.1.2.zip": "pkg:sourceforge/pwiki?download_url=http://master.dl.sourceforge.net/project/pwiki/pwiki/0.1.2/0.1.2.zip",
"http://master.dl.sourceforge.net/project/iswraid/iswraid/0.1.4.3/2.4.28-pre3-iswraid.patch.gz": "pkg:sourceforge/iswraid?download_url=http://master.dl.sourceforge.net/project/iswraid/iswraid/0.1.4.3/2.4.28-pre3-iswraid.patch.gz",
"http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%20stable/0.1a1_stable_AloysCore.zip": "pkg:sourceforge/aloyscore?download_url=http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%2520stable/0.1a1_stable_AloysCore.zip",
"http://master.dl.sourceforge.net/project/myenterprise/OldFiles/1.0.0.2.MyEnterprise.Source.zip": "pkg:sourceforge/myenterprise?download_url=http://master.dl.sourceforge.net/project/myenterprise/OldFiles/1.0.0.2.MyEnterprise.Source.zip",
"http://master.dl.sourceforge.net/project/wxhaskell/wxhaskell/wxhaskell-0.9/wxhaskell-src-0.9.zip": "pkg:sourceforge/wxhaskell?download_url=http://master.dl.sourceforge.net/project/wxhaskell/wxhaskell/wxhaskell-0.9/wxhaskell-src-0.9.zip",
"http://master.dl.sourceforge.net/project/a2freedom/A2/1.2/a2freedom-1.2.zip": "pkg:sourceforge/a2freedom?download_url=http://master.dl.sourceforge.net/project/a2freedom/A2/1.2/a2freedom-1.2.zip",
"http://master.dl.sourceforge.net/project/tinyos/OldFiles/tinyos/1.1.0/tinyos-1.1.0.tar.gz": "pkg:sourceforge/tinyos/tinyos@1.1.0",
"http://master.dl.sourceforge.net/project/urlchecker/lu/ng/urlchecker/urlchecker/1.7/urlchecker-1.7-javadoc.jar": "pkg:sourceforge/urlchecker?download_url=http://master.dl.sourceforge.net/project/urlchecker/lu/ng/urlchecker/urlchecker/1.7/urlchecker-1.7-javadoc.jar",
"http://master.dl.sourceforge.net/project/zclasspath/maven2/org/zclasspath/zclasspath/1.5/zclasspath-1.5.jar": "pkg:sourceforge/zclasspath?download_url=http://master.dl.sourceforge.net/project/zclasspath/maven2/org/zclasspath/zclasspath/1.5/zclasspath-1.5.jar",
"http://master.dl.sourceforge.net/project/googleimagedown/project/v1.1/GoogleImageDownloader-v1.1-src.tar.bz2": "pkg:sourceforge/googleimagedown?download_url=http://master.dl.sourceforge.net/project/googleimagedown/project/v1.1/GoogleImageDownloader-v1.1-src.tar.bz2",
"https://sourceforge.net/projects/scribus/files/scribus/1.6.0/scribus-1.6.0.tar.gz/download": "pkg:sourceforge/scribus@1.6.0",
"https://sourceforge.net/projects/turbovnc/files/3.1/turbovnc-3.1.tar.gz/download": "pkg:sourceforge/turbovnc@3.1",
"https://sourceforge.net/projects/ventoy/files/v1.0.96/Ventoy%201.0.96%20release%20source%20code.tar.gz/download": "pkg:sourceforge/ventoy@1.0.96",
"https://sourceforge.net/projects/geoserver/files/GeoServer/2.23.4/geoserver-2.23.4-war.zip/download": "pkg:sourceforge/geoserver@2.23.4",
"https://sourceforge.net/projects/spacesniffer/files/spacesniffer_1_3_0_2.zip/download": "pkg:sourceforge/spacesniffer@1_3_0_2",
"https://crates.io/api/v1/crates/rand/0.7.2/download": "pkg:cargo/rand@0.7.2",
"https://crates.io/api/v1/crates/clap/2.33.0/download": "pkg:cargo/clap@2.33.0",
"https://crates.io/api/v1/crates/structopt/0.3.11/download": "pkg:cargo/structopt@0.3.11",
Expand Down Expand Up @@ -253,17 +228,6 @@
"https://bitbucket.org/multicoreware/x265/downloads/x265_2.6.tar.gz": "pkg:bitbucket/multicoreware/x265?download_url=https://bitbucket.org/multicoreware/x265/downloads/x265_2.6.tar.gz",
"https://bitbucket.org/robeden/trove/downloads/trove-3.0.3.zip": "pkg:bitbucket/robeden/trove?download_url=https://bitbucket.org/robeden/trove/downloads/trove-3.0.3.zip",
"https://bitbucket.org/efotinis/deskpins/downloads/DeskPins-1.31-setup.exe": "pkg:bitbucket/efotinis/deskpins?download_url=https://bitbucket.org/efotinis/deskpins/downloads/DeskPins-1.31-setup.exe",
"https://gitlab.com/TG1999/firebase/-/tree/1a122122/views": "pkg:gitlab/tg1999/firebase@1a122122#views",
"https://gitlab.com/tg1999/firebase": "pkg:gitlab/tg1999/firebase",
"https://gitlab.com/TG1999/firebase/-/": "pkg:gitlab/tg1999/firebase",
"https://gitlab.com/TG1999/firebase/-/tree": "pkg:gitlab/tg1999/firebase",
"https://gitlab.com/TG1999/firebase/-/master": "pkg:gitlab/tg1999/firebase",
"https://gitlab.com/TG1999/firebase/tree/": "pkg:gitlab/tg1999/firebase",
"https://gitlab.com/TG1999/firebase/master": "pkg:gitlab/tg1999/firebase@master",
"https://gitlab.com/TG1999/firebase/-/tree/master": "pkg:gitlab/tg1999/firebase@master",
"https://gitlab.com/tg1999/Firebase/-/tree/master": "pkg:gitlab/tg1999/firebase@master",
"https://gitlab.com/TG1999/FIREBASE": "pkg:gitlab/tg1999/firebase",
"https://gitlab.com/hoppr/hoppr/-/archive/v1.11.1-dev.2/hoppr-v1.11.1-dev.2.tar.gz": "pkg:gitlab/hoppr/hoppr@v1.11.1-dev.2",
"https://hackage.haskell.org/package/a50-0.5/a50-0.5.tar.gz": "pkg:hackage/a50@0.5",
"https://hackage.haskell.org/package/AC-HalfInteger-1.2.1/AC-HalfInteger-1.2.1.tar.gz": "pkg:hackage/AC-HalfInteger@1.2.1",
"https://hackage.haskell.org/package/3d-graphics-examples-0.0.0.2/3d-graphics-examples-0.0.0.2.tar.gz": "pkg:hackage/3d-graphics-examples@0.0.0.2",
Expand All @@ -276,6 +240,5 @@
"https://cran.r-project.org/src/contrib/jsonlite_1.8.8.tar.gz": "pkg:cran/jsonlite@1.8.8",
"https://packagemanager.rstudio.com/cran/2022-06-23/src/contrib/curl_4.3.2.tar.gz": "pkg:cran/curl@4.3.2?download_url=https://packagemanager.rstudio.com/cran/2022-06-23/src/contrib/curl_4.3.2.tar.gz",
"https://github.com/TG1999/first_repo/commit/98e516011d6e096e25247b82fc5f196bbeecff10": "pkg:github/tg1999/first_repo@98e516011d6e096e25247b82fc5f196bbeecff10",
"https://gitlab.com/TG1999/first_repo/-/commit/bf04e5f289885cf2f20a92b387bcc6df33e30809": "pkg:gitlab/tg1999/first_repo@bf04e5f289885cf2f20a92b387bcc6df33e30809",
"https://bitbucket.org/TG1999/first_repo/commits/16a60c4a74ef477cd8c16ca82442eaab2fbe8c86": "pkg:bitbucket/tg1999/first_repo@16a60c4a74ef477cd8c16ca82442eaab2fbe8c86"
}
Loading