diff --git a/AUTHORS.rst b/AUTHORS.rst index 9a8224dc7d..d84a325506 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -27,6 +27,7 @@ The following organizations or individuals have contributed to ScanCode: - Daniel Eder @daniel-eder - Dan Kegel @dankegel - Dennis Clark @DennisClark +- Diana Galeana @diana-galeana - Divyansh Sharma @Divyansh2512 - Duncan Howe @Duncan-Howe - Felix Kauselmann @selmf @@ -61,6 +62,7 @@ The following organizations or individuals have contributed to ScanCode: - Nisha Kumar @nishakm - Nishchith Shetty @inishchith - Nitish Sharma @nitish81299 +- Octavio Valdivia Mendoza @OctavioValdiviaMendoza - Paul Gier @pgier - Philippe Ombredanne @pombredanne - Pi Delport @PiDelport diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d9a5a6b402..67ccd1e5f4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,9 @@ Next release ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 +- Add a package data handler for compiled Linux Kernel Module (``.ko``) + files that extracts metadata from the ELF ``.modinfo`` section. + v33.0.0rc1 - 2026-05-14 ------------------------ diff --git a/pyproject-scancode-toolkit.toml b/pyproject-scancode-toolkit.toml index 407d65b9c4..37f90b388e 100644 --- a/pyproject-scancode-toolkit.toml +++ b/pyproject-scancode-toolkit.toml @@ -88,6 +88,7 @@ dependencies = [ "plugincode >= 32.0.0", "publicsuffix2", "pyahocorasick >= 2.3.0", + "pyelftools >= 0.32", "pygmars >= 1.0.0", "pygments >= 1.0.0", "pymaven_patch >= 0.2.8", diff --git a/requirements.txt b/requirements.txt index e534652d52..05175e3b58 100644 --- a/requirements.txt +++ b/requirements.txt @@ -63,6 +63,7 @@ publicsuffix2==2.20191221 py==1.11.0 pyahocorasick==2.3.1 pycparser==2.23 +pyelftools==0.32 pygmars==1.0.0 Pygments==2.13.0 pymaven-patch==0.3.2 diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index fc1e490eef..0c8e75c6cd 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -26,6 +26,7 @@ from packagedcode import godeps from packagedcode import golang from packagedcode import haxe +from packagedcode import lkm from packagedcode import maven from packagedcode import misc from packagedcode import npm @@ -233,8 +234,10 @@ debian.DebianInstalledMd5sumFilelistHandler, debian.DebianInstalledStatusDatabaseHandler, + lkm.LinuxKernelModuleHandler, + rpm.RpmLicenseFilesHandler, - rpm.RpmMarinerContainerManifestHandler + rpm.RpmMarinerContainerManifestHandler, ] if on_linux: diff --git a/src/packagedcode/lkm.py b/src/packagedcode/lkm.py new file mode 100644 index 0000000000..6204e745ec --- /dev/null +++ b/src/packagedcode/lkm.py @@ -0,0 +1,208 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os + +from typing import Dict +from typing import Iterator +from typing import List +from typing import Optional + +from elftools.common.exceptions import ELFError +from elftools.elf.elffile import ELFFile +from packageurl import PackageURL + +from packagedcode.models import DatafileHandler +from packagedcode.models import DependentPackage +from packagedcode.models import PackageData +from packagedcode.models import Party + + +class LinuxKernelModuleHandler(DatafileHandler): + """ + Extract package metadata from compiled Linux Kernel Module ELF files. + """ + datasource_id = 'linux_kernel_module' + datasource_type = 'sys' + supported_oses = ('linux',) + default_package_type = 'linux-kernel-module' + path_patterns = ('*.ko',) + description = 'Linux Kernel Module' + documentation_url = 'https://docs.kernel.org/kbuild/modules.html' + + @classmethod + def parse(cls, location: str, package_only=False) -> Iterator[PackageData]: + metadata = cls.extract_modinfo(location) + + if not metadata: + return + + yield cls.build_package_data( + metadata=metadata, + location=location, + package_only=package_only, + ) + + + @staticmethod + def extract_modinfo(location: str) -> Dict[str, List[str]]: + """ + Return .modinfo metadata as a mapping of keys to lists of values. + + Multiple values are preserved because fields such as 'author', 'alias', + and 'firmware' may appear more than once. + """ + + metadata: Dict[str, List[str]] = {} + + try: + with open(location, 'rb') as module_file: + elf_file = ELFFile(module_file) + + # Restrict parsing to .modinfo instead of searching arbitrary binary data. + modinfo_section = elf_file.get_section_by_name('.modinfo') + if modinfo_section is None: + return {} + + # Read only the raw bytes stored in the ELF .modinfo section. + raw_bytes = modinfo_section.data() + + except (ELFError, OSError): + return {} + + # Entries in .modinfo are NUL-terminated key=value strings. + for raw_entry in raw_bytes.split(b'\x00'): + if not raw_entry: + continue + + entry = raw_entry.decode('utf-8', errors='replace') + + if '=' not in entry: + continue + + key, value = entry.split('=', 1) + + if not key: + continue + + metadata.setdefault(key, []).append(value) + + return metadata + + @staticmethod + def get_first(metadata: Dict[str, List[str]], key: str) -> Optional[str]: + values = metadata.get(key) or [] + return values[0] if values else None + + @staticmethod + def get_dependency_names( + metadata: Dict[str, List[str]] + ) -> List[str]: + dependencies = [] + + # The depends field lists modules required when this module is loaded. + for depends_entry in metadata.get('depends', []): + dependencies.extend( + dependency.strip() + for dependency in depends_entry.split(',') + if dependency.strip() + ) + + return dependencies + + @classmethod + def get_dependent_packages( + cls, + metadata: Dict[str, List[str]], + ) -> List[DependentPackage]: + dependency_names = cls.get_dependency_names(metadata) + + return [ + DependentPackage( + purl=PackageURL( + type=cls.default_package_type, + name=dependency_name, + ).to_string(), + extracted_requirement=None, + scope='runtime', + is_runtime=True, + is_optional=False, + ) + for dependency_name in dependency_names + ] + + @classmethod + def build_package_data( + cls, + metadata: Dict[str, List[str]], + location: str, + package_only: bool = False, + ) -> PackageData: + """ + Return PackageData built from raw .modinfo metadata. + """ + + # Represent every declared author as a package party. + parties = [] + for author in metadata.get('author', []): + author = author.strip() + + if author: + parties.append( + Party( + type='person', + role='author', + name=author, + ) + ) + + filename = os.path.basename(location) + name = filename[:-3] if filename.endswith('.ko') else filename + + normalized_keys = { + 'author', + 'description', + 'license', + 'version', + 'depends', + } + + #Preserve additional metadata that is not mapped to PackageData fields. + extra_data = { + key: values + for key, values in metadata.items() + if key not in normalized_keys + } + + # PackageData stores only the first value for these normalized fields. + # Preserve all values in extra_data when a field occurs more than once. + for key in ('description', 'license', 'version'): + values = metadata.get(key, []) + if len(values) > 1: + extra_data[key] = values + + dependencies = cls.get_dependent_packages(metadata) + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + name=name, + version=cls.get_first(metadata, 'version'), + description=cls.get_first(metadata, 'description'), + extracted_license_statement=cls.get_first(metadata, 'license'), + parties=parties, + dependencies=dependencies, + extra_data=extra_data, + ) + + return PackageData.from_data( + package_data=package_data, + package_only=package_only, + ) + \ No newline at end of file diff --git a/tests/packagedcode/test_lkm.py b/tests/packagedcode/test_lkm.py new file mode 100644 index 0000000000..f3f6db3d78 --- /dev/null +++ b/tests/packagedcode/test_lkm.py @@ -0,0 +1,262 @@ +import shutil +import os +from pathlib import Path +from xml.sax import handler + +import pytest + +from packagedcode import lkm +from packagedcode.models import DatafileHandler +from packagedcode.models import PackageData +from packagedcode.models import Party +from packagedcode.models import DependentPackage +from packagedcode import HANDLER_BY_DATASOURCE_ID + + +class TestLinuxKernelModule: + + test_data_dir = Path(__file__).resolve().parents[2] / "tests" / "licensedcode" / "data" / "query" + + def test_is_linux_kernel_module(self): + test_file = self.test_data_dir / "eeepc_acpi.ko" + + assert lkm.LinuxKernelModuleHandler.is_datafile( + str(test_file) + ) + + def test_extract_modinfo(self): + test_file = self.test_data_dir / "eeepc_acpi.ko" + + metadata = lkm.LinuxKernelModuleHandler.extract_modinfo( + str(test_file) + ) + + assert "license" in metadata + assert "description" in metadata + assert "author" in metadata + + def test_get_first(self): + metadata = { + "license": [ + "GPL", + "MIT", + ] + } + + result = lkm.LinuxKernelModuleHandler.get_first( + metadata, + "license", + ) + + assert result == "GPL" + + def test_get_dependency_names(self): + metadata = { + "depends": [ + "usbcore,cfg80211,mac80211" + ] + } + + result = lkm.LinuxKernelModuleHandler.get_dependency_names( + metadata + ) + + assert result == [ + "usbcore", + "cfg80211", + "mac80211", + ] + + def test_get_dependency_names_handles_multiple_entries(self): + metadata = { + "depends": [ + "usbcore,cfg80211", + "mac80211,netdev", + ] + } + + result = lkm.LinuxKernelModuleHandler.get_dependency_names( + metadata + ) + + assert result == [ + "usbcore", + "cfg80211", + "mac80211", + "netdev", + ] + + def test_get_first_missing_optional_value_returns_none(self): + metadata = { + "license": [ + "GPL", + ] + } + + result = lkm.LinuxKernelModuleHandler.get_first( + metadata, + "version", + ) + + assert result is None + + def test_parse_elf_without_modinfo_section_returns_empty(self, tmp_path): + if os.name == 'nt': + pytest.skip("Test currently depends on the Unix /bin/ls ELF binary'") + test_file = tmp_path / "no-modinfo.ko" + shutil.copyfile("/bin/ls", test_file) + + package = list( + lkm.LinuxKernelModuleHandler.parse( + str(test_file) + ) + ) + + assert package == [] + + def test_parse_invalid_lkm(self, tmp_path): + test_file = tmp_path / "fake.ko" + test_file.write_bytes(b"not an elf file") + + package = list( + lkm.LinuxKernelModuleHandler.parse( + str(test_file) + ) + ) + + assert package == [] + + def test_parse_linux_kernel_module(self): + test_file = self.test_data_dir / 'eeepc_acpi.ko' + + packages = list( + lkm.LinuxKernelModuleHandler.parse( + str(test_file) + ) + ) + + assert len(packages) == 1 + + package = packages[0] + + assert package.datasource_id == 'linux_kernel_module' + assert package.type == 'linux-kernel-module' + assert package.name == 'eeepc_acpi' + assert package.extracted_license_statement + assert package.description + assert package.parties + + def test_get_dependent_packages(self): + metadata = { + 'depends': [ + 'usbcore,cfg80211', + ] + } + + dependencies = ( + lkm.LinuxKernelModuleHandler.get_dependent_packages(metadata) + ) + + assert len(dependencies) == 2 + + assert dependencies[0].purl == ( + 'pkg:linux-kernel-module/usbcore' + ) + assert dependencies[0].scope == 'runtime' + assert dependencies[0].is_runtime is True + assert dependencies[0].is_optional is False + assert dependencies[0].extracted_requirement is None + + assert dependencies[1].purl == ( + 'pkg:linux-kernel-module/cfg80211' + ) + + def test_build_package_data_includes_dependencies(self): + metadata = { + 'license': ['GPL'], + 'depends': ['usbcore,cfg80211'], + } + + package_data = ( + lkm.LinuxKernelModuleHandler.build_package_data( + metadata=metadata, + location='/tmp/example.ko', + ) + ) + + assert package_data.name == 'example' + assert len(package_data.dependencies) == 2 + assert package_data.dependencies[0].purl == ( + 'pkg:linux-kernel-module/usbcore' + ) + assert package_data.dependencies[1].purl == ( + 'pkg:linux-kernel-module/cfg80211' + ) + + def test_build_package_data_preserves_unknown_metadata(self): + metadata = { + 'license': ['GPL'], + 'vermagic': ['6.8.0 SMP preempt mod_unload'], + 'srcversion': ['ABC123'], + 'firmware': ['example.bin'], + } + + package_data = ( + lkm.LinuxKernelModuleHandler.build_package_data( + metadata=metadata, + location='/tmp/example.ko', + ) + ) + + assert package_data.extra_data["vermagic"] == [ + "6.8.0 SMP preempt mod_unload" + ] + + assert package_data.extra_data["srcversion"] == [ + "ABC123" + ] + + assert package_data.extra_data["firmware"] == [ + "example.bin" + ] + + def test_build_package_data_handles_multiple_authors(self): + metadata = { + 'author': [ + 'Alice', + 'Bob', + ], + } + + package_data = ( + lkm.LinuxKernelModuleHandler.build_package_data( + metadata=metadata, + location='/tmp/example.ko', + ) + ) + + assert [party.name for party in package_data.parties] == [ + 'Alice', + 'Bob', + ] + + def test_datasource_id_is_registered(self): + handler = HANDLER_BY_DATASOURCE_ID[ + lkm.LinuxKernelModuleHandler.datasource_id + ] + + assert handler is lkm.LinuxKernelModuleHandler + + def test_build_package_data_multiple_alias_entries(self): + metadata = { + 'license': ['GPL'], + 'alias': ['usbcore', 'cfg80211'], + } + + package_data = lkm.LinuxKernelModuleHandler.build_package_data( + metadata=metadata, + location='/tmp/example.ko', + ) + + assert package_data.extra_data['alias'] == ['usbcore', 'cfg80211'] + assert 'depends' not in package_data.extra_data \ No newline at end of file