#

# Copyright (c) nexB Inc. and others. All rights reserved.

# ScanCode is a trademark of nexB Inc.

# SPDX-License-Identifier: Apache-2.0

# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.

# See https://github.com/nexB/scancode-toolkit for support or download.

# See https://aboutcode.org for more information about nexB OSS projects.

#

import ast

import base64

import copy

import json

import logging

import os

import posixpath

import re

import sys

import tempfile

import zipfile

from configparser import ConfigParser

from fnmatch import fnmatchcase

from pathlib import Path

from typing import NamedTuple

import dparse2

# NOTE: we always want to use the external library rather than the built-in for now

import importlib_metadata

import packvers as packaging

import pip_requirements_parser

import pkginfo2

from commoncode import fileutils

from commoncode.fileutils import as_posixpath

from commoncode.resource import Resource

from packageurl import PackageURL

from packvers.specifiers import SpecifierSet

from packvers import markers

from packvers.requirements import Requirement

from packvers.utils import canonicalize_name

from packagedcode import models

from packagedcode.utils import build_description

from packagedcode.utils import parse_maintainer_name_email

from packagedcode.utils import yield_dependencies_from_package_data

from packagedcode.utils import yield_dependencies_from_package_resource

from packagedcode.utils import get_base_purl

# tomli was added to the stdlib as tomllib in Python 3.11.

# It's the same code.

# Still, prefer tomli if it's installed, as on newer Python versions, it is

# compiled with mypyc and is more performant.

try:

import tomli as tomllib

except ImportError:

import tomllib

try:

from zipfile import Path as ZipPath

except ImportError:

from zipp import Path as ZipPath

"""

Detect and collect Python packages information.

"""

# TODO: add support for poetry and setup.cfg and metadata.json

# TODO: add support for pex, pyz, etc.

# TODO: Add missing ABOUT file for Pyserial code

TRACE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False)

def logger_debug(*args):

pass

logger = logging.getLogger(__name__)

if TRACE:

logging.basicConfig(stream=sys.stdout)

logger.setLevel(logging.DEBUG)

def logger_debug(*args):

return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))

class PythonEggPkgInfoFile(models.DatafileHandler):

datasource_id = 'pypi_egg_pkginfo'

default_package_type = 'pypi'

default_primary_language = 'Python'

path_patterns = ('*/EGG-INFO/PKG-INFO',)

description = 'PyPI extracted egg PKG-INFO'

documentation_url = 'https://peps.python.org/pep-0376/'

@classmethod

def parse(cls, location, package_only=False):

"""

Parse package data from a PKG-INFO file and other manifests present in

neighboring files as needed when an installed layout is found.

"""

yield parse_metadata(

location=location,

datasource_id=cls.datasource_id,

package_type=cls.default_package_type,

package_only=package_only,

)

@classmethod

def assign_package_to_resources(cls, package, resource, codebase, package_adder):

# two levels up

root = resource.parent(codebase).parent(codebase)

if root:

return models.DatafileHandler.assign_package_to_resources(package, root, codebase, package_adder)

class PythonEditableInstallationPkgInfoFile(models.DatafileHandler):

datasource_id = 'pypi_editable_egg_pkginfo'

default_package_type = 'pypi'

default_primary_language = 'Python'

path_patterns = ('*.egg-info/PKG-INFO',)

description = 'PyPI editable local installation PKG-INFO'

documentation_url = 'https://peps.python.org/pep-0376/'

@classmethod

def parse(cls, location, package_only=False):

"""

Parse package data from a PKG-INFO file and other manifests present in

neighboring files as needed when an installed layout is found.

"""

yield parse_metadata(

location=location,

datasource_id=cls.datasource_id,

package_type=cls.default_package_type,

package_only=package_only,

)

@classmethod

def assign_package_to_resources(cls, package, resource, codebase, package_adder):

# only the parent for now... though it can be more complex

return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase, package_adder)

def create_package_from_package_data(package_data, datafile_path):

package = models.Package.from_package_data(

package_data=package_data,

datafile_path=datafile_path,

)

package.populate_license_fields()

return package

def is_egg_info_directory(resource):

"""

Return True if `resource` is a Python .egg-info directory

"""

return (

isinstance(resource, Resource)

and resource.path.endswith('.egg-info')

)

class BaseExtractedPythonLayout(models.DatafileHandler):

"""

Base class for development repos, sdist tarballs and other related extracted

layourt for Python packages that can use and mix multiple datafiles.

"""

@classmethod

def assemble(cls, package_data, resource, codebase, package_adder):

# a source distribution can have many manifests

datafile_name_patterns = (

PipfileHandler.path_patterns + PipfileLockHandler.path_patterns

+ PipRequirementsFileHandler.path_patterns + PyprojectTomlHandler.path_patterns

)

is_datafile_pypi = any(fnmatchcase(resource.path, pat) for pat in datafile_name_patterns)

package_resource = None

if resource.name == 'PKG-INFO':

# Initially use current Resource as `package_resource`.

# We'll want update `package_resource` with the Resource of a

# PKG-INFO file that's in an .egg-info Directory.

package_resource = resource

# We want to use the PKG-INFO file from an .egg-info directory, as

# the package info collected from a *.egg_info/PKG-INFO file has

# dependency information that a PKG-INFO from the root of a Python

# project lacks.

parent_resource = resource.parent(codebase)

if not is_egg_info_directory(parent_resource):

# If we are not in an .egg-info directory, we assume we are at

# the root of a Python codebase and we want to find the

# .egg_info dir

egg_info_dir = None

for sibling in resource.siblings(codebase):

if sibling.path.endswith('.egg-info'):

egg_info_dir = sibling

break

# If we find the .egg_info dir, then we look for the PKG-INFO

# file in it and use that as our package_resource

if egg_info_dir:

for child in egg_info_dir.children(codebase):

if not child.name == 'PKG-INFO':

continue

package_resource = child

break

elif is_datafile_pypi:

if resource.has_parent():

siblings = resource.siblings(codebase)

package_resources = [r for r in siblings if r.name == 'PKG-INFO']

if package_resource:

package_resource = package_resources[0]

package = None

if package_resource:

pkg_data = package_resource.package_data[0]

pkg_data = models.PackageData.from_dict(pkg_data)

if pkg_data.purl:

# We yield only the package and the resource, and not dependencies because

# PKG-INFO also has the dependencies from

package = create_package_from_package_data(

package_data=pkg_data,

datafile_path=package_resource.path

)

yield package

package_adder(package.package_uid, package_resource, codebase)

yield package_resource

else:

setup_resources = []

if resource.has_parent():

siblings = resource.siblings(codebase)

setup_resources = [

r for r in siblings

if r.name in ('setup.py', 'setup.cfg')

and r.package_data

]

if setup_resources:

setup_package_data = [

(setup_resource, models.PackageData.from_dict(setup_resource.package_data[0]))

for setup_resource in setup_resources

]

setup_package_data = sorted(setup_package_data, key=lambda s: bool(s[1].purl), reverse=True)

for setup_resource, setup_pkg_data in setup_package_data:

if setup_pkg_data.purl:

if not package:

package = create_package_from_package_data(

package_data=setup_pkg_data,

datafile_path=setup_resource.path,

)

yield package

package_resource = setup_resource

else:

package.update(setup_pkg_data, setup_resource.path)

if package:

for setup_resource, setup_pkg_data in setup_package_data:

package_adder(package.package_uid, setup_resource, codebase)

yield setup_resource

yield from yield_dependencies_from_package_data(

package_data=setup_pkg_data,

datafile_path=setup_resource.path,

package_uid=package.package_uid

)

else:

package_resource = resource

pkg_data = package_resource.package_data[0]

pkg_data = models.PackageData.from_dict(pkg_data)

if pkg_data.purl:

package = create_package_from_package_data(

package_data=pkg_data,

datafile_path=package_resource.path

)

yield package

package_adder(package.package_uid, package_resource, codebase)

yield package_resource

yield from yield_dependencies_from_package_data(

package_data=pkg_data,

datafile_path=package_resource.path,

package_uid=package.package_uid

)

if package:

package.populate_license_fields()

package_uid = package.package_uid

package_resource_parent = package_resource.parent(codebase)

if is_egg_info_directory(package_resource_parent):

root = package_resource_parent.parent(codebase)

else:

# We're assuming that our package resource is already at the

# root

root = package_resource_parent

if root:

for py_res in cls.walk_pypi(resource=root, codebase=codebase):

if py_res.is_dir:

continue

if package_uid and package_uid not in py_res.for_packages:

package_adder(package_uid, py_res, codebase)

yield py_res

elif codebase.has_single_resource:

if package_uid and package_uid not in package_resource.for_packages:

package_adder(package_uid, package_resource, codebase)

else:

package_uid = None

# Yield dependencies from sibling manifests

if package_resource:

for sibling in package_resource.siblings(codebase):

if not sibling:

continue

is_sibling_pypi_manifest = any(

fnmatchcase(sibling.path, pat)

for pat in datafile_name_patterns

)

if is_sibling_pypi_manifest:

yield from yield_dependencies_from_package_resource(

resource=sibling,

package_uid=package_uid,

)

if package_uid and package_uid not in sibling.for_packages:

package_adder(package_uid, sibling, codebase)

yield sibling

@classmethod

def walk_pypi(cls, resource, codebase):

"""

Walk the ``codebase`` Codebase top-down, breadth-first starting from the

``resource`` Resource.

Skip the directory named "site-packages": this avoids

reporting nested vendored packages as being part of their parent.

Instead they will be reported on their own.

"""

for child in resource.children(codebase):

if child.name == 'site-packages':

continue

yield child

if child.is_dir:

for subchild in cls.walk_pypi(child, codebase):

yield subchild

class PythonSdistPkgInfoFile(BaseExtractedPythonLayout):

datasource_id = 'pypi_sdist_pkginfo'

default_package_type = 'pypi'

default_primary_language = 'Python'

path_patterns = ('*/PKG-INFO',)

description = 'PyPI extracted sdist PKG-INFO'

documentation_url = 'https://peps.python.org/pep-0314/'

@classmethod

def is_datafile(cls, location):

return (

super().is_datafile(location) and

not PythonEggPkgInfoFile.is_datafile(location) and

not PythonEditableInstallationPkgInfoFile.is_datafile(location)

)

@classmethod

def parse(cls, location, package_only=False):

yield parse_metadata(

location=location,

datasource_id=cls.datasource_id,

package_type=cls.default_package_type,

package_only=package_only,

)

class PythonInstalledWheelMetadataFile(models.DatafileHandler):

datasource_id = 'pypi_wheel_metadata'

path_patterns = ('*.dist-info/METADATA',)

default_package_type = 'pypi'

default_primary_language = 'Python'

description = 'PyPI installed wheel METADATA'

documentation_url = 'https://packaging.python.org/en/latest/specifications/core-metadata/'

@classmethod

def parse(cls, location, package_only=False):

yield parse_metadata(

location=location,

datasource_id=cls.datasource_id,

package_type=cls.default_package_type,

package_only=package_only,

)

@classmethod

def assign_package_to_resources(cls, package, resource, codebase, package_adder):

"""

Assign files to package for an installed wheel. This requires a bit

of navigation around as the files can be in multiple places.

"""

dist_info_dir = resource.parent(codebase)

if not dist_info_dir:

return

# This could be a regular directory too which is not `site-packages`

site_packages = dist_info_dir.parent(codebase)

if not site_packages:

return

package_data = resource.package_data

assert len(resource.package_data) == 1, (

f'Unsupported Pypi METADATA wheel structure: {resource.path!r} '

f'with multiple {package_data!r}'

)

package_data = models.PackageData.from_dict(package_data[0])

package_uid = package.package_uid

if package_uid:

# save thyself!

package_adder(package_uid, resource, codebase)

# collect actual paths based on the file references

for file_ref in package_data.file_references:

path_ref = file_ref.path

if path_ref.startswith('..'):

# relative paths need special treatment

# most of thense are references to bin ../../../bin/wheel

cannot_resolve = False

ref_resource = site_packages

# note that resolving leading ".." always stays in the codebase

while path_ref.startswith('..'):

_, _, path_ref = path_ref.partition('../')

ref_resource = ref_resource.parent(codebase)

if not ref_resource:

cannot_resolve = True

break

if cannot_resolve or not ref_resource:

# TODO:w e should log these kind of things

continue

else:

ref_resource = codebase.get_resource(

path=posixpath.join(ref_resource.path, path_ref)

)

if ref_resource and package_uid:

package_adder(package_uid, ref_resource, codebase)

else:

# These are absolute paths from the site-packages directory

ref_resource = codebase.get_resource(

path=posixpath.join(site_packages.path, path_ref)

)

if ref_resource and package_uid:

package_adder(package_uid, ref_resource, codebase)

class PyprojectTomlHandler(BaseExtractedPythonLayout):

datasource_id = 'pypi_pyproject_toml'

path_patterns = ('*pyproject.toml',)

default_package_type = 'pypi'

default_primary_language = 'Python'

description = 'Python pyproject.toml'

documentation_url = 'https://packaging.python.org/en/latest/specifications/pyproject-toml/'

@classmethod

def is_datafile(cls, location, filetypes=tuple()):

return (

super().is_datafile(location, filetypes=filetypes)

and not is_poetry_pyproject_toml(location)

)

@classmethod

def parse(cls, location, package_only=False):

with open(location, "rb") as fp:

package_data = tomllib.load(fp)

project_data = package_data.get("project")

if not project_data:

return

name = project_data.get('name')

version = project_data.get('version')

description = project_data.get('description') or ''

description = description.strip()

urls, extra_data = get_urls(metainfo=project_data, name=name, version=version)

extracted_license_statement, license_file = get_declared_license(project_data)

if license_file:

extra_data['license_file'] = license_file

dependencies = []

parsed_dependencies = get_requires_dependencies(

requires=project_data.get("dependencies", []),

)

dependencies.extend(parsed_dependencies)

for dep_type, deps in project_data.get("optional-dependencies", {}).items():

parsed_dependencies = get_requires_dependencies(

requires=deps,

default_scope=dep_type,

is_optional=True,

)

dependencies.extend(parsed_dependencies)

package_data = dict(

datasource_id=cls.datasource_id,

type=cls.default_package_type,

primary_language='Python',

name=name,

version=version,

extracted_license_statement=extracted_license_statement,

description=description,

keywords=get_keywords(project_data),

parties=get_pyproject_toml_parties(project_data),

dependencies=dependencies,

extra_data=extra_data,

**urls,

)

yield models.PackageData.from_data(package_data, package_only)

def is_poetry_pyproject_toml(location):

with open(location, 'r') as file:

data = file.read()

if "tool.poetry" in data:

return True

else:

return False

class BasePoetryPythonLayout(BaseExtractedPythonLayout):

"""

Base class for poetry python projects.

"""

@classmethod

def assemble(cls, package_data, resource, codebase, package_adder):

package_resource = None

if resource.name == 'pyproject.toml':

package_resource = resource

elif resource.name == 'poetry.lock':

if resource.has_parent():

siblings = resource.siblings(codebase)

package_resource = [r for r in siblings if r.name == 'pyproject.toml']

if package_resource:

package_resource = package_resource[0]

if not package_resource:

# we do not have a pyproject.toml

yield from yield_dependencies_from_package_resource(resource)

return

if codebase.has_single_resource:

yield from models.DatafileHandler.assemble(package_data, resource, codebase, package_adder)

return

assert len(package_resource.package_data) == 1, f'Invalid pyproject.toml for {package_resource.path}'

pkg_data = package_resource.package_data[0]

pkg_data = models.PackageData.from_dict(pkg_data)

if pkg_data.purl:

package = models.Package.from_package_data(

package_data=pkg_data,

datafile_path=package_resource.path,

)

package_uid = package.package_uid

package.populate_license_fields()

yield package

root = package_resource.parent(codebase)

if root:

for pypi_res in cls.walk_pypi(resource=root, codebase=codebase):

if package_uid and package_uid not in pypi_res.for_packages:

package_adder(package_uid, pypi_res, codebase)

yield pypi_res

yield package_resource

else:

# we have no package, so deps are not for a specific package uid

package_uid = None

# in all cases yield possible dependencies

yield from yield_dependencies_from_package_data(pkg_data, package_resource.path, package_uid)

# we yield this as we do not want this further processed

yield package_resource

for lock_file in package_resource.siblings(codebase):

if lock_file.name == 'poetry.lock':

yield from yield_dependencies_from_package_resource(lock_file, package_uid)

if package_uid and package_uid not in lock_file.for_packages:

package_adder(package_uid, lock_file, codebase)

yield lock_file

class PoetryPyprojectTomlHandler(BasePoetryPythonLayout):

datasource_id = 'pypi_poetry_pyproject_toml'

path_patterns = ('*pyproject.toml',)

default_package_type = 'pypi'

default_primary_language = 'Python'

description = 'Python poetry pyproject.toml'

documentation_url = 'https://packaging.python.org/en/latest/specifications/pyproject-toml/'

@classmethod

def is_datafile(cls, location, filetypes=tuple()):

return (

super().is_datafile(location, filetypes=filetypes)

and is_poetry_pyproject_toml(location)

)

@classmethod

def parse_non_group_dependencies(

cls,

dependencies,

scope="install",

is_optional=False,

is_runtime=True,

):

dependency_mappings = []

for dep_name, requirement in dependencies.items():

if is_runtime and dep_name == "python":

continue

purl = PackageURL(

type=cls.default_package_type,

name=dep_name,

)

extra_data = {}

if isinstance(requirement, str):

extracted_requirement = requirement

elif isinstance(requirement, dict):

extracted_requirement = requirement.get("version")

is_optional = requirement.get("optional", False)

python_version = requirement.get("python")

if python_version:

extra_data["python_version"] = python_version

dependency = models.DependentPackage(

purl=purl.to_string(),

extracted_requirement=extracted_requirement,

scope=scope,

is_runtime=is_runtime,

is_optional=is_optional,

is_direct=True,

is_pinned=False,

extra_data=extra_data,

)

dependency_mappings.append(dependency.to_dict())

return dependency_mappings

@classmethod

def parse(cls, location, package_only=False):

with open(location, "rb") as fp:

toml_data = tomllib.load(fp)

tool_data = toml_data.get('tool')

if not tool_data:

return

poetry_data = tool_data.get('poetry')

if not poetry_data:

return

name = poetry_data.get('name')

version = poetry_data.get('version')

description = poetry_data.get('description') or ''

description = description.strip()

urls, extra_data = get_urls(metainfo=poetry_data, name=name, version=version, poetry=True)

extracted_license_statement, license_file = get_declared_license(poetry_data)

if license_file:

extra_data['license_file'] = license_file

dependencies = []

parsed_deps = cls.parse_non_group_dependencies(

dependencies=poetry_data.get("dependencies", {}),

scope="dependencies",

)

dependencies.extend(parsed_deps)

parsed_deps = cls.parse_non_group_dependencies(

dependencies=poetry_data.get("dev-dependencies", {}),

scope="dev-dependencies",

is_runtime=False,

)

dependencies.extend(parsed_deps)

for group_name, group_deps in poetry_data.get("group", {}).items():

for name, requirement in group_deps.get("dependencies", {}).items():

purl = PackageURL(

type=cls.default_package_type,

name=name,

)

dependency = models.DependentPackage(

purl=purl.to_string(),

extracted_requirement=requirement,

scope=group_name,

is_runtime=False,

is_optional=False,

is_direct=True,

is_pinned=False,

)

dependencies.append(dependency.to_dict())

package_data = dict(

datasource_id=cls.datasource_id,

type=cls.default_package_type,

primary_language='Python',

name=name,

version=version,

extracted_license_statement=extracted_license_statement,

description=description,

keywords=get_keywords(poetry_data),

parties=get_pyproject_toml_parties(poetry_data),

extra_data=extra_data,

dependencies=dependencies,

**urls,

)

yield models.PackageData.from_data(package_data, package_only)

class PoetryLockHandler(BasePoetryPythonLayout):

datasource_id = 'pypi_poetry_lock'

path_patterns = ('*poetry.lock',)

default_package_type = 'pypi'

default_primary_language = 'Python'

description = 'Python poetry lockfile'

documentation_url = 'https://python-poetry.org/docs/basic-usage/#installing-with-poetrylock'

@classmethod

def parse(cls, location, package_only=False):

with open(location, "rb") as fp:

toml_data = tomllib.load(fp)

packages = toml_data.get('package')

if not packages:

return

metadata = toml_data.get('metadata')

dependencies = []

for package in packages:

dependencies_for_resolved = []

deps = package.get("dependencies") or {}

for name, requirement in deps.items():

purl = PackageURL(

type=cls.default_package_type,

name=name,

)

dependency = models.DependentPackage(

purl=purl.to_string(),

extracted_requirement=requirement,

scope="dependencies",

is_runtime=True,

is_optional=False,

is_direct=True,

is_pinned=False,

)

dependencies_for_resolved.append(dependency.to_dict())

extra_deps = package.get("extras") or {}

for group_name, group_deps in extra_deps.items():

for dep in group_deps:

if " (" in dep and ")" in dep:

name, requirement = dep.split(" (")

requirement = requirement.rstrip(")")

else:

requirement = None

name = dep

purl = PackageURL(

type=cls.default_package_type,

name=name,

)

dependency = models.DependentPackage(

purl=purl.to_string(),

extracted_requirement=requirement,

scope=group_name,

is_runtime=False,

is_optional=False,

is_direct=True,

is_pinned=False,

)

dependencies_for_resolved.append(dependency.to_dict())

name = package.get('name')

version = package.get('version')

urls = get_pypi_urls(name, version)

package_data = dict(

datasource_id=cls.datasource_id,

type=cls.default_package_type,

primary_language='Python',

name=name,

version=version,

description=metadata.get('description'),

is_virtual=True,

dependencies=dependencies_for_resolved,

**urls,

)

resolved_package = models.PackageData.from_data(package_data, package_only)

is_optional = package.get("is_optional") or True

dependency = models.DependentPackage(

purl=resolved_package.purl,

extracted_requirement=None,

scope=None,

is_runtime=True,

is_optional=is_optional,

is_direct=False,

is_pinned=True,

resolved_package=resolved_package.to_dict()

)

dependencies.append(dependency.to_dict())

extra_data = {}

extra_data['python_version'] = metadata.get("python-versions")

extra_data['lock_version'] = metadata.get("lock-version")

package_data = dict(

datasource_id=cls.datasource_id,

type=cls.default_package_type,

primary_language='Python',

extra_data=extra_data,

dependencies=dependencies,

)

yield models.PackageData.from_data(package_data, package_only)

class PipInspectDeplockHandler(models.DatafileHandler):

datasource_id = 'pypi_inspect_deplock'

path_patterns = ('*pip-inspect.deplock',)

default_package_type = 'pypi'

default_primary_language = 'Python'

description = 'Python poetry pyproject.toml'

# These are files generated by deplock, see https://github.com/nexB/dependency-inspector

documentation_url = 'https://pip.pypa.io/en/stable/cli/pip_inspect/'

@classmethod

def get_resolved_package_from_metadata(cls, metadata, package_only=False):

requires_dist = metadata.get('requires_dist')

dependencies_for_resolved = get_requires_dependencies(

requires=requires_dist,

)

package_data = dict(

datasource_id=cls.datasource_id,

type=cls.default_package_type,

primary_language='Python',

name=metadata.get('name'),

version=metadata.get('version'),

extracted_license_statement=metadata.get('license'),

description=metadata.get('description'),

keywords=metadata.get('keywords'),

is_virtual=True,

dependencies=[

dep.to_dict()

for dep in dependencies_for_resolved

],

)

return models.PackageData.from_data(package_data, package_only)

@classmethod

def parse(cls, location, package_only=False):

with open(location) as f:

content = f.read()

data = json.loads(content)

installed_packages = data.get('installed')

if not installed_packages:

return

main_package_metadata = {}

dependencies = []

direct_deps_of_main_package = []

for package_metadata in installed_packages:

package_metadata_dep = package_metadata.get('metadata')

is_requested = package_metadata.get('requested')

# `direct_url` is only present for root package

# `requested` is true for root package and direct dependencies only

if is_requested and 'direct_url' in package_metadata:

main_package_metadata = package_metadata_dep

main_package_requires = main_package_metadata.get('requires_dist')

dependencies_for_main = get_requires_dependencies(

requires=main_package_requires,

)

direct_deps_of_main_package.extend([

get_base_purl(dep.purl)

for dep in dependencies_for_main

])

continue

package_data_dep = cls.get_resolved_package_from_metadata(

metadata=package_metadata_dep,

package_only=package_only,

)

dep_purl = package_data_dep.purl

dependency = models.DependentPackage(

purl=dep_purl,

extracted_requirement=None,

scope=None,

is_runtime=True,

is_optional=False,

is_direct=False,

is_pinned=True,

resolved_package=package_data_dep.to_dict()

)

if is_requested:

dependency.is_direct = True

dependencies.append(dependency)

dependency_mappings = []

resolved_main_dependencies = []

# Update is_direct for direct dependencies

for dep in dependencies:

base_purl = get_base_purl(dep.purl)

if base_purl in direct_deps_of_main_package:

dep.is_direct = True

resolved_main_dependencies.append(base_purl)

dependency_mappings.append(dep.to_dict())

pip_version = data.get('pip_version')

inspect_version = data.get('version')

extra_data = {

"pip_version": pip_version,

"inspect_version": inspect_version,

}

package_data_main = cls.get_resolved_package_from_metadata(

metadata=main_package_metadata,

package_only=package_only,

)

main_dependencies = []

for dep in package_data_main.dependencies:

base_purl = get_base_purl(purl=dep.get('purl'))

if base_purl not in resolved_main_dependencies:

main_dependencies.append(dep)

package_data_main.dependencies = dependencies

package_data_main.dependencies.extend(main_dependencies)

package_data_main.extra_data = extra_data

yield package_data_main

META_DIR_SUFFIXES = '.dist-info', '.egg-info', 'EGG-INFO',

def parse_metadata(location, datasource_id, package_type, package_only=False):

"""

Return a PackageData object from a PKG-INFO or METADATA file at ``location``

which is a path string or pathlib.Path-like object (including a possible zip

file ZipPath for a wheel)

Looks in neighboring files as needed when an installed layout is found.

"""

path = location

if not isinstance(location, (Path, ZipPath)):

path = Path(location)

# build from dir if we are an installed distro

parent = path.parent

if parent.name.endswith(META_DIR_SUFFIXES):

path = parent

dist = importlib_metadata.PathDistribution(path)

meta = dist.metadata

name = get_attribute(meta, 'Name')

version = get_attribute(meta, 'Version')

urls, extra_data = get_urls(metainfo=meta, name=name, version=version)

extracted_license_statement, license_file = get_declared_license(metainfo=meta)

if license_file:

extra_data['license_file'] = license_file

# FIXME: We are getting dependencies from other sibling files, this is duplicated

# data at the package_data level, is this necessary? We also have the entire dependency

# relationships here at requires.txt present in ``.egg-info`` should we store these

# nicely?

dependencies = get_dist_dependencies(dist)

file_references = list(get_file_references(dist))

package_data = dict(

datasource_id=datasource_id,

type=package_type,

primary_language='Python',