# Copyright (c) nexB Inc. and others. All rights reserved.

# ScanCode is a trademark of nexB Inc.

# SPDX-License-Identifier: Apache-2.0

# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.

# See https://github.com/nexB/scancode-toolkit for support or download.

# See https://aboutcode.org for more information about nexB OSS projects.

#

import base64

import io

import fnmatch

import os

import logging

import json

import re

import sys

import urllib.parse

from functools import partial

from itertools import islice

from packageurl import PackageURL

from packagedcode import models

from packagedcode.utils import normalize_vcs_url

from packagedcode.utils import yield_dependencies_from_package_data

from packagedcode.utils import yield_dependencies_from_package_resource

from packagedcode.utils import update_dependencies_as_resolved

from packagedcode.utils import is_simple_path

from packagedcode.utils import is_simple_path_pattern

import saneyaml

"""

Handle Node.js npm packages

per https://docs.npmjs.com/files/package.json

"""

"""

To check https://github.com/npm/normalize-package-data

"""

SCANCODE_DEBUG_PACKAGE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False)

SCANCODE_DEBUG_PACKAGE_NPM = os.environ.get('SCANCODE_DEBUG_PACKAGE_NPM', False)

TRACE = SCANCODE_DEBUG_PACKAGE

TRACE_NPM = SCANCODE_DEBUG_PACKAGE_NPM

def logger_debug(*args):

pass

logger = logging.getLogger(__name__)

if TRACE or TRACE_NPM:

logging.basicConfig(stream=sys.stdout)

logger.setLevel(logging.DEBUG)

def logger_debug(*args):

return logger.debug(

' '.join(isinstance(a, str) and a or repr(a) for a in args)

)

# TODO: add os and engines from package.json??

# TODO: add new yarn v2 lock file format

# TODO: add pnp.js and pnpm-lock.yaml https://pnpm.io/

# TODO: add support for "lockfileVersion": 2 for package-lock.json and lockfileVersion: 3

class BaseNpmHandler(models.DatafileHandler):

lockfile_names = {

'package-lock.json',

'.package-lock.json',

'npm-shrinkwrap.json',

'yarn.lock',

'shrinkwrap.yaml',

'pnpm-lock.yaml'

}

@classmethod

def assemble(cls, package_data, resource, codebase, package_adder):

"""

If ``resource``, or one of its siblings, is a package.json file, use it

to create and yield the package, the package dependencies, and the

package resources.

When reporting the resources of a package, we walk the codebase, skipping

the node_modules directory, assign resources to the package and yield

resources.

For each lock file, assign dependencies to package instances and yield dependencies.

If there is no package.json, we do not have a package instance. In this

case, we yield each of the dependencies in each lock file.

"""

package_resource = None

if resource.name == 'package.json':

package_resource = resource

elif resource.name in cls.lockfile_names:

if resource.has_parent():

siblings = resource.siblings(codebase)

package_resource = [r for r in siblings if r.name == 'package.json']

if package_resource:

package_resource = package_resource[0]

if not package_resource:

# we do not have a package.json

yield from yield_dependencies_from_package_resource(resource)

return

if codebase.has_single_resource:

yield from models.DatafileHandler.assemble(package_data, resource, codebase, package_adder)

return

# We do not have any package data detected here

if not package_resource.package_data:

return

assert len(package_resource.package_data) == 1, f'Invalid package.json for {package_resource.path}'

pkg_data = package_resource.package_data[0]

pkg_data = models.PackageData.from_dict(pkg_data)

workspace_root = package_resource.parent(codebase)

workspace_root_path = None

if workspace_root:

workspace_root_path = workspace_root.path

workspaces = pkg_data.extra_data.get('workspaces') or []

# Also look for pnpm workspaces

pnpm_workspace = None

if not workspaces and workspace_root:

pnpm_workspace_path = os.path.join(workspace_root_path, 'pnpm-workspace.yaml')

pnpm_workspace = codebase.get_resource(path=pnpm_workspace_path)

if pnpm_workspace:

pnpm_workspace_pkg_data = pnpm_workspace.package_data

if pnpm_workspace_pkg_data:

workspace_package = pnpm_workspace_pkg_data[0]

extra_data = workspace_package.get('extra_data')

workspaces = extra_data.get('workspaces')

workspace_members = cls.get_workspace_members(

workspaces=workspaces,

codebase=codebase,

workspace_root_path=workspace_root_path,

)

cls.update_workspace_members(workspace_members, codebase)

# do we have enough to create a package?

if pkg_data.purl and not workspaces:

package = models.Package.from_package_data(

package_data=pkg_data,

datafile_path=package_resource.path,

)

package_uid = package.package_uid

package.populate_license_fields()

# Always yield the package resource in all cases and first!

yield package

if workspace_root:

for npm_res in cls.walk_npm(resource=workspace_root, codebase=codebase):

if package_uid and package_uid not in npm_res.for_packages:

package_adder(package_uid, npm_res, codebase)

yield npm_res

yield package_resource

elif workspaces:

yield from cls.create_packages_from_workspaces(

workspace_members=workspace_members,

workspace_root=workspace_root,

codebase=codebase,

package_adder=package_adder,

pnpm=pnpm_workspace and pkg_data.purl,

)

package_uid = None

if pnpm_workspace and pkg_data.purl:

package = models.Package.from_package_data(

package_data=pkg_data,

datafile_path=package_resource.path,

)

package_uid = package.package_uid

package.populate_license_fields()

# Always yield the package resource in all cases and first!

yield package

if workspace_root:

for npm_res in cls.walk_npm(resource=workspace_root, codebase=codebase):

if package_uid and not npm_res.for_packages:

package_adder(package_uid, npm_res, codebase)

yield npm_res

yield package_resource

else:

# we have no package, so deps are not for a specific package uid

package_uid = None

yield from cls.yield_npm_dependencies_and_resources(

package_resource=package_resource,

package_data=pkg_data,

package_uid=package_uid,

codebase=codebase,

package_adder=package_adder,

)

@classmethod

def yield_npm_dependencies_and_resources(cls, package_resource, package_data, package_uid, codebase, package_adder):

# in all cases yield possible dependencies

yield from yield_dependencies_from_package_data(package_data, package_resource.path, package_uid)

# we yield this as we do not want this further processed

yield package_resource

for lock_file in package_resource.siblings(codebase):

if lock_file.name in cls.lockfile_names:

yield from yield_dependencies_from_package_resource(lock_file, package_uid)

if package_uid and package_uid not in lock_file.for_packages:

package_adder(package_uid, lock_file, codebase)

yield lock_file

@classmethod

def create_packages_from_workspaces(

cls,

workspace_members,

workspace_root,

codebase,

package_adder,

pnpm=False,

):

workspace_package_uids = []

for workspace_member in workspace_members:

if not workspace_member.package_data:

continue

pkg_data = workspace_member.package_data[0]

pkg_data = models.PackageData.from_dict(pkg_data)

package = models.Package.from_package_data(

package_data=pkg_data,

datafile_path=workspace_member.path,

)

package_uid = package.package_uid

workspace_package_uids.append(package_uid)

package.populate_license_fields()

# Always yield the package resource in all cases and first!

yield package

member_root = workspace_member.parent(codebase)

package_adder(package_uid, member_root, codebase)

for npm_res in cls.walk_npm(resource=member_root, codebase=codebase):

if package_uid and package_uid not in npm_res.for_packages:

package_adder(package_uid, npm_res, codebase)

yield npm_res

yield from cls.yield_npm_dependencies_and_resources(

package_resource=workspace_member,

package_data=pkg_data,

package_uid=package_uid,

codebase=codebase,

package_adder=package_adder,

)

# All resources which are not part of a workspace package exclusively

# are a part of all packages (this is skipped if we have a root pnpm

# package)

if pnpm:

return

for npm_res in cls.walk_npm(resource=workspace_root, codebase=codebase):

if npm_res.for_packages:

continue

for package_uid in workspace_package_uids:

package_adder(package_uid, npm_res, codebase)

npm_res.save(codebase)

@classmethod

def walk_npm(cls, resource, codebase, depth=0):

"""

Walk the ``codebase`` Codebase top-down, breadth-first starting from the

``resource`` Resource.

Skip a first level child directory named "node_modules": this avoids

reporting nested vendored packages as being part of their parent.

Instead they will be reported on their own.

"""

for child in resource.children(codebase):

if depth == 0 and child.name == 'node_modules':

continue

yield child

if child.is_dir:

depth += 1

for subchild in cls.walk_npm(child, codebase, depth=depth):

yield subchild

@classmethod

def update_dependencies_by_purl(

cls,

dependencies,

scope,

dependencies_by_purl,

is_runtime=False,

is_optional=False,

is_pinned=False,

is_direct=True,

):

"""

Update the `dependencies_by_purl` mapping (which contains the cumulative

dependencies for a package metadata) from a list of `dependencies` which

have new dependencies or metadata for already existing dependencies.

"""

# npm/pnpm Dependency scopes which contain metadata for dependencies

# see documentation below for more details

# https://docs.npmjs.com/cli/v10/configuring-npm/package-json#peerdependenciesmeta

# https://pnpm.io/package_json#dependenciesmeta

metadata_deps = ['peerDependenciesMeta', 'dependenciesMeta']

if isinstance(dependencies, list):

for subdep in dependencies:

sdns, _ , sdname = subdep.rpartition('/')

dep_purl = PackageURL(

type=cls.default_package_type,

namespace=sdns,

name=sdname

).to_string()

dep_package = models.DependentPackage(

purl=dep_purl,

scope=scope,

is_runtime=is_runtime,

is_optional=is_optional,

is_pinned=is_pinned,

is_direct=is_direct,

)

dependencies_by_purl[dep_purl] = dep_package

elif isinstance(dependencies, dict):

for subdep, metadata in dependencies.items():

sdns, _ , sdname = subdep.rpartition('/')

dep_purl = PackageURL(

type=cls.default_package_type,

namespace=sdns,

name=sdname

).to_string()

if scope in metadata_deps :

dep_package = dependencies_by_purl.get(dep_purl)

if dep_package:

dep_package.is_optional = metadata.get("optional")

else:

dep_package = models.DependentPackage(

purl=dep_purl,

scope=scope,

is_runtime=is_runtime,

is_optional=metadata.get("optional"),

is_pinned=is_pinned,

is_direct=is_direct,

)

dependencies_by_purl[dep_purl] = dep_package

continue

# pnpm has peer dependencies also sometimes in version?

# dependencies:

# '@react-spring/animated': [email protected]

# TODO: store this relation too?

requirement = metadata

if 'pnpm' in cls.datasource_id:

if '_' in metadata:

requirement, _extra = metadata.split('_')

if ':' in requirement and '@' in requirement:

# dependencies with requirements like this are aliases and should be reported

aliased_package, _, constraint = requirement.rpartition('@')

_, _, aliased_package_name = aliased_package.rpartition(':')

sdns, _ , sdname = aliased_package_name.rpartition('/')

dep_purl = PackageURL(

type=cls.default_package_type,

namespace=sdns,

name=sdname

).to_string()

requirement = constraint

dep_package = models.DependentPackage(

purl=dep_purl,

scope=scope,

extracted_requirement=requirement,

is_runtime=is_runtime,

is_optional=is_optional,

is_pinned=is_pinned,

is_direct=is_direct,

)

dependencies_by_purl[dep_purl] = dep_package

@classmethod

def get_workspace_members(cls, workspaces, codebase, workspace_root_path):

"""

Given the workspaces, a list of paths/glob path patterns for npm

workspaces present in package.json, the codebase, and the

workspace_root_path, which is the parent directory of the

package.json which contains the workspaces, get a list of

workspace member package.json resources.

"""

workspace_members = []

for workspace_path in workspaces:

# Case 1: A definite path, instead of a pattern (only one package.json)

if is_simple_path(workspace_path):

workspace_dir_path = os.path.join(workspace_root_path, workspace_path)

workspace_member_path = os.path.join(workspace_dir_path, 'package.json')

workspace_member = codebase.get_resource(path=workspace_member_path)

if workspace_member and workspace_member.package_data:

workspace_members.append(workspace_member)

# Case 2: we have glob path which is a directory, relative to the workspace root

# Here we have only one * at the last (This is an optimization, this is a very

# commonly encountered subcase of case 3)

elif is_simple_path_pattern(workspace_path):

workspace_pattern_prefix = workspace_path.rstrip('*')

workspace_dir_path = os.path.join(workspace_root_path, workspace_pattern_prefix)

workspace_search_dir = codebase.get_resource(path=workspace_dir_path)

if not workspace_search_dir:

continue

for resource in workspace_search_dir.walk(codebase):

if resource.package_data and NpmPackageJsonHandler.is_datafile(

location=resource.location,

):

workspace_members.append(resource)

# Case 3: This is a complex glob pattern, we are doing a full codebase walk

# and glob matching each resource

else:

workspace_root = codebase.get_resource(path=workspace_root_path)

for resource in workspace_root.walk(codebase):

if NpmPackageJsonHandler.is_datafile(resource.location) and fnmatch.fnmatch(

name=resource.location, pat=workspace_path,

):

workspace_members.append(resource)

return workspace_members

@classmethod

def update_workspace_members(cls, workspace_members, codebase):

"""

Update all version requirements referencing workspace level

package versions with data from all the `workspace_members`.

Example: "ruru-components@workspace:^"

"""

# Collect info needed from all workspace member

workspace_package_versions_by_base_purl = {}

workspace_dependencies_by_base_purl = {}

for workspace_manifest in workspace_members:

workspace_package_data = workspace_manifest.package_data[0]

dependencies = workspace_package_data.get('dependencies')

for dependency in dependencies:

dep_purl = dependency.get('purl')

workspace_dependencies_by_base_purl[dep_purl] = dependency

is_private = workspace_package_data.get("is_private")

package_url = workspace_package_data.get('purl')

if is_private or not package_url:

continue

purl = PackageURL.from_string(package_url)

base_purl = PackageURL(

type=purl.type,

namespace=purl.namespace,

name=purl.name,

).to_string()

version = workspace_package_data.get('version')

if purl and version:

workspace_package_versions_by_base_purl[base_purl] = version

# Update workspace member package information from

# workspace level data

for base_purl, dependency in workspace_dependencies_by_base_purl.items():

extracted_requirement = dependency.get('extracted_requirement')

if 'workspace' in extracted_requirement:

version = workspace_package_versions_by_base_purl.get(base_purl)

if version:

new_requirement = extracted_requirement.replace('workspace', version)

dependency['extracted_requirement'] = new_requirement

for member in workspace_members:

member.save(codebase)

def get_urls(namespace, name, version, **kwargs):

return dict(

repository_homepage_url=npm_homepage_url(namespace, name, registry='https://www.npmjs.com/package'),

repository_download_url=npm_download_url(namespace, name, version, registry='https://registry.npmjs.org'),

api_data_url=npm_api_url(namespace, name, version, registry='https://registry.npmjs.org'),

)

class NpmPackageJsonHandler(BaseNpmHandler):

datasource_id = 'npm_package_json'

path_patterns = ('*/package.json',)

default_package_type = 'npm'

default_primary_language = 'JavaScript'

description = 'npm package.json'

documentation_url = 'https://docs.npmjs.com/cli/v8/configuring-npm/package-json'

@classmethod

def _parse(cls, json_data, package_only=False):

name = json_data.get('name')

version = json_data.get('version')

homepage_url = json_data.get('homepage', '')

# a package.json without name and version can be a private package

if homepage_url and isinstance(homepage_url, list):

# TODO: should we keep other URLs

homepage_url = homepage_url[0]

homepage_url = homepage_url.strip() or None

namespace, name = split_scoped_package_name(name)

is_private = False

if json_data.get('private'):

is_private = True

if is_private:

urls = {}

else:

urls = get_urls(namespace, name, version)

package_data = dict(

datasource_id=cls.datasource_id,

type=cls.default_package_type,

primary_language=cls.default_primary_language,

namespace=namespace or None,

name=name,

version=version or None,

description=json_data.get('description', '').strip() or None,

homepage_url=homepage_url,

is_private=is_private,

**urls,

)

package = models.PackageData.from_data(package_data, package_only)

vcs_revision = json_data.get('gitHead') or None

# mapping of top level package.json items to a function accepting as

# arguments the package.json element value and returning an iterable of (key,

# values) to update on a package

field_mappers = [

('author', partial(party_mapper, party_type='author')),

('contributors', partial(party_mapper, party_type='contributor')),

('maintainers', partial(party_mapper, party_type='maintainer')),

('dependencies', partial(deps_mapper, field_name='dependencies')),

('devDependencies', partial(deps_mapper, field_name='devDependencies')),

('peerDependencies', partial(deps_mapper, field_name='peerDependencies')),

('optionalDependencies', partial(deps_mapper, field_name='optionalDependencies')),

('bundledDependencies', bundle_deps_mapper),

('resolutions', partial(deps_mapper, field_name='resolutions')),

('repository', partial(vcs_repository_mapper, vcs_revision=vcs_revision)),

('keywords', keywords_mapper,),

('bugs', bugs_mapper),

('dist', dist_mapper),

]

extra_data = {}

extra_data_fields = ['workspaces', 'engines', 'packageManager']

for extra_data_field in extra_data_fields:

value = json_data.get(extra_data_field)

if value:

extra_data[extra_data_field] = value

package.extra_data = extra_data

for source, func in field_mappers:

value = json_data.get(source) or None

if value:

if isinstance(value, str):

value = value.strip()

if value:

func(value, package)

if not package.download_url:

# Only add a synthetic download URL if there is none from the dist mapping.

package.download_url = npm_download_url(package.namespace, package.name, package.version)

# licenses are a tad special with many different data structures

lic = json_data.get('license')

lics = json_data.get('licenses')

package = licenses_mapper(lic, lics, package)

if not package_only:

package.populate_license_fields()

if TRACE:

logger_debug(f'NpmPackageJsonHandler: parse: package: {package.to_dict()}')

return package

@classmethod

def parse(cls, location, package_only=False):

with io.open(location, encoding='utf-8') as loc:

json_data = json.load(loc)

yield cls._parse(json_data, package_only)

class BaseNpmLockHandler(BaseNpmHandler):

@classmethod

def parse(cls, location, package_only=False):

with io.open(location, encoding='utf-8') as loc:

package_data = json.load(loc)

# we have two formats: v1 and v2

lockfile_version = package_data.get('lockfileVersion', 1)

root_name = package_data.get('name')

root_version = package_data.get('version')

root_ns, _ , root_name = root_name.rpartition('/')

extra_data = dict(lockfile_version=lockfile_version)

# this is the top level element that we return

root_package_mapping = dict(

datasource_id=cls.datasource_id,

type=cls.default_package_type,

primary_language=cls.default_primary_language,

namespace=root_ns,

name=root_name,

version=root_version,

extra_data=extra_data,

**get_urls(root_ns, root_name, root_version)

)

root_package_data = models.PackageData.from_data(root_package_mapping, package_only)

# https://docs.npmjs.com/cli/v8/configuring-npm/package-lock-json#lockfileversion

if lockfile_version == 1:

deps_key = 'dependencies'

else:

# v2 and may be v3???

deps_key = 'packages'

deps_mapping = package_data.get(deps_key) or {}

# Top level package metadata is present here

root_pkg = deps_mapping.get("")

if root_pkg:

pkg_name = root_pkg.get('name')

pkg_ns, _ , pkg_name = pkg_name.rpartition('/')

pkg_version = root_pkg.get('version')

pkg_purl = PackageURL(

type=cls.default_package_type,

namespace=pkg_ns,

name=pkg_name,

version=pkg_version,

).to_string()

if pkg_purl != root_package_data.purl:

if TRACE_NPM:

logger_debug(f'BaseNpmLockHandler: parse: purl mismatch: {pkg_purl} vs {root_package_data.purl}')

else:

extracted_license_statement = root_pkg.get('license')

if extracted_license_statement:

root_package_data.extracted_license_statement = extracted_license_statement

root_package_data.populate_license_fields()

deps_mapper(

deps=root_pkg.get('devDependencies') or {},

package=root_package_data,

field_name='devDependencies',

)

deps_mapper(

deps=root_pkg.get('optionalDependencies') or {},

package=root_package_data,

field_name='optionalDependencies',

)

dependencies = []

for dep, dep_data in deps_mapping.items():

is_dev = dep_data.get('dev', False)

is_optional = dep_data.get('optional', False)

is_devoptional = dep_data.get('devOptional', False)

if is_dev or is_devoptional:

is_runtime = False

is_optional = True

scope = 'devDependencies'

else:

is_runtime = True

is_optional = is_optional

scope = 'dependencies'

if not dep:

# in v2 format the first dep is the same as the top level

# package and has no name

continue

# only present for first top level

# otherwise get name from dep

name = dep_data.get('name')

if not name:

if 'node_modules/' in dep:

# the name is the last segment as the dep can be:

# "node_modules/ansi-align/node_modules/ansi-regex"

_, _, name = dep.rpartition('node_modules/')

else:

name = dep

ns, _ , name = name.rpartition('/')

version = dep_data.get('version')

dep_purl = PackageURL(

type=cls.default_package_type,

namespace=ns,

name=name,

version=version,

).to_string()

dependency = models.DependentPackage(

purl=dep_purl,

extracted_requirement=version,

scope=scope,

is_runtime=is_runtime,

is_optional=is_optional,

is_pinned=True,

is_direct=False,

)

# URLs and checksums

misc = get_urls(ns, name, version)

resolved = dep_data.get('resolved')

misc.update(get_checksum_and_url(resolved).items())

integrity = dep_data.get('integrity')

misc.update(get_algo_hexsum(integrity).items())

resolved_package_mapping = dict(

datasource_id=cls.datasource_id,

type=cls.default_package_type,

primary_language=cls.default_primary_language,

namespace=ns,

name=name,

version=version,

**misc,

is_virtual=True,

)

resolved_package = models.PackageData.from_data(resolved_package_mapping, package_only)

# these are paths t the root of the installed package in v2

if dep:

resolved_package.file_references = [models.FileReference(path=dep)],

# v1 as name/constraint pairs

subrequires = dep_data.get('requires') or {}

# in v1 these are further nested dependencies (TODO: handle these with tests)

# in v2 these are name/constraint pairs like v1 requires

subdependencies = dep_data.get('dependencies')

# v2? ignored for now

engines = dep_data.get('engines')

funding = dep_data.get('funding')

if lockfile_version == 1:

subdeps_data = subrequires

else:

subdeps_data = subdependencies

subdeps_data = subdeps_data or {}

sub_deps_by_purl = {}

cls.update_dependencies_by_purl(

dependencies=subdeps_data,

scope=scope,

dependencies_by_purl=sub_deps_by_purl,

is_runtime=is_runtime,

is_optional=is_optional,

is_pinned=False,

is_direct=True,

)

resolved_package.dependencies = [

sub_dep.to_dict()

for sub_dep in sub_deps_by_purl.values()

]

dependency.resolved_package = resolved_package.to_dict()

dependencies.append(dependency.to_dict())

update_dependencies_as_resolved(dependencies=dependencies)

root_package_data.dependencies = dependencies

yield root_package_data

class NpmPackageLockJsonHandler(BaseNpmLockHandler):

# Note that there are multiple lockfileVersion 1 and 2 (and even 3)

# and each have a different layout

datasource_id = 'npm_package_lock_json'

path_patterns = (

'*/package-lock.json',

'*/.package-lock.json',

)

default_package_type = 'npm'

default_primary_language = 'JavaScript'

is_lockfile = True

description = 'npm package-lock.json lockfile'

documentation_url = 'https://docs.npmjs.com/cli/v8/configuring-npm/package-lock-json'

class NpmShrinkwrapJsonHandler(BaseNpmLockHandler):

datasource_id = 'npm_shrinkwrap_json'

path_patterns = ('*/npm-shrinkwrap.json',)

default_package_type = 'npm'

default_primary_language = 'JavaScript'

is_lockfile = True

description = 'npm shrinkwrap.json lockfile'

documentation_url = 'https://docs.npmjs.com/cli/v8/configuring-npm/npm-shrinkwrap-json'

class UnknownYarnLockFormat(Exception):

pass

def is_yarn_v2(location):

"""

Return True if this is a yarn.lock format version 2 and False if this is

version 1. Raise an UnknownYarnLockFormat exception if neither v1 or v2.

v1 is a custom, almost-like-YAMl format. v2 is a proper subset of YAML.

The start of v1 file has this:

# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.

# yarn lockfile v1

The start of v2 file has this:

# This file is generated by running "yarn install" inside your project.

# Manual changes might be lost - proceed with caution!

__metadata:

"""

with open(location) as ylf:

# check only in the first 10 lines

for line in islice(ylf, 0, 10):

if '__metadata:' in line:

return True

if 'yarn lockfile v1' in line:

return False

raise UnknownYarnLockFormat(location)

class YarnLockV2Handler(BaseNpmHandler):

"""

Handle yarn.lock v2 format, which is YAML

"""

datasource_id = 'yarn_lock_v2'

path_patterns = ('*/yarn.lock',)

default_package_type = 'npm'

default_primary_language = 'JavaScript'

is_lockfile = True

description = 'yarn.lock lockfile v2 format'

documentation_url = 'https://classic.yarnpkg.com/lang/en/docs/yarn-lock/'

@classmethod

def is_datafile(cls, location, filetypes=tuple()):

return super().is_datafile(location, filetypes=filetypes) and is_yarn_v2(location)

@classmethod

def parse(cls, location, package_only=False):

"""

Parse a bew yarn.lock v2 YAML format which looks like this:

"@algolia/cache-browser-local-storage@npm:4.2.0":

version: 4.2.0

resolution: "@algolia/cache-browser-local-storage@npm:4.2.0"

dependencies:

"@algolia/cache-common": 4.2.0

checksum: 72ac158925eb5a51e015aa22df5d2026fc0c0b6b58eb8c1290712e0

languageName: node

linkType: hard

Yield a single PackageData

"""

with open(location) as yl:

lock_data = saneyaml.load(yl.read())

top_dependencies = []

for spec, details in lock_data.items():

if spec == '__metadata':

continue

version = details.get('version')

resolution = details.get('resolution')

ns_name, _, version = resolution.rpartition('@')

ns, _, name = ns_name.rpartition('/')

if version.startswith('npm:'):

_npm, _, version = version.partition(':')

purl = PackageURL(

type=cls.default_package_type,

namespace=ns,

name=name,

version=version,

)

# TODO: what type of checksum is this? ... this is a complex one

# See https://github.com/yarnpkg/berry/blob/f1edfae49d1bab7679ce3061e2749113dc3b80e8/packages/yarnpkg-core/sources/tgzUtils.ts

checksum = details.get('checksum')

dependencies = details.get('dependencies') or {}

peer_dependencies = details.get('peerDependencies') or {}

dependencies_meta = details.get('dependenciesMeta') or {}

# these are file references

bin = details.get('bin') or []

deps_for_resolved_by_purl = {}

cls.update_dependencies_by_purl(

dependencies=dependencies,

scope="dependencies",

dependencies_by_purl=deps_for_resolved_by_purl,

)

cls.update_dependencies_by_purl(

dependencies=peer_dependencies,

scope="peerDependencies",

dependencies_by_purl=deps_for_resolved_by_purl,

)

cls.update_dependencies_by_purl(

dependencies=dependencies_meta,

scope="dependenciesMeta",

dependencies_by_purl=deps_for_resolved_by_purl,

)

dependencies_for_resolved = [

dep_package.to_dict()

for dep_package in deps_for_resolved_by_purl.values()

]

resolved_package_mapping = dict(

datasource_id=cls.datasource_id,

type=cls.default_package_type,

primary_language=cls.default_primary_language,

namespace=ns,

name=name,

version=version,

dependencies=dependencies_for_resolved,

is_virtual=True,

)

resolved_package = models.PackageData.from_data(resolved_package_mapping)

# These are top level dependencies which do not have a

# scope defined there, so we are assigning the default

# scope, this would be merged with the dependency having

# correct scope value when resolved

dependency = models.DependentPackage(

purl=str(purl),

extracted_requirement=version,

is_pinned=True,

resolved_package=resolved_package.to_dict(),

scope='dependencies',

is_optional=False,

is_runtime=True,

)

top_dependencies.append(dependency.to_dict())

update_dependencies_as_resolved(dependencies=top_dependencies)

package_data = dict(

datasource_id=cls.datasource_id,

type=cls.default_package_type,

primary_language=cls.default_primary_language,

dependencies=top_dependencies,

)

yield models.PackageData.from_data(package_data, package_only)

class YarnLockV1Handler(BaseNpmHandler):

"""

Handle yarn.lock v1 format, which is more or less but not quite like YAML

"""

datasource_id = 'yarn_lock_v1'

path_patterns = ('*/yarn.lock',)

default_package_type = 'npm'

default_primary_language = 'JavaScript'

is_lockfile = True

description = 'yarn.lock lockfile v1 format'

documentation_url = 'https://classic.yarnpkg.com/lang/en/docs/yarn-lock/'

@classmethod

def is_datafile(cls, location, filetypes=tuple()):

return super().is_datafile(location, filetypes=filetypes) and not is_yarn_v2(location)

@classmethod

def parse(cls, location, package_only=False):

"""

Parse a classic yarn.lock format which looks like this:

"@babel/core@^7.1.0", "@babel/core@^7.3.4":

version "7.3.4"

resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.3.4.tgz#921a5a13746c21e32445bf0798680e9d11a6530b"

integrity sha512-jRsuseXBo9pN197KnDwhhaaBzyZr2oIcLHHTt2oDdQrej5Qp57dCCJafWx5ivU8/alEYDpssYqv1MUqcxwQlrA==