Skip to content

Commit bcf0f2f

Browse files
anderruizclaude
anderruiz
and
claude
committed
feat(lockfile): add BlockLocation to yarn-lock extractor
Add lockfile position tracking for both yarn v1-v3 (text format) and v4+ (JSON format). For text format, track line numbers during block grouping. For JSON format, scan lines for entry keys within "entries" object. Buffer full content for both formats to enable line scanning. Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
1 parent ae7573b commit bcf0f2f

2 files changed

Lines changed: 165 additions & 32 deletions

File tree

‎pkg/lockfile/javascript/parse-yarn-lock.go‎

Lines changed: 164 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package javascript
33
import (
44
"bufio"
55
"encoding/json"
6-
"errors"
76
"fmt"
87
"io"
98
"net/url"
@@ -12,6 +11,7 @@ import (
1211
"strings"
1312

1413
"github.com/DataDog/datadog-sbom-generator/internal/cachedregexp"
14+
"github.com/DataDog/datadog-sbom-generator/internal/utility/fileposition"
1515
"github.com/DataDog/datadog-sbom-generator/pkg/lockfile"
1616
"github.com/DataDog/datadog-sbom-generator/pkg/models"
1717
)
@@ -53,12 +53,42 @@ func parseYarnPackageBlock(block []string) []YarnPackage {
5353
return packages
5454
}
5555

56-
func groupYarnPackageLines(scanner *bufio.Scanner) []YarnPackage {
56+
// findLastNonEmptyLineInRange finds the 1-indexed line number of the last non-empty line
57+
// within the 0-indexed range [startIdx, endIdx].
58+
func findLastNonEmptyLineInRange(lines []string, startIdx, endIdx int) int {
59+
if endIdx >= len(lines) {
60+
endIdx = len(lines) - 1
61+
}
62+
63+
for i := endIdx; i >= startIdx; i-- {
64+
if strings.TrimSpace(lines[i]) != "" {
65+
return i + 1 // 1-indexed
66+
}
67+
}
68+
69+
return startIdx + 1
70+
}
71+
72+
// buildYarnBlockPosition creates a FilePosition from 1-indexed start and end line numbers.
73+
func buildYarnBlockPosition(lines []string, startLine, endLine int) models.FilePosition {
74+
colStart := fileposition.GetFirstNonEmptyCharacterIndexInLine(lines[startLine-1])
75+
colEnd := fileposition.GetLastNonEmptyCharacterIndexInLine(lines[endLine-1])
76+
77+
return models.FilePosition{
78+
Line: models.Position{Start: startLine, End: endLine},
79+
Column: models.Position{Start: colStart, End: colEnd},
80+
}
81+
}
82+
83+
func groupYarnPackageLines(scanner *bufio.Scanner, lines []string) []YarnPackage {
5784
var groups []YarnPackage
5885
var group []string
86+
var blockStartLine int // 1-indexed line number of block start
5987

88+
lineNum := 0
6089
var line string
6190
for scanner.Scan() {
91+
lineNum++
6292
line = scanner.Text()
6393

6494
if shouldSkipYarnLine(line) {
@@ -69,16 +99,26 @@ func groupYarnPackageLines(scanner *bufio.Scanner) []YarnPackage {
6999
if !strings.HasPrefix(line, " ") {
70100
if len(group) > 0 {
71101
packages := parseYarnPackageBlock(group)
102+
// Set BlockLocation on each package
103+
blockEndLine := findLastNonEmptyLineInRange(lines, blockStartLine-1, lineNum-2)
104+
for i := range packages {
105+
packages[i].BlockLocation = buildYarnBlockPosition(lines, blockStartLine, blockEndLine)
106+
}
72107
groups = append(groups, packages...)
73108
}
74109
group = make([]string, 0)
110+
blockStartLine = lineNum
75111
}
76112

77113
group = append(group, line)
78114
}
79115

80116
if len(group) > 0 {
81117
packages := parseYarnPackageBlock(group)
118+
blockEndLine := findLastNonEmptyLineInRange(lines, blockStartLine-1, len(lines)-1)
119+
for i := range packages {
120+
packages[i].BlockLocation = buildYarnBlockPosition(lines, blockStartLine, blockEndLine)
121+
}
82122
groups = append(groups, packages...)
83123
}
84124

@@ -336,7 +376,7 @@ func buildDependencyTree(rootPkgName, rootPkgTargetVersion, rootPkgRegistry stri
336376
return results
337377
}
338378

339-
func parseYarnPackage(dependency YarnPackage) lockfile.PackageDetails {
379+
func parseYarnPackage(dependency YarnPackage, filePath string) lockfile.PackageDetails {
340380
if dependency.Version == "" {
341381
_, _ = fmt.Fprintf(
342382
os.Stderr,
@@ -350,6 +390,9 @@ func parseYarnPackage(dependency YarnPackage) lockfile.PackageDetails {
350390
nameLocation = &models.FilePosition{Filename: dependency.WorkspacePath}
351391
}
352392

393+
blockLocation := dependency.BlockLocation
394+
blockLocation.Filename = filePath
395+
353396
return lockfile.PackageDetails{
354397
Name: dependency.Name,
355398
Version: dependency.Version,
@@ -358,6 +401,7 @@ func parseYarnPackage(dependency YarnPackage) lockfile.PackageDetails {
358401
Ecosystem: models.EcosystemNPM,
359402
Commit: tryExtractCommit(dependency.Resolution),
360403
NameLocation: nameLocation,
404+
BlockLocation: blockLocation,
361405
}
362406
}
363407

@@ -450,12 +494,110 @@ func isJSONFormat(content []byte) bool {
450494
//
451495
// Returns a slice of YarnPackage structs compatible with the existing YAML parser output,
452496
// allowing the rest of the extraction logic to work identically for both formats.
453-
func parseYarnBerryJSON(content []byte) ([]YarnPackage, error) {
497+
// extractYarnBerryJSONPositions scans JSON lines for entry keys within the "entries" object.
498+
// Entry keys appear as " \"package@npm:^1.0.0\": {" at 4-space indent inside "entries".
499+
func extractYarnBerryJSONPositions(lines []string) map[string]models.FilePosition {
500+
positions := make(map[string]models.FilePosition)
501+
502+
inEntries := false
503+
var currentKey string
504+
braceDepth := 0
505+
506+
for i, line := range lines {
507+
lineNum := i + 1
508+
trimmed := strings.TrimSpace(line)
509+
510+
// Detect "entries": {
511+
if !inEntries && strings.Contains(trimmed, `"entries"`) && strings.HasSuffix(trimmed, "{") {
512+
inEntries = true
513+
braceDepth = 1
514+
515+
continue
516+
}
517+
518+
if !inEntries {
519+
continue
520+
}
521+
522+
// Track brace depth
523+
for _, ch := range trimmed {
524+
if ch == '{' {
525+
braceDepth++
526+
} else if ch == '}' {
527+
braceDepth--
528+
}
529+
}
530+
531+
if braceDepth <= 0 {
532+
// Close last entry
533+
if currentKey != "" {
534+
closeBerryEntry(positions, currentKey, i, lines)
535+
currentKey = ""
536+
}
537+
538+
inEntries = false
539+
540+
continue
541+
}
542+
543+
// Entry key at depth 1 (exactly 4-space indent, opens an object): " \"package@npm:^1.0.0\": {"
544+
// Require HasSuffix("{") to avoid false positives on internal fields like "checksum": "..."
545+
// which also sit at depth 2 but do not open a new brace.
546+
if braceDepth == 2 && strings.HasSuffix(trimmed, "{") && strings.HasPrefix(line, " ") {
547+
// This is a new entry key
548+
if currentKey != "" {
549+
closeBerryEntry(positions, currentKey, i, lines)
550+
}
551+
552+
// Extract the key between quotes
553+
firstQuote := strings.Index(trimmed, `"`)
554+
lastQuote := strings.Index(trimmed[firstQuote+1:], `"`)
555+
556+
if firstQuote >= 0 && lastQuote >= 0 {
557+
key := trimmed[firstQuote+1 : firstQuote+1+lastQuote]
558+
currentKey = key
559+
560+
colStart := fileposition.GetFirstNonEmptyCharacterIndexInLine(line)
561+
562+
positions[currentKey] = models.FilePosition{
563+
Line: models.Position{Start: lineNum, End: 0},
564+
Column: models.Position{Start: colStart, End: 0},
565+
}
566+
}
567+
}
568+
}
569+
570+
if currentKey != "" {
571+
closeBerryEntry(positions, currentKey, len(lines), lines)
572+
}
573+
574+
return positions
575+
}
576+
577+
func closeBerryEntry(positions map[string]models.FilePosition, key string, beforeIndex int, lines []string) {
578+
pos := positions[key]
579+
lastNonEmpty := beforeIndex - 1
580+
for lastNonEmpty >= 0 && strings.TrimSpace(lines[lastNonEmpty]) == "" {
581+
lastNonEmpty--
582+
}
583+
584+
if lastNonEmpty >= 0 {
585+
pos.Line.End = lastNonEmpty + 1
586+
pos.Column.End = fileposition.GetLastNonEmptyCharacterIndexInLine(lines[lastNonEmpty])
587+
} else {
588+
pos.Line.End = pos.Line.Start
589+
}
590+
591+
positions[key] = pos
592+
}
593+
594+
func parseYarnBerryJSON(content []byte, lines []string) ([]YarnPackage, error) {
454595
var berryJSON YarnBerryJSON
455596
if err := json.Unmarshal(content, &berryJSON); err != nil {
456597
return nil, fmt.Errorf("failed to parse yarn.lock JSON: %w", err)
457598
}
458599

600+
positions := extractYarnBerryJSONPositions(lines)
459601
packages := make([]YarnPackage, 0, len(berryJSON.Entries))
460602

461603
for entryKey, entry := range berryJSON.Entries {
@@ -484,6 +626,12 @@ func parseYarnBerryJSON(content []byte) ([]YarnPackage, error) {
484626
})
485627
}
486628

629+
// Look up position by entry key
630+
var blockPos models.FilePosition
631+
if pos, ok := positions[entryKey]; ok {
632+
blockPos = pos
633+
}
634+
487635
// Create one YarnPackage per target version
488636
for _, targetVersion := range targetVersions {
489637
packages = append(packages, YarnPackage{
@@ -493,6 +641,7 @@ func parseYarnBerryJSON(content []byte) ([]YarnPackage, error) {
493641
Resolution: resolution,
494642
Dependencies: dependencies,
495643
WorkspacePath: workspacePath,
644+
BlockLocation: blockPos,
496645
})
497646
}
498647
}
@@ -501,41 +650,24 @@ func parseYarnBerryJSON(content []byte) ([]YarnPackage, error) {
501650
}
502651

503652
func (e YarnLockExtractor) Extract(f lockfile.DepFile, context lockfile.ScanContext) ([]lockfile.PackageDetails, error) {
504-
// Peek first bytes to detect format without loading entire file into memory
505-
buf := make([]byte, 200)
506-
n, err := f.Read(buf)
507-
if err != nil && !errors.Is(err, io.EOF) {
653+
content, err := io.ReadAll(f)
654+
if err != nil {
508655
return []lockfile.PackageDetails{}, fmt.Errorf("error reading yarn.lock: %w", err)
509656
}
510657

511-
// Try to reset file position for streaming
512-
var reader io.Reader = f
513-
if seeker, ok := f.(io.Seeker); ok {
514-
if _, err := seeker.Seek(0, 0); err != nil {
515-
return []lockfile.PackageDetails{}, fmt.Errorf("error seeking yarn.lock: %w", err)
516-
}
517-
} else {
518-
// If we can't seek, prepend the peeked bytes back to the reader
519-
reader = io.MultiReader(strings.NewReader(string(buf[:n])), f)
520-
}
658+
lines := fileposition.BytesToLines(content)
521659

522660
var yarnPackages []YarnPackage
523-
if isJSONFormat(buf[:n]) {
661+
if isJSONFormat(content) {
524662
// Parse JSON format (Yarn v4+)
525-
// JSON requires loading entire file into memory for parsing
526-
content, err := io.ReadAll(reader)
527-
if err != nil {
528-
return []lockfile.PackageDetails{}, fmt.Errorf("error reading yarn.lock JSON: %w", err)
529-
}
530-
yarnPackages, err = parseYarnBerryJSON(content)
663+
yarnPackages, err = parseYarnBerryJSON(content, lines)
531664
if err != nil {
532665
return []lockfile.PackageDetails{}, err
533666
}
534667
} else {
535-
// Parse YAML format (Yarn v1-3) using streaming scanner
536-
// This avoids loading the entire file into memory
537-
scanner := bufio.NewScanner(reader)
538-
yarnPackages = groupYarnPackageLines(scanner)
668+
// Parse YAML-like format (Yarn v1-3)
669+
scanner := bufio.NewScanner(strings.NewReader(string(content)))
670+
yarnPackages = groupYarnPackageLines(scanner, lines)
539671

540672
if err := scanner.Err(); err != nil {
541673
return []lockfile.PackageDetails{}, fmt.Errorf("error while scanning %s: %w", f.Path(), err)
@@ -561,7 +693,7 @@ func (e YarnLockExtractor) Extract(f lockfile.DepFile, context lockfile.ScanCont
561693
}
562694

563695
dependencyWorkspaces := createDependencyWorkspaceMap(workspaces, allResolvedPackages)
564-
packages := createPackageDetails(allResolvedPackages, dependencyWorkspaces)
696+
packages := createPackageDetails(allResolvedPackages, dependencyWorkspaces, f.Path())
565697

566698
pkgIndex := indexByNameAndVersions(packages)
567699
for index, pkg := range packages {
@@ -608,12 +740,12 @@ func createDependencyWorkspaceMap(workspaces []YarnPackage, allResolvedPackages
608740
return dependencyWorkspaces
609741
}
610742

611-
func createPackageDetails(allResolvedPackages []YarnPackage, dependencyWorkspaces map[string][]string) []lockfile.PackageDetails {
743+
func createPackageDetails(allResolvedPackages []YarnPackage, dependencyWorkspaces map[string][]string, filePath string) []lockfile.PackageDetails {
612744
packages := make([]lockfile.PackageDetails, 0, len(allResolvedPackages))
613745

614746
// Create lockfile.PackageDetails for regular packages, with workspace information where applicable
615747
for _, yarnPackage := range allResolvedPackages {
616-
basePackage := parseYarnPackage(yarnPackage)
748+
basePackage := parseYarnPackage(yarnPackage, filePath)
617749
depKey := getWorkspaceDependencyKey(yarnPackage.Name, yarnPackage.Version, yarnPackage.TargetVersion)
618750

619751
if workspacePaths, exists := dependencyWorkspaces[depKey]; exists {

‎pkg/lockfile/javascript/types.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,7 @@ type YarnPackage struct {
248248
Resolution string
249249
Dependencies []YarnDependency
250250
WorkspacePath string
251+
BlockLocation models.FilePosition
251252
}
252253

253254
type YarnLockExtractor struct {

0 commit comments

Comments
 (0)