Skip to content

feat(vulnfeeds): group extracted_events provenance with source, strategy, and ref tags #6063

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jess-lowe wants to merge 22 commits into
base: master
Choose a base branch
from
Open

feat(vulnfeeds): group extracted_events provenance with source, strategy, and ref tags #6063

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
08f3db6
Strategy based refactor
jess-lowe Sep 2, 2026
1e829b0
linux
jess-lowe Aug 17, 2026
7a775d6
fix some artifacts
jess-lowe Sep 2, 2026
beeb0df
cleanup
jess-lowe Sep 2, 2026
16bdbc5
remove wordpress for now
jess-lowe Sep 2, 2026
0e462d6
lint and snaps
jess-lowe Sep 3, 2026
d44272e
put strategies in their own folders
jess-lowe Sep 21, 2026
a476031
Merge remote-tracking branch 'upstream/master' into refactor/strategies
jess-lowe Sep 21, 2026
b66b69b
fix instances of addnote
jess-lowe Sep 21, 2026
f8d1747
fix test
jess-lowe Sep 21, 2026
d3803eb
refactor(vulnfeeds): move CVE5 version extraction strategies to Affec…
jess-lowe Sep 22, 2026
2d63651
add missing testcase
jess-lowe Sep 22, 2026
4f092eb
Merge branch 'refactor/strategies' into refactor/strat/versions-combined
jess-lowe Sep 22, 2026
db8ba4e
Merge branch 'master' into refactor/strategies
jess-lowe Sep 23, 2026
9a1e5d5
provenance is gorgeous
jess-lowe Sep 24, 2026
97ead91
Merge branch 'master' into refactor/provenance
jess-lowe Sep 29, 2026
5314a6e
be more go idiomatic
jess-lowe Oct 6, 2026
903736a
make tests us protojson
jess-lowe Oct 6, 2026
5730ee3
be more go idiomatic
jess-lowe Oct 6, 2026
6e0f1a2
refactor duplicate code
jess-lowe Oct 6, 2026
df2c2a8
update comment
jess-lowe Oct 6, 2026
2f3270d
lint
jess-lowe Oct 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 61 additions & 5 deletions vulnfeeds/cmd/combine-to-osv/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -607,12 +607,41 @@ func getExtractedEvents(r *osvschema.Range) []*structpb.Value {
return val.GetListValue().GetValues()
}

func parseExtractedEvent(v *structpb.Value) ExtractedEvent {
func parseExtractedEvent(v *structpb.Value) []ExtractedEvent {
s := v.GetStructValue()
if s == nil {
return ExtractedEvent{}
return nil
}
fields := s.GetFields()

if rangeVal, ok := fields["range"]; ok && rangeVal.GetListValue() != nil {
var events []ExtractedEvent
for _, evVal := range rangeVal.GetListValue().GetValues() {
evS := evVal.GetStructValue()
if evS == nil {
continue
}
evFields := evS.GetFields()
var ev ExtractedEvent
if intro, ok := evFields["introduced"]; ok {
ev.Introduced = intro.GetStringValue()
}
if fixed, ok := evFields["fixed"]; ok {
ev.Fixed = fixed.GetStringValue()
}
if la, ok := evFields["last_affected"]; ok {
ev.LastAffected = la.GetStringValue()
}
if lim, ok := evFields["limit"]; ok {
ev.Limit = lim.GetStringValue()
}
events = append(events, ev)
}

return events
}

// Fallback to old flat structure (single event)
var ev ExtractedEvent
if intro, ok := fields["introduced"]; ok {
ev.Introduced = intro.GetStringValue()
Expand All @@ -627,17 +656,21 @@ func parseExtractedEvent(v *structpb.Value) ExtractedEvent {
ev.Limit = lim.GetStringValue()
}

return ev
if ev != (ExtractedEvent{}) {
return []ExtractedEvent{ev}
}

return nil
}

func parseExtractedEvents(r *osvschema.Range) []ExtractedEvent {
rawValues := getExtractedEvents(r)
if len(rawValues) == 0 {
return nil
}
events := make([]ExtractedEvent, 0, len(rawValues))
var events []ExtractedEvent
for _, val := range rawValues {
events = append(events, parseExtractedEvent(val))
events = append(events, parseExtractedEvent(val)...)
}

return events
Expand Down Expand Up @@ -688,6 +721,29 @@ func isCPERange(r *osvschema.Range) bool {
if fields == nil {
return false
}

// Check new location inside extracted_events first
if extractedEventsVal, ok := fields["extracted_events"]; ok && extractedEventsVal.GetListValue() != nil {
for _, groupVal := range extractedEventsVal.GetListValue().GetValues() {
groupS := groupVal.GetStructValue()
if groupS == nil {
continue
}
groupFields := groupS.GetFields()
if sourceVal, ok := groupFields["source"]; ok {
if sourceVal.GetStringValue() == "CPE_RANGE" {
return true
}
}
if strategyVal, ok := groupFields["strategy"]; ok {
if strategyVal.GetStringValue() == "CPEApplicability" {
return true
}
}
}
}

// Fallback to old location
val, ok := fields["source"]
if !ok {
return false
Expand Down
80 changes: 60 additions & 20 deletions vulnfeeds/conversion/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package conversion

import (
"bytes"
"cmp"
"encoding/csv"
"encoding/json"
"errors"
Expand Down Expand Up @@ -293,16 +294,7 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri
}
successfulRepos = append(successfulRepos, repo)
if len(vr.Range.GetEvents()) > 0 {
dbSpecificMap := map[string]any{
"extracted_events": vr.Range.GetEvents(),
}
if vr.Metadata.CPE != "" {
dbSpecificMap["cpe"] = vr.Metadata.CPE
}
if string(vr.Metadata.Source) != "" {
dbSpecificMap["source"] = string(vr.Metadata.Source)
}
databaseSpecific, err := utility.NewStructpbFromMap(dbSpecificMap)
databaseSpecific, err := buildRangeDatabaseSpecific(vr)
if err != nil {
metrics.AddNotef("failed to make database specific: %v", err)
} else {
Expand All @@ -324,6 +316,29 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri
return newVersionRanges, unresolvedRanges, successfulRepos
}

func buildRangeDatabaseSpecific(vr models.RangeWithMetadata) (*structpb.Struct, error) {
extractedEventGroup := map[string]any{
"range": vr.Range.GetEvents(),
}
if vr.Metadata.CPE != "" {
extractedEventGroup["cpe"] = vr.Metadata.CPE
}
if string(vr.Metadata.Source) != "" {
extractedEventGroup["source"] = string(vr.Metadata.Source)
}
if vr.Metadata.Strategy != "" {
extractedEventGroup["strategy"] = vr.Metadata.Strategy
}
if vr.Metadata.OriginalTag != "" {
extractedEventGroup["original_tag"] = vr.Metadata.OriginalTag
}
dbSpecificMap := map[string]any{
"extracted_events": []any{extractedEventGroup},

@michaelkedar michaelkedar Oct 6, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is there a reason that this is a slice containing a single map, rather than having extracted_events just be the map?

@jess-lowe jess-lowe Oct 6, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When multiple ranges or commits are merged downstream (e.g. combining NVD and CVE5 data for the same repo), their provenance maps get appended into this list.

Starting it as a 1-element slice keeps the schema consistently a list (*structpb.ListValue) so consumers don't have to handle polymorphic map vs. []map types.

}

return utility.NewStructpbFromMap(dbSpecificMap)
}

// BuildVersionRange is a helper function that adds 'introduced', 'fixed', or 'last_affected'
// events to an OSV version range. If 'intro' is empty, it defaults to "0".
func BuildVersionRange(intro string, lastAff string, fixed string) *osvschema.Range {
Expand Down Expand Up @@ -555,7 +570,9 @@ func CreateUnresolvedRanges(unresolvedRanges []models.RangeWithMetadata) *struct

type key struct {
Source string
Strategy string
VendorProduct string
OriginalTag string
}

rangesByKey := make(map[key][]models.RangeWithMetadata)
Expand All @@ -570,19 +587,25 @@ func CreateUnresolvedRanges(unresolvedRanges []models.RangeWithMetadata) *struct
vendorProduct = ur.Metadata.CPE
}
}
k := key{Source: string(ur.Metadata.Source), VendorProduct: vendorProduct}
k := key{
Source: string(ur.Metadata.Source),
Strategy: ur.Metadata.Strategy,
VendorProduct: vendorProduct,
OriginalTag: ur.Metadata.OriginalTag,
}
if _, ok := rangesByKey[k]; !ok {
keys = append(keys, k)
}
rangesByKey[k] = append(rangesByKey[k], ur)
}

slices.SortFunc(keys, func(a, b key) int {
if a.Source != b.Source {
return strings.Compare(a.Source, b.Source)
}

return strings.Compare(a.VendorProduct, b.VendorProduct)
return cmp.Or(
cmp.Compare(a.Source, b.Source),
cmp.Compare(a.Strategy, b.Strategy),
cmp.Compare(a.VendorProduct, b.VendorProduct),
cmp.Compare(a.OriginalTag, b.OriginalTag),
)
})

listElements := make([]any, 0, len(keys))
Expand Down Expand Up @@ -623,14 +646,24 @@ func CreateUnresolvedRanges(unresolvedRanges []models.RangeWithMetadata) *struct
if k.VendorProduct != "" {
unresolvedRangesMap["vendor_product"] = k.VendorProduct
}
if k.Source != "" {
unresolvedRangesMap["source"] = k.Source
}
if len(cpes) > 0 {
unresolvedRangesMap["cpes"] = cpes
}

unresolvedRangesMap["extracted_events"] = events
extractedEventGroup := map[string]any{
"range": events,
}
if k.Source != "" {
extractedEventGroup["source"] = k.Source
}
if k.Strategy != "" {
extractedEventGroup["strategy"] = k.Strategy
}
if k.OriginalTag != "" {
extractedEventGroup["original_tag"] = k.OriginalTag
}

unresolvedRangesMap["extracted_events"] = []any{extractedEventGroup}
listElements = append(listElements, unresolvedRangesMap)
}

Expand Down Expand Up @@ -807,6 +840,13 @@ func ProcessRanges(ranges []models.RangeWithMetadata, repos []string, metrics *m
if repo != "" {
vr.Range.Repo = repo
vr.Range.Type = osvschema.Range_GIT
if len(vr.Range.GetEvents()) > 0 && vr.Range.GetDatabaseSpecific() == nil {
if databaseSpecific, err := buildRangeDatabaseSpecific(vr); err != nil {
metrics.AddNotef("failed to make database specific: %v", err)
} else {
vr.Range.DatabaseSpecific = databaseSpecific
}
}
resolvedRanges = append(resolvedRanges, vr)
successfulRepos = append(successfulRepos, repo)
} else {
Expand Down
32 changes: 28 additions & 4 deletions vulnfeeds/conversion/common_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,6 @@ func TestCreateUnresolvedRanges(t *testing.T) {
StructValue: &structpb.Struct{
Fields: map[string]*structpb.Value{
"vendor_product": structpb.NewStringValue("another:app"),
"source": structpb.NewStringValue(string(models.VersionSourceCPE)),
"cpes": structpb.NewListValue(&structpb.ListValue{
Values: []*structpb.Value{
structpb.NewStringValue("cpe:2.3:a:another:app:*:*:*:*:*:*:*:*"),
Expand All @@ -406,7 +405,20 @@ func TestCreateUnresolvedRanges(t *testing.T) {
Kind: &structpb.Value_StructValue{
StructValue: &structpb.Struct{
Fields: map[string]*structpb.Value{
"fixed": structpb.NewStringValue("2.0"),
"source": structpb.NewStringValue(string(models.VersionSourceCPE)),
"range": structpb.NewListValue(&structpb.ListValue{
Values: []*structpb.Value{
{
Kind: &structpb.Value_StructValue{
StructValue: &structpb.Struct{
Fields: map[string]*structpb.Value{
"fixed": structpb.NewStringValue("2.0"),
},
},
},
},
},
}),
},
},
},
Expand All @@ -424,7 +436,6 @@ func TestCreateUnresolvedRanges(t *testing.T) {
StructValue: &structpb.Struct{
Fields: map[string]*structpb.Value{
"vendor_product": structpb.NewStringValue("example:app"),
"source": structpb.NewStringValue(string(models.VersionSourceDescription)),
"cpes": structpb.NewListValue(&structpb.ListValue{
Values: []*structpb.Value{
structpb.NewStringValue("cpe:2.3:a:example:app:*:*:*:*:*:*:*:*"),
Expand All @@ -438,7 +449,20 @@ func TestCreateUnresolvedRanges(t *testing.T) {
Kind: &structpb.Value_StructValue{
StructValue: &structpb.Struct{
Fields: map[string]*structpb.Value{
"introduced": structpb.NewStringValue("1.0"),
"source": structpb.NewStringValue(string(models.VersionSourceDescription)),
"range": structpb.NewListValue(&structpb.ListValue{
Values: []*structpb.Value{
{
Kind: &structpb.Value_StructValue{
StructValue: &structpb.Struct{
Fields: map[string]*structpb.Value{
"introduced": structpb.NewStringValue("1.0"),
},
},
},
},
},
}),
},
},
},
Expand Down
Loading
Loading