feat(vulnfeeds): group extracted_events provenance with source, strategy, and ref tags

vulnfeeds/cmd/combine-to-osv/main.go

@@ -607,12 +607,41 @@ func getExtractedEvents(r *osvschema.Range) []*structpb.Value {
 	return val.GetListValue().GetValues()
 }
 
-func parseExtractedEvent(v *structpb.Value) ExtractedEvent {
+func parseExtractedEvent(v *structpb.Value) []ExtractedEvent {
 	s := v.GetStructValue()
 	if s == nil {
-		return ExtractedEvent{}
+		return nil
 	}
 	fields := s.GetFields()
+
+	if rangeVal, ok := fields["range"]; ok && rangeVal.GetListValue() != nil {
+		var events []ExtractedEvent
+		for _, evVal := range rangeVal.GetListValue().GetValues() {
+			evS := evVal.GetStructValue()
+			if evS == nil {
+				continue
+			}
+			evFields := evS.GetFields()
+			var ev ExtractedEvent
+			if intro, ok := evFields["introduced"]; ok {
+				ev.Introduced = intro.GetStringValue()
+			}
+			if fixed, ok := evFields["fixed"]; ok {
+				ev.Fixed = fixed.GetStringValue()
+			}
+			if la, ok := evFields["last_affected"]; ok {
+				ev.LastAffected = la.GetStringValue()
+			}
+			if lim, ok := evFields["limit"]; ok {
+				ev.Limit = lim.GetStringValue()
+			}
+			events = append(events, ev)
+		}
+
+		return events
+	}
+
+	// Fallback to old flat structure (single event)
 	var ev ExtractedEvent
 	if intro, ok := fields["introduced"]; ok {
 		ev.Introduced = intro.GetStringValue()
@@ -627,17 +656,21 @@ func parseExtractedEvent(v *structpb.Value) ExtractedEvent {
 		ev.Limit = lim.GetStringValue()
 	}
 
-	return ev
+	if ev != (ExtractedEvent{}) {
+		return []ExtractedEvent{ev}
+	}
+
+	return nil
 }
 
 func parseExtractedEvents(r *osvschema.Range) []ExtractedEvent {
 	rawValues := getExtractedEvents(r)
 	if len(rawValues) == 0 {
 		return nil
 	}
-	events := make([]ExtractedEvent, 0, len(rawValues))
+	var events []ExtractedEvent
 	for _, val := range rawValues {
-		events = append(events, parseExtractedEvent(val))
+		events = append(events, parseExtractedEvent(val)...)
 	}
 
 	return events
@@ -688,6 +721,29 @@ func isCPERange(r *osvschema.Range) bool {
 	if fields == nil {
 		return false
 	}
+
+	// Check new location inside extracted_events first
+	if extractedEventsVal, ok := fields["extracted_events"]; ok && extractedEventsVal.GetListValue() != nil {
+		for _, groupVal := range extractedEventsVal.GetListValue().GetValues() {
+			groupS := groupVal.GetStructValue()
+			if groupS == nil {
+				continue
+			}
+			groupFields := groupS.GetFields()
+			if sourceVal, ok := groupFields["source"]; ok {
+				if sourceVal.GetStringValue() == "CPE_RANGE" {
+					return true
+				}
+			}
+			if strategyVal, ok := groupFields["strategy"]; ok {
+				if strategyVal.GetStringValue() == "CPEApplicability" {
+					return true
+				}
+			}
+		}
+	}
+
+	// Fallback to old location
 	val, ok := fields["source"]
 	if !ok {
 		return false

vulnfeeds/conversion/common.go

@@ -293,16 +293,7 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri
 				}
 				successfulRepos = append(successfulRepos, repo)
 				if len(vr.Range.GetEvents()) > 0 {
-					dbSpecificMap := map[string]any{
-						"extracted_events": vr.Range.GetEvents(),
-					}
-					if vr.Metadata.CPE != "" {
-						dbSpecificMap["cpe"] = vr.Metadata.CPE
-					}
-					if string(vr.Metadata.Source) != "" {
-						dbSpecificMap["source"] = string(vr.Metadata.Source)
-					}
-					databaseSpecific, err := utility.NewStructpbFromMap(dbSpecificMap)
+					databaseSpecific, err := buildRangeDatabaseSpecific(vr)
 					if err != nil {
 						metrics.AddNotef("failed to make database specific: %v", err)
 					} else {
@@ -324,6 +315,29 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri
 	return newVersionRanges, unresolvedRanges, successfulRepos
 }
 
+func buildRangeDatabaseSpecific(vr models.RangeWithMetadata) (*structpb.Struct, error) {
+	extractedEventGroup := map[string]any{
+		"range": vr.Range.GetEvents(),
+	}
+	if vr.Metadata.CPE != "" {
+		extractedEventGroup["cpe"] = vr.Metadata.CPE
+	}
+	if string(vr.Metadata.Source) != "" {
+		extractedEventGroup["source"] = string(vr.Metadata.Source)
+	}
+	if vr.Metadata.Strategy != "" {
+		extractedEventGroup["strategy"] = vr.Metadata.Strategy
+	}
+	if vr.Metadata.OriginalTag != "" {
+		extractedEventGroup["original_tag"] = vr.Metadata.OriginalTag
+	}
+	dbSpecificMap := map[string]any{
+		"extracted_events": []any{extractedEventGroup},
+	}
+
+	return utility.NewStructpbFromMap(dbSpecificMap)
+}
+
 // BuildVersionRange is a helper function that adds 'introduced', 'fixed', or 'last_affected'
 // events to an OSV version range. If 'intro' is empty, it defaults to "0".
 func BuildVersionRange(intro string, lastAff string, fixed string) *osvschema.Range {
@@ -555,7 +569,9 @@ func CreateUnresolvedRanges(unresolvedRanges []models.RangeWithMetadata) *struct
 
 	type key struct {
 		Source        string
+		Strategy      string
 		VendorProduct string
+		OriginalTag   string
 	}
 
 	rangesByKey := make(map[key][]models.RangeWithMetadata)
@@ -570,7 +586,12 @@ func CreateUnresolvedRanges(unresolvedRanges []models.RangeWithMetadata) *struct
 				vendorProduct = ur.Metadata.CPE
 			}
 		}
-		k := key{Source: string(ur.Metadata.Source), VendorProduct: vendorProduct}
+		k := key{
+			Source:        string(ur.Metadata.Source),
+			Strategy:      ur.Metadata.Strategy,
+			VendorProduct: vendorProduct,
+			OriginalTag:   ur.Metadata.OriginalTag,
+		}
 		if _, ok := rangesByKey[k]; !ok {
 			keys = append(keys, k)
 		}
@@ -581,8 +602,14 @@ func CreateUnresolvedRanges(unresolvedRanges []models.RangeWithMetadata) *struct
 		if a.Source != b.Source {
 			return strings.Compare(a.Source, b.Source)
 		}
+		if a.Strategy != b.Strategy {
+			return strings.Compare(a.Strategy, b.Strategy)
+		}
+		if a.VendorProduct != b.VendorProduct {
+			return strings.Compare(a.VendorProduct, b.VendorProduct)
+		}
 
-		return strings.Compare(a.VendorProduct, b.VendorProduct)
+		return strings.Compare(a.OriginalTag, b.OriginalTag)
 	})
 
 	listElements := make([]any, 0, len(keys))
@@ -623,14 +650,24 @@ func CreateUnresolvedRanges(unresolvedRanges []models.RangeWithMetadata) *struct
 		if k.VendorProduct != "" {
 			unresolvedRangesMap["vendor_product"] = k.VendorProduct
 		}
-		if k.Source != "" {
-			unresolvedRangesMap["source"] = k.Source
-		}
 		if len(cpes) > 0 {
 			unresolvedRangesMap["cpes"] = cpes
 		}
 
-		unresolvedRangesMap["extracted_events"] = events
+		extractedEventGroup := map[string]any{
+			"range": events,
+		}
+		if k.Source != "" {
+			extractedEventGroup["source"] = k.Source
+		}
+		if k.Strategy != "" {
+			extractedEventGroup["strategy"] = k.Strategy
+		}
+		if k.OriginalTag != "" {
+			extractedEventGroup["original_tag"] = k.OriginalTag
+		}
+
+		unresolvedRangesMap["extracted_events"] = []any{extractedEventGroup}
 		listElements = append(listElements, unresolvedRangesMap)
 	}
 
@@ -807,6 +844,13 @@ func ProcessRanges(ranges []models.RangeWithMetadata, repos []string, metrics *m
 			if repo != "" {
 				vr.Range.Repo = repo
 				vr.Range.Type = osvschema.Range_GIT
+				if len(vr.Range.GetEvents()) > 0 && vr.Range.GetDatabaseSpecific() == nil {
+					if databaseSpecific, err := buildRangeDatabaseSpecific(vr); err != nil {
+						metrics.AddNotef("failed to make database specific: %v", err)
+					} else {
+						vr.Range.DatabaseSpecific = databaseSpecific
+					}
+				}
 				resolvedRanges = append(resolvedRanges, vr)
 				successfulRepos = append(successfulRepos, repo)
 			} else {

vulnfeeds/conversion/common_test.go

@@ -392,7 +392,6 @@ func TestCreateUnresolvedRanges(t *testing.T) {
 							StructValue: &structpb.Struct{
 								Fields: map[string]*structpb.Value{
 									"vendor_product": structpb.NewStringValue("another:app"),
-									"source":         structpb.NewStringValue(string(models.VersionSourceCPE)),
 									"cpes": structpb.NewListValue(&structpb.ListValue{
 										Values: []*structpb.Value{
 											structpb.NewStringValue("cpe:2.3:a:another:app:*:*:*:*:*:*:*:*"),
@@ -406,7 +405,20 @@ func TestCreateUnresolvedRanges(t *testing.T) {
 														Kind: &structpb.Value_StructValue{
 															StructValue: &structpb.Struct{
 																Fields: map[string]*structpb.Value{
-																	"fixed": structpb.NewStringValue("2.0"),
+																	"source": structpb.NewStringValue(string(models.VersionSourceCPE)),
+																	"range": structpb.NewListValue(&structpb.ListValue{
+																		Values: []*structpb.Value{
+																			{
+																				Kind: &structpb.Value_StructValue{
+																					StructValue: &structpb.Struct{
+																						Fields: map[string]*structpb.Value{
+																							"fixed": structpb.NewStringValue("2.0"),
+																						},
+																					},
+																				},
+																			},
+																		},
+																	}),
 																},
 															},
 														},
@@ -424,7 +436,6 @@ func TestCreateUnresolvedRanges(t *testing.T) {
 							StructValue: &structpb.Struct{
 								Fields: map[string]*structpb.Value{
 									"vendor_product": structpb.NewStringValue("example:app"),
-									"source":         structpb.NewStringValue(string(models.VersionSourceDescription)),
 									"cpes": structpb.NewListValue(&structpb.ListValue{
 										Values: []*structpb.Value{
 											structpb.NewStringValue("cpe:2.3:a:example:app:*:*:*:*:*:*:*:*"),
@@ -438,7 +449,20 @@ func TestCreateUnresolvedRanges(t *testing.T) {
 														Kind: &structpb.Value_StructValue{
 															StructValue: &structpb.Struct{
 																Fields: map[string]*structpb.Value{
-																	"introduced": structpb.NewStringValue("1.0"),
+																	"source": structpb.NewStringValue(string(models.VersionSourceDescription)),
+																	"range": structpb.NewListValue(&structpb.ListValue{
+																		Values: []*structpb.Value{
+																			{
+																				Kind: &structpb.Value_StructValue{
+																					StructValue: &structpb.Struct{
+																						Fields: map[string]*structpb.Value{
+																							"introduced": structpb.NewStringValue("1.0"),
+																						},
+																					},
+																				},
+																			},
+																		},
+																	}),
 																},
 															},
 														},