From 1756e9496ef1c761ff9d5eef83cfaf385c85af01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Sobo=C5=84?= Date: Thu, 23 Jul 2026 19:49:13 +0200 Subject: [PATCH] [Cloud Asset Inventory] Fix entity.attribute mapping bug by renaming it to entity.Details (#7473) ### Summary of your changes For: #incident-3395-failing-quality-gate-for-entity-store-synthetics Incident Slack: https://elastic.slack.com/archives/C0BJQCSBT47 Publishing asset details as `entity.attributes` breaks Entity Store extraction. This, along with related integrations PR, fixes the issue by using `entity.Details` instead. Integrations PR contains an ingest pipeline that ensures even old Cloudbeat versions that publish `entity.attributes` get remapped to `Details`. This allows extraction to work as intended. ### Related Issues Towards https://github.com/elastic/security-team/issues/18408 (cherry picked from commit 815379b795bbc99eb0d6cf9e0b700756e64aa2cb) --- internal/inventory/asset.go | 26 ++++++++--------- .../awsfetcher/fetcher_ec2_instance.go | 28 +++++++++---------- .../awsfetcher/fetcher_ec2_instance_test.go | 4 +-- internal/inventory/awsfetcher/fetcher_eks.go | 22 +++++++-------- .../inventory/awsfetcher/fetcher_eks_test.go | 4 +-- internal/inventory/awsfetcher/fetcher_elb.go | 16 +++++------ .../inventory/awsfetcher/fetcher_elb_test.go | 4 +-- internal/inventory/awsfetcher/fetcher_rds.go | 8 +++--- .../inventory/awsfetcher/fetcher_rds_test.go | 4 +-- .../inventory/awsfetcher/fetcher_route53.go | 28 +++++++++---------- .../awsfetcher/fetcher_route53_test.go | 4 +-- 11 files changed, 74 insertions(+), 74 deletions(-) diff --git a/internal/inventory/asset.go b/internal/inventory/asset.go index 859dec963d..0fe6ba8d05 100644 --- a/internal/inventory/asset.go +++ b/internal/inventory/asset.go @@ -182,11 +182,11 @@ type URL struct { // Entity contains the identifiers of the asset type Entity struct { - Id string `json:"id"` - Name string `json:"name"` - Source *string `json:"source"` - Raw *any `json:"raw"` - Attributes map[string]any `json:"attributes,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + Source *string `json:"source"` + Raw *any `json:"raw"` + Details map[string]any `json:"Details,omitempty"` AssetClassification // non exported fields @@ -451,28 +451,28 @@ func WithContainer(container Container) AssetEnricher { } } -// WithEntityAttributes sets non-ECS resource-specific attributes on the entity. +// WithEntityDetails sets non-ECS resource-specific attributes on the entity (entity.Details). // Keys should use UpperCamelCase per the non-ECS field naming convention. // A nil or empty map is a no-op. -func WithEntityAttributes(attrs map[string]any) AssetEnricher { +func WithEntityDetails(details map[string]any) AssetEnricher { return func(a *AssetEvent) { - if len(attrs) == 0 { + if len(details) == 0 { return } - a.Entity.Attributes = attrs + a.Entity.Details = details } } -// WithCreatedAt sets the resource creation timestamp in entity.attributes["CreatedAt"]. +// WithCreatedAt sets the resource creation timestamp in entity.Details["CreatedAt"]. // A nil time is a no-op. func WithCreatedAt(t *time.Time) AssetEnricher { return func(a *AssetEvent) { if t == nil { return } - if a.Entity.Attributes == nil { - a.Entity.Attributes = make(map[string]any) + if a.Entity.Details == nil { + a.Entity.Details = make(map[string]any) } - a.Entity.Attributes["CreatedAt"] = t + a.Entity.Details["CreatedAt"] = t } } diff --git a/internal/inventory/awsfetcher/fetcher_ec2_instance.go b/internal/inventory/awsfetcher/fetcher_ec2_instance.go index a81dfd537a..6d59401cf9 100644 --- a/internal/inventory/awsfetcher/fetcher_ec2_instance.go +++ b/internal/inventory/awsfetcher/fetcher_ec2_instance.go @@ -128,7 +128,7 @@ func (e *ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inve IP: buildIPs(i.PublicIpAddress, i.PrivateIpAddress), MacAddress: i.GetResourceMacAddresses(), }), - inventory.WithEntityAttributes(e.buildAttributes(i, tags, roleArnCache)), + inventory.WithEntityDetails(e.buildDetails(i, tags, roleArnCache)), inventory.WithCreatedAt(i.LaunchTime), iamFetcher, ) @@ -177,43 +177,43 @@ func profileNameFromArn(arn string) string { return arn[idx+len(marker):] } -// buildAttributes collects non-ECS, resource-specific EC2 fields into entity.attributes, +// buildDetails collects non-ECS, resource-specific EC2 fields into entity.Details, // using UpperCamelCase keys. Empty values are omitted so events stay clean and struct // comparison in tests is stable. -func (e *ec2InstanceFetcher) buildAttributes(i *ec2.Ec2Instance, tags map[string]string, roleArnCache map[string]string) map[string]any { - attrs := map[string]any{} +func (e *ec2InstanceFetcher) buildDetails(i *ec2.Ec2Instance, tags map[string]string, roleArnCache map[string]string) map[string]any { + details := map[string]any{} if v := pointers.Deref(i.ImageId); v != "" { - attrs["ImageId"] = v + details["ImageId"] = v } if v := string(i.Platform); v != "" { - attrs["Platform"] = v + details["Platform"] = v } if v := pointers.Deref(i.VpcId); v != "" { - attrs["VpcId"] = v + details["VpcId"] = v } if v := pointers.Deref(i.SubnetId); v != "" { - attrs["SubnetId"] = v + details["SubnetId"] = v } if i.State != nil { if v := string(i.State.Name); v != "" { - attrs["State"] = v + details["State"] = v } } if i.IamInstanceProfile != nil { if profileArn := pointers.Deref(i.IamInstanceProfile.Arn); profileArn != "" { - attrs["InstanceProfileArn"] = profileArn + details["InstanceProfileArn"] = profileArn if roleArn, ok := roleArnCache[profileArn]; ok && roleArn != "" { - attrs["RoleArn"] = roleArn + details["RoleArn"] = roleArn } } } if v := awslib.LookupTag(tags, "owner"); v != "" { - attrs["Owner"] = v + details["Owner"] = v } if v := awslib.LookupTag(tags, "costcenter", "cost-center", "cost_center"); v != "" { - attrs["CostCenter"] = v + details["CostCenter"] = v } - return attrs + return details } // buildIPs collects non-empty IP address strings into a slice, returning nil when none exist. diff --git a/internal/inventory/awsfetcher/fetcher_ec2_instance_test.go b/internal/inventory/awsfetcher/fetcher_ec2_instance_test.go index b4d022b025..32d6d2f779 100644 --- a/internal/inventory/awsfetcher/fetcher_ec2_instance_test.go +++ b/internal/inventory/awsfetcher/fetcher_ec2_instance_test.go @@ -124,7 +124,7 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) { IP: []string{"public-ip-addr", "private-ip-addre"}, MacAddress: []string{"mac1", "mac2"}, }), - inventory.WithEntityAttributes(map[string]any{ + inventory.WithEntityDetails(map[string]any{ "ImageId": "image-id", "Platform": "linux", "VpcId": "vpc-id", @@ -213,7 +213,7 @@ func TestEC2InstanceFetcher_Fetch_ResolverError(t *testing.T) { IP: []string{"public-ip-addr", "private-ip-addre"}, MacAddress: []string{"mac1", "mac2"}, }), - inventory.WithEntityAttributes(map[string]any{ + inventory.WithEntityDetails(map[string]any{ "ImageId": "image-id", "Platform": "linux", "VpcId": "vpc-id", diff --git a/internal/inventory/awsfetcher/fetcher_eks.go b/internal/inventory/awsfetcher/fetcher_eks.go index 4b13fc070f..0b62537534 100644 --- a/internal/inventory/awsfetcher/fetcher_eks.go +++ b/internal/inventory/awsfetcher/fetcher_eks.go @@ -78,37 +78,37 @@ func (f *eksFetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.As AccountName: f.accountName, ServiceName: "AWS EKS", }), - inventory.WithEntityAttributes(buildEKSAttributes(cluster)), + inventory.WithEntityDetails(buildEKSDetails(cluster)), inventory.WithCreatedAt(cluster.CreatedAt), ) } } -// buildEKSAttributes maps a cluster's non-ECS fields into entity.attributes using +// buildEKSDetails maps a cluster's non-ECS fields into entity.Details using // UpperCamelCase keys. The endpoint-access booleans are always included as they are // meaningful even when false; other empty values are omitted. -func buildEKSAttributes(cluster eks.Cluster) map[string]any { - attrs := map[string]any{ +func buildEKSDetails(cluster eks.Cluster) map[string]any { + details := map[string]any{ "EndpointPublicAccess": cluster.EndpointPublicAccess, "EndpointPrivateAccess": cluster.EndpointPrivateAccess, } if cluster.Status != "" { - attrs["Status"] = cluster.Status + details["Status"] = cluster.Status } if cluster.Version != "" { - attrs["Version"] = cluster.Version + details["Version"] = cluster.Version } if cluster.Endpoint != "" { - attrs["Endpoint"] = cluster.Endpoint + details["Endpoint"] = cluster.Endpoint } if cluster.RoleArn != "" { - attrs["RoleArn"] = cluster.RoleArn + details["RoleArn"] = cluster.RoleArn } if cluster.PlatformVersion != "" { - attrs["PlatformVersion"] = cluster.PlatformVersion + details["PlatformVersion"] = cluster.PlatformVersion } if v := cluster.GetOwnerTag(); v != "" { - attrs["OwnerTag"] = v + details["OwnerTag"] = v } - return attrs + return details } diff --git a/internal/inventory/awsfetcher/fetcher_eks_test.go b/internal/inventory/awsfetcher/fetcher_eks_test.go index bae2a1fa6c..c37998a1d4 100644 --- a/internal/inventory/awsfetcher/fetcher_eks_test.go +++ b/internal/inventory/awsfetcher/fetcher_eks_test.go @@ -68,7 +68,7 @@ func TestEKSFetcher_Fetch(t *testing.T) { AccountName: "alias", ServiceName: "AWS EKS", }), - inventory.WithEntityAttributes(map[string]any{ + inventory.WithEntityDetails(map[string]any{ "EndpointPublicAccess": true, "EndpointPrivateAccess": false, "Status": "ACTIVE", @@ -91,7 +91,7 @@ func TestEKSFetcher_Fetch(t *testing.T) { AccountName: "alias", ServiceName: "AWS EKS", }), - inventory.WithEntityAttributes(map[string]any{ + inventory.WithEntityDetails(map[string]any{ "EndpointPublicAccess": false, "EndpointPrivateAccess": false, }), diff --git a/internal/inventory/awsfetcher/fetcher_elb.go b/internal/inventory/awsfetcher/fetcher_elb.go index 18c57c82e8..e1aa0d912c 100644 --- a/internal/inventory/awsfetcher/fetcher_elb.go +++ b/internal/inventory/awsfetcher/fetcher_elb.go @@ -94,27 +94,27 @@ func (f *elbFetcher) fetch(ctx context.Context, resourceName string, function el } for _, item := range awsResources { - var attrs map[string]any + var details map[string]any var createdAt *time.Time if r, ok := item.(elbInventoryResource); ok { - attrs = map[string]any{ + details = map[string]any{ "DNSName": r.GetDNSName(), "PubliclyAccessible": r.IsPubliclyAccessible(), } if f.AccountId != "" { - attrs["AccountID"] = f.AccountId + details["AccountID"] = f.AccountId } if v := r.GetLoadBalancerType(); v != "" { - attrs["LoadBalancerType"] = v + details["LoadBalancerType"] = v } if v := r.GetState(); v != "" { - attrs["State"] = v + details["State"] = v } if v := r.GetIPAddresses(); len(v) > 0 { - attrs["IPAddresses"] = v + details["IPAddresses"] = v } if v := r.GetOwnerTag(); v != "" { - attrs["OwnerTag"] = v + details["OwnerTag"] = v } createdAt = r.GetCreatedAt() } @@ -131,7 +131,7 @@ func (f *elbFetcher) fetch(ctx context.Context, resourceName string, function el AccountName: f.AccountName, ServiceName: "AWS Networking", }), - inventory.WithEntityAttributes(attrs), + inventory.WithEntityDetails(details), inventory.WithCreatedAt(createdAt), ) } diff --git a/internal/inventory/awsfetcher/fetcher_elb_test.go b/internal/inventory/awsfetcher/fetcher_elb_test.go index eee28cc2fd..2f4dbc2be1 100644 --- a/internal/inventory/awsfetcher/fetcher_elb_test.go +++ b/internal/inventory/awsfetcher/fetcher_elb_test.go @@ -79,7 +79,7 @@ func TestELBv1Fetcher_Fetch(t *testing.T) { AccountName: "alias", ServiceName: "AWS Networking", }), - inventory.WithEntityAttributes(map[string]any{ + inventory.WithEntityDetails(map[string]any{ "DNSName": "internal-my-elb-v1.us-east-1.elb.amazonaws.com", "PubliclyAccessible": false, // scheme is "internal" "AccountID": "123", @@ -134,7 +134,7 @@ func TestELBv2Fetcher_Fetch(t *testing.T) { AccountName: "alias", ServiceName: "AWS Networking", }), - inventory.WithEntityAttributes(map[string]any{ + inventory.WithEntityDetails(map[string]any{ "DNSName": "internal-my-elb-v2.us-east-1.elb.amazonaws.com", "PubliclyAccessible": false, // scheme is internal "AccountID": "123", diff --git a/internal/inventory/awsfetcher/fetcher_rds.go b/internal/inventory/awsfetcher/fetcher_rds.go index c436dd02ba..ad6a81cc01 100644 --- a/internal/inventory/awsfetcher/fetcher_rds.go +++ b/internal/inventory/awsfetcher/fetcher_rds.go @@ -70,14 +70,14 @@ func (s *rdsFetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.As }) for _, item := range rdsInstances { - attrs := map[string]any{ + details := map[string]any{ "PubliclyAccessible": item.PubliclyAccessible, } if item.Engine != "" { - attrs["Engine"] = item.Engine + details["Engine"] = item.Engine } if item.EngineVersion != "" { - attrs["EngineVersion"] = item.EngineVersion + details["EngineVersion"] = item.EngineVersion } assetChannel <- inventory.NewAssetEvent( @@ -93,7 +93,7 @@ func (s *rdsFetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.As AccountName: s.AccountName, ServiceName: "AWS RDS", }), - inventory.WithEntityAttributes(attrs), + inventory.WithEntityDetails(details), inventory.WithCreatedAt(item.CreatedAt), ) } diff --git a/internal/inventory/awsfetcher/fetcher_rds_test.go b/internal/inventory/awsfetcher/fetcher_rds_test.go index 0ad735df19..89d2deb49a 100644 --- a/internal/inventory/awsfetcher/fetcher_rds_test.go +++ b/internal/inventory/awsfetcher/fetcher_rds_test.go @@ -103,7 +103,7 @@ func TestRDSInstanceFetcher_Fetch(t *testing.T) { AccountName: "alias", ServiceName: "AWS RDS", }), - inventory.WithEntityAttributes(map[string]any{ + inventory.WithEntityDetails(map[string]any{ "PubliclyAccessible": false, "Engine": "postgres", "EngineVersion": "15.4", @@ -122,7 +122,7 @@ func TestRDSInstanceFetcher_Fetch(t *testing.T) { AccountName: "alias", ServiceName: "AWS RDS", }), - inventory.WithEntityAttributes(map[string]any{ + inventory.WithEntityDetails(map[string]any{ "PubliclyAccessible": true, "Engine": "mysql", "EngineVersion": "8.0.35", diff --git a/internal/inventory/awsfetcher/fetcher_route53.go b/internal/inventory/awsfetcher/fetcher_route53.go index 8e8e6ac26e..5412b552dd 100644 --- a/internal/inventory/awsfetcher/fetcher_route53.go +++ b/internal/inventory/awsfetcher/fetcher_route53.go @@ -78,41 +78,41 @@ func (f *route53Fetcher) Fetch(ctx context.Context, assetChannel chan<- inventor AccountName: f.accountName, ServiceName: "AWS Route 53", }), - inventory.WithEntityAttributes(buildRoute53Attributes(record)), + inventory.WithEntityDetails(buildRoute53Details(record)), ) } } -// buildRoute53Attributes maps a record's non-ECS fields into entity.attributes using +// buildRoute53Details maps a record's non-ECS fields into entity.Details using // UpperCamelCase keys. Empty values are omitted. -func buildRoute53Attributes(record route53.Record) map[string]any { - attrs := map[string]any{} +func buildRoute53Details(record route53.Record) map[string]any { + details := map[string]any{} if record.Type != "" { - attrs["Type"] = record.Type + details["Type"] = record.Type } if len(record.Records) > 0 { - attrs["ResourceRecords"] = record.Records + details["ResourceRecords"] = record.Records } if record.Weight != nil { - attrs["Weight"] = *record.Weight + details["Weight"] = *record.Weight } if record.RoutingRegion != "" { - attrs["Region"] = record.RoutingRegion + details["Region"] = record.RoutingRegion } if record.ZoneID != "" { - attrs["ZoneID"] = record.ZoneID + details["ZoneID"] = record.ZoneID } if record.ZoneName != "" { - attrs["ZoneName"] = record.ZoneName + details["ZoneName"] = record.ZoneName } if record.AliasTargetDNS != "" { - attrs["AliasTargetDNS"] = record.AliasTargetDNS + details["AliasTargetDNS"] = record.AliasTargetDNS } if record.AliasTargetZoneID != "" { - attrs["AliasTargetZoneId"] = record.AliasTargetZoneID + details["AliasTargetZoneId"] = record.AliasTargetZoneID } if record.HealthCheckID != "" { - attrs["HealthCheckId"] = record.HealthCheckID + details["HealthCheckId"] = record.HealthCheckID } - return attrs + return details } diff --git a/internal/inventory/awsfetcher/fetcher_route53_test.go b/internal/inventory/awsfetcher/fetcher_route53_test.go index ab531d5daa..424f0148c7 100644 --- a/internal/inventory/awsfetcher/fetcher_route53_test.go +++ b/internal/inventory/awsfetcher/fetcher_route53_test.go @@ -68,7 +68,7 @@ func TestRoute53Fetcher_Fetch(t *testing.T) { AccountName: "alias", ServiceName: "AWS Route 53", }), - inventory.WithEntityAttributes(map[string]any{ + inventory.WithEntityDetails(map[string]any{ "Type": "A", "ResourceRecords": []string{"203.0.113.10"}, "Weight": int64(10), @@ -92,7 +92,7 @@ func TestRoute53Fetcher_Fetch(t *testing.T) { AccountName: "alias", ServiceName: "AWS Route 53", }), - inventory.WithEntityAttributes(map[string]any{ + inventory.WithEntityDetails(map[string]any{ "Type": "NS", "ZoneID": "Z123", "ZoneName": "example.com.",