Skip to content

Commit a9fff18

Browse files
bhshkhmeredithslotakolea2
authored
feat(datastore): SUM and AVG aggregations (#8307)
* feat(datastore): SUM and AVG aggregations * feat(datastore): Fixing integration tests for SUM AVG * feat(datastore): Fixing integration tests * feat(datastore): Fixing integration tests * feat(datastore): Updating protos * feat(datastore): updating protos * feat(datastore): Undo go.work.sum changes * feat(datastore): Used new protos * feat(datastore): Use latest protos --------- Co-authored-by: meredithslota <[email protected]> Co-authored-by: kolea2 <[email protected]>
1 parent fa6e827 commit a9fff18

File tree

5 files changed

+174
-41
lines changed

5 files changed

+174
-41
lines changed

‎datastore/go.mod‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,20 @@ module cloud.google.com/go/datastore
33
go 1.19
44

55
require (
6-
cloud.google.com/go v0.110.2
7-
cloud.google.com/go/longrunning v0.5.0
6+
cloud.google.com/go v0.110.7
7+
cloud.google.com/go/longrunning v0.5.1
88
github.com/golang/protobuf v1.5.3
99
github.com/google/go-cmp v0.5.9
1010
github.com/googleapis/gax-go/v2 v2.12.0
1111
google.golang.org/api v0.128.0
12-
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc
13-
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc
14-
google.golang.org/grpc v1.56.1
12+
google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93
13+
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5
14+
google.golang.org/grpc v1.57.0
1515
google.golang.org/protobuf v1.31.0
1616
)
1717

1818
require (
19-
cloud.google.com/go/compute v1.19.3 // indirect
19+
cloud.google.com/go/compute v1.23.0 // indirect
2020
cloud.google.com/go/compute/metadata v0.2.3 // indirect
2121
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
2222
github.com/google/s2a-go v0.1.4 // indirect
@@ -25,9 +25,10 @@ require (
2525
golang.org/x/crypto v0.9.0 // indirect
2626
golang.org/x/net v0.10.0 // indirect
2727
golang.org/x/oauth2 v0.8.0 // indirect
28+
golang.org/x/sync v0.2.0 // indirect
2829
golang.org/x/sys v0.8.0 // indirect
2930
golang.org/x/text v0.9.0 // indirect
3031
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
3132
google.golang.org/appengine v1.6.7 // indirect
32-
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
33+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect
3334
)

‎datastore/go.sum‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
22
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
3-
cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
4-
cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
5-
cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
6-
cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
3+
cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o=
4+
cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
5+
cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY=
6+
cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
77
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
88
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
9-
cloud.google.com/go/longrunning v0.5.0 h1:DK8BH0+hS+DIvc9a2TPnteUievsTCH4ORMAASSb7JcQ=
10-
cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc=
9+
cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI=
10+
cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc=
1111
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
1212
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
1313
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -118,6 +118,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
118118
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
119119
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
120120
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
121+
golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
121122
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
122123
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
123124
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -162,12 +163,12 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA
162163
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
163164
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
164165
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
165-
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
166-
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
167-
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
168-
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
169-
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
170-
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
166+
google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93 h1:zv6ieVm8jNcN33At1+APsRISkRgynuWUxUhv6G123jY=
167+
google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
168+
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44=
169+
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q=
170+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg=
171+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I=
171172
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
172173
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
173174
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
@@ -176,8 +177,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
176177
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
177178
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
178179
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
179-
google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
180-
google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
180+
google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
181+
google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
181182
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
182183
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
183184
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

‎datastore/integration_test.go‎

Lines changed: 92 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import (
3737
"google.golang.org/grpc"
3838
"google.golang.org/grpc/codes"
3939
"google.golang.org/grpc/status"
40+
"google.golang.org/protobuf/types/known/structpb"
4041
)
4142

4243
// TODO(djd): Make test entity clean up more robust: some test entities may
@@ -52,6 +53,7 @@ var suffix string
5253
const (
5354
replayFilename = "datastore.replay"
5455
envDatabases = "GCLOUD_TESTS_GOLANG_DATASTORE_DATABASES"
56+
keyPrefix = "TestIntegration_"
5557
)
5658

5759
type replayInfo struct {
@@ -471,6 +473,8 @@ func TestIntegration_NilKey(t *testing.T) {
471473
type SQChild struct {
472474
I, J int
473475
T, U int64
476+
V float64
477+
W string
474478
}
475479

476480
type SQTestCase struct {
@@ -701,17 +705,17 @@ func TestIntegration_AggregationQueries(t *testing.T) {
701705
client := newTestClient(ctx, t)
702706
defer client.Close()
703707

704-
parent := NameKey("SQParent", "TestIntegration_Filters"+suffix, nil)
708+
parent := NameKey("SQParent", keyPrefix+"AggregationQueries"+suffix, nil)
705709
now := timeNow.Truncate(time.Millisecond).Unix()
706710
children := []*SQChild{
707-
{I: 0, T: now, U: now},
708-
{I: 1, T: now, U: now},
709-
{I: 2, T: now, U: now},
710-
{I: 3, T: now, U: now},
711-
{I: 4, T: now, U: now},
712-
{I: 5, T: now, U: now},
713-
{I: 6, T: now, U: now},
714-
{I: 7, T: now, U: now},
711+
{I: 0, T: now, U: now, V: 1.5, W: "str"},
712+
{I: 1, T: now, U: now, V: 1.5, W: "str"},
713+
{I: 2, T: now, U: now, V: 1.5, W: "str"},
714+
{I: 3, T: now, U: now, V: 1.5, W: "str"},
715+
{I: 4, T: now, U: now, V: 1.5, W: "str"},
716+
{I: 5, T: now, U: now, V: 1.5, W: "str"},
717+
{I: 6, T: now, U: now, V: 1.5, W: "str"},
718+
{I: 7, T: now, U: now, V: 1.5, W: "str"},
715719
}
716720

717721
keys := make([]*Key, len(children))
@@ -729,7 +733,6 @@ func TestIntegration_AggregationQueries(t *testing.T) {
729733
}
730734
}()
731735

732-
baseQuery := NewQuery("SQChild").Ancestor(parent)
733736
testCases := []struct {
734737
desc string
735738
aggQuery *AggregationQuery
@@ -738,21 +741,91 @@ func TestIntegration_AggregationQueries(t *testing.T) {
738741
wantAggResult AggregationResult
739742
}{
740743
{
741-
desc: "Count Failure - Missing index",
742-
aggQuery: baseQuery.Filter("T>=", now).NewAggregationQuery().WithCount("count"),
743-
wantFailure: true,
744-
wantErrMsg: "no matching index found",
745-
wantAggResult: nil,
744+
desc: "Count Failure - Missing index",
745+
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T>=", now).
746+
NewAggregationQuery().
747+
WithCount("count"),
748+
wantFailure: true,
749+
wantErrMsg: "no matching index found",
746750
},
747751
{
748-
desc: "Count Success",
749-
aggQuery: baseQuery.Filter("T=", now).Filter("I>=", 3).NewAggregationQuery().WithCount("count"),
750-
wantFailure: false,
751-
wantErrMsg: "",
752+
desc: "Count Success",
753+
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Filter("I>=", 3).
754+
NewAggregationQuery().
755+
WithCount("count"),
752756
wantAggResult: map[string]interface{}{
753757
"count": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: 5}},
754758
},
755759
},
760+
{
761+
desc: "Multiple aggregations",
762+
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).
763+
NewAggregationQuery().
764+
WithSum("I", "i_sum").
765+
WithAvg("I", "avg").
766+
WithSum("V", "v_sum"),
767+
wantAggResult: map[string]interface{}{
768+
"i_sum": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: 28}},
769+
"v_sum": &pb.Value{ValueType: &pb.Value_DoubleValue{DoubleValue: 12}},
770+
"avg": &pb.Value{ValueType: &pb.Value_DoubleValue{DoubleValue: 3.5}},
771+
},
772+
},
773+
{
774+
desc: "Multiple aggregations with limit ",
775+
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Limit(2).
776+
NewAggregationQuery().
777+
WithSum("I", "sum").
778+
WithAvg("I", "avg"),
779+
wantAggResult: map[string]interface{}{
780+
"sum": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: 1}},
781+
"avg": &pb.Value{ValueType: &pb.Value_DoubleValue{DoubleValue: 0.5}},
782+
},
783+
},
784+
{
785+
desc: "Multiple aggregations on non-numeric field",
786+
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Limit(2).
787+
NewAggregationQuery().
788+
WithSum("W", "sum").
789+
WithAvg("W", "avg"),
790+
wantAggResult: map[string]interface{}{
791+
"sum": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: int64(0)}},
792+
"avg": &pb.Value{ValueType: &pb.Value_NullValue{NullValue: structpb.NullValue_NULL_VALUE}},
793+
},
794+
},
795+
{
796+
desc: "Sum aggregation without alias",
797+
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).
798+
NewAggregationQuery().
799+
WithSum("I", ""),
800+
wantAggResult: map[string]interface{}{
801+
"property_1": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: 28}},
802+
},
803+
},
804+
{
805+
desc: "Average aggregation without alias",
806+
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).
807+
NewAggregationQuery().
808+
WithAvg("I", ""),
809+
wantAggResult: map[string]interface{}{
810+
"property_1": &pb.Value{ValueType: &pb.Value_DoubleValue{DoubleValue: 3.5}},
811+
},
812+
},
813+
{
814+
desc: "Sum aggregation on '__key__'",
815+
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).
816+
NewAggregationQuery().
817+
WithSum("__key__", ""),
818+
wantFailure: true,
819+
wantErrMsg: "Aggregations are not supported for the property",
820+
},
821+
{
822+
desc: "Average aggregation on '__key__'",
823+
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).
824+
NewAggregationQuery().
825+
WithAvg("__key__", ""),
826+
wantFailure: true,
827+
wantErrMsg: "Aggregations are not supported for the property",
828+
},
756829
}
757830

758831
for _, testCase := range testCases {

‎datastore/query.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1057,5 +1057,49 @@ func (aq *AggregationQuery) WithCount(alias string) *AggregationQuery {
10571057
return aq
10581058
}
10591059

1060+
// WithSum specifies that the aggregation query should provide a sum of the values
1061+
// of the provided field in the results returned by the underlying Query.
1062+
// The alias argument can be empty or a valid Datastore entity property name. It can be used
1063+
// as key in the AggregationResult to get the sum value. If alias is empty, Datastore
1064+
// will autogenerate a key.
1065+
func (aq *AggregationQuery) WithSum(fieldName string, alias string) *AggregationQuery {
1066+
aqpb := &pb.AggregationQuery_Aggregation{
1067+
Alias: alias,
1068+
Operator: &pb.AggregationQuery_Aggregation_Sum_{
1069+
Sum: &pb.AggregationQuery_Aggregation_Sum{
1070+
Property: &pb.PropertyReference{
1071+
Name: fieldName,
1072+
},
1073+
},
1074+
},
1075+
}
1076+
1077+
aq.aggregationQueries = append(aq.aggregationQueries, aqpb)
1078+
1079+
return aq
1080+
}
1081+
1082+
// WithAvg specifies that the aggregation query should provide an average of the values
1083+
// of the provided field in the results returned by the underlying Query.
1084+
// The alias argument can be empty or a valid Datastore entity property name. It can be used
1085+
// as key in the AggregationResult to get the sum value. If alias is empty, Datastore
1086+
// will autogenerate a key.
1087+
func (aq *AggregationQuery) WithAvg(fieldName string, alias string) *AggregationQuery {
1088+
aqpb := &pb.AggregationQuery_Aggregation{
1089+
Alias: alias,
1090+
Operator: &pb.AggregationQuery_Aggregation_Avg_{
1091+
Avg: &pb.AggregationQuery_Aggregation_Avg{
1092+
Property: &pb.PropertyReference{
1093+
Name: fieldName,
1094+
},
1095+
},
1096+
},
1097+
}
1098+
1099+
aq.aggregationQueries = append(aq.aggregationQueries, aqpb)
1100+
1101+
return aq
1102+
}
1103+
10601104
// AggregationResult contains the results of an aggregation query.
10611105
type AggregationResult map[string]interface{}

‎datastore/testdata/index.yaml‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,4 +38,18 @@ indexes:
3838
properties:
3939
- name: T
4040
- name: J
41-
- name: U
41+
- name: U
42+
43+
- kind: SQChild
44+
ancestor: yes
45+
properties:
46+
- name: T
47+
- name: I
48+
- name: V
49+
50+
- kind: SQChild
51+
ancestor: yes
52+
properties:
53+
- name: T
54+
- name: W
55+

0 commit comments

Comments
 (0)