Skip to content

Commit 4c25d13

Browse files
shollymanshollyman
committed
bigquery: add policytag support
This PR adds support for policyTags. It doesn't add an integration test currently as there's issues defining the policy tags in the data catalog API, but this was verified manually using some predefined tags. Also, this cleans up some unnecessary debug statements in an unrelated integration test. Change-Id: I5447bd8aaa7850ccfc82b2f503cb579ff7307291 Reviewed-on: https://code-review.googlesource.com/c/gocloud/+/54130 Reviewed-by: kokoro <[email protected]> Reviewed-by: Alex Hong <[email protected]>
1 parent 29a0e82 commit 4c25d13

4 files changed

Lines changed: 101 additions & 50 deletions

File tree

‎bigquery/integration_test.go‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -699,12 +699,7 @@ func TestIntegration_DatasetUpdateAccess(t *testing.T) {
699699
t.Log("could not restore dataset access list")
700700
}
701701
}()
702-
for _, v := range md.Access {
703-
fmt.Printf("md %+v\n", v)
704-
}
705-
for _, v := range newAccess {
706-
fmt.Printf("newAccess %+v\n", v)
707-
}
702+
708703
if diff := testutil.Diff(md.Access, newAccess, cmpopts.SortSlices(lessAccessEntries)); diff != "" {
709704
t.Fatalf("got=-, want=+:\n%s", diff)
710705
}

‎bigquery/schema.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ type FieldSchema struct {
6464
// The field data type. If Type is Record, then this field contains a nested schema,
6565
// which is described by Schema.
6666
Type FieldType
67+
68+
// Annotations for enforcing column-level security constraints.
69+
PolicyTags *PolicyTagList
70+
6771
// Describes the nested schema if Type is set to Record.
6872
Schema Schema
6973
}
@@ -73,6 +77,7 @@ func (fs *FieldSchema) toBQ() *bq.TableFieldSchema {
7377
Description: fs.Description,
7478
Name: fs.Name,
7579
Type: string(fs.Type),
80+
PolicyTags: fs.PolicyTags.toBQ(),
7681
}
7782

7883
if fs.Repeated {
@@ -88,6 +93,30 @@ func (fs *FieldSchema) toBQ() *bq.TableFieldSchema {
8893
return tfs
8994
}
9095

96+
// PolicyTagList represents the annotations on a schema column for enforcing column-level security.
97+
// For more information, see https://cloud.google.com/bigquery/docs/column-level-security-intro
98+
type PolicyTagList struct {
99+
Names []string
100+
}
101+
102+
func (ptl *PolicyTagList) toBQ() *bq.TableFieldSchemaPolicyTags {
103+
if ptl == nil {
104+
return nil
105+
}
106+
return &bq.TableFieldSchemaPolicyTags{
107+
Names: ptl.Names,
108+
}
109+
}
110+
111+
func bqToPolicyTagList(pt *bq.TableFieldSchemaPolicyTags) *PolicyTagList {
112+
if pt == nil {
113+
return nil
114+
}
115+
return &PolicyTagList{
116+
Names: pt.Names,
117+
}
118+
}
119+
91120
func (s Schema) toBQ() *bq.TableSchema {
92121
var fields []*bq.TableFieldSchema
93122
for _, f := range s {
@@ -103,6 +132,7 @@ func bqToFieldSchema(tfs *bq.TableFieldSchema) *FieldSchema {
103132
Repeated: tfs.Mode == "REPEATED",
104133
Required: tfs.Mode == "REQUIRED",
105134
Type: FieldType(tfs.Type),
135+
PolicyTags: bqToPolicyTagList(tfs.PolicyTags),
106136
}
107137

108138
for _, f := range tfs.Fields {

‎bigquery/schema_test.go‎

Lines changed: 66 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -42,22 +42,36 @@ func (fs *FieldSchema) GoString() string {
4242
)
4343
}
4444

45-
func bqTableFieldSchema(desc, name, typ, mode string) *bq.TableFieldSchema {
45+
func bqTableFieldSchema(desc, name, typ, mode string, tags []string) *bq.TableFieldSchema {
46+
var policy *bq.TableFieldSchemaPolicyTags
47+
if tags != nil {
48+
policy = &bq.TableFieldSchemaPolicyTags{
49+
Names: tags,
50+
}
51+
}
4652
return &bq.TableFieldSchema{
4753
Description: desc,
4854
Name: name,
4955
Mode: mode,
5056
Type: typ,
57+
PolicyTags: policy,
5158
}
5259
}
5360

54-
func fieldSchema(desc, name, typ string, repeated, required bool) *FieldSchema {
61+
func fieldSchema(desc, name, typ string, repeated, required bool, tags []string) *FieldSchema {
62+
var policy *PolicyTagList
63+
if tags != nil {
64+
policy = &PolicyTagList{
65+
Names: tags,
66+
}
67+
}
5568
return &FieldSchema{
5669
Description: desc,
5770
Name: name,
5871
Repeated: repeated,
5972
Required: required,
6073
Type: FieldType(typ),
74+
PolicyTags: policy,
6175
}
6276
}
6377

@@ -160,113 +174,125 @@ func TestSchemaConversion(t *testing.T) {
160174
// required
161175
bqSchema: &bq.TableSchema{
162176
Fields: []*bq.TableFieldSchema{
163-
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
177+
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED", nil),
164178
},
165179
},
166180
schema: Schema{
167-
fieldSchema("desc", "name", "STRING", false, true),
181+
fieldSchema("desc", "name", "STRING", false, true, nil),
168182
},
169183
},
170184
{
171185
// repeated
172186
bqSchema: &bq.TableSchema{
173187
Fields: []*bq.TableFieldSchema{
174-
bqTableFieldSchema("desc", "name", "STRING", "REPEATED"),
188+
bqTableFieldSchema("desc", "name", "STRING", "REPEATED", nil),
175189
},
176190
},
177191
schema: Schema{
178-
fieldSchema("desc", "name", "STRING", true, false),
192+
fieldSchema("desc", "name", "STRING", true, false, nil),
179193
},
180194
},
181195
{
182196
// nullable, string
183197
bqSchema: &bq.TableSchema{
184198
Fields: []*bq.TableFieldSchema{
185-
bqTableFieldSchema("desc", "name", "STRING", ""),
199+
bqTableFieldSchema("desc", "name", "STRING", "", nil),
186200
},
187201
},
188202
schema: Schema{
189-
fieldSchema("desc", "name", "STRING", false, false),
203+
fieldSchema("desc", "name", "STRING", false, false, nil),
190204
},
191205
},
192206
{
193207
// integer
194208
bqSchema: &bq.TableSchema{
195209
Fields: []*bq.TableFieldSchema{
196-
bqTableFieldSchema("desc", "name", "INTEGER", ""),
210+
bqTableFieldSchema("desc", "name", "INTEGER", "", nil),
197211
},
198212
},
199213
schema: Schema{
200-
fieldSchema("desc", "name", "INTEGER", false, false),
214+
fieldSchema("desc", "name", "INTEGER", false, false, nil),
201215
},
202216
},
203217
{
204218
// float
205219
bqSchema: &bq.TableSchema{
206220
Fields: []*bq.TableFieldSchema{
207-
bqTableFieldSchema("desc", "name", "FLOAT", ""),
221+
bqTableFieldSchema("desc", "name", "FLOAT", "", nil),
208222
},
209223
},
210224
schema: Schema{
211-
fieldSchema("desc", "name", "FLOAT", false, false),
225+
fieldSchema("desc", "name", "FLOAT", false, false, nil),
212226
},
213227
},
214228
{
215229
// boolean
216230
bqSchema: &bq.TableSchema{
217231
Fields: []*bq.TableFieldSchema{
218-
bqTableFieldSchema("desc", "name", "BOOLEAN", ""),
232+
bqTableFieldSchema("desc", "name", "BOOLEAN", "", nil),
219233
},
220234
},
221235
schema: Schema{
222-
fieldSchema("desc", "name", "BOOLEAN", false, false),
236+
fieldSchema("desc", "name", "BOOLEAN", false, false, nil),
223237
},
224238
},
225239
{
226240
// timestamp
227241
bqSchema: &bq.TableSchema{
228242
Fields: []*bq.TableFieldSchema{
229-
bqTableFieldSchema("desc", "name", "TIMESTAMP", ""),
243+
bqTableFieldSchema("desc", "name", "TIMESTAMP", "", nil),
230244
},
231245
},
232246
schema: Schema{
233-
fieldSchema("desc", "name", "TIMESTAMP", false, false),
247+
fieldSchema("desc", "name", "TIMESTAMP", false, false, nil),
234248
},
235249
},
236250
{
237251
// civil times
238252
bqSchema: &bq.TableSchema{
239253
Fields: []*bq.TableFieldSchema{
240-
bqTableFieldSchema("desc", "f1", "TIME", ""),
241-
bqTableFieldSchema("desc", "f2", "DATE", ""),
242-
bqTableFieldSchema("desc", "f3", "DATETIME", ""),
254+
bqTableFieldSchema("desc", "f1", "TIME", "", nil),
255+
bqTableFieldSchema("desc", "f2", "DATE", "", nil),
256+
bqTableFieldSchema("desc", "f3", "DATETIME", "", nil),
243257
},
244258
},
245259
schema: Schema{
246-
fieldSchema("desc", "f1", "TIME", false, false),
247-
fieldSchema("desc", "f2", "DATE", false, false),
248-
fieldSchema("desc", "f3", "DATETIME", false, false),
260+
fieldSchema("desc", "f1", "TIME", false, false, nil),
261+
fieldSchema("desc", "f2", "DATE", false, false, nil),
262+
fieldSchema("desc", "f3", "DATETIME", false, false, nil),
249263
},
250264
},
251265
{
252266
// numeric
253267
bqSchema: &bq.TableSchema{
254268
Fields: []*bq.TableFieldSchema{
255-
bqTableFieldSchema("desc", "n", "NUMERIC", ""),
269+
bqTableFieldSchema("desc", "n", "NUMERIC", "", nil),
270+
},
271+
},
272+
schema: Schema{
273+
fieldSchema("desc", "n", "NUMERIC", false, false, nil),
274+
},
275+
},
276+
{
277+
// geography
278+
bqSchema: &bq.TableSchema{
279+
Fields: []*bq.TableFieldSchema{
280+
bqTableFieldSchema("geo", "g", "GEOGRAPHY", "", nil),
256281
},
257282
},
258283
schema: Schema{
259-
fieldSchema("desc", "n", "NUMERIC", false, false),
284+
fieldSchema("geo", "g", "GEOGRAPHY", false, false, nil),
260285
},
261286
},
262287
{
288+
// policy tags
263289
bqSchema: &bq.TableSchema{
264290
Fields: []*bq.TableFieldSchema{
265-
bqTableFieldSchema("geo", "g", "GEOGRAPHY", ""),
291+
bqTableFieldSchema("some pii", "restrictedfield", "STRING", "", []string{"tag1"}),
266292
},
267293
},
268294
schema: Schema{
269-
fieldSchema("geo", "g", "GEOGRAPHY", false, false),
295+
fieldSchema("some pii", "restrictedfield", "STRING", false, false, []string{"tag1"}),
270296
},
271297
},
272298
{
@@ -279,7 +305,7 @@ func TestSchemaConversion(t *testing.T) {
279305
Mode: "REQUIRED",
280306
Type: "RECORD",
281307
Fields: []*bq.TableFieldSchema{
282-
bqTableFieldSchema("inner field", "inner", "STRING", ""),
308+
bqTableFieldSchema("inner field", "inner", "STRING", "", nil),
283309
},
284310
},
285311
},
@@ -1019,17 +1045,17 @@ func TestSchemaFromJSON(t *testing.T) {
10191045
{"name":"flat_geography","type":"GEOGRAPHY","mode":"REQUIRED","description":"Flat required GEOGRAPHY"}
10201046
]`),
10211047
expectedSchema: Schema{
1022-
fieldSchema("Flat nullable string", "flat_string", "STRING", false, false),
1023-
fieldSchema("Flat required BYTES", "flat_bytes", "BYTES", false, true),
1024-
fieldSchema("Flat nullable INTEGER", "flat_integer", "INTEGER", false, false),
1025-
fieldSchema("Flat required FLOAT", "flat_float", "FLOAT", false, true),
1026-
fieldSchema("Flat nullable BOOLEAN", "flat_boolean", "BOOLEAN", false, false),
1027-
fieldSchema("Flat required TIMESTAMP", "flat_timestamp", "TIMESTAMP", false, true),
1028-
fieldSchema("Flat required DATE", "flat_date", "DATE", false, false),
1029-
fieldSchema("Flat nullable TIME", "flat_time", "TIME", false, true),
1030-
fieldSchema("Flat required DATETIME", "flat_datetime", "DATETIME", false, false),
1031-
fieldSchema("Flat nullable NUMERIC", "flat_numeric", "NUMERIC", false, true),
1032-
fieldSchema("Flat required GEOGRAPHY", "flat_geography", "GEOGRAPHY", false, true),
1048+
fieldSchema("Flat nullable string", "flat_string", "STRING", false, false, nil),
1049+
fieldSchema("Flat required BYTES", "flat_bytes", "BYTES", false, true, nil),
1050+
fieldSchema("Flat nullable INTEGER", "flat_integer", "INTEGER", false, false, nil),
1051+
fieldSchema("Flat required FLOAT", "flat_float", "FLOAT", false, true, nil),
1052+
fieldSchema("Flat nullable BOOLEAN", "flat_boolean", "BOOLEAN", false, false, nil),
1053+
fieldSchema("Flat required TIMESTAMP", "flat_timestamp", "TIMESTAMP", false, true, nil),
1054+
fieldSchema("Flat required DATE", "flat_date", "DATE", false, false, nil),
1055+
fieldSchema("Flat nullable TIME", "flat_time", "TIME", false, true, nil),
1056+
fieldSchema("Flat required DATETIME", "flat_datetime", "DATETIME", false, false, nil),
1057+
fieldSchema("Flat nullable NUMERIC", "flat_numeric", "NUMERIC", false, true, nil),
1058+
fieldSchema("Flat required GEOGRAPHY", "flat_geography", "GEOGRAPHY", false, true, nil),
10331059
},
10341060
},
10351061
{
@@ -1040,7 +1066,7 @@ func TestSchemaFromJSON(t *testing.T) {
10401066
{"name":"nested_record","type":"RECORD","mode":"NULLABLE","description":"Nested nullable RECORD","fields":[{"name":"record_field_1","type":"STRING","mode":"NULLABLE","description":"First nested record field"},{"name":"record_field_2","type":"INTEGER","mode":"REQUIRED","description":"Second nested record field"}]}
10411067
]`),
10421068
expectedSchema: Schema{
1043-
fieldSchema("Flat nullable string", "flat_string", "STRING", false, false),
1069+
fieldSchema("Flat nullable string", "flat_string", "STRING", false, false, nil),
10441070
&FieldSchema{
10451071
Description: "Nested nullable RECORD",
10461072
Name: "nested_record",
@@ -1071,7 +1097,7 @@ func TestSchemaFromJSON(t *testing.T) {
10711097
{"name":"nested_record","type":"RECORD","mode":"REPEATED","description":"Nested nullable RECORD","fields":[{"name":"record_field_1","type":"STRING","mode":"NULLABLE","description":"First nested record field"},{"name":"record_field_2","type":"INTEGER","mode":"REQUIRED","description":"Second nested record field"}]}
10721098
]`),
10731099
expectedSchema: Schema{
1074-
fieldSchema("Flat nullable string", "flat_string", "STRING", false, false),
1100+
fieldSchema("Flat nullable string", "flat_string", "STRING", false, false, nil),
10751101
&FieldSchema{
10761102
Description: "Nested nullable RECORD",
10771103
Name: "nested_record",

‎bigquery/table_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ func TestBQToTableMetadata(t *testing.T) {
125125
func TestTableMetadataToBQ(t *testing.T) {
126126
aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
127127
aTimeMillis := aTime.UnixNano() / 1e6
128-
sc := Schema{fieldSchema("desc", "name", "STRING", false, true)}
128+
sc := Schema{fieldSchema("desc", "name", "STRING", false, true, nil)}
129129

130130
for _, test := range []struct {
131131
in *TableMetadata
@@ -148,7 +148,7 @@ func TestTableMetadataToBQ(t *testing.T) {
148148
Description: "d",
149149
Schema: &bq.TableSchema{
150150
Fields: []*bq.TableFieldSchema{
151-
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
151+
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED", nil),
152152
},
153153
},
154154
ExpirationTime: aTimeMillis,
@@ -307,13 +307,13 @@ func TestTableMetadataToUpdateToBQ(t *testing.T) {
307307
},
308308
{
309309
tm: TableMetadataToUpdate{
310-
Schema: Schema{fieldSchema("desc", "name", "STRING", false, true)},
310+
Schema: Schema{fieldSchema("desc", "name", "STRING", false, true, nil)},
311311
ExpirationTime: aTime,
312312
},
313313
want: &bq.Table{
314314
Schema: &bq.TableSchema{
315315
Fields: []*bq.TableFieldSchema{
316-
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
316+
bqTableFieldSchema("desc", "name", "STRING", "REQUIRED", nil),
317317
},
318318
},
319319
ExpirationTime: aTime.UnixNano() / 1e6,

0 commit comments

Comments
 (0)