Skip to content
This repository was archived by the owner on Mar 23, 2026. It is now read-only.

test: add integration test for Column ACLs #1266

Merged
gcf-merge-on-green merged 7 commits into from
Apr 30, 2021
Merged

test: add integration test for Column ACLs #1266

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6c523fc
test: add integration test for Column ACLs
stephaniewang526 Apr 29, 2021
41e4846
declare additional dep
stephaniewang526 Apr 29, 2021
1955415
declare additional dep
stephaniewang526 Apr 29, 2021
c416d48
Merge branch 'master' into add-it-column-acl
stephaniewang526 Apr 29, 2021
8ab4813
address dependency comparison issue
stephaniewang526 Apr 30, 2021
70ea98a
add exclusion to synth.py
stephaniewang526 Apr 30, 2021
1ce6dd0
update comment
stephaniewang526 Apr 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .kokoro/dependencies.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,9 @@ function completenessCheck() {

# Output dep list generated using the flattened pom (only 'compile' and 'runtime' scopes)
msg "Generating dependency list using flattened pom..."
mvn dependency:list -f .flattened-pom.xml -DincludeScope=runtime -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' >.new-list.txt
# Excluding commons-codec,commons-logging from the comparison as a temp fix
# Explanation and issue filed in maven-dependency-plugin: https://issues.apache.org/jira/browse/MDEP-737
mvn dependency:list -f .flattened-pom.xml -DexcludeArtifactIds=commons-codec,commons-logging -DincludeScope=runtime -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' >.new-list.txt

# Compare two dependency lists
msg "Comparing dependency lists..."
Expand Down
8 changes: 8 additions & 0 deletions google-cloud-bigquery/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,14 @@
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client-jackson2</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-datacatalog</artifactId>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-cloud-datacatalog-v1</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
Expand Down
100 changes: 81 additions & 19 deletions google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import com.google.cloud.bigquery.ExternalTableDefinition;
import com.google.cloud.bigquery.ExtractJobConfiguration;
import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.FieldList;
import com.google.cloud.bigquery.FieldValue;
import com.google.cloud.bigquery.FieldValueList;
import com.google.cloud.bigquery.FormatOptions;
Expand Down Expand Up @@ -98,6 +99,12 @@
import com.google.cloud.bigquery.ViewDefinition;
import com.google.cloud.bigquery.WriteChannelConfiguration;
import com.google.cloud.bigquery.testing.RemoteBigQueryHelper;
import com.google.cloud.datacatalog.v1.CreatePolicyTagRequest;
import com.google.cloud.datacatalog.v1.CreateTaxonomyRequest;
import com.google.cloud.datacatalog.v1.PolicyTag;
import com.google.cloud.datacatalog.v1.PolicyTagManagerClient;
import com.google.cloud.datacatalog.v1.Taxonomy;
import com.google.cloud.datacatalog.v1.Taxonomy.PolicyType;
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.BucketInfo;
import com.google.cloud.storage.Storage;
Expand Down Expand Up @@ -155,10 +162,6 @@ public class ITBigQueryTest {
ImmutableMap.of(
"example-label1", "example-value1",
"example-label2", "example-value2");
private static final String sampleTag =
String.format("projects/%s/locations/us/taxonomies/1/policyTags/2", PROJECT_ID);
private static final PolicyTags POLICY_TAGS =
PolicyTags.newBuilder().setNames(ImmutableList.of(sampleTag)).build();
private static final Field TIMESTAMP_FIELD_SCHEMA =
Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP)
.setMode(Field.Mode.NULLABLE)
Expand Down Expand Up @@ -241,12 +244,6 @@ public class ITBigQueryTest {
.setMode(Field.Mode.NULLABLE)
.setDescription("BigNumeric4Description")
.build();
private static final Field STRING_FIELD_SCHEMA_WITH_POLICY =
Field.newBuilder("StringFieldWithPolicy", LegacySQLTypeName.STRING)
.setMode(Field.Mode.NULLABLE)
.setDescription("field has a policy")
.setPolicyTags(POLICY_TAGS)
.build();
private static final Schema TABLE_SCHEMA =
Schema.of(
TIMESTAMP_FIELD_SCHEMA,
Expand Down Expand Up @@ -297,8 +294,6 @@ public class ITBigQueryTest {
.build());

private static final Schema SIMPLE_SCHEMA = Schema.of(STRING_FIELD_SCHEMA);
private static final Schema POLICY_SCHEMA =
Schema.of(STRING_FIELD_SCHEMA, STRING_FIELD_SCHEMA_WITH_POLICY, INTEGER_FIELD_SCHEMA);
private static final Schema QUERY_RESULT_SCHEMA =
Schema.of(
Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP)
Expand Down Expand Up @@ -705,18 +700,85 @@ public void testCreateTableWithRangePartitioning() {
}
}

public void testCreateTableWithPolicyTags() {
String tableName = "test_create_table_policytags";
TableId tableId = TableId.of(DATASET, tableName);
try {
@Test
public void testCreateAndUpdateTableWithPolicyTags() throws IOException {
// Set up policy tags in the datacatalog service
try (PolicyTagManagerClient policyTagManagerClient = PolicyTagManagerClient.create()) {
CreateTaxonomyRequest createTaxonomyRequest =
CreateTaxonomyRequest.newBuilder()
.setParent(String.format("projects/%s/locations/%s", PROJECT_ID, "us"))
.setTaxonomy(
Taxonomy.newBuilder()
.setDisplayName("testing taxonomy")
.setDescription("taxonomy created for integration tests")
.addActivatedPolicyTypes(PolicyType.FINE_GRAINED_ACCESS_CONTROL)
.build())
.build();
Taxonomy taxonomyResponse = policyTagManagerClient.createTaxonomy(createTaxonomyRequest);
String taxonomyId = taxonomyResponse.getName();

CreatePolicyTagRequest createPolicyTagRequest =
CreatePolicyTagRequest.newBuilder()
.setParent(taxonomyId)
.setPolicyTag(PolicyTag.newBuilder().setDisplayName("ExamplePolicyTag").build())
.build();
PolicyTag policyTagResponse = policyTagManagerClient.createPolicyTag(createPolicyTagRequest);
String policyTagId = policyTagResponse.getName();
PolicyTags policyTags =
PolicyTags.newBuilder().setNames(ImmutableList.of(policyTagId)).build();
Field stringFieldWithPolicy =
Field.newBuilder("StringFieldWithPolicy", LegacySQLTypeName.STRING)
.setMode(Field.Mode.NULLABLE)
.setDescription("field has a policy")
.setPolicyTags(policyTags)
.build();
Schema policySchema =
Schema.of(STRING_FIELD_SCHEMA, stringFieldWithPolicy, INTEGER_FIELD_SCHEMA);

// Test: Amend an existing schema with a policy tag.
String tableNameForUpdate = "test_update_table_policytags";
TableId tableIdForUpdate = TableId.of(DATASET, tableNameForUpdate);
TableInfo tableInfo =
TableInfo.newBuilder(tableIdForUpdate, StandardTableDefinition.of(TABLE_SCHEMA))
.setDescription("policy tag update test table")
.build();
Table createdTableForUpdate = bigquery.create(tableInfo);
assertNotNull(createdTableForUpdate);
Schema schema = createdTableForUpdate.getDefinition().getSchema();
FieldList fields = schema.getFields();
// Create a new schema adding the current fields, plus the new policy tag field
List<Field> fieldList = new ArrayList<>();
for (Field field : fields) {
fieldList.add(field);
}
fieldList.add(stringFieldWithPolicy);
Schema updatedSchemaWithPolicyTag = Schema.of(fieldList);
Table updatedTable =
createdTableForUpdate
.toBuilder()
.setDefinition(StandardTableDefinition.of(updatedSchemaWithPolicyTag))
.build();
updatedTable.update();
Table remoteUpdatedTable = bigquery.getTable(DATASET, tableNameForUpdate);
assertEquals(
updatedSchemaWithPolicyTag,
remoteUpdatedTable.<StandardTableDefinition>getDefinition().getSchema());
bigquery.delete(tableIdForUpdate);

// Test: Create a new table with a policy tag defined.
String tableName = "test_create_table_policytags";
TableId tableId = TableId.of(DATASET, tableName);
StandardTableDefinition tableDefinition =
StandardTableDefinition.newBuilder().setSchema(POLICY_SCHEMA).build();
StandardTableDefinition.newBuilder().setSchema(policySchema).build();
Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition));
assertNotNull(createdTable);
Table remoteTable = bigquery.getTable(DATASET, tableName);
assertEquals(POLICY_SCHEMA, remoteTable.<StandardTableDefinition>getDefinition().getSchema());
} finally {
assertEquals(policySchema, remoteTable.<StandardTableDefinition>getDefinition().getSchema());
bigquery.delete(tableId);

// Clean up policy tags
policyTagManagerClient.deletePolicyTag(policyTagId);
policyTagManagerClient.deleteTaxonomy(taxonomyId);
}
}

Expand Down
12 changes: 12 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,18 @@
<version>1.113.16</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-datacatalog</artifactId>
<version>1.3.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-cloud-datacatalog-v1</artifactId>
<version>1.3.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
Expand Down
1 change: 1 addition & 0 deletions synth.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,6 @@
'.kokoro/nightly/samples.cfg',
'.kokoro/presubmit/java8-samples.cfg',
'.kokoro/presubmit/java11-samples.cfg',
'.kokoro/dependencies.sh',
'codecov.yaml'
])