feat: Add resource_record_sets to GCP plugin (#10917)

paulopontesm · web-flow · commit f750cd2fbfcf · 2023-05-26T15:51:50.000Z
#### Summary

This PR adds the `gcp_dns_resource_record_sets` table to the GCP plugin.

These are the columns for the new table: 

```csv
_cq_source_name,_cq_sync_time,_cq_id,_cq_parent_id,project_id,kind,name,routing_policy,rrdatas,signature_rrdatas,ttl,type
```

Tested this on my GCP project and seems to be working ok. Let me know if there's something I'm missing.

&lt;!--
diff --git a/plugins/source/gcp/CONTRIBUTING.md b/plugins/source/gcp/CONTRIBUTING.md
@@ -5,7 +5,7 @@ Thanks for contributing to CloudQuery! You are awesome. This document serves as
 There are two main steps to adding a new GCP resource:
 
 1. [Add a code generation recipe](#1-add-a-code-generation-recipe)
-2. [Writing the resolver function to fetch the resource using the AWS SDK](#2-setting-up-the-resource)
+2. [Writing the resolver function to fetch the resource using the GCP SDK](#2-setting-up-the-resource)
 
 ## 1. Add a Code Generation Recipe
 
diff --git a/plugins/source/gcp/client/services.go b/plugins/source/gcp/client/services.go
diff --git a/plugins/source/gcp/docs/tables/README.md b/plugins/source/gcp/docs/tables/README.md
@@ -121,6 +121,7 @@
 - [gcp_container_clusters](../../../../../website/tables/gcp/gcp_container_clusters.md)
 - [gcp_containeranalysis_occurrences](../../../../../website/tables/gcp/gcp_containeranalysis_occurrences.md)
 - [gcp_dns_managed_zones](../../../../../website/tables/gcp/gcp_dns_managed_zones.md)
+  - [gcp_dns_resource_record_sets](../../../../../website/tables/gcp/gcp_dns_resource_record_sets.md)
 - [gcp_dns_policies](../../../../../website/tables/gcp/gcp_dns_policies.md)
 - [gcp_domains_registrations](../../../../../website/tables/gcp/gcp_domains_registrations.md)
 - [gcp_functions_functions](../../../../../website/tables/gcp/gcp_functions_functions.md)
diff --git a/plugins/source/gcp/resources/services/dns/managed_zones.go b/plugins/source/gcp/resources/services/dns/managed_zones.go
@@ -22,5 +22,8 @@ func ManagedZones() *schema.Table {
 				Resolver: client.ResolveProject,
 			},
 		},
+		Relations: []*schema.Table{
+			resourceRecordSets(),
+		},
 	}
 }
diff --git a/plugins/source/gcp/resources/services/dns/managed_zones_mock_test.go b/plugins/source/gcp/resources/services/dns/managed_zones_mock_test.go
@@ -16,15 +16,39 @@ type MockManagedZonesResult struct {
 	ManagedZones []*dns.ManagedZone `json:"managedzones,omitempty"`
 }
 
+type MockResourceRecordSetsResult struct {
+	Rrsets []*dns.ResourceRecordSet `json:"rrsets,omitempty"`
+}
+
 func createManagedZones(mux *httprouter.Router) error {
-	var item dns.ManagedZone
-	if err := faker.FakeObject(&item); err != nil {
+	var managedZoneItem dns.ManagedZone
+	if err := faker.FakeObject(&managedZoneItem); err != nil {
 		return err
 	}
 
-	mux.GET("/*filepath", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+	mux.GET("/dns/v1/projects/testProject/managedZones", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		resp := &MockManagedZonesResult{
-			ManagedZones: []*dns.ManagedZone{&item},
+			ManagedZones: []*dns.ManagedZone{&managedZoneItem},
+		}
+		b, err := json.Marshal(resp)
+		if err != nil {
+			http.Error(w, "unable to marshal request: "+err.Error(), http.StatusBadRequest)
+			return
+		}
+		if _, err := w.Write(b); err != nil {
+			http.Error(w, "failed to write", http.StatusBadRequest)
+			return
+		}
+	})
+
+	var recordSetItem dns.ResourceRecordSet
+	if err := faker.FakeObject(&recordSetItem); err != nil {
+		return err
+	}
+
+	mux.GET("/dns/v1/projects/testProject/managedZones/test string/rrsets", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		resp := &MockResourceRecordSetsResult{
+			Rrsets: []*dns.ResourceRecordSet{&recordSetItem},
 		}
 		b, err := json.Marshal(resp)
 		if err != nil {
@@ -36,6 +60,7 @@ func createManagedZones(mux *httprouter.Router) error {
 			return
 		}
 	})
+
 	return nil
 }
 
diff --git a/plugins/source/gcp/resources/services/dns/resource_record_sets.go b/plugins/source/gcp/resources/services/dns/resource_record_sets.go
@@ -0,0 +1,32 @@
+package dns
+
+import (
+	"github.com/apache/arrow/go/v13/arrow"
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugin-sdk/v3/transformers"
+	"github.com/cloudquery/plugins/source/gcp/client"
+	"google.golang.org/api/dns/v1"
+)
+
+func resourceRecordSets() *schema.Table {
+	return &schema.Table{
+		Name:        "gcp_dns_resource_record_sets",
+		Description: `https://cloud.google.com/dns/docs/reference/v1/resourceRecordSets`,
+		Resolver:    fetchResourceRecordSets,
+		Multiplex:   client.ProjectMultiplexEnabledServices("dns.googleapis.com"),
+		Transform:   client.TransformWithStruct(&dns.ResourceRecordSet{}, transformers.WithPrimaryKeys("Name", "Type")),
+		Columns: []schema.Column{
+			{
+				Name:       "project_id",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   client.ResolveProject,
+				PrimaryKey: true,
+			}, {
+				Name:       "managed_zone_name",
+				Type:       arrow.BinaryTypes.String,
+				Resolver:   schema.ParentColumnResolver("name"),
+				PrimaryKey: true,
+			},
+		},
+	}
+}
diff --git a/plugins/source/gcp/resources/services/dns/resource_record_sets_fetch.go b/plugins/source/gcp/resources/services/dns/resource_record_sets_fetch.go
@@ -0,0 +1,40 @@
+package dns
+
+import (
+	"context"
+
+	"github.com/cloudquery/plugin-sdk/v3/schema"
+	"github.com/cloudquery/plugins/source/gcp/client"
+	"google.golang.org/api/dns/v1"
+)
+
+func fetchResourceRecordSets(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
+	c := meta.(*client.Client)
+	nextPageToken := ""
+	dnsClient, err := dns.NewService(ctx, c.ClientOptions...)
+	if err != nil {
+		return err
+	}
+
+	for {
+		managedZone := parent.Item.(*dns.ManagedZone)
+		call := dnsClient.ResourceRecordSets.List(c.ProjectId, managedZone.Name)
+
+		if nextPageToken != "" {
+			call = call.PageToken(nextPageToken)
+		}
+
+		output, err := call.Do()
+
+		if err != nil {
+			return err
+		}
+		res <- output.Rrsets
+
+		if output.NextPageToken == "" {
+			break
+		}
+		nextPageToken = output.NextPageToken
+	}
+	return nil
+}
diff --git a/website/pages/docs/plugins/sources/gcp/tables.md b/website/pages/docs/plugins/sources/gcp/tables.md
diff --git a/website/tables/gcp/gcp_dns_managed_zones.md b/website/tables/gcp/gcp_dns_managed_zones.md
@@ -6,6 +6,11 @@ https://cloud.google.com/dns/docs/reference/v1/managedZones#resource
 
 The primary key for this table is **id**.
 
+## Relations
+
+The following tables depend on gcp_dns_managed_zones:
+  - [gcp_dns_resource_record_sets](gcp_dns_resource_record_sets)
+
 ## Columns
 
 | Name          | Type          |
diff --git a/website/tables/gcp/gcp_dns_resource_record_sets.md b/website/tables/gcp/gcp_dns_resource_record_sets.md
@@ -0,0 +1,29 @@
+# Table: gcp_dns_resource_record_sets
+
+This table shows data for GCP DNS Resource Record Sets.
+
+https://cloud.google.com/dns/docs/reference/v1/resourceRecordSets
+
+The composite primary key for this table is (**project_id**, **managed_zone_name**, **name**, **type**).
+
+## Relations
+
+This table depends on [gcp_dns_managed_zones](gcp_dns_managed_zones).
+
+## Columns
+
+| Name          | Type          |
+| ------------- | ------------- |
+|_cq_source_name|utf8|
+|_cq_sync_time|timestamp[us, tz=UTC]|
+|_cq_id|uuid|
+|_cq_parent_id|uuid|
+|project_id (PK)|utf8|
+|managed_zone_name (PK)|utf8|
+|kind|utf8|
+|name (PK)|utf8|
+|routing_policy|json|
+|rrdatas|list<item: utf8, nullable>|
+|signature_rrdatas|list<item: utf8, nullable>|
+|ttl|int64|
+|type (PK)|utf8|

Original file line number	Diff line number	Diff line change
`@@ -22,5 +22,8 @@ func ManagedZones() *schema.Table {`
`22`	`22`	`Resolver: client.ResolveProject,`
`23`	`23`	`},`
`24`	`24`	`},`
	`25`	`+ Relations: []*schema.Table{`
	`26`	`+ resourceRecordSets(),`
	`27`	`+ },`
`25`	`28`	`}`
`26`	`29`	`}`