Skip to content

Commit e5b9ec7

Browse files
disqdisq
authored
feat(azure): Add policy docs (#10253)
Including views (just like #10250) Looks like: https://cloudquery-rhkamisgy-cloudquery.vercel.app/docs/plugins/sources/azure/policies This PR also removes the static `policies.mdx` file.
1 parent a197db5 commit e5b9ec7

File tree

4 files changed

+167
-61
lines changed

4 files changed

+167
-61
lines changed

‎plugins/source/azure/Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ test:
55

66
.PHONY: gen-docs
77
gen-docs:
8+
go run main.go doc --format json docs
9+
go run scripts/policy_docs/main.go azure policies ../../../website/pages/docs/plugins/sources/azure/policies.md docs/__tables.json
10+
rm docs/__tables.json
11+
812
rm -rf ../../../website/tables/azure
913
go run main.go doc ../../../website/tables/azure
1014
sed 's_(\(.*\))_(../../../../../website/tables/azure/\1)_' ../../../website/tables/azure/README.md > ./docs/tables/README.md

‎plugins/source/azure/scripts/policy_docs/main.go‎

Lines changed: 84 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,18 @@ import (
1212
"sort"
1313
"strings"
1414
"text/template"
15+
16+
"golang.org/x/exp/maps"
1517
)
1618

1719
//go:embed templates/*.go.tpl
1820
var templatesFS embed.FS
1921

20-
var (
21-
reInline = regexp.MustCompile(`(?im)^\\ir (.+)`)
22-
reTable = regexp.MustCompile(`(?i)(?:from|join)\s+(\w+)`)
23-
reTitle = regexp.MustCompile(`(?i)\:'check_id'(?:\s+as\s+check_id\s*)?,\s+'(.+)'(?:\s+as\s+title\s*)?,`)
24-
)
22+
var reInline = regexp.MustCompile(`(?im)^\\ir (.+)`)
23+
var reTable = regexp.MustCompile(`(?i)(?:from|join)\s+(\w+)`)
24+
var reTitle = regexp.MustCompile(`(?i)\:'check_id'(?:\s+as\s+check_id\s*)?,\s+'(.+)'(?:\s+as\s+title\s*)?,`)
25+
var reViewName = regexp.MustCompile(`(?i)create\s+(or\s+replace\s+)view\s+(.+)\s+as`)
26+
var reIsPolicyQuery *regexp.Regexp
2527

2628
type Index struct {
2729
Policies []Policy `json:"policies"`
@@ -50,6 +52,10 @@ type PolicyInfo struct {
5052
Name string
5153
Queries []Query
5254
Tables []string
55+
56+
CreatedViews map[string]bool
57+
DependentViews []string
58+
UnusedViews []string
5359
}
5460

5561
func newPolicyInfo(name string, queries []Query, allTables []Table) *PolicyInfo {
@@ -63,28 +69,62 @@ func newPolicyInfo(name string, queries []Query, allTables []Table) *PolicyInfo
6369

6470
// Tables returns the unique set of tables needed to run all queries
6571
func (pi *PolicyInfo) setTables(allTables Tables) {
66-
t := map[string]struct{}{}
72+
t, v := map[string]struct{}{}, map[string]struct{}{}
73+
pi.CreatedViews = map[string]bool{}
74+
75+
// Add all views detected from CREATE VIEW statements
6776
for _, q := range pi.Queries {
77+
if q.View {
78+
pi.CreatedViews[q.ViewName] = true
79+
}
80+
}
81+
82+
// Accessed views from queries, should be a subset of the above
83+
for _, q := range pi.Queries {
84+
for _, vi := range q.Views {
85+
v[vi] = struct{}{}
86+
}
87+
}
88+
89+
for _, q := range pi.Queries {
90+
// Add all tables from queries *and* views
6891
for _, table := range q.Tables {
92+
if pi.CreatedViews[table] {
93+
// This is a view, not a table
94+
v[table] = struct{}{}
95+
}
96+
6997
t[table] = struct{}{}
7098
ancestors := allTables.FindAncestors(table)
7199
for _, a := range ancestors {
72100
t[a.Name] = struct{}{}
73101
}
74102
}
75103
}
76-
var final []string
77-
for k := range t {
78-
final = append(final, k)
104+
105+
pi.Tables, pi.DependentViews = maps.Keys(t), maps.Keys(v)
106+
107+
unused := make(map[string]struct{}, len(pi.CreatedViews))
108+
for k := range pi.CreatedViews {
109+
if _, ok := v[k]; !ok {
110+
unused[k] = struct{}{}
111+
}
79112
}
80-
sort.Strings(final)
81-
pi.Tables = final
113+
pi.UnusedViews = maps.Keys(unused)
114+
115+
sort.Strings(pi.Tables)
116+
sort.Strings(pi.DependentViews)
117+
sort.Strings(pi.UnusedViews)
82118
}
83119

84120
type Query struct {
85-
Title string
121+
Title string // Empty for views
86122
Path string
87123
Tables []string
124+
Views []string
125+
126+
View bool `json:",omitempty"`
127+
ViewName string `json:",omitempty"`
88128
}
89129

90130
func removeDuplicates(queries []Query) []Query {
@@ -115,26 +155,40 @@ func extractQueries(prefix, sqlPath string) ([]Query, error) {
115155
}
116156
content := string(b)
117157
var queries []Query
118-
tableMatches := reTable.FindAllStringSubmatch(content, -1)
119-
reIsPolicyQuery := regexp.MustCompile(`(?i)insert\s+into\s+` + prefix + `_policy_results`)
120-
isPolicyQuery := reIsPolicyQuery.MatchString(content)
121-
if isPolicyQuery && len(tableMatches) > 0 {
122-
q := Query{}
158+
q := Query{}
159+
if tableMatches := reTable.FindAllStringSubmatch(content, -1); len(tableMatches) > 0 {
123160
for _, m := range tableMatches {
124-
if !strings.HasPrefix(m[1], prefix) {
125-
continue
161+
switch {
162+
case strings.HasPrefix(m[1], prefix):
163+
// This could be a table or still a view, will sort out later
164+
q.Tables = append(q.Tables, m[1])
165+
q.Path = sqlPath
166+
case strings.HasPrefix(m[1], "view_"+prefix):
167+
q.Views = append(q.Views, m[1])
168+
q.Path = sqlPath
126169
}
127-
q.Tables = append(q.Tables, m[1])
128-
q.Path = sqlPath
129170
}
171+
}
130172

173+
isPolicyQuery := reIsPolicyQuery.MatchString(content)
174+
if isPolicyQuery && (len(q.Tables)+len(q.Views)) > 0 {
131175
titleMatches := reTitle.FindAllStringSubmatch(content, -1)
132176
if len(titleMatches) == 0 {
133177
return nil, fmt.Errorf("failed to find title for query in %v", sqlPath)
134178
} else if len(titleMatches) >= 1 {
135179
q.Title = titleMatches[0][1]
136180
}
137181
queries = append(queries, q)
182+
} else {
183+
mv := reViewName.FindStringSubmatch(content)
184+
if len(mv) > 0 {
185+
q.View = true
186+
q.ViewName = mv[2]
187+
if !strings.HasPrefix(q.ViewName, prefix+"_") && !strings.HasPrefix(q.ViewName, "view_"+prefix) {
188+
return nil, fmt.Errorf("view %q in %s does not start with `%s_` or `view_%s`", q.ViewName, sqlPath, prefix, prefix)
189+
}
190+
queries = append(queries, q)
191+
}
138192
}
139193

140194
// recurse to find queries in inlined files
@@ -151,7 +205,13 @@ func extractQueries(prefix, sqlPath string) ([]Query, error) {
151205
}
152206

153207
func writePolicyDocs(info []*PolicyInfo, outputPath string) error {
154-
t, err := template.New("policies.md.go.tpl").ParseFS(templatesFS, "templates/policies.md.go.tpl")
208+
t, err := template.New("policies.md.go.tpl").
209+
Funcs(template.FuncMap{
210+
"add": func(a int, b int) int {
211+
return a + b
212+
},
213+
}).
214+
ParseFS(templatesFS, "templates/policies.md.go.tpl")
155215
if err != nil {
156216
return fmt.Errorf("failed to parse template for README.md: %v", err)
157217
}
@@ -214,6 +274,8 @@ func main() {
214274
log.Fatalf("error reading tables JSON: %v", err)
215275
}
216276

277+
reIsPolicyQuery = regexp.MustCompile(`(?i)insert\s+into\s+` + prefix + `_policy_results`)
278+
217279
var info []*PolicyInfo
218280
for _, p := range index.Policies {
219281
pi, err := getPolicyInfo(prefix, tables, dir, p)

‎plugins/source/azure/scripts/policy_docs/templates/policies.md.go.tpl‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,36 @@ tables:
1818
```
1919

2020
### Queries
21+
2122
{{ .Name }} performs the following checks:
22-
{{- range $query := .Queries }}
23+
{{- range $query := .Queries }}{{ if not $query.View }}
2324
- {{ $query.Title }}
25+
{{- end }}{{- end }}
26+
27+
{{- if .DependentViews }}
28+
29+
### Dependent Views
30+
31+
{{ .Name }} depends on the following views:
32+
{{$createdViews := .CreatedViews }}
33+
{{- $num_created := 0}}
34+
{{- range $v := .DependentViews }}
35+
- {{ $v }}{{ if index $createdViews $v }}<sup>*</sup>{{ $num_created = add $num_created 1 }}{{end}}
36+
{{- end }}
37+
38+
{{- if gt $num_created 0}}
39+
40+
<sup>*</sup> {{if eq $num_created 1}}This view is{{else}}These views are{{end}} automatically created or updated by this policy.
41+
{{- end}}
42+
{{- end }}
43+
{{- if .UnusedViews }}
44+
45+
### Unused Views
46+
47+
{{ .Name }} creates {{if eq (len .UnusedViews) 1}}this view but does not use it:{{else}}these views but does not use them:{{end}}
48+
{{range $v := .UnusedViews }}
49+
- {{ $v }}
2450
{{- end }}
51+
{{end}}
2552

2653
{{- end }}

website/pages/docs/plugins/sources/azure/policies.mdx renamed to website/pages/docs/plugins/sources/azure/policies.md

Lines changed: 51 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -8,36 +8,35 @@ This page documents the available CloudQuery SQL Policies for Azure. See the [re
88
### Requirements
99
Azure CIS v1.3.0 requires the following tables to be synced before the policy is executed:
1010

11-
```yaml copy
11+
```yaml
1212
tables:
13+
- azure_appservice_web_app_auth_settings
14+
- azure_appservice_web_apps
1315
- azure_compute_disks
1416
- azure_compute_virtual_machines
15-
- azure_keyvault_keys
16-
- azure_keyvault_secrets
17-
- azure_keyvault_vaults
17+
- azure_keyvault_keyvault
18+
- azure_keyvault_keyvault_keys
19+
- azure_keyvault_keyvault_secrets
1820
- azure_mysql_servers
19-
- azure_postgresql_configurations
20-
- azure_postgresql_firewall_rules
21+
- azure_policy_assignments
22+
- azure_postgresql_server_configurations
23+
- azure_postgresql_server_firewall_rules
2124
- azure_postgresql_servers
2225
- azure_security_auto_provisioning_settings
23-
- azure_security_contacts
2426
- azure_security_pricings
25-
- azure_security_settings
26-
- azure_sql_database_blob_auditing_policies
27-
- azure_sql_database_threat_detection_policies
28-
- azure_sql_databases
29-
- azure_sql_encryption_protectors
3027
- azure_sql_server_admins
3128
- azure_sql_server_blob_auditing_policies
29+
- azure_sql_server_database_blob_auditing_policies
30+
- azure_sql_server_database_threat_protections
31+
- azure_sql_server_databases
32+
- azure_sql_server_encryption_protectors
3233
- azure_sql_server_vulnerability_assessments
3334
- azure_sql_servers
3435
- azure_sql_transparent_data_encryptions
35-
- azure_web_apps
36-
- azure_web_publishing_profiles
37-
- azure_web_site_auth_settings
3836
```
3937
4038
### Queries
39+
4140
Azure CIS v1.3.0 performs the following checks:
4241
- Ensure that Azure Defender is set to On for Servers (Automatic)
4342
- Ensure that Azure Defender is set to On for App Service (Automatic)
@@ -47,11 +46,8 @@ Azure CIS v1.3.0 performs the following checks:
4746
- Ensure that Azure Defender is set to On for Kubernetes (Automatic)
4847
- Ensure that Azure Defender is set to On for Container Registries (Automatic)
4948
- Ensure that Azure Defender is set to On for Key Vault (Manual)
50-
- Ensure that Microsoft Cloud App Security (MCAS) integration with Security Center is selected (Automatic)
5149
- Ensure that "Automatic provisioning of monitoring agent" is set to "On" (Automated)
5250
- Ensure any of the ASC Default policy setting is not set to "Disabled" (Automated)
53-
- Ensure "Additional email addresses" is configured with a security contact email (Automated)
54-
- Ensure that "Notify about alerts with the following severity" is set to "High" (Automated)
5551
- Ensure that "Auditing" is set to "On" (Automated)
5652
- Ensure that "Data encryption" is set to "On" on a SQL Database (Automated)
5753
- Ensure that "Auditing" Retention is "greater than 90 days" (Automated)
@@ -79,31 +75,40 @@ Azure CIS v1.3.0 performs the following checks:
7975
- Ensure the key vault is recoverable (Automated)
8076
- Ensure App Service Authentication is set on Azure App Service (Automated)
8177
- Ensure web app redirects all HTTP traffic to HTTPS in Azure App Service (Automated)
78+
- Ensure web app is using the latest version of TLS encryption (Automated)
8279
- Ensure the web app has ''Client Certificates (Incoming client certificates)'' set to ''On'' (Automated)
8380
- Ensure that Register with Azure Active Directory is enabled on App Service (Automated)
84-
- Ensure FTP deployments are disabled (Automated)
81+
82+
### Dependent Views
83+
84+
Azure CIS v1.3.0 depends on the following views:
85+
86+
- view_azure_security_policy_parameters<sup>*</sup>
87+
88+
<sup>*</sup> This view is automatically created or updated by this policy.
8589
## Azure HIPAA HITRUST v9.2
8690
8791
### Requirements
8892
Azure HIPAA HITRUST v9.2 requires the following tables to be synced before the policy is executed:
8993
90-
```yaml copy
94+
```yaml
9195
tables:
96+
- azure_appservice_web_app_vnet_connections
97+
- azure_appservice_web_apps
9298
- azure_authorization_role_assignments
9399
- azure_authorization_role_definitions
94-
- azure_batch_accounts
100+
- azure_batch_account
95101
- azure_compute_virtual_machine_extensions
96102
- azure_compute_virtual_machine_scale_sets
97103
- azure_compute_virtual_machines
98-
- azure_container_managed_clusters
99-
- azure_container_registries
100-
- azure_cosmosdb_accounts
101-
- azure_datalake_store_accounts
104+
- azure_containerregistry_registries
105+
- azure_containerservice_managed_clusters
106+
- azure_cosmos_database_accounts
107+
- azure_datalakestore_accounts
108+
- azure_eventhub_namespace_network_rule_sets
102109
- azure_eventhub_namespaces
103-
- azure_eventhub_network_rule_sets
104-
- azure_keyvault_managed_hsms
105-
- azure_keyvault_vaults
106-
- azure_logic_diagnostic_settings
110+
- azure_keyvault_keyvault
111+
- azure_keyvault_keyvault_managed_hsms
107112
- azure_logic_workflows
108113
- azure_mariadb_servers
109114
- azure_monitor_activity_log_alerts
@@ -122,27 +127,27 @@ tables:
122127
- azure_security_assessments
123128
- azure_security_auto_provisioning_settings
124129
- azure_security_jit_network_access_policies
125-
- azure_sql_backup_long_term_retention_policies
126-
- azure_sql_database_vulnerability_assessment_scans
127-
- azure_sql_databases
128-
- azure_sql_encryption_protectors
129130
- azure_sql_managed_instance_encryption_protectors
130131
- azure_sql_managed_instance_vulnerability_assessments
131132
- azure_sql_managed_instances
132133
- azure_sql_server_blob_auditing_policies
134+
- azure_sql_server_database_long_term_retention_policies
135+
- azure_sql_server_database_vulnerability_assessment_scans
136+
- azure_sql_server_database_vulnerability_assessments
137+
- azure_sql_server_databases
138+
- azure_sql_server_encryption_protectors
139+
- azure_sql_server_virtual_network_rules
133140
- azure_sql_server_vulnerability_assessments
134141
- azure_sql_servers
135142
- azure_sql_transparent_data_encryptions
136-
- azure_sql_virtual_network_rules
137143
- azure_storage_accounts
138144
- azure_streamanalytics_streaming_jobs
139-
- azure_subscriptions
140-
- azure_subscriptions_locations
141-
- azure_web_apps
142-
- azure_web_vnet_connections
145+
- azure_subscription_subscription_locations
146+
- azure_subscription_subscriptions
143147
```
144148
145149
### Queries
150+
146151
Azure HIPAA HITRUST v9.2 performs the following checks:
147152
- MFA should be enabled on accounts with owner permissions on your subscription
148153
- MFA should be enabled on accounts with write permissions on your subscription
@@ -224,3 +229,11 @@ Azure HIPAA HITRUST v9.2 performs the following checks:
224229
- Audit virtual machines without disaster recovery configured.
225230
- Azure Key Vault Managed HSM should have purge protection enabled
226231
- Ensure the key vault is recoverable (Automated)
232+
233+
### Dependent Views
234+
235+
Azure HIPAA HITRUST v9.2 depends on the following views:
236+
237+
- view_azure_nsg_rules<sup>*</sup>
238+
239+
<sup>*</sup> This view is automatically created or updated by this policy.

0 commit comments

Comments
 (0)