Skip to content

Commit 9ee960b

Browse files
fbrydenfbryden
authored
Merge branch 'main' into fbryden/online_validation_v2
2 parents c8fcade + 1efa83d commit 9ee960b

68 files changed

Lines changed: 2304 additions & 1906 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ indexmap = { version = "2.11", features = ["serde"] }
3333
itertools = "0.14.0"
3434
derive_builder = "0.12"
3535
serde = { version = "1", features = ["derive"] }
36-
serde_json = "1.0"
36+
serde_json = { version = "1.0", features = ["raw_value"] }
3737
serde-sarif = "0.4"
3838
serde_yaml = "0.9.21"
3939
sha2 = "0.10.9"

‎README.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ The map in the `arguments` field uses an argument's name as its key, and the val
184184
An annotated example of a configuration file:
185185

186186
```yaml
187-
# This is a "v1" configuration file.
187+
# This is a legacy "v1" configuration file.
188188
schema-version: v1
189189
# The list of rulesets to enable for this repository.
190190
rulesets:
@@ -275,14 +275,14 @@ max-file-size-kb: 256
275275
276276
## Configuration file schema
277277
278-
There is a JSON Schema definition for the `static-analysis.datadog.yml` in the `schema` subdirectory.
278+
There is a JSON Schema definition for the `static-analysis.datadog.yml` in the `schema/legacy` subdirectory.
279279

280280
You can use it to check the syntax of your configuration file:
281281

282-
1. Execute `npx --yes [email protected] validate -s schema/schema.json -r 'schema/*/*.json' -d path/to/your/static-analysis.datadog.yml`
282+
1. Execute `npx --yes [email protected] validate -s schema/legacy/schema.json -r 'schema/legacy/examples/*/*.json' -d path/to/your/static-analysis.datadog.yml`
283283

284-
There are some examples of valid and invalid configuration files in the [`schema/v1/examples/valid`](schema/v1/examples/valid)
285-
and [`schema/v1/examples/invalid`](schema/v1/examples/invalid) subdirectories, respectively. If you make changes to the JSON
284+
There are some examples of valid and invalid configuration files in the [`schema/legacy/examples/valid`](schema/legacy/examples/valid)
285+
and [`schema/legacy/examples/invalid`](schema/legacy/examples/invalid) subdirectories, respectively. If you make changes to the JSON
286286
Schema, you can test them against our examples:
287287

288288
1. Execute `make -C schema`

‎crates/bins/src/bin/datadog-static-analyzer-git-hook.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ use itertools::Itertools;
2626
use kernel::analysis::ddsa_lib::v8_platform::{initialize_v8, Initialized, V8Platform};
2727
use kernel::classifiers::ArtifactClassification;
2828
use kernel::config::common::{ConfigMethod, PathConfig};
29-
use kernel::config::file_v2;
29+
use kernel::config::file_v1;
3030
use kernel::constants::{CARGO_VERSION, VERSION};
3131
use kernel::model::common::OutputFormat::Json;
3232
use kernel::model::rule::{Rule, RuleResult};
@@ -257,7 +257,7 @@ fn main() -> Result<()> {
257257
let configuration_file_and_method = get_config(directory_to_analyze.as_str(), use_debug);
258258

259259
let (configuration_file, configuration_method): (
260-
Option<file_v2::ConfigFile>,
260+
Option<file_v1::ConfigFile>,
261261
Option<ConfigMethod>,
262262
) = match configuration_file_and_method {
263263
Ok(cfg) => match cfg {

‎crates/bins/src/bin/datadog-static-analyzer-server.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
// Copyright 2024 Datadog, Inc.
44

55
use crate::datadog_static_analyzer_server::rule_cache::RuleCache;
6+
use crate::datadog_static_analyzer_server::secret_scanner_cache::SecretScannerCache;
67
use kernel::analysis::ddsa_lib::v8_platform::{initialize_v8, Initialized, V8Platform};
78
use std::sync::OnceLock;
89

@@ -11,6 +12,7 @@ mod datadog_static_analyzer_server;
1112
pub(crate) static V8_PLATFORM: OnceLock<V8Platform<Initialized>> = OnceLock::new();
1213
pub(crate) static RAYON_POOL: OnceLock<rayon::ThreadPool> = OnceLock::new();
1314
pub(crate) static RULE_CACHE: OnceLock<RuleCache> = OnceLock::new();
15+
pub(crate) static SECRET_SCANNER_CACHE: OnceLock<SecretScannerCache> = OnceLock::new();
1416

1517
#[rocket::main]
1618
async fn main() {
@@ -30,5 +32,9 @@ async fn main() {
3032
.set(rayon_pool)
3133
.expect("cell should have been unset");
3234

35+
SECRET_SCANNER_CACHE
36+
.set(SecretScannerCache::new())
37+
.expect("cell should have been unset");
38+
3339
datadog_static_analyzer_server::start().await;
3440
}

‎crates/bins/src/bin/datadog-static-analyzer.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ use kernel::analysis::ddsa_lib::v8_platform::{initialize_v8, Initialized, V8Plat
3535
use kernel::analysis::generated_content::DEFAULT_IGNORED_GLOBS;
3636
use kernel::classifiers::ArtifactClassification;
3737
use kernel::config::common::{ConfigMethod, PathConfig};
38-
use kernel::config::file_v2;
38+
use kernel::config::file_v1;
3939
use kernel::constants::{CARGO_VERSION, VERSION};
4040
use kernel::model::common::OutputFormat;
4141
use kernel::model::rule::{Rule, RuleSeverity};
@@ -264,7 +264,7 @@ fn main() -> Result<()> {
264264
let configuration_file_and_method = get_config(directory_to_analyze.as_str(), use_debug);
265265

266266
let (configuration_file, configuration_method): (
267-
Option<file_v2::ConfigFile>,
267+
Option<file_v1::ConfigFile>,
268268
Option<ConfigMethod>,
269269
) = match configuration_file_and_method {
270270
Ok(cfg) => match cfg {

‎crates/bins/src/bin/datadog_static_analyzer_server/endpoints.rs‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use std::path::Path;
33

44
use crate::datadog_static_analyzer_server::fairings::TraceSpan;
55
use crate::datadog_static_analyzer_server::rule_cache::cached_analysis_request;
6-
use crate::{RAYON_POOL, RULE_CACHE, V8_PLATFORM};
6+
use crate::{RAYON_POOL, RULE_CACHE, SECRET_SCANNER_CACHE, V8_PLATFORM};
77
use kernel::analysis::ddsa_lib::JsRuntime;
88
use rocket::{
99
fs::NamedFile,
@@ -153,7 +153,10 @@ async fn analyze(
153153
.unwrap()
154154
}
155155

156-
fn process_secret_scan_request(request: SecretScanRequest) -> Result<Vec<SecretResult>, String> {
156+
fn process_secret_scan_request(
157+
request: SecretScanRequest,
158+
cache: Option<&super::secret_scanner_cache::SecretScannerCache>,
159+
) -> Result<Vec<SecretResult>, String> {
157160
// Maximum number of rules per request to prevent excessive CPU usage.
158161
const MAX_RULES_COUNT: usize = 1000;
159162

@@ -174,16 +177,20 @@ fn process_secret_scan_request(request: SecretScanRequest) -> Result<Vec<SecretR
174177
return Err("Invalid filename: path traversal detected".to_string());
175178
}
176179

177-
// Deserialize rules from JSON
178-
let rules: Vec<secrets::model::secret_rule::SecretRule> = request
179-
.rules
180-
.iter()
181-
.map(|r| serde_json::from_value(r.clone()))
182-
.collect::<Result<Vec<_>, _>>()
183-
.map_err(|e| format!("Failed to parse rules: {}", e))?;
184-
185-
// Build the scanner with the provided rules
186-
let scanner = secrets::scanner::build_sds_scanner(&rules, request.use_debug)?;
180+
// Get scanner + parsed rules (from cache or fresh build)
181+
let (scanner, rules) = if let Some(cache) = cache {
182+
cache.get_or_build(&request.rules, request.use_debug)?
183+
} else {
184+
// No cache - build from scratch
185+
let rules: Vec<secrets::model::secret_rule::SecretRule> = request
186+
.rules
187+
.iter()
188+
.map(|r| serde_json::from_str(r.get()))
189+
.collect::<Result<Vec<_>, _>>()
190+
.map_err(|e| format!("Failed to parse rules: {}", e))?;
191+
let scanner = secrets::scanner::build_sds_scanner(&rules, request.use_debug)?;
192+
(std::sync::Arc::new(scanner), std::sync::Arc::new(rules))
193+
};
187194

188195
// Configure analysis options
189196
let options = common::analysis_options::AnalysisOptions {
@@ -210,10 +217,11 @@ async fn scan_secrets(span: TraceSpan, request: Json<SecretScanRequest>) -> Valu
210217

211218
rocket::tokio::task::spawn_blocking(move || {
212219
let request = request.into_inner();
213-
let (rule_responses, errors) = match process_secret_scan_request(request) {
214-
Ok(resp) => (resp, vec![]),
215-
Err(err) => (vec![], vec![err]),
216-
};
220+
let (rule_responses, errors) =
221+
match process_secret_scan_request(request, SECRET_SCANNER_CACHE.get()) {
222+
Ok(resp) => (resp, vec![]),
223+
Err(err) => (vec![], vec![err]),
224+
};
217225

218226
json!(SecretScanResponse {
219227
rule_responses,

0 commit comments

Comments
 (0)