Skip to content

Commit cf0f6c3

Browse files
authored
fix(prom): return empty remote read results for missing metrics (#9613)
* fix: return empty remote read results for missing metrics Signed-off-by: fys <[email protected]> * test(prom): cover empty reads for missing databases Signed-off-by: fys <[email protected]> * test(prom): simplify remote read regression cases Signed-off-by: fys <[email protected]> * fix(prom): log missing tables during remote read Signed-off-by: fys <[email protected]> --------- Signed-off-by: fys <[email protected]>
1 parent 60097a0 commit cf0f6c3

4 files changed

Lines changed: 180 additions & 11 deletions

File tree

‎src/frontend/src/instance/prom_store.rs‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ use tracing::instrument;
6161

6262
use crate::error::{
6363
AmbiguousValueColumnSnafu, CatalogSnafu, ColumnNotFoundSnafu, ExecLogicalPlanSnafu,
64-
PromStoreRemoteQueryPlanSnafu, ReadTableSnafu, Result, TableNotFoundSnafu,
64+
PromStoreRemoteQueryPlanSnafu, ReadTableSnafu, Result,
6565
};
6666
use crate::instance::Instance;
6767

@@ -207,15 +207,21 @@ impl Instance {
207207
schema_name: &str,
208208
table_name: &str,
209209
query: &Query,
210-
) -> Result<RemoteQueryOutput> {
211-
let table = self
210+
) -> Result<Option<RemoteQueryOutput>> {
211+
let Some(table) = self
212212
.catalog_manager
213213
.table(catalog_name, schema_name, table_name, Some(ctx))
214214
.await
215215
.context(CatalogSnafu)?
216-
.with_context(|| TableNotFoundSnafu {
217-
table_name: format_full_table_name(catalog_name, schema_name, table_name),
218-
})?;
216+
else {
217+
debug!(
218+
catalog = catalog_name,
219+
schema = schema_name,
220+
table = table_name,
221+
"Table not found during remote read, returning empty result"
222+
);
223+
return Ok(None);
224+
};
219225

220226
let timestamp_column_name = table
221227
.schema()
@@ -250,12 +256,12 @@ impl Instance {
250256
.await
251257
.context(ExecLogicalPlanSnafu)?;
252258

253-
Ok(RemoteQueryOutput {
259+
Ok(Some(RemoteQueryOutput {
254260
table_name: table_name.to_string(),
255261
timestamp_column_name,
256262
value_column_name,
257263
output,
258-
})
264+
}))
259265
}
260266

261267
#[tracing::instrument(skip_all)]
@@ -264,7 +270,7 @@ impl Instance {
264270
ctx: QueryContextRef,
265271
queries: &[Query],
266272
query_targets: &[PermissionTableTarget],
267-
) -> ServerResult<Vec<RemoteQueryOutput>> {
273+
) -> ServerResult<Vec<Option<RemoteQueryOutput>>> {
268274
let mut results = Vec::with_capacity(queries.len());
269275

270276
for (query, target) in queries.iter().zip(query_targets) {
@@ -599,6 +605,10 @@ impl PromStoreProtocolHandler for Instance {
599605

600606
let response_type = negotiate_response_type(&request.accepted_response_types)?;
601607

608+
for query in &request.queries {
609+
prom_store::validate_matchers(&query.matchers)?;
610+
}
611+
602612
// TODO(dennis): use read_hints to speedup query if possible
603613
let results = self
604614
.handle_remote_queries(ctx, &request.queries, &query_targets)
@@ -609,6 +619,10 @@ impl PromStoreProtocolHandler for Instance {
609619
let mut query_results = Vec::with_capacity(results.len());
610620
let mut map = HashMap::new();
611621
for result in results {
622+
let Some(result) = result else {
623+
query_results.push(QueryResult { timeseries: vec![] });
624+
continue;
625+
};
612626
let RemoteQueryOutput {
613627
table_name,
614628
timestamp_column_name,

‎src/servers/src/prom_store.rs‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use std::hash::{Hash, Hasher};
2121
use std::sync::Arc;
2222

2323
use api::prom_store::remote::label_matcher::Type as MatcherType;
24-
use api::prom_store::remote::{Label, Query, Sample, TimeSeries, WriteRequest};
24+
use api::prom_store::remote::{Label, LabelMatcher, Query, Sample, TimeSeries, WriteRequest};
2525
use api::v1::RowInsertRequests;
2626
use arrow::array::{
2727
Array, ArrayRef, AsArray, DictionaryArray, LargeStringArray, StringArray, StringViewArray,
@@ -138,6 +138,30 @@ pub fn extract_schema_from_query(query: &Query) -> Option<String> {
138138
.map(|matcher| matcher.value.clone())
139139
}
140140

141+
/// Validate matcher syntax independently of whether the metric exists.
142+
pub fn validate_matchers(matchers: &[LabelMatcher]) -> Result<()> {
143+
for matcher in matchers {
144+
let matcher_type = MatcherType::try_from(matcher.r#type).map_err(|e| {
145+
error::InvalidPromRemoteRequestSnafu {
146+
msg: format!("invalid LabelMatcher type, decode error: {e}"),
147+
}
148+
.build()
149+
})?;
150+
if matches!(matcher_type, MatcherType::Re | MatcherType::Nre) {
151+
// Validate using the same regex engine as query execution.
152+
// Its syntax differs from Prometheus's; e.g. `(?x) a` is accepted here
153+
// but rejected by Prometheus.
154+
regex::Regex::new(&matcher.value).map_err(|e| {
155+
error::InvalidPromRemoteRequestSnafu {
156+
msg: format!("invalid regex for label '{}': {e}", matcher.name),
157+
}
158+
.build()
159+
})?;
160+
}
161+
}
162+
Ok(())
163+
}
164+
141165
/// Create a DataFrame from a remote Query
142166
#[tracing::instrument(skip_all)]
143167
pub fn query_to_plan(

‎tests-integration/src/prom_store.rs‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ mod tests {
3434
use frontend::instance::Instance;
3535
use prost::Message;
3636
use servers::http::prom_store::PHYSICAL_TABLE_PARAM;
37+
use servers::http::test_helpers::{TestClient, TestResponse};
38+
use servers::http::{HttpOptions, HttpServerBuilder};
3739
use servers::prom_store;
3840
use servers::prom_store::to_grpc_row_insert_requests;
3941
use servers::query_handler::PromStoreProtocolHandler;
@@ -663,10 +665,135 @@ mod tests {
663665
]
664666
);
665667

668+
instance
669+
.do_query(
670+
"CREATE TABLE invalid_metric (ts TIMESTAMP TIME INDEX, a DOUBLE, b DOUBLE)",
671+
ctx.clone(),
672+
)
673+
.await[0]
674+
.as_ref()
675+
.unwrap();
676+
let server = HttpServerBuilder::new(HttpOptions::default())
677+
.with_prom_handler(instance.clone(), None, true, Default::default(), None)
678+
.build();
679+
let client = TestClient::new(server.build(server.make_app()).unwrap()).await;
680+
681+
// Cover missing schemas/catalogs and each database label alias.
682+
for (selector, database) in [
683+
("__schema__", "unknown_schema"),
684+
("__database__", "unknown_catalog-prometheus"),
685+
("x_greptime_database", "unknown_catalog-information_schema"),
686+
] {
687+
let mut query = remote_query("missing_metric");
688+
query.matchers.push(LabelMatcher {
689+
name: selector.to_string(),
690+
value: database.to_string(),
691+
r#type: MatcherType::Eq as i32,
692+
});
693+
let response = post_remote_read(&client, vec![query]).await;
694+
assert_eq!(response.status(), 200, "{selector}={database}");
695+
let body = prom_store::snappy_decompress(&response.bytes().await).unwrap();
696+
let response = ReadResponse::decode(&body[..]).unwrap();
697+
assert_eq!(response.results.len(), 1);
698+
assert!(response.results[0].timeseries.is_empty());
699+
}
700+
701+
// Matcher syntax must be checked even when the metric does not exist.
702+
for metric in ["metric1", "missing_metric"] {
703+
for (matcher_type, value, expected_status) in [
704+
(99, "spark", 400),
705+
(MatcherType::Re as i32, "[", 400),
706+
(MatcherType::Nre as i32, "[", 400),
707+
(MatcherType::Eq as i32, "[", 200),
708+
(MatcherType::Neq as i32, "[", 200),
709+
(MatcherType::Re as i32, "spark", 200),
710+
(MatcherType::Nre as i32, "api.*", 200),
711+
] {
712+
let mut query = remote_query(metric);
713+
query.matchers.push(LabelMatcher {
714+
name: "job".to_string(),
715+
value: value.to_string(),
716+
r#type: matcher_type,
717+
});
718+
let response = post_remote_read(&client, vec![query]).await;
719+
assert_eq!(
720+
response.status().as_u16(),
721+
expected_status,
722+
"metric={metric}, type={matcher_type}, value={value}"
723+
);
724+
}
725+
}
726+
727+
// Missing metrics must retain their result slots in multi-query responses.
728+
for metrics in [
729+
vec!["missing_metric"],
730+
vec!["missing_metric", "another_missing_metric"],
731+
vec![
732+
"missing_metric",
733+
"metric1",
734+
"missing_metric",
735+
"metric3",
736+
"missing_metric",
737+
],
738+
vec!["missing_metric", "invalid_metric"],
739+
] {
740+
let queries = metrics.iter().map(|metric| remote_query(metric)).collect();
741+
let response = post_remote_read(&client, queries).await;
742+
if metrics.contains(&"invalid_metric") {
743+
assert!(!response.status().is_success());
744+
assert!(response.text().await.contains("Ambiguous value column"));
745+
continue;
746+
}
747+
assert_eq!(response.status(), 200, "{}", response.text().await);
748+
assert_eq!(response.headers()["content-type"], "application/x-protobuf");
749+
assert_eq!(response.headers()["content-encoding"], "snappy");
750+
let body = prom_store::snappy_decompress(&response.bytes().await).unwrap();
751+
let response = ReadResponse::decode(&body[..]).unwrap();
752+
assert_eq!(response.results.len(), metrics.len());
753+
for (result, metric) in response.results.iter().zip(metrics) {
754+
if metric.ends_with("missing_metric") {
755+
assert!(result.timeseries.is_empty());
756+
continue;
757+
}
758+
assert!(!result.timeseries.is_empty());
759+
for series in &result.timeseries {
760+
assert!(series.labels.iter().any(|label| {
761+
label.name == prom_store::METRIC_NAME_LABEL && label.value == metric
762+
}));
763+
assert_eq!(series.samples.len(), 2);
764+
}
765+
}
766+
}
767+
666768
// check physical table if provided
667769
if let Some(physical_table) = physical_table {
668770
let sql = format!("DESC TABLE {physical_table};");
669771
instance.do_query(&sql, ctx).await[0].as_ref().unwrap();
670772
}
671773
}
774+
775+
fn remote_query(metric: &str) -> Query {
776+
Query {
777+
start_timestamp_ms: 1000,
778+
end_timestamp_ms: 2000,
779+
matchers: vec![LabelMatcher {
780+
name: prom_store::METRIC_NAME_LABEL.to_string(),
781+
value: metric.to_string(),
782+
r#type: MatcherType::Eq as i32,
783+
}],
784+
..Default::default()
785+
}
786+
}
787+
788+
async fn post_remote_read(client: &TestClient, queries: Vec<Query>) -> TestResponse {
789+
let request = ReadRequest {
790+
queries,
791+
..Default::default()
792+
};
793+
client
794+
.post("/v1/prometheus/read?db=prometheus")
795+
.body(prom_store::snappy_compress(&request.encode_to_vec()).unwrap())
796+
.send()
797+
.await
798+
}
672799
}

‎tests-integration/tests/http.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5463,7 +5463,11 @@ pub async fn test_prometheus_remote_schema_labels(store_type: StorageType) {
54635463
.body(compressed_unknown_read_request)
54645464
.send()
54655465
.await;
5466-
assert_eq!(unknown_result.status(), StatusCode::BAD_REQUEST);
5466+
assert_eq!(unknown_result.status(), StatusCode::OK);
5467+
let body = prom_store::snappy_decompress(&unknown_result.bytes().await).unwrap();
5468+
let response = ReadResponse::decode(&body[..]).unwrap();
5469+
assert_eq!(response.results.len(), 1);
5470+
assert!(response.results[0].timeseries.is_empty());
54675471

54685472
guard.remove_all().await;
54695473
}

0 commit comments

Comments
 (0)