Skip to content

Commit 8e9ccb0

Browse files
committed
Evaluate MemberAccess of constants in filter translators
Fixes microsoft#11673
1 parent 8b8ccd6 commit 8e9ccb0

11 files changed

Lines changed: 166 additions & 42 deletions

File tree

‎dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchFilterTranslator.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ internal string Translate(LambdaExpression lambdaExpression, VectorStoreRecordMo
3131
Debug.Assert(lambdaExpression.Parameters.Count == 1);
3232
this._recordParameter = lambdaExpression.Parameters[0];
3333

34-
var preprocessor = new FilterTranslationPreprocessor { InlineCapturedVariables = true };
35-
var preprocessedExpression = preprocessor.Visit(lambdaExpression.Body);
34+
var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false };
35+
var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body);
3636

3737
this.Translate(preprocessedExpression);
3838

‎dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBFilterTranslator.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ internal BsonDocument Translate(LambdaExpression lambdaExpression, VectorStoreRe
2727
Debug.Assert(lambdaExpression.Parameters.Count == 1);
2828
this._recordParameter = lambdaExpression.Parameters[0];
2929

30-
var preprocessor = new FilterTranslationPreprocessor { InlineCapturedVariables = true };
31-
var preprocessedExpression = preprocessor.Visit(lambdaExpression.Body);
30+
var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false };
31+
var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body);
3232

3333
return this.Translate(preprocessedExpression);
3434
}

‎dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSqlFilterTranslator.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ internal class AzureCosmosDBNoSqlFilterTranslator
3131
Debug.Assert(lambdaExpression.Parameters.Count == 1);
3232
this._recordParameter = lambdaExpression.Parameters[0];
3333

34-
var preprocessor = new FilterTranslationPreprocessor { InlineCapturedVariables = false };
35-
var preprocessedExpression = preprocessor.Visit(lambdaExpression);
34+
var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = true };
35+
var preprocessedExpression = preprocessor.Preprocess(lambdaExpression);
3636

3737
this.Translate(lambdaExpression.Body);
3838

‎dotnet/src/Connectors/Connectors.Memory.Common/SqlFilterTranslator.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ internal void Translate(bool appendWhere)
4242
this._sql.Append("WHERE ");
4343
}
4444

45-
var preprocessor = new FilterTranslationPreprocessor { TransformCapturedVariablesToQueryParameterExpressions = true };
46-
var preprocessedExpression = preprocessor.Visit(this._lambdaExpression.Body);
45+
var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = true };
46+
var preprocessedExpression = preprocessor.Preprocess(this._lambdaExpression.Body);
4747

4848
this.Translate(preprocessedExpression, isSearchCondition: true);
4949
}

‎dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBFilterTranslator.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ internal BsonDocument Translate(LambdaExpression lambdaExpression, VectorStoreRe
2727
Debug.Assert(lambdaExpression.Parameters.Count == 1);
2828
this._recordParameter = lambdaExpression.Parameters[0];
2929

30-
var preprocessor = new FilterTranslationPreprocessor { InlineCapturedVariables = true };
31-
var preprocessedExpression = preprocessor.Visit(lambdaExpression.Body);
30+
var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false };
31+
var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body);
3232

3333
return this.Translate(preprocessedExpression);
3434
}

‎dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeFilterTranslator.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ internal Metadata Translate(LambdaExpression lambdaExpression, VectorStoreRecord
3030
Debug.Assert(lambdaExpression.Parameters.Count == 1);
3131
this._recordParameter = lambdaExpression.Parameters[0];
3232

33-
var preprocessor = new FilterTranslationPreprocessor { InlineCapturedVariables = true };
34-
var preprocessedExpression = preprocessor.Visit(lambdaExpression.Body);
33+
var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false };
34+
var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body);
3535

3636
return this.Translate(preprocessedExpression);
3737
}

‎dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantFilterTranslator.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ internal Filter Translate(LambdaExpression lambdaExpression, VectorStoreRecordMo
2727
Debug.Assert(lambdaExpression.Parameters.Count == 1);
2828
this._recordParameter = lambdaExpression.Parameters[0];
2929

30-
var preprocessor = new FilterTranslationPreprocessor { InlineCapturedVariables = true };
31-
var preprocessedExpression = preprocessor.Visit(lambdaExpression.Body);
30+
var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false };
31+
var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body);
3232

3333
return this.Translate(preprocessedExpression);
3434
}

‎dotnet/src/Connectors/Connectors.Memory.Redis/RedisFilterTranslator.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ internal string Translate(LambdaExpression lambdaExpression, VectorStoreRecordMo
2727
Debug.Assert(lambdaExpression.Parameters.Count == 1);
2828
this._recordParameter = lambdaExpression.Parameters[0];
2929

30-
var preprocessor = new FilterTranslationPreprocessor { InlineCapturedVariables = true };
31-
var preprocessedExpression = preprocessor.Visit(lambdaExpression.Body);
30+
var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false };
31+
var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body);
3232

3333
this.Translate(preprocessedExpression);
3434
return this._filter.ToString();

‎dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateFilterTranslator.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ internal string Translate(LambdaExpression lambdaExpression, VectorStoreRecordMo
2929
Debug.Assert(lambdaExpression.Parameters.Count == 1);
3030
this._recordParameter = lambdaExpression.Parameters[0];
3131

32-
var preprocessor = new FilterTranslationPreprocessor { InlineCapturedVariables = true };
33-
var preprocessedExpression = preprocessor.Visit(lambdaExpression.Body);
32+
var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false };
33+
var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body);
3434

3535
this.Translate(preprocessedExpression);
3636
return this._filter.ToString();
Lines changed: 92 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

3-
using System;
3+
using System.Collections.Generic;
44
using System.Diagnostics.CodeAnalysis;
55
using System.Linq.Expressions;
66
using System.Reflection;
7-
using System.Runtime.CompilerServices;
87

98
namespace Microsoft.Extensions.VectorData.ConnectorSupport.Filter;
109

@@ -15,40 +14,110 @@ namespace Microsoft.Extensions.VectorData.ConnectorSupport.Filter;
1514
[Experimental("MEVD9001")]
1615
public class FilterTranslationPreprocessor : ExpressionVisitor
1716
{
17+
private List<string>? _parameterNames;
18+
1819
/// <summary>
19-
/// Whether to inline captured variables in the filter expression (when the database doesn't support parameters).
20+
/// Whether the connector supports parameterization.
2021
/// </summary>
21-
public bool InlineCapturedVariables { get; init; }
22+
/// <remarks>
23+
/// If <see langword="false"/>, the visitor will inline captured variables and constant member accesses as simple constant nodes.
24+
/// If <see langword="true"/>, these will instead be replaced with <see cref="QueryParameterExpression"/> nodes.
25+
/// </remarks>
26+
public required bool SupportsParameterization { get; init; }
2227

2328
/// <summary>
24-
/// Whether to transform captured variables in the filter expression to <see cref="QueryParameterExpression"/> (when the database supports parameters).
29+
/// Preprocesses the filter expression before translation.
2530
/// </summary>
26-
public bool TransformCapturedVariablesToQueryParameterExpressions { get; init; }
31+
public Expression Preprocess(Expression node)
32+
{
33+
if (this.SupportsParameterization)
34+
{
35+
this._parameterNames = [];
36+
}
37+
38+
return this.Visit(node);
39+
}
2740

2841
/// <inheritdoc />
2942
protected override Expression VisitMember(MemberExpression node)
3043
{
31-
// This identifies compiler-generated closure types which contain captured variables.
44+
var visited = (MemberExpression)base.VisitMember(node);
45+
46+
// This identifies field and property access over constants, which can be evaluated immediately.
47+
// This covers captured variables, since those are actually memeber accesses over compiled-generated closure types:
48+
// var x = 8;
49+
// _ = await collection.SearchAsync(vector, top: 3, new() { Filter = r => r.Int == x });
50+
//
51+
// This also covers member variables:
52+
// _ = await collection.SearchAsync(vector, top: 3, new() { Filter = r => r.Int == this._x });
53+
// ... as "this" here is represented by a ConstantExpression node in the tree.
54+
//
3255
// Some databases - mostly relational ones - support out-of-band parameters which can be referenced via placeholders
33-
// from the query itself. For those databases, we transform the captured variable to QueryParameterExpression (this simplifies things for those
56+
// from the query itself. For those databases, we transform the member access to QueryParameterExpression (this simplifies things for those
3457
// connectors, and centralizes the pattern matching in a single centralized place).
35-
// For databases which don't support parameters, we simply inline the captured variable as a constant in the tree, so that translators don't
36-
// even need to be aware of the captured variable.
37-
// For all other databases, we simply inline the captured variable as a constant in the tree.
38-
if (node is MemberExpression { Expression: ConstantExpression constant, Member: FieldInfo fieldInfo }
39-
&& constant.Type.Attributes.HasFlag(TypeAttributes.NestedPrivate)
40-
&& Attribute.IsDefined(constant.Type, typeof(CompilerGeneratedAttribute), inherit: true))
58+
// For databases which don't support parameters, we simply inline the evaluated member access as a constant in the tree, so that translators don't
59+
// even need to be aware of it.
60+
61+
// Evaluate the MemberExpression to get the actual value, either for instance members (expression is a ConstantExpression) or for
62+
// static members (expression is null).
63+
object? baseValue;
64+
switch (visited.Expression)
65+
{
66+
// Member access over constant (i.e. instance members)
67+
case ConstantExpression { Value: var v }:
68+
baseValue = v;
69+
break;
70+
71+
// Member constant over null (i.e. static members)
72+
case null:
73+
baseValue = null;
74+
break;
75+
76+
// Member constant over something that has already been parameterized (i.e. nested member access, e.g. r=> r.Int == this.SomeWrapper.Something)
77+
case QueryParameterExpression p:
78+
baseValue = p.Value;
79+
80+
// The previous parameter is getting replaced by the new one we're creating here, so remove its name from the list of parameter names.
81+
this._parameterNames!.Remove(p.Name);
82+
break;
83+
84+
default:
85+
return visited;
86+
}
87+
88+
object? evaluatedValue;
89+
90+
var memberInfo = visited.Member;
91+
92+
switch (memberInfo)
93+
{
94+
case FieldInfo fieldInfo:
95+
evaluatedValue = fieldInfo.GetValue(baseValue);
96+
break;
97+
case PropertyInfo propertyInfo:
98+
evaluatedValue = propertyInfo.GetValue(baseValue);
99+
break;
100+
default:
101+
return visited;
102+
}
103+
104+
// Inline the evaluated value (if the connector doesn't support parameterization, or if the field is readonly),
105+
if (!this.SupportsParameterization)
106+
{
107+
return Expression.Constant(evaluatedValue, visited.Type);
108+
}
109+
110+
// Otherwise, transform the node to a QueryParameterExpression which the connector will then translate to a parameter (e.g. SqlParameter).
111+
112+
// Make sure parameter names are unique.
113+
var origName = memberInfo.Name;
114+
var name = origName;
115+
for (var i = 0; this._parameterNames!.Contains(name); i++)
41116
{
42-
return (this.InlineCapturedVariables, this.TransformCapturedVariablesToQueryParameterExpressions) switch
43-
{
44-
(true, false) => Expression.Constant(fieldInfo.GetValue(constant.Value), node.Type),
45-
(false, true) => new QueryParameterExpression(fieldInfo.Name, fieldInfo.GetValue(constant.Value), node.Type),
46-
47-
(true, true) => throw new InvalidOperationException("InlineCapturedVariables and TransformCapturedVariablesToQueryParameterExpressions cannot both be true."),
48-
(false, false) => base.VisitMember(node)
49-
};
117+
name = $"{origName}_{i}";
50118
}
119+
this._parameterNames.Add(name);
51120

52-
return base.VisitMember(node);
121+
return new QueryParameterExpression(name, evaluatedValue, visited.Type);
53122
}
54123
}

0 commit comments

Comments
 (0)