Skip to content

Commit ecad048

Browse files
UlimoUlimo
authored
Add pushdown on inner join that equals should not be null (#811)
* Add pushdown on inner join that equals should not be null * remove commented code * Add push down on left join and right join with not null * Fix so all conditions in AND expression is checked as well
1 parent 781a708 commit ecad048

File tree

2 files changed

+127
-67
lines changed

2 files changed

+127
-67
lines changed

‎src/FlowtideDotNet.Core/Optimizer/FIlterPushdown/JoinFilterPushdownVisitor.cs

+126-66
Original file line numberDiff line numberDiff line change
@@ -23,56 +23,115 @@ namespace FlowtideDotNet.Core.Optimizer.FilterPushdown
2323
/// </summary>
2424
internal class JoinFilterPushdownVisitor : OptimizerBaseVisitor
2525
{
26+
private static void TestPushdownNotNull(JoinRelation joinRelation, Expression expression, List<Expression> leftPushdowns, List<Expression> rightPushdowns)
27+
{
28+
if ((joinRelation.Type == JoinType.Inner || joinRelation.Type == JoinType.Left || joinRelation.Type == JoinType.Right) &&
29+
MergeJoinFindVisitor.Check(joinRelation, expression, out var leftKey, out var rightKey))
30+
{
31+
if (joinRelation.Type == JoinType.Inner || joinRelation.Type == JoinType.Right)
32+
{
33+
if (leftKey.ReferenceSegment is StructReferenceSegment leftStruct)
34+
{
35+
leftPushdowns.Add(new ScalarFunction()
36+
{
37+
Arguments = new List<Expression>()
38+
{
39+
new DirectFieldReference()
40+
{
41+
ReferenceSegment = new StructReferenceSegment()
42+
{
43+
Field = leftStruct.Field,
44+
Child = leftStruct.Child
45+
}
46+
}
47+
},
48+
ExtensionUri = FunctionsComparison.Uri,
49+
ExtensionName = FunctionsComparison.IsNotNull
50+
});
51+
}
52+
}
53+
54+
if (joinRelation.Type == JoinType.Inner || joinRelation.Type == JoinType.Left)
55+
{
56+
if (rightKey.ReferenceSegment is StructReferenceSegment rightStruct)
57+
{
58+
rightPushdowns.Add(new ScalarFunction()
59+
{
60+
Arguments = new List<Expression>()
61+
{
62+
new DirectFieldReference()
63+
{
64+
ReferenceSegment = new StructReferenceSegment()
65+
{
66+
Field = rightStruct.Field,
67+
Child = rightStruct.Child
68+
}
69+
}
70+
},
71+
ExtensionUri = FunctionsComparison.Uri,
72+
ExtensionName = FunctionsComparison.IsNotNull
73+
});
74+
}
75+
}
76+
}
77+
}
78+
2679
public override Relation VisitJoinRelation(JoinRelation joinRelation, object state)
2780
{
2881
// Check root expression
2982
var visitor = new JoinExpressionVisitor(joinRelation.Left.OutputLength);
3083
visitor.Visit(joinRelation.Expression!, state);
84+
85+
List<Expression> leftPushdowns = new List<Expression>();
86+
List<Expression> rightPushdowns = new List<Expression>();
87+
88+
if (joinRelation.Expression != null)
89+
{
90+
TestPushdownNotNull(joinRelation, joinRelation.Expression, leftPushdowns, rightPushdowns);
91+
92+
if (joinRelation.Expression is ScalarFunction scalarFunc &&
93+
scalarFunc.ExtensionUri == FunctionsBoolean.Uri && scalarFunc.ExtensionName == FunctionsBoolean.And)
94+
{
95+
for (int i = 0; i < scalarFunc.Arguments.Count; i++)
96+
{
97+
TestPushdownNotNull(joinRelation, scalarFunc.Arguments[i], leftPushdowns, rightPushdowns);
98+
}
99+
}
100+
}
101+
31102
if (!visitor.unknownCase)
32103
{
33104
// Only fields from left is used
34105
if (visitor.fieldInLeft && !visitor.fieldInRight && joinRelation.Type == JoinType.Inner)
35106
{
36-
joinRelation.Left = new FilterRelation()
37-
{
38-
Condition = joinRelation.Expression!,
39-
Input = joinRelation.Left
40-
};
41-
joinRelation.Expression = new BoolLiteral() { Value = true };
107+
leftPushdowns.Add(new BoolLiteral() { Value = true });
42108
}
43109
// Only field in right is used
44110
else if (!visitor.fieldInLeft && visitor.fieldInRight && joinRelation.Type == JoinType.Inner)
45111
{
46-
joinRelation.Right = new FilterRelation()
47-
{
48-
Condition = joinRelation.Expression!,
49-
Input = joinRelation.Right
50-
};
51-
joinRelation.Expression = new BoolLiteral() { Value = true };
112+
rightPushdowns.Add(new BoolLiteral() { Value = true });
52113
}
53114
}
54115

55116
if (joinRelation.Expression is ScalarFunction andFunctionScalar &&
56117
andFunctionScalar.ExtensionUri == FunctionsBoolean.Uri &&
57118
andFunctionScalar.ExtensionName == FunctionsBoolean.And)
58119
{
59-
List<Expression> leftPushDown = new List<Expression>();
60-
List<Expression> rightPushDown = new List<Expression>();
61120
for (int i = 0; i < andFunctionScalar.Arguments.Count; i++)
62121
{
63122
var expr = andFunctionScalar.Arguments[i];
64123
var andVisitor = new JoinExpressionVisitor(joinRelation.Left.OutputLength);
65124
andVisitor.Visit(expr, state);
66125
if (andVisitor.fieldInLeft && !andVisitor.fieldInRight && joinRelation.Type == JoinType.Inner)
67126
{
68-
leftPushDown.Add(expr);
127+
leftPushdowns.Add(expr);
69128
andFunctionScalar.Arguments.RemoveAt(i);
70129
i--;
71130
}
72131
// Only field in right is used
73132
else if (!andVisitor.fieldInLeft && andVisitor.fieldInRight && joinRelation.Type == JoinType.Inner)
74133
{
75-
rightPushDown.Add(expr);
134+
rightPushdowns.Add(expr);
76135
andFunctionScalar.Arguments.RemoveAt(i);
77136
i--;
78137
}
@@ -85,64 +144,65 @@ public override Relation VisitJoinRelation(JoinRelation joinRelation, object sta
85144
joinRelation.Expression = new BoolLiteral() { Value = true };
86145
}
87146
}
88-
if (leftPushDown.Count > 0)
89-
{
147+
}
90148

91-
if (leftPushDown.Count == 1)
92-
{
93-
joinRelation.Left = new FilterRelation()
94-
{
95-
Condition = leftPushDown[0],
96-
Input = joinRelation.Left
97-
};
98-
}
99-
else
149+
if (leftPushdowns.Count > 0)
150+
{
151+
152+
if (leftPushdowns.Count == 1)
153+
{
154+
joinRelation.Left = new FilterRelation()
100155
{
101-
joinRelation.Left = new FilterRelation()
102-
{
103-
Condition = new ScalarFunction() { ExtensionUri = FunctionsBoolean.Uri, ExtensionName = FunctionsBoolean.And, Arguments = leftPushDown },
104-
Input = joinRelation.Left
105-
};
106-
}
156+
Condition = leftPushdowns[0],
157+
Input = joinRelation.Left
158+
};
107159
}
108-
if (rightPushDown.Count > 0)
160+
else
109161
{
110-
// Find used fields
111-
var usageVisitor = new ExpressionFieldUsageVisitor(joinRelation.Left.OutputLength);
112-
foreach (var expr in rightPushDown)
162+
joinRelation.Left = new FilterRelation()
113163
{
114-
usageVisitor.Visit(expr, default);
115-
}
116-
var rightUsageFields = usageVisitor.UsedFieldsRight.Distinct().ToList();
164+
Condition = new ScalarFunction() { ExtensionUri = FunctionsBoolean.Uri, ExtensionName = FunctionsBoolean.And, Arguments = leftPushdowns },
165+
Input = joinRelation.Left
166+
};
167+
}
168+
}
169+
if (rightPushdowns.Count > 0)
170+
{
171+
// Find used fields
172+
var usageVisitor = new ExpressionFieldUsageVisitor(joinRelation.Left.OutputLength);
173+
foreach (var expr in rightPushdowns)
174+
{
175+
usageVisitor.Visit(expr, default);
176+
}
177+
var rightUsageFields = usageVisitor.UsedFieldsRight.Distinct().ToList();
117178

118-
// Build lookup table from old to new field id
119-
Dictionary<int, int> oldToNew = new Dictionary<int, int>();
120-
foreach (var usedField in rightUsageFields)
121-
{
122-
oldToNew.Add(usedField, usedField - joinRelation.Left.OutputLength);
123-
}
124-
// Replace old ids with the new ids
125-
var replaceVisitor = new ExpressionFieldReplaceVisitor(oldToNew);
126-
foreach (var expr in rightPushDown)
127-
{
128-
replaceVisitor.Visit(expr, default);
129-
}
130-
if (rightPushDown.Count == 1)
179+
// Build lookup table from old to new field id
180+
Dictionary<int, int> oldToNew = new Dictionary<int, int>();
181+
foreach (var usedField in rightUsageFields)
182+
{
183+
oldToNew.Add(usedField, usedField - joinRelation.Left.OutputLength);
184+
}
185+
// Replace old ids with the new ids
186+
var replaceVisitor = new ExpressionFieldReplaceVisitor(oldToNew);
187+
foreach (var expr in rightPushdowns)
188+
{
189+
replaceVisitor.Visit(expr, default);
190+
}
191+
if (rightPushdowns.Count == 1)
192+
{
193+
joinRelation.Right = new FilterRelation()
131194
{
132-
joinRelation.Right = new FilterRelation()
133-
{
134-
Condition = rightPushDown[0],
135-
Input = joinRelation.Right
136-
};
137-
}
138-
else
195+
Condition = rightPushdowns[0],
196+
Input = joinRelation.Right
197+
};
198+
}
199+
else
200+
{
201+
joinRelation.Right = new FilterRelation()
139202
{
140-
joinRelation.Right = new FilterRelation()
141-
{
142-
Condition = new ScalarFunction() { ExtensionUri = FunctionsBoolean.Uri, ExtensionName = FunctionsBoolean.And, Arguments = rightPushDown },
143-
Input = joinRelation.Right
144-
};
145-
}
203+
Condition = new ScalarFunction() { ExtensionUri = FunctionsBoolean.Uri, ExtensionName = FunctionsBoolean.And, Arguments = rightPushdowns },
204+
Input = joinRelation.Right
205+
};
146206
}
147207
}
148208

‎src/FlowtideDotNet.Core/Optimizer/MergeJoinFindVisitor.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ namespace FlowtideDotNet.Core.Optimizer
1919
{
2020
internal class MergeJoinFindVisitor : OptimizerBaseVisitor
2121
{
22-
private bool Check(JoinRelation joinRelation, Expression? expression, [NotNullWhen(true)] out DirectFieldReference? leftKey, [NotNullWhen(true)] out DirectFieldReference? rightKey)
22+
internal static bool Check(JoinRelation joinRelation, Expression? expression, [NotNullWhen(true)] out DirectFieldReference? leftKey, [NotNullWhen(true)] out DirectFieldReference? rightKey)
2323
{
2424
if (expression is ScalarFunction booleanComparison &&
2525
booleanComparison.ExtensionUri == FunctionsComparison.Uri &&

0 commit comments

Comments
 (0)