Skip to content

Commit 83db407

Browse files
Fix the endless loop in Join and Merge when both suffixes are the same (dotnet#7692)
Fixes dotnet#6128 `Join` and `Merge` rename a column that appears in both frames . the existing one gets `leftSuffix` , the new one gets `rightSuffix` , and it repeats until the two names are different . when both suffixes are the same string the names stay equal after every round , so nothing ever ends the loop and both names keep growing . the reporter hit it by passing the join column name where the suffix goes . michaelgsharp already named the cause on the issue in 2022 , this is just the patch for it . it now throws an `ArgumentException` naming the two parameters and the column . the check is inside `SetSuffixForDuplicatedColumnNames` rather than up front in `Join` and `Merge` , because equal suffixes are fine when the two frames share no column name . nothing gets renamed in that case and it works today , so an up front check would break it . i measured that before deciding where to put it : ``` join , shared column name , same suffix hangs , all four algorithms join , no shared column name , same suffix fine merge , shared column name , same suffix hangs merge , default suffixes fine ``` three tests added . the first covers all four `JoinAlgorithm` values with `"_same"/"_same"` and with `""/""` , then checks `"_left"/"_right"` still gives the full column set . the second goes the other way and pins that equal suffixes are still allowed when nothing collides . the third is the same thing for `Merge` . worth saying how the without-the-fix run looks , since it is not an ordinary failure . the defect is an endless loop , so the tests do not go red , they never finish . i took the fix out and the run was still going after 150 seconds with no result , at about 92 percent cpu . with the fix the same six tests finish in 43 ms . `Microsoft.Data.Analysis.Tests` is 483 passing , 0 failures , 5 skipped . the tree without this runs 477 , so the only difference is the six new ones . build is clean with `EnforceCodeStyleInBuild`. i also grepped every `.Join(` and `.Merge(` caller in `src` and `test` , none of them pass equal suffixes , so the new guard cannot fire on existing code .
1 parent e1e858a commit 83db407

5 files changed

Lines changed: 67 additions & 0 deletions

File tree

‎src/Microsoft.Data.Analysis/DataFrame.Join.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,15 @@ public partial class DataFrame
2323
private void SetSuffixForDuplicatedColumnNames(DataFrame dataFrame, DataFrameColumn column, string leftSuffix, string rightSuffix)
2424
{
2525
int index = dataFrame._columnCollection.IndexOf(column.Name);
26+
27+
// The loop below appends leftSuffix to the existing column and rightSuffix to the new one until the two
28+
// names differ. If both suffixes are the same the names stay equal however many times they are appended,
29+
// so there is no name that ends the loop.
30+
if (index != -1 && leftSuffix == rightSuffix)
31+
{
32+
throw new ArgumentException(string.Format(Strings.SuffixesMustBeDifferent, nameof(leftSuffix), nameof(rightSuffix), column.Name), nameof(rightSuffix));
33+
}
34+
2635
while (index != -1)
2736
{
2837
// Pre-existing column. Change name

‎src/Microsoft.Data.Analysis/Strings.Designer.cs‎

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/Microsoft.Data.Analysis/Strings.resx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,9 @@
261261
<data name="StreamDoesntSupportReading" xml:space="preserve">
262262
<value>Stream doesn't support reading</value>
263263
</data>
264+
<data name="SuffixesMustBeDifferent" xml:space="preserve">
265+
<value>{0} and {1} must be different when both DataFrames contain a column called '{2}'</value>
266+
</data>
264267
<data name="ValueNameAlreadyExists" xml:space="preserve">
265268
<value>Value name '{0}' matches an existing column name</value>
266269
</data>

‎test/Microsoft.Data.Analysis.Tests/DataFrameTests.Join.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,40 @@ public void TestJoin()
8282
VerifyJoin(join, left, right, JoinAlgorithm.Inner);
8383
}
8484

85+
[Theory]
86+
[InlineData(JoinAlgorithm.Left)]
87+
[InlineData(JoinAlgorithm.Right)]
88+
[InlineData(JoinAlgorithm.FullOuter)]
89+
[InlineData(JoinAlgorithm.Inner)]
90+
public void TestJoin_ColumnNamesCollide_ThrowsOnSameSuffixOnBothSides(JoinAlgorithm joinAlgorithm)
91+
{
92+
DataFrame left = MakeDataFrameWithNumericColumns(3, false);
93+
DataFrame right = MakeDataFrameWithNumericColumns(4, false);
94+
95+
// Both frames have a column called "Int", and one suffix cannot make the two names different,
96+
// so the join used to keep renaming for ever instead of coming back
97+
Assert.Throws<ArgumentException>(() => left.Join(right, "_same", "_same", joinAlgorithm));
98+
Assert.Throws<ArgumentException>(() => left.Join(right, "", "", joinAlgorithm));
99+
100+
// different suffixes still work
101+
DataFrame join = left.Join(right, "_left", "_right", joinAlgorithm);
102+
Assert.Equal(left.Columns.Count + right.Columns.Count, join.Columns.Count);
103+
}
104+
105+
[Fact]
106+
public void TestJoin_SameSuffixWithNoSharedColumnNames()
107+
{
108+
DataFrame left = new DataFrame(new Int32DataFrameColumn("Left", new int?[] { 0, 1, 2 }));
109+
DataFrame right = new DataFrame(new Int32DataFrameColumn("Right", new int?[] { 0, 1, 2 }));
110+
111+
// no name is shared, so nothing is renamed and the suffixes are never used
112+
DataFrame join = left.Join(right, "_same", "_same");
113+
114+
Assert.Equal(2, join.Columns.Count);
115+
Assert.Equal("Left", join.Columns[0].Name);
116+
Assert.Equal("Right", join.Columns[1].Name);
117+
}
118+
85119
private void VerifyJoin(DataFrame join, DataFrame left, DataFrame right, JoinAlgorithm joinAlgorithm)
86120
{
87121
Int64DataFrameColumn mapIndices = new Int64DataFrameColumn("map", join.Rows.Count);

‎test/Microsoft.Data.Analysis.Tests/DataFrameTests.Merge.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1013,6 +1013,18 @@ public void TestMerge_CorrectColumnTypes()
10131013
Assert.NotNull(merge.Columns.GetDateTimeColumn("DateTime_right"));
10141014
}
10151015

1016+
[Fact]
1017+
public void TestMerge_ColumnNamesCollide_ThrowsOnSameSuffixOnBothSides()
1018+
{
1019+
DataFrame left = MakeDataFrameWithNumericColumns(3, false);
1020+
DataFrame right = MakeDataFrameWithNumericColumns(4, false);
1021+
1022+
Assert.Throws<ArgumentException>(() => left.Merge<int>(right, "Int", "Int", "_same", "_same"));
1023+
1024+
DataFrame merge = left.Merge<int>(right, "Int", "Int");
1025+
Assert.Equal(left.Columns.Count + right.Columns.Count, merge.Columns.Count);
1026+
}
1027+
10161028
private void VerifyMerge(DataFrame merge, DataFrame left, DataFrame right, JoinAlgorithm joinAlgorithm)
10171029
{
10181030
if (joinAlgorithm == JoinAlgorithm.Left || joinAlgorithm == JoinAlgorithm.Inner)

0 commit comments

Comments
 (0)