Add support for list sort ascending null last

This fixes #768

Author: Ulimo

docs/docs/expressions/scalarfunctions/list.md

@@ -0,0 +1,21 @@
+---
+sidebar_position: 8
+---
+
+# List Functions
+
+## List Sort Ascending Null Last
+
+*This function has no substrait equivalent*
+
+Sorts a list of values in ascending order, placing any null values at the end of the result.
+
+Values are ordered according to their natural ascending order (e.g., numerically or lexicographically).
+Nulls are not compared to other values directly; they are always considered greater for the purpose of ordering.
+Any value that is not a list will return the result as null.
+
+### SQL Usage
+
+```sql
+SELECT list_sort_asc_null_last(list(orderkey, userkey)) FROM ...
+```

src/FlowtideDotNet.Core/Compute/Columnar/Functions/BuiltInListFunctions.cs

@@ -0,0 +1,50 @@
+// Licensed under the Apache License, Version 2.0 (the "License")
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//  
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using FlowtideDotNet.Core.ColumnStore;
+using FlowtideDotNet.Core.ColumnStore.Comparers;
+using FlowtideDotNet.Core.ColumnStore.DataValues;
+using FlowtideDotNet.Substrait.FunctionExtensions;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace FlowtideDotNet.Core.Compute.Columnar.Functions
+{
+    internal static class BuiltInListFunctions
+    {
+        public static void AddBuiltInListFunctions(IFunctionsRegister functionsRegister)
+        {
+            functionsRegister.RegisterScalarMethod(FunctionsList.Uri, FunctionsList.ListSortAscendingNullLast, typeof(BuiltInListFunctions), nameof(ListSortAscendingNullLast));
+        }
+
+        private static IDataValue ListSortAscendingNullLast<T>(T value)
+            where T : IDataValue
+        {
+            if (value.Type == ArrowTypeId.List)
+            {
+                var list = value.AsList;
+                IDataValue[] newList = new IDataValue[list.Count];
+                for (int i = 0; i < list.Count; i++)
+                {
+                    newList[i] = list.GetAt(i);
+                }
+                
+                Array.Sort(newList, SortFieldCompareCompiler.CompareAscendingNullsLastImplementation);
+                return new ListValue(newList);
+            }
+            return NullValue.Instance;
+        }
+    }
+}

src/FlowtideDotNet.Core/Compute/Internal/BuiltinFunctions.cs

@@ -30,6 +30,7 @@ public static void RegisterFunctions(FunctionsRegister functionsRegister)
             Columnar.Functions.BuiltInRoundingFunctions.AddRoundingFunctions(functionsRegister);
             BuiltInCheckFunctions.RegisterCheckFunctions(functionsRegister);
             Columnar.Functions.BuiltInStructFunctions.AddBuiltInStructFunctions(functionsRegister);
+            Columnar.Functions.BuiltInListFunctions.AddBuiltInListFunctions(functionsRegister);
 
             BuiltInComparisonFunctions.AddComparisonFunctions(functionsRegister);
             BuiltInBooleanFunctions.AddBooleanFunctions(functionsRegister);

src/FlowtideDotNet.Substrait/FunctionExtensions/FunctionsList.cs

@@ -17,5 +17,6 @@ public static class FunctionsList
         public const string Uri = "/functions_list.yaml";
         public const string ListAgg = "list_agg";
         public const string ListUnionDistinctAgg = "list_union_distinct_agg";
+        public const string ListSortAscendingNullLast = "list_sort_asc_null_last";
     }
 }

src/FlowtideDotNet.Substrait/Sql/Internal/BuiltInSqlFunctions.cs

@@ -875,6 +875,8 @@ public static void AddBuiltInFunctions(SqlFunctionRegister sqlFunctionRegister)
 
             RegisterOneVariableScalarFunction(sqlFunctionRegister, "floor_timestamp_day", FunctionsDatetime.Uri, FunctionsDatetime.FloorTimestampDay);
 
+            RegisterOneVariableScalarFunction(sqlFunctionRegister, "list_sort_asc_null_last", FunctionsList.Uri, FunctionsList.ListSortAscendingNullLast);
+
             // Table functions
             UnnestSqlFunction.AddUnnest(sqlFunctionRegister);
 

tests/FlowtideDotNet.AcceptanceTests/ListTests.cs

@@ -189,5 +189,26 @@ FROM testdata
                 new {list = new List<object>(){ 2, "a", "c" }}
             });
         }
+
+        [Fact]
+        public async Task ListSortAscendingNullLast()
+        {
+            GenerateData();
+            await StartStream(@"
+                INSERT INTO output 
+                SELECT 
+                    list_sort_asc_null_last(list(orderkey, userkey))
+                FROM orders
+                ");
+
+            await WaitForUpdate();
+
+            var expectedList = Orders.Select(x => new
+            {
+                list = (new List<int>() { x.OrderKey, x.UserKey }).OrderBy(x => x).ToList()
+            } );
+
+            AssertCurrentDataEqual(expectedList);
+        }
     }
 }

tests/FlowtideDotNet.ComputeTests/Internal/Parser/Tests/DataTypeValueWriter.cs

@@ -20,7 +20,7 @@ internal static class DataTypeValueWriter
     {
         public static IDataValue WriteDataValue(string value, string dataType)
         {
-            if (value == "Null")
+            if (value.Equals("Null", StringComparison.OrdinalIgnoreCase))
             {
                 return NullValue.Instance;
             }

tests/FlowtideDotNet.ComputeTests/OtherCases/list/list_sort_asc_null_last.test

@@ -0,0 +1,10 @@
+### SUBSTRAIT_SCALAR_TEST: v1.0
+### SUBSTRAIT_INCLUDE: '/extensions/functions_list.yaml'
+
+# basic: Basic examples without any special cases
+list_sort_asc_null_last([3, 2, 1]::LIST<i32>) = [1, 2, 3]::LIST<i32>
+list_sort_asc_null_last(['c', 'a']::LIST<str>) = ['a', 'c']::LIST<str>
+
+# null_input: Examples with null as input
+list_sort_asc_null_last([3, 2, null]::LIST<i32>) = [2, 3, null]::LIST<i32>
+list_sort_asc_null_last([null, 3, 2]::LIST<i32>) = [2, 3, null]::LIST<i32>