Refactor "In"-Expression

src-console/ConsoleApp_net10/Program.cs

@@ -38,10 +38,24 @@ public class GroupedSalesData
     public int GroupLevel { get; set; }
 }
 
+class MyEntity
+{
+    // Factory method to create a list of MyEntity objects from a given list of ids
+    public static IEnumerable<MyEntity> CreateList(IEnumerable<int> ids)
+    {
+        foreach (var id in ids) yield return new MyEntity { Id = id };
+    }
+
+    public int Id { get; set; }
+}
+
 class Program
 {
     static void Main(string[] args)
     {
+        Issue987();
+        return;
+
         Issue918();
         return;
 
@@ -72,6 +86,52 @@ static void Main(string[] args)
         Dynamic();
     }
 
+    private static void Issue987()
+    {
+        var list = new List<MyEntity>();
+        for (int i = 0; i < 10000; i++)
+            list.Add(new MyEntity { Id = i });
+
+        var test1 = list.AsQueryable()
+            .Where("Id in (9495, 9496, 9498, 9500, 9501, 9503, 9505, 9508, 9509, 9510, 9511, 9514, 9515, 9517, 9518, 9519, 9520, 9521, 9523, 9524, 9525, 9526, 9527, 9528, 9529, 9530, 9531, 9532, 9533, 9534, 9535, 9536, 9538, 9539, 9540, 9541, 9542, 9543, 9544, 9545, 9546, 9547, 9548, 9549, 9550, 9552, 9554, 9556, 9557, 9558, 9559, 9560, 9561, 9562, 9563, 9565, 9567, 9569, 9570, 9575, 9576, 9577, 9578, 9579, 9580, 9581, 9582, 9583, 9584, 9585, 9586, 9587, 9588, 9589, 9590, 9591, 9592, 9593, 9594, 9595, 9596, 9597, 9598, 9599, 9600, 9601, 9602, 9603, 9604, 9605, 9606, 9607, 9608, 9609, 9610, 9611, 9612, 9613, 9614, 9615, 9616, 9617, 9618, 9619, 9620, 9621, 9622, 9623, 9624, 9625, 9626, 9627, 9628, 9629)")
+            .ToList();
+
+        Console.WriteLine("Number of elements : " + test1.Count);
+
+        //the list of ids that were actually used in our application and resulted in the discovery of this bug
+        var originalIdList = new List<int>() { 9495, 9496, 9498, 9500, 9501, 9503, 9505, 9508, 9509, 9510, 9511, 9514, 9515, 9517, 9518, 9519, 9520, 9521, 9523, 9524, 9525, 9526, 9527, 9528, 9529, 9530, 9531, 9532, 9533, 9534, 9535, 9536, 9538, 9539, 9540, 9541, 9542, 9543, 9544, 9545, 9546, 9547, 9548, 9549, 9550, 9552, 9554, 9556, 9557, 9558, 9559, 9560, 9561, 9562, 9563, 9565, 9567, 9569, 9570, 9575, 9576, 9577, 9578, 9579, 9580, 9581, 9582, 9583, 9584, 9585, 9586, 9587, 9588, 9589, 9590, 9591, 9592, 9593, 9594, 9595, 9596, 9597, 9598, 9599, 9600, 9601, 9602, 9603, 9604, 9605, 9606, 9607, 9608, 9609, 9610, 9611, 9612, 9613, 9614, 9615, 9616, 9617, 9618, 9619, 9620, 9621, 9622, 9623, 9624, 9625, 9626, 9627, 9628, 9629 };
+        //list of ids also starting at 9495, but without gaps
+        var adjacentIdList = Enumerable.Range(9495, 114);
+        //original list starting at Id1
+        var originalIdListStartingAt1 = originalIdList.Select(id => id - 9494);
+        //list with gaps of 1
+        var listWithGapsOf1 = Enumerable.Range(1, 114).Select(id => id * 2);
+        //list with gaps of 2
+        var listWithGapsOf2 = Enumerable.Range(1, 114).Select(id => id * 3);
+        //list with gaps of 3
+        var listWithGapsOf3 = Enumerable.Range(1, 114).Select(id => id * 4);
+        //list with gaps of 4
+        var listWithGapsOf4 = Enumerable.Range(1, 114).Select(id => id * 5);
+
+        //list of 10.000 entities , with ids starting at 0
+        var entityList = MyEntity.CreateList(Enumerable.Range(1, 10_000));
+
+        //filter the list of entities by the list of ids using dynamic linq and write the number of elements in the filtered list to the console
+        static void Filter(IEnumerable<MyEntity> entities, IEnumerable<int> ids)
+        {
+            var filtered = entities.AsQueryable().Where($"Id in ({string.Join(',', ids)})").ToList();
+            Console.WriteLine("Number of elements : " + filtered.Count);
+        }
+
+        Filter(entityList, originalIdList);
+        Filter(entityList, adjacentIdList);
+        Filter(entityList, originalIdListStartingAt1);
+        Filter(entityList, listWithGapsOf1);
+        Filter(entityList, listWithGapsOf2);
+        Filter(entityList, listWithGapsOf3);
+        Filter(entityList, listWithGapsOf4);
+    }
+
     private static void Issue918()
     {
         var persons = new DataTable();

src/System.Linq.Dynamic.Core/DynamicQueryableExtensions.cs

@@ -24,7 +24,7 @@ namespace System.Linq.Dynamic.Core
     [SuppressMessage("ReSharper", "PossibleMultipleEnumeration")]
     public static class DynamicQueryableExtensions
     {
-#if !(SILVERLIGHT)
+#if !SILVERLIGHT
         private static readonly TraceSource TraceSource = new(nameof(DynamicQueryableExtensions));
 #endif
 
@@ -34,7 +34,7 @@ private static Expression OptimizeExpression(Expression expression)
             {
                 var optimized = ExtensibilityPoint.QueryOptimizer(expression);
 
-#if !(SILVERLIGHT)
+#if !SILVERLIGHT
                 if (optimized != expression)
                 {
                     TraceSource.TraceEvent(TraceEventType.Verbose, 0, "Expression before : {0}", expression);
@@ -2094,7 +2094,7 @@ public static IQueryable SelectMany(
             string collectionParameterName,
             string resultParameterName,
             object?[]? collectionSelectorArgs = null,
-            params object[]? resultSelectorArgs)
+            params object?[]? resultSelectorArgs)
         {
             Check.NotNull(source);
             Check.NotNull(config);
@@ -2682,7 +2682,6 @@ public static IQueryable Where(this IQueryable source, ParsingConfig config, str
 
             bool createParameterCtor = SupportsLinqToObjects(config, source);
             LambdaExpression lambda = DynamicExpressionParser.ParseLambda(config, createParameterCtor, source.ElementType, null, predicate, args);
-
             var optimized = OptimizeExpression(Expression.Call(typeof(Queryable), nameof(Queryable.Where), [source.ElementType], source.Expression, Expression.Quote(lambda)));
             return source.Provider.CreateQuery(optimized);
         }

src/System.Linq.Dynamic.Core/Parser/ExpressionHelper.cs

@@ -156,6 +156,37 @@ public Expression GenerateNotEqual(Expression left, Expression right)
         return Expression.NotEqual(left, right);
     }
 
+    public Expression GenerateBinaryOrElseTree(IList<Expression> expressions)
+    {
+        Check.NotNullOrEmpty(expressions);
+
+        if (expressions.Count == 1)
+        {
+            return expressions[0];
+        }
+
+        var currentLevel = new List<Expression>(expressions);
+        while (currentLevel.Count > 1)
+        {
+            var nextLevel = new List<Expression>((currentLevel.Count + 1) / 2);
+            for (var i = 0; i < currentLevel.Count; i += 2)
+            {
+                if (i + 1 < currentLevel.Count)
+                {
+                    nextLevel.Add(Expression.OrElse(currentLevel[i], currentLevel[i + 1]));
+                }
+                else
+                {
+                    nextLevel.Add(currentLevel[i]);
+                }
+            }
+
+            currentLevel = nextLevel;
+        }
+
+        return currentLevel[0];
+    }
+
     public Expression GenerateGreaterThan(Expression left, Expression right)
     {
         TryConvertTypes(ref left, ref right);

src/System.Linq.Dynamic.Core/Parser/ExpressionParser.cs

@@ -367,6 +367,12 @@ private Expression ParseIn()
 
             if (_textParser.CurrentToken.Id == TokenId.OpenParen) // literals (or other inline list)
             {
+                var values = new List<Expression>();
+                var comparisons = new List<Expression>();
+                Expression? containsLeft = null;
+                string? containsLeftText = null;
+                var canUseContains = true;
+
                 while (_textParser.CurrentToken.Id != TokenId.CloseParen)
                 {
                     _textParser.NextToken();
@@ -393,13 +399,26 @@ private Expression ParseIn()
                         CheckAndPromoteOperands(typeof(IEqualitySignatures), TokenId.DoubleEqual, "==", ref left, ref right, token.Pos);
                     }
 
-                    if (accumulate.Type != typeof(bool))
+                    var equalsExpression = _expressionHelper.GenerateEqual(left, right);
+                    comparisons.Add(equalsExpression);
+
+                    if (canUseContains && equalsExpression is BinaryExpression binaryExpression && binaryExpression.NodeType == ExpressionType.Equal)
                     {
-                        accumulate = _expressionHelper.GenerateEqual(left, right);
+                        containsLeft ??= binaryExpression.Left;
+                        containsLeftText ??= binaryExpression.Left.ToString();
+
+                        if (containsLeft.Type != binaryExpression.Left.Type || !string.Equals(containsLeftText, binaryExpression.Left.ToString(), StringComparison.Ordinal) || binaryExpression.Right.Type != containsLeft.Type)
+                        {
+                            canUseContains = false;
+                        }
+                        else
+                        {
+                            values.Add(binaryExpression.Right);
+                        }
                     }
                     else
                     {
-                        accumulate = Expression.OrElse(accumulate, _expressionHelper.GenerateEqual(left, right));
+                        canUseContains = false;
                     }
 
                     if (_textParser.CurrentToken.Id == TokenId.End)
@@ -408,6 +427,17 @@ private Expression ParseIn()
                     }
                 }
 
+                if (canUseContains && containsLeft != null)
+                {
+                    var typeArgs = new[] { containsLeft.Type };
+                    var args = new Expression[] { Expression.NewArrayInit(containsLeft.Type, values), containsLeft };
+                    accumulate = Expression.Call(typeof(Enumerable), nameof(Enumerable.Contains), typeArgs, args);
+                }
+                else
+                {
+                    accumulate = _expressionHelper.GenerateBinaryOrElseTree(comparisons);
+                }
+
                 // Since this started with an open paren, make sure to move off the close
                 _textParser.NextToken();
             }
@@ -1514,7 +1544,7 @@ private Expression ParseNew()
                 {
                     if (!propertyNames.Add(propName!))
                     {
-                        throw ParseError(exprPos, Res.DuplicateIdentifier, propName);
+                        throw ParseError(exprPos, Res.DuplicateIdentifier, propName!);
                     }
 
                     properties.Add(new DynamicProperty(propName!, expr.Type));

src/System.Linq.Dynamic.Core/Parser/IExpressionHelper.cs

@@ -1,4 +1,5 @@
-using System.Diagnostics.CodeAnalysis;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq.Expressions;
 
 namespace System.Linq.Dynamic.Core.Parser;
@@ -21,6 +22,8 @@ internal interface IExpressionHelper
 
     Expression GenerateNotEqual(Expression left, Expression right);
 
+    Expression GenerateBinaryOrElseTree(IList<Expression> expressions);
+
     Expression GenerateStringConcat(Expression left, Expression right);
 
     Expression GenerateSubtract(Expression left, Expression right);

src/System.Linq.Dynamic.Core/Parser/SupportedMethods/MethodData.cs

@@ -1,12 +1,13 @@
 using System.Linq.Expressions;
 using System.Reflection;
 
-namespace System.Linq.Dynamic.Core.Parser.SupportedMethods
+namespace System.Linq.Dynamic.Core.Parser.SupportedMethods;
+
+internal class MethodData
 {
-    internal class MethodData
-    {
-        public MethodBase MethodBase { get; set; }
-        public ParameterInfo[] Parameters { get; set; }
-        public Expression[] Args { get; set; }
-    }
-}
+    public MethodBase MethodBase { get; set; }
+
+    public ParameterInfo[] Parameters { get; set; }
+
+    public Expression[] Args { get; set; }
+}

src/System.Linq.Dynamic.Core/Validation/Check.cs

@@ -10,7 +10,7 @@ internal static class Check
 {
     private const string ParsingConfigError = "The ParsingConfig should be provided as first argument to this method.";
 
-    public static object?[]? Args(object?[]? args, [CallerArgumentExpression("args")] string? parameterName = null)
+    public static object?[]? Args(object?[]? args, [CallerArgumentExpression(nameof(args))] string? parameterName = null)
     {
         if (args?.Any(a => a is ParsingConfig) == true)
         {
@@ -20,7 +20,7 @@ internal static class Check
         return args;
     }
 
-    public static T Condition<T>(T value, Predicate<T> predicate, [CallerArgumentExpression("value")] string? parameterName = null)
+    public static T Condition<T>(T value, Predicate<T> predicate, [CallerArgumentExpression(nameof(value))] string? parameterName = null)
     {
         NotNull(predicate);
 
@@ -34,7 +34,7 @@ public static T Condition<T>(T value, Predicate<T> predicate, [CallerArgumentExp
         return value;
     }
 
-    public static T NotNull<T>(T value, [CallerArgumentExpression("value")] string? parameterName = null)
+    public static T NotNull<T>(T value, [CallerArgumentExpression(nameof(value))] string? parameterName = null)
     {
         if (value is null)
         {
@@ -59,7 +59,7 @@ public static T NotNull<T>(T value, string parameterName, string propertyName)
         return value;
     }
 
-    public static IEnumerable<T> NotNullOrEmpty<T>(IEnumerable<T> value, [CallerArgumentExpression("value")] string? parameterName = null)
+    public static IEnumerable<T> NotNullOrEmpty<T>(IEnumerable<T> value, [CallerArgumentExpression(nameof(value))] string? parameterName = null)
     {
         IEnumerable<T> result = NotNull(value, parameterName);
 
@@ -75,10 +75,10 @@ public static IEnumerable<T> NotNullOrEmpty<T>(IEnumerable<T> value, [CallerArgu
         return result;
     }
 
-    public static string NotEmpty(string? value, [CallerArgumentExpression("value")] string? parameterName = null) =>
+    public static string NotEmpty(string? value, [CallerArgumentExpression(nameof(value))] string? parameterName = null) =>
         NotNullOrWhiteSpace(value, parameterName);
 
-    public static string NotNullOrEmpty(string? value, [CallerArgumentExpression("value")] string? parameterName = null)
+    public static string NotNullOrEmpty(string? value, [CallerArgumentExpression(nameof(value))] string? parameterName = null)
     {
         if (value is null)
         {
@@ -95,7 +95,7 @@ public static string NotNullOrEmpty(string? value, [CallerArgumentExpression("va
         return value;
     }
 
-    public static string NotNullOrWhiteSpace(string? value, [CallerArgumentExpression("value")] string? parameterName = null)
+    public static string NotNullOrWhiteSpace(string? value, [CallerArgumentExpression(nameof(value))] string? parameterName = null)
     {
         if (value is null)
         {
@@ -112,7 +112,7 @@ public static string NotNullOrWhiteSpace(string? value, [CallerArgumentExpressio
         return value;
     }
 
-    public static IEnumerable<T> HasNoNulls<T>(IEnumerable<T> value, [CallerArgumentExpression("value")] string? parameterName = null)
+    public static IEnumerable<T> HasNoNulls<T>(IEnumerable<T> value, [CallerArgumentExpression(nameof(value))] string? parameterName = null)
     {
         if (value is null)
         {

test/System.Linq.Dynamic.Core.Tests/DynamicExpressionParserTests.cs

@@ -87,7 +87,7 @@ private class ComplexParseLambda1Result
     {
         public int? Age;
         public int TotalIncome;
-        public string Name;
+        public string? Name;
     }
 
     [DynamicLinqType]

test/System.Linq.Dynamic.Core.Tests/EntitiesTests.In.cs

@@ -1,4 +1,7 @@
-#if EFCORE
+using System.Linq.Dynamic.Core.Tests.Helpers.Entities;
+
+#if EFCORE
+using System.Collections.Generic;
 using Microsoft.EntityFrameworkCore;
 #else
 using System.Data.Entity;
@@ -25,4 +28,40 @@ public void Entities_Where_In_And()
         // Assert
         Assert.Equal(expected, test);
     }
+
+    [Fact]
+    public void Entities_Where_In_DifferentTypes()
+    {
+        // Arrange
+        var expected = _context.Blogs.Include(b => b.Posts).Where(b => new long[] { 1000, 1001, 1002 }.Contains(b.BlogLongId)).ToArray();
+
+        // Act
+        var test = _context.Blogs.Include(b => b.Posts).Where(@"BlogLongId in (1000, 1001, 1002)").ToArray();
+
+        // Assert
+        Assert.Equal(expected, test);
+    }
+
+    [Fact]
+    public void Entities_Where_In_Issue987()
+    {
+        // Arrange
+        for (int i = 0; i < 10000; i++)
+        {
+            var blogText = new BlogText
+            {
+                Id = i
+            };
+            _context.BlogTexts.Add(blogText);   
+        }
+        _context.SaveChanges();
+
+        // Act
+        var test = _context.BlogTexts
+            .Where("Id in (9495, 9496, 9498, 9500, 9501, 9503, 9505, 9508, 9509, 9510, 9511, 9514, 9515, 9517, 9518, 9519, 9520, 9521, 9523, 9524, 9525, 9526, 9527, 9528, 9529, 9530, 9531, 9532, 9533, 9534, 9535, 9536, 9538, 9539, 9540, 9541, 9542, 9543, 9544, 9545, 9546, 9547, 9548, 9549, 9550, 9552, 9554, 9556, 9557, 9558, 9559, 9560, 9561, 9562, 9563, 9565, 9567, 9569, 9570, 9575, 9576, 9577, 9578, 9579, 9580, 9581, 9582, 9583, 9584, 9585, 9586, 9587, 9588, 9589, 9590, 9591, 9592, 9593, 9594, 9595, 9596, 9597, 9598, 9599, 9600, 9601, 9602, 9603, 9604, 9605, 9606, 9607, 9608, 9609, 9610, 9611, 9612, 9613, 9614, 9615, 9616, 9617, 9618, 9619, 9620, 9621, 9622, 9623, 9624, 9625, 9626, 9627, 9628, 9629)")
+            .ToList();
+
+        // Assert
+        Assert.Equal(114, test.Count);
+    }
 }

test/System.Linq.Dynamic.Core.Tests/EntitiesTests.TakeWhile.cs

@@ -9,9 +9,6 @@ public partial class EntitiesTests
     [Fact(Skip = "not supported")]
     public void Entities_TakeWhile()
     {
-        // Arrange
-        const int total = 33;
-
         // Act
         var expected = _context.Blogs.OrderBy(b => b.BlogId).TakeWhile(b => b.BlogId > 5).ToArray();
         var result = _context.Blogs.OrderBy("BlogId").TakeWhile("b.BlogId > 5").ToDynamicArray<Blog>();