Fix Enum.min/2,3 and Enum.max/2,3 with a custom sorter on ranges

lib/elixir/lib/enum.ex

@@ -4309,11 +4309,24 @@ defmodule Enum do
         empty.()
 
       _ ->
-        last = last - rem(last - first, step)
-
-        case fun.(first, last) do
-          true -> first
-          false -> last
+        # The endpoint shortcut is only valid for sorters consistent with
+        # the natural integer order of the range elements, which is known
+        # to hold for the default sorters; any other sorter traverses the
+        # elements, seeded with the first one since the range is not empty
+        if fun == (&<=/2) or fun == (&>=/2) do
+          last = last - rem(last - first, step)
+
+          case fun.(first, last) do
+            true -> first
+            false -> last
+          end
+        else
+          reduce_range(first + step, last, step, first, fn element, acc ->
+            case fun.(acc, element) do
+              true -> acc
+              false -> element
+            end
+          end)
         end
     end
   end

lib/elixir/test/elixir/enum_test.exs

@@ -2096,6 +2096,14 @@ defmodule EnumTest.Range do
     assert Enum.max(1..2, fn -> 0 end) === 2
   end
 
+  test "max/2 with custom sorter" do
+    sorter = fn a, b -> rem(a, 5) >= rem(b, 5) end
+    assert Enum.max(1..10, sorter) == 4
+    assert Enum.max(1..10//2, sorter) == 9
+    assert Enum.max(10..1//-1, sorter) == 9
+    assert Enum.max(1..0//1, sorter, fn -> :empty end) == :empty
+  end
+
   test "max_by/2" do
     assert Enum.max_by(1..1, fn x -> :math.pow(-2, x) end) == 1
     assert Enum.max_by(1..3, fn x -> :math.pow(-2, x) end) == 2
@@ -2139,6 +2147,14 @@ defmodule EnumTest.Range do
     assert Enum.min(1..2, fn -> 0 end) === 1
   end
 
+  test "min/2 with custom sorter" do
+    sorter = fn a, b -> rem(a, 5) <= rem(b, 5) end
+    assert Enum.min(1..10, sorter) == 5
+    assert Enum.min(1..10//2, sorter) == 5
+    assert Enum.min(10..1//-1, sorter) == 10
+    assert Enum.min(1..0//1, sorter, fn -> :empty end) == :empty
+  end
+
   test "min_by/2" do
     assert Enum.min_by(1..1, fn x -> :math.pow(-2, x) end) == 1
     assert Enum.min_by(1..3, fn x -> :math.pow(-2, x) end) == 3