fix special function ::Real fallback stack overflow (#57790)

Fixes #57789

(cherry picked from commit 6817691)

base/math.jl

@@ -1542,7 +1542,7 @@ for f in (:sin, :cos, :tan, :asin, :atan, :acos,
           :exponent, :sqrt, :cbrt, :sinpi, :cospi, :sincospi, :tanpi)
     @eval function ($f)(x::Real)
         xf = float(x)
-        x === xf && throw(MethodError($f, (x,)))
+        xf isa typeof(x) && throw(MethodError($f, (x,)))
         return ($f)(xf)
     end
     @eval $(f)(::Missing) = missing

test/math.jl

@@ -1549,6 +1549,14 @@ end
     end
 end
 
+@testset "special function `::Real` fallback shouldn't recur without bound, issue #57789" begin
+    mutable struct Issue57789 <: Real end
+    Base.float(::Issue57789) = Issue57789()
+    for f ∈ (sin, sinpi, log, exp)
+        @test_throws MethodError f(Issue57789())
+    end
+end
+
 # Test that sqrt behaves correctly and doesn't exhibit fp80 double rounding.
 # This happened on old glibc versions.
 # Test case from https://sourceware.org/bugzilla/show_bug.cgi?id=14032.