RAI: --safe-crash-log-file flag

Author: d-netto

base/options.jl

@@ -64,6 +64,7 @@ struct JLOptions
     trim::Int8
     task_metrics::Int8
     timeout_for_safepoint_straggler_s::Int16
+    safe_crash_log_file::Ptr{UInt8}
 end
 
 # This runs early in the sysimage != is not defined yet

src/init.c

@@ -757,6 +757,15 @@ JL_DLLEXPORT void jl_init_(jl_image_buf_t sysimage)
     if (jl_options.handle_signals == JL_OPTIONS_HANDLE_SIGNALS_ON)
         jl_install_default_signal_handlers();
 
+#if (defined(_OS_LINUX_) && defined(_CPU_X86_64_)) || (defined(_OS_DARWIN_) && defined(_CPU_AARCH64_))
+    if (jl_options.safe_crash_log_file != NULL) {
+        jl_sig_fd = open(jl_options.safe_crash_log_file, O_WRONLY | O_CREAT | O_APPEND, 0600);
+        if (jl_sig_fd == -1) {
+            jl_error("fatal error: could not open safe crash log file for writing");
+        }
+    }
+#endif
+
     jl_gc_init();
 
     arraylist_new(&jl_linkage_blobs, 0);

src/jl_uv.c

@@ -15,6 +15,7 @@
 #include "errno.h"
 #include <unistd.h>
 #include <sys/socket.h>
+#include <sys/time.h>
 #endif
 
 #include "julia.h"
@@ -812,6 +813,56 @@ JL_DLLEXPORT int jl_printf(uv_stream_t *s, const char *format, ...)
     return c;
 }
 
+STATIC_INLINE void print_error_msg_as_json(char *buf) JL_NOTSAFEPOINT
+{
+    // Our telemetry on SPCS expects a JSON object per line
+    // The following lines prepare the timestamp string and the JSON object
+    struct timeval tv;
+    struct tm* tm_info;
+    char timestamp_buffer[50];
+    // Get current time
+    gettimeofday(&tv, NULL);
+    tm_info = gmtime(&tv.tv_sec);
+    // Format time
+    int offset = strftime(timestamp_buffer, 25, "%Y-%m-%dT%H:%M:%S", tm_info);
+    // Append milliseconds
+    snprintf(timestamp_buffer + offset, 25, ".%03d", tv.tv_usec / 1000);
+    const char *json_preamble_p1 = "\n{\"level\":\"Error\", \"timestamp\":\"";
+    const char *json_preamble_p2 = "\", \"message\": \"";
+    const char *json_postamble = "\"}\n";
+    // Ignore write failures because there is nothing we can do
+    write(jl_sig_fd, json_preamble_p1, strlen(json_preamble_p1));
+    write(jl_sig_fd, timestamp_buffer, strlen(timestamp_buffer));
+    write(jl_sig_fd, json_preamble_p2, strlen(json_preamble_p2));
+    // JSON escape the input string
+    for(size_t i = 0; i < strlen(buf); i += 1) {
+        switch (buf[i]) {
+            case '"':
+                write(jl_sig_fd, "\\\"", 2);
+                break;
+            case '\b':
+                write(jl_sig_fd, "\\b", 2);
+                break;
+            case '\n':
+                write(jl_sig_fd, "\\n", 2);
+                break;
+            case '\r':
+                write(jl_sig_fd, "\\r", 2);
+                break;
+            case '\t':
+                write(jl_sig_fd, "\\t", 2);
+                break;
+            case '\\':
+                write(jl_sig_fd, "\\\\", 2);
+                break;
+            default:
+                write(jl_sig_fd, buf + i, 1);
+        }
+    }
+    write(jl_sig_fd, json_postamble, strlen(json_postamble));
+    fdatasync(jl_sig_fd);
+}
+
 JL_DLLEXPORT void jl_safe_printf(const char *fmt, ...)
 {
     static char buf[1000];
@@ -828,6 +879,9 @@ JL_DLLEXPORT void jl_safe_printf(const char *fmt, ...)
     va_end(args);
 
     buf[999] = '\0';
+    if (jl_inside_signal_handler() && jl_sig_fd != 0) {
+        print_error_msg_as_json(buf);
+    }
     if (write(STDERR_FILENO, buf, strlen(buf)) < 0) {
         // nothing we can do; ignore the failure
     }

src/jloptions.c

@@ -155,6 +155,7 @@ JL_DLLEXPORT void jl_init_options(void)
                         JL_TRIM_NO, // trim
                         0, // task_metrics
                         -1, // timeout_for_safepoint_straggler_s
+                        NULL, // safe_crash_log_file
     };
     jl_options_initialized = 1;
 }
@@ -384,6 +385,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
            opt_permalloc_pkgimg,
            opt_trim,
            opt_experimental_features,
+           opt_safe_crash_log_file,
     };
     static const char* const shortopts = "+vhqH:e:E:L:J:C:it:p:O:g:m:";
     static const struct option longopts[] = {
@@ -452,6 +454,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
         { "permalloc-pkgimg",required_argument, 0, opt_permalloc_pkgimg },
         { "heap-size-hint",  required_argument, 0, opt_heap_size_hint },
         { "trim",  optional_argument, 0, opt_trim },
+        { "safe-crash-log-file",   required_argument, 0, opt_safe_crash_log_file },
         { 0, 0, 0, 0 }
     };
 
@@ -1008,6 +1011,10 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
                 jl_options.task_metrics = JL_OPTIONS_TASK_METRICS_ON;
             else
                 jl_errorf("julia: invalid argument to --task-metrics={yes|no} (%s)", optarg);
+        case opt_safe_crash_log_file:
+            jl_options.safe_crash_log_file = strdup(optarg);
+            if (jl_options.safe_crash_log_file == NULL)
+                jl_error("julia: failed to allocate memory for --safe-crash-log-file");
             break;
         default:
             jl_errorf("julia: unhandled option -- %c\n"

src/jloptions.h

@@ -68,6 +68,7 @@ typedef struct {
     int8_t trim;
     int8_t task_metrics;
     int16_t timeout_for_safepoint_straggler_s;
+    const char *safe_crash_log_file;
 } jl_options_t;
 
 #endif

src/julia_internal.h

@@ -785,6 +785,32 @@ JL_CALLABLE(jl_f_opaque_closure_call);
 void jl_install_default_signal_handlers(void);
 void restore_signals(void);
 void jl_install_thread_signal_handler(jl_ptls_t ptls);
+extern const size_t sig_stack_size;
+STATIC_INLINE int is_addr_on_sigstack(jl_ptls_t ptls, void *ptr)
+{
+    // One guard page for signal_stack.
+    return !((char*)ptr < (char*)ptls->signal_stack - jl_page_size ||
+             (char*)ptr > (char*)ptls->signal_stack + sig_stack_size);
+}
+STATIC_INLINE int jl_inside_signal_handler(void)
+{
+#if (defined(_OS_LINUX_) && defined(_CPU_X86_64_)) || (defined(_OS_DARWIN_) && defined(_CPU_AARCH64_))
+    // Read the stack pointer
+    size_t sp;
+#if defined(_OS_LINUX_) && defined(_CPU_X86_64_)
+    __asm__ __volatile__("movq %%rsp, %0" : "=r"(sp));
+#elif defined(_OS_DARWIN_) && defined(_CPU_AARCH64_)
+    __asm__ __volatile__("mov %0, sp" : "=r"(sp));
+#endif
+    // Check if the stack pointer is within the signal stack
+    jl_ptls_t ptls = jl_current_task->ptls;
+    return is_addr_on_sigstack(ptls, (void*)sp);
+#else
+    return 0;
+#endif
+}
+// File-descriptor for safe logging on signal handling
+extern int jl_sig_fd;
 
 JL_DLLEXPORT jl_fptr_args_t jl_get_builtin_fptr(jl_datatype_t *dt);
 

src/signal-handling.c

@@ -30,6 +30,8 @@ static const uint64_t GIGA = 1000000000ULL;
 // Timers to take samples at intervals
 JL_DLLEXPORT void jl_profile_stop_timer(void);
 JL_DLLEXPORT int jl_profile_start_timer(uint8_t);
+// File-descriptor for safe logging on signal handling
+int jl_sig_fd;
 
 ///////////////////////
 // Utility functions //

src/signals-unix.c

@@ -42,7 +42,7 @@
 #endif
 
 // 8M signal stack, same as default stack size (though we barely use this)
-static const size_t sig_stack_size = 8 * 1024 * 1024;
+const size_t sig_stack_size = 8 * 1024 * 1024;
 
 #include "julia_assert.h"
 
@@ -102,14 +102,6 @@ static inline uintptr_t jl_get_rsp_from_ctx(const void *_ctx)
 #endif
 }
 
-static int is_addr_on_sigstack(jl_ptls_t ptls, void *ptr) JL_NOTSAFEPOINT
-{
-    // One guard page for signal_stack.
-    return ptls->signal_stack == NULL ||
-           ((char*)ptr >= (char*)ptls->signal_stack - jl_page_size &&
-            (char*)ptr <= (char*)ptls->signal_stack + (ptls->signal_stack_size ? ptls->signal_stack_size : sig_stack_size));
-}
-
 // Modify signal context `_ctx` so that `fptr` will execute when the signal returns
 // The function `fptr` itself must not return.
 JL_NO_ASAN static void jl_call_in_ctx(jl_ptls_t ptls, void (*fptr)(void), int sig, void *_ctx)

src/signals-win.c

@@ -4,7 +4,7 @@
 // Note that this file is `#include`d by "signal-handling.c"
 #include <mmsystem.h> // hidden by LEAN_AND_MEAN
 
-static const size_t sig_stack_size = 131072; // 128k reserved for backtrace_fiber for stack overflow handling
+const size_t sig_stack_size = 131072; // 128k reserved for backtrace_fiber for stack overflow handling
 
 // Copied from MINGW_FLOAT_H which may not be found due to a collision with the builtin gcc float.h
 // eventually we can probably integrate this into OpenLibm.