8368365: ASAN errors should produce hs-err files and core dumps

Reviewed-by: mbaesken, asmehra

src/hotspot/share/logging/logTag.hpp

@@ -41,6 +41,7 @@ class outputStream;
   LOG_TAG(aot) \
   LOG_TAG(arguments) \
   LOG_TAG(array) \
+  LOG_TAG(asan) \
   LOG_TAG(attach) \
   LOG_TAG(barrier) \
   LOG_TAG(blocks) \

src/hotspot/share/runtime/globals.hpp

@@ -502,7 +502,7 @@ const int ObjectAlignmentInBytes = 8;
           "If > 0, provokes an error after VM initialization; the value "   \
           "determines which error to provoke. See controlled_crash() "      \
           "in vmError.cpp.")                                                \
-          range(0, 17)                                                      \
+          range(0, 18)                                                      \
                                                                             \
   develop(uint, TestCrashInErrorHandler, 0,                                 \
           "If > 0, provokes an error inside VM error handler (a secondary " \

src/hotspot/share/runtime/threads.cpp

@@ -97,6 +97,7 @@
 #include "runtime/trimNativeHeap.hpp"
 #include "runtime/vm_version.hpp"
 #include "runtime/vmOperations.hpp"
+#include "sanitizers/address.hpp"
 #include "services/attachListener.hpp"
 #include "services/management.hpp"
 #include "services/threadIdTable.hpp"
@@ -702,6 +703,10 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
   // No more stub generation allowed after that point.
   StubCodeDesc::freeze();
 
+#ifdef ADDRESS_SANITIZER
+  Asan::initialize();
+#endif
+
   // Set flag that basic initialization has completed. Used by exceptions and various
   // debug stuff, that does not work until all basic classes have been initialized.
   set_init_completed();

src/hotspot/share/sanitizers/address.cpp

@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifdef ADDRESS_SANITIZER
+
+#include "logging/log.hpp"
+#include "sanitizers/address.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/vmError.hpp"
+
+#include <dlfcn.h>
+#include <stdio.h>
+
+typedef void (*callback_setter_t) (void (*callback)(const char *));
+static callback_setter_t g_callback_setter = nullptr;
+static const char* g_report = nullptr;
+
+extern "C" void asan_error_callback(const char* report_text) {
+  // Please keep things very short and simple here and use as little
+  // as possible of any hotspot infrastructure. However shaky the JVM,
+  // we should always at least get the ASAN report on stderr.
+
+  // Note: this is threadsafe since ASAN synchronizes error reports
+  g_report = report_text;
+
+  // First, print off the bare error to stderr
+  fprintf(stderr, "JVM caught ASAN Error\n");
+  fprintf(stderr, "%s\n", report_text);
+
+  // Then, let normal JVM error handling run its due course.
+  fatal("ASAN Error");
+}
+
+void Asan::initialize() {
+
+  // For documentation of __asan_set_error_report_callback() see asan_interface.h .
+  g_callback_setter = (callback_setter_t) dlsym(RTLD_DEFAULT, "__asan_set_error_report_callback");
+  if (g_callback_setter == nullptr) {
+    log_info(asan)("*** Failed to install JVM callback for ASAN. ASAN errors will not generate hs-err files. ***");
+    return;
+  }
+
+  g_callback_setter(asan_error_callback);
+  log_info(asan)("JVM callback for ASAN errors successfully installed");
+
+  // Controlling core dump behavior:
+  //
+  // In hotspot, CreateCoredumpOnCrash decides whether to create a core dump (on Posix, whether to
+  // end the process with abort(3) or exit(3)).
+  //
+  // Core generation in the default ASAN reporter is controlled by two options:
+  // - "abort_on_error=0" (default) - end with exit(3), "abort_on_error=1" end with abort(3)
+  // - "disable_coredump=1" (default) disables cores by imposing a near-zero core soft limit.
+  // By default both options are set to prevent cores. That default makes sense since ASAN cores
+  // can get very large (due to the shadow map) and very numerous (ASAN is typically ran for
+  // large-scale integration tests, not targeted micro-tests).
+  //
+  // In hotspot ASAN builds, we replace the default ASAN reporter. The soft limit imposed by
+  // "disable_coredump=1" is still in effect. But "abort_on_error" is not honored. Since we'd
+  // like to exhibit exactly the same behavior as the standard ASAN error reporter, we disable
+  // core files if ASAN would inhibit them (we just switch off CreateCoredumpOnCrash).
+  //
+  // Thus:
+  //     abort_on_error      disable_coredump       core file?
+  //         0                   0                  No  (enforced by ergo-setting CreateCoredumpOnCrash=0)
+  // (*)     0                   1                  No  (enforced by ASAN-imposed soft limit)
+  //         1                   0                  Yes, unless -XX:-CreateCoredumpOnCrash set on command line
+  //         1                   1                  No  (enforced by ASAN-imposed soft limit)
+  // (*) is the default if no ASAN options are specified.
+
+  const char* const asan_options = getenv("ASAN_OPTIONS");
+  const bool asan_inhibits_cores = (asan_options == nullptr) ||
+                                   (::strstr(asan_options, "abort_on_error=1") == nullptr) ||
+                                   (::strstr(asan_options, "disable_coredump=0") == nullptr);
+  if (asan_inhibits_cores) {
+    if (CreateCoredumpOnCrash) {
+      log_info(asan)("CreateCoredumpOnCrash overruled by%s asan options. Core generation disabled.",
+                        asan_options != nullptr ? "" : " default setting for");
+      log_info(asan)("Use 'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1' "
+                     "to enable core generation.");
+    }
+    FLAG_SET_ERGO(CreateCoredumpOnCrash, false);
+  }
+}
+
+bool Asan::had_error() {
+  return g_report != nullptr;
+}
+
+void Asan::report(outputStream* st) {
+  if (had_error()) {
+    // Use raw print here to avoid truncation.
+    st->print_raw(g_report);
+    st->cr();
+    st->cr();
+  }
+}
+
+#endif // ADDRESS_SANITIZER

src/hotspot/share/sanitizers/address.hpp

@@ -26,6 +26,8 @@
 #define SHARE_SANITIZERS_ADDRESS_HPP
 
 #ifdef ADDRESS_SANITIZER
+#include "memory/allStatic.hpp"
+
 #include <sanitizer/asan_interface.h>
 #endif
 
@@ -74,4 +76,14 @@
   } while (false)
 #endif
 
+class outputStream;
+
+#ifdef ADDRESS_SANITIZER
+struct Asan : public AllStatic {
+  static void initialize();
+  static bool had_error();
+  static void report(outputStream* st);
+};
+#endif
+
 #endif // SHARE_SANITIZERS_ADDRESS_HPP

src/hotspot/share/utilities/vmError.cpp

@@ -60,6 +60,7 @@
 #include "runtime/vm_version.hpp"
 #include "runtime/vmOperations.hpp"
 #include "runtime/vmThread.hpp"
+#include "sanitizers/address.hpp"
 #include "sanitizers/ub.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/decoder.hpp"
@@ -910,7 +911,16 @@ void VMError::report(outputStream* st, bool _verbose) {
   STEP_IF("printing date and time", _verbose)
     os::print_date_and_time(st, buf, sizeof(buf));
 
-  STEP_IF("printing thread", _verbose)
+#ifdef ADDRESS_SANITIZER
+  STEP_IF("printing ASAN error information", _verbose && Asan::had_error())
+    st->cr();
+    st->print_cr("------------------  A S A N ----------------");
+    st->cr();
+    Asan::report(st);
+    st->cr();
+#endif // ADDRESS_SANITIZER
+
+    STEP_IF("printing thread", _verbose)
     st->cr();
     st->print_cr("---------------  T H R E A D  ---------------");
     st->cr();
@@ -2186,6 +2196,14 @@ void VMError::controlled_crash(int how) {
         fatal("Force crash with a nested ThreadsListHandle.");
       }
     }
+    case 18: {
+      // Trigger an error that should cause ASAN to report a double free or use-after-free.
+      // Please note that this is not 100% bullet-proof since it assumes that this block
+      // is not immediately repurposed by some other thread after free.
+      void* const p = os::malloc(4096, mtTest);
+      os::free(p);
+      os::free(p);
+    }
     default:
       // If another number is given, give a generic crash.
       fatal("Crashing with number %d", how);

test/hotspot/jtreg/runtime/ErrorHandling/AsanReportTest.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2025, IBM Corporation. All rights reserved.
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @summary Test that we get ASAN-reports and hs-err files on ASAN error
+ * @library /test/lib
+ * @requires vm.asan
+ * @requires vm.flagless
+ * @requires vm.debug == true & os.family == "linux"
+ * @modules java.base/jdk.internal.misc
+ *          java.management
+ * @run driver AsanReportTest
+ */
+
+// Note: this test can only run on debug since it relies on VMError::controlled_crash() which
+// only exists in debug builds.
+import java.io.File;
+import java.util.regex.Pattern;
+
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+
+public class AsanReportTest {
+
+    private static void do_test() throws Exception {
+
+        ProcessBuilder pb = ProcessTools.createLimitedTestJavaProcessBuilder(
+                "-Xmx64M", "-XX:CompressedClassSpaceSize=64M",
+                // Default ASAN options should prevent core file generation, which should overrule +CreateCoredumpOnCrash.
+                // We test below.
+                "-XX:+CreateCoredumpOnCrash",
+                "-Xlog:asan",
+                // Switch off NMT since it can alter the error ASAN sees; we want the pure double free error
+                "-XX:NativeMemoryTracking=off",
+                // Causes double-free in controlled_crash
+                "-XX:ErrorHandlerTest=18",
+                "-version");
+
+        OutputAnalyzer output = new OutputAnalyzer(pb.start());
+
+        output.shouldNotHaveExitValue(0);
+
+        // ASAN error should appear on stderr
+        output.shouldContain("CreateCoredumpOnCrash overruled");
+        output.shouldContain("JVM caught ASAN Error");
+        output.shouldMatch("AddressSanitizer.*double-free");
+        output.shouldMatch("# +A fatal error has been detected by the Java Runtime Environment");
+        output.shouldMatch("# +fatal error: ASAN");
+        output.shouldNotContain("Aborted (core dumped)");
+
+        File hs_err_file = HsErrFileUtils.openHsErrFileFromOutput(output);
+        Pattern[] pat = new Pattern[] {
+                Pattern.compile(".*A S A N.*"),
+                Pattern.compile(".*AddressSanitizer.*double-free.*"),
+                Pattern.compile(".*(crash_with_segfault|controlled_crash).*")
+        };
+        HsErrFileUtils.checkHsErrFileContent(hs_err_file, pat, false);
+    }
+
+    public static void main(String[] args) throws Exception {
+        do_test();
+    }
+
+}
+