From c8216fbd7276b029c4b3f2ce07712ffcbc5da5c7 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Fri, 20 Mar 2026 17:22:12 +0100 Subject: [PATCH 01/49] exps --- src/hotspot/cpu/x86/interp_masm_x86.cpp | 35 ++++++++++++++++++++--- src/hotspot/cpu/x86/interp_masm_x86.hpp | 3 +- src/hotspot/cpu/x86/templateTable_x86.cpp | 4 +-- src/hotspot/share/c1/c1_LIRGenerator.cpp | 5 ++-- src/hotspot/share/ci/ciMethod.cpp | 2 +- src/hotspot/share/ci/ciMethodData.cpp | 5 ++-- src/hotspot/share/ci/ciMethodData.hpp | 13 +++++++-- src/hotspot/share/oops/methodData.cpp | 2 +- src/hotspot/share/oops/methodData.hpp | 25 ++++++---------- 9 files changed, 60 insertions(+), 34 deletions(-) diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index d5e7b22b16a..826006c2fb6 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -1644,15 +1644,42 @@ template void InterpreterMacroAssembler::profile_array_type(Re } } -template void InterpreterMacroAssembler::profile_array_type(Register mdp, - Register array, - Register tmp); template void InterpreterMacroAssembler::profile_array_type(Register mdp, Register array, Register tmp); -void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) { +void InterpreterMacroAssembler::profile_multiple_array_types(Register mdp, + Register array, + Register tmp) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + load_klass(tmp, array, rscratch1); + profile_receiver_type(array, mdp, 0); + + Label not_flat; + test_non_flat_array_oop(array, tmp, not_flat); + + set_mdp_flag_at(mdp, ArrayLoadData::flat_array_byte_constant()); + + bind(not_flat); + + Label not_null_free; + test_non_null_free_array_oop(array, tmp, not_null_free); + + set_mdp_flag_at(mdp, ArrayLoadData::null_free_array_byte_constant()); + + bind(not_null_free); + + bind(profile_continue); + } +} + +void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp) { if (ProfileInterpreter) { Label profile_continue; diff --git a/src/hotspot/cpu/x86/interp_masm_x86.hpp b/src/hotspot/cpu/x86/interp_masm_x86.hpp index ed4b8003ed4..8c7a040d813 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.hpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.hpp @@ -261,8 +261,9 @@ class InterpreterMacroAssembler: public MacroAssembler { void profile_switch_case(Register index_in_scratch, Register mdp, Register scratch2); template void profile_array_type(Register mdp, Register array, Register tmp); + void profile_multiple_array_types(Register mdp, Register array, Register tmp); - void profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2); + void profile_multiple_element_types(Register mdp, Register element, Register tmp); void profile_element_type(Register mdp, Register element, Register tmp); void profile_acmp(Register mdp, Register left, Register right, Register tmp); diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp index 4c1ccde0e52..1371490a6bf 100644 --- a/src/hotspot/cpu/x86/templateTable_x86.cpp +++ b/src/hotspot/cpu/x86/templateTable_x86.cpp @@ -782,7 +782,7 @@ void TemplateTable::aaload() { Register index = rax; index_check(array, index); // kills rbx - __ profile_array_type(rbx, array, rcx); + __ profile_multiple_array_types(rbx, array, rcx); if (UseArrayFlattening) { Label is_flat_array, done; __ test_flat_array_oop(array, rbx, is_flat_array); @@ -1093,7 +1093,7 @@ void TemplateTable::aastore() { index_check_without_pop(rdx, rcx); // kills rbx __ profile_array_type(rdi, rdx, rbx); - __ profile_multiple_element_types(rdi, rax, rbx, rcx); + __ profile_multiple_element_types(rdi, rax, rbx); __ testptr(rax, rax); __ jcc(Assembler::zero, is_null); diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp index c3bb4d9e379..ef57dd5b1ce 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.cpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp @@ -3017,8 +3017,9 @@ void LIRGenerator::profile_null_free_array(LIRItem array, ciMethodData* md, ciPr template void LIRGenerator::profile_array_type(AccessIndexed* x, ciMethodData*& md, ArrayData*& load_store) { assert(compilation()->profile_array_accesses(), "array access profiling is disabled"); LIR_Opr mdp = LIR_OprFact::illegalOpr; - profile_type(md, md->byte_offset_of_slot(load_store, ArrayData::array_offset()), 0, - load_store->array()->type(), x->array(), mdp, true, nullptr, nullptr); + ShouldNotReachHere(); + // profile_type(md, md->byte_offset_of_slot(load_store, ArrayData::array_offset()), 0, + // load_store->array()->type(), x->array(), mdp, true, nullptr, nullptr); } void LIRGenerator::profile_element_type(Value element, ciMethodData* md, ciArrayLoadData* load_data) { diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp index 7b4220ec636..5ad75f69821 100644 --- a/src/hotspot/share/ci/ciMethod.cpp +++ b/src/hotspot/share/ci/ciMethod.cpp @@ -679,7 +679,7 @@ bool ciMethod::array_access_profiled_type(int bci, ciKlass*& array_type, ciKlass if (data != nullptr) { if (data->is_ArrayLoadData()) { ciArrayLoadData* array_access = (ciArrayLoadData*) data->as_ArrayLoadData(); - array_type = array_access->array()->valid_type(); + array_type = nullptr; //array_access->array()->valid_type(); element_type = array_access->element()->valid_type(); element_ptr = array_access->element()->ptr_kind(); flat_array = array_access->flat_array(); diff --git a/src/hotspot/share/ci/ciMethodData.cpp b/src/hotspot/share/ci/ciMethodData.cpp index fe1cce9dade..b6c8598c146 100644 --- a/src/hotspot/share/ci/ciMethodData.cpp +++ b/src/hotspot/share/ci/ciMethodData.cpp @@ -834,8 +834,7 @@ void ciMethodData::dump_replay_data(outputStream* out) { dump_replay_data_receiver_type_helper(out, round, count, array_store_data); } else if (pdata->is_ArrayLoadData()) { ciArrayLoadData* array_load_data = (ciArrayLoadData*)pdata; - dump_replay_data_type_helper(out, round, count, array_load_data, ciArrayLoadData::array_offset(), - array_load_data->array()->valid_type()); + dump_replay_data_receiver_type_helper(out, round, count, array_load_data); dump_replay_data_type_helper(out, round, count, array_load_data, ciArrayLoadData::element_offset(), array_load_data->element()->valid_type()); } else if (pdata->is_ACmpData()) { @@ -1019,7 +1018,7 @@ void ciArrayLoadData::print_data_on(outputStream* st, const char* extra) const { st->cr(); tab(st, true); st->print("array"); - array()->print_data_on(st); + rtd_super()->print_receiver_data_on(st); tab(st, true); st->print("element"); element()->print_data_on(st); diff --git a/src/hotspot/share/ci/ciMethodData.hpp b/src/hotspot/share/ci/ciMethodData.hpp index fca4f07099f..8b62d8779b1 100644 --- a/src/hotspot/share/ci/ciMethodData.hpp +++ b/src/hotspot/share/ci/ciMethodData.hpp @@ -385,18 +385,25 @@ class ciArrayStoreData : public ArrayStoreData { }; class ciArrayLoadData : public ArrayLoadData { + // Fake multiple inheritance... It's a ciReceiverTypeData also. + ciReceiverTypeData* rtd_super() const { return (ciReceiverTypeData*) this; } + public: ciArrayLoadData(DataLayout* layout) : ArrayLoadData(layout) {} - ciSingleTypeEntry* array() const { return (ciSingleTypeEntry*)ArrayLoadData::array(); } + // ciSingleTypeEntry* array() const { return (ciSingleTypeEntry*)ArrayLoadData::array(); } ciSingleTypeEntry* element() const { return (ciSingleTypeEntry*)ArrayLoadData::element(); } virtual void translate_from(const ProfileData* data) { - array()->translate_type_data_from(data->as_ArrayLoadData()->array()); + rtd_super()->translate_receiver_data_from(data); element()->translate_type_data_from(data->as_ArrayLoadData()->element()); } -#ifndef PRODUCT + ciKlass* receiver(uint row) { + return rtd_super()->receiver(row); + } + + #ifndef PRODUCT void print_data_on(outputStream* st, const char* extra = nullptr) const; #endif }; diff --git a/src/hotspot/share/oops/methodData.cpp b/src/hotspot/share/oops/methodData.cpp index f3ff17cd9ca..7e46110b94e 100644 --- a/src/hotspot/share/oops/methodData.cpp +++ b/src/hotspot/share/oops/methodData.cpp @@ -732,7 +732,7 @@ void ArrayLoadData::print_data_on(outputStream* st, const char* extra) const { st->cr(); tab(st, true); st->print("array"); - _array.print_data_on(st); + print_receiver_data_on(st); tab(st, true); st->print("element"); _element.print_data_on(st); diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 19ea9ac3ea1..74c34a379f4 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_OOPS_METHODDATA_HPP #define SHARE_OOPS_METHODDATA_HPP +#include "methodData.hpp" #include "interpreter/bytecodes.hpp" #include "interpreter/invocationCounter.hpp" #include "oops/metadata.hpp" @@ -1174,7 +1175,8 @@ class ReceiverTypeData : public CounterData { assert(layout->tag() == DataLayout::receiver_type_data_tag || layout->tag() == DataLayout::virtual_call_data_tag || layout->tag() == DataLayout::virtual_call_type_data_tag || - layout->tag() == DataLayout::array_store_data_tag, "wrong type"); + layout->tag() == DataLayout::array_store_data_tag || + layout->tag() == DataLayout::array_load_data_tag, "wrong type"); } virtual bool is_ReceiverTypeData() const { return true; } @@ -1976,30 +1978,23 @@ class ArrayStoreData : public ReceiverTypeData { virtual void print_data_on(outputStream* st, const char* extra = nullptr) const; }; -class ArrayLoadData : public BitData { +class ArrayLoadData : public ReceiverTypeData { private: enum { flat_array_flag = BitData::last_bit_data_flag, null_free_array_flag = flat_array_flag + 1, }; - SingleTypeEntry _array; SingleTypeEntry _element; public: ArrayLoadData(DataLayout* layout) : - BitData(layout), - _array(0), + ReceiverTypeData(layout), _element(SingleTypeEntry::static_cell_count()) { assert(layout->tag() == DataLayout::array_load_data_tag, "wrong type"); - _array.set_profile_data(this); _element.set_profile_data(this); } - const SingleTypeEntry* array() const { - return &_array; - } - const SingleTypeEntry* element() const { return &_element; } @@ -2007,7 +2002,7 @@ class ArrayLoadData : public BitData { virtual bool is_ArrayLoadData() const { return true; } static int static_cell_count() { - return SingleTypeEntry::static_cell_count() * 2; + return SingleTypeEntry::static_cell_count() + ReceiverTypeData::static_cell_count(); } virtual int cell_count() const { @@ -2029,21 +2024,17 @@ class ArrayLoadData : public BitData { return flag_number_to_constant(null_free_array_flag); } - static ByteSize array_offset() { - return cell_offset(0); - } - static ByteSize element_offset() { return cell_offset(SingleTypeEntry::static_cell_count()); } virtual void clean_weak_klass_links(bool always_clean) { - _array.clean_weak_klass_links(always_clean); + ReceiverTypeData::clean_weak_klass_links(always_clean); _element.clean_weak_klass_links(always_clean); } virtual void metaspace_pointers_do(MetaspaceClosure* it) { - _array.metaspace_pointers_do(it); + ReceiverTypeData::metaspace_pointers_do(it); _element.metaspace_pointers_do(it); } From 3d2f5b3e05e4039a8e2b9493f6955a4490d44312 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Mon, 23 Mar 2026 09:39:14 +0100 Subject: [PATCH 02/49] more --- src/hotspot/cpu/x86/interp_masm_x86.cpp | 2 +- src/hotspot/share/oops/methodData.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index 826006c2fb6..32874052cd7 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -1659,7 +1659,7 @@ void InterpreterMacroAssembler::profile_multiple_array_types(Register mdp, test_method_data_pointer(mdp, profile_continue); load_klass(tmp, array, rscratch1); - profile_receiver_type(array, mdp, 0); + profile_receiver_type(tmp, mdp, 0); Label not_flat; test_non_flat_array_oop(array, tmp, not_flat); diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 74c34a379f4..c44bab6ffb5 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -2025,7 +2025,7 @@ class ArrayLoadData : public ReceiverTypeData { } static ByteSize element_offset() { - return cell_offset(SingleTypeEntry::static_cell_count()); + return cell_offset(ReceiverTypeData::static_cell_count()); } virtual void clean_weak_klass_links(bool always_clean) { From 3230215d3fb1d1cc135ff45a32c911019684b316 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Mon, 23 Mar 2026 10:35:37 +0100 Subject: [PATCH 03/49] more --- src/hotspot/share/oops/methodData.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index c44bab6ffb5..699ba8b70e2 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -1990,7 +1990,7 @@ class ArrayLoadData : public ReceiverTypeData { public: ArrayLoadData(DataLayout* layout) : ReceiverTypeData(layout), - _element(SingleTypeEntry::static_cell_count()) { + _element(ReceiverTypeData::static_cell_count()) { assert(layout->tag() == DataLayout::array_load_data_tag, "wrong type"); _element.set_profile_data(this); } From 68d22be91b704f21e08f4ec2df0895ce6bf35ba7 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Mon, 23 Mar 2026 14:41:52 +0100 Subject: [PATCH 04/49] more --- src/hotspot/cpu/x86/interp_masm_x86.cpp | 9 +- src/hotspot/share/oops/methodData.cpp | 2 + src/hotspot/share/oops/methodData.hpp | 23 +++- .../inlinetypes/TestArrayLoadProfiling.java | 117 ++++++++++++++++++ 4 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index 32874052cd7..9f2530f85b2 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -1658,16 +1658,19 @@ void InterpreterMacroAssembler::profile_multiple_array_types(Register mdp, // If no method data exists, go to profile_continue. test_method_data_pointer(mdp, profile_continue); + Label not_flat, null_free_check; + test_non_flat_array_oop(array, tmp, not_flat); + load_klass(tmp, array, rscratch1); profile_receiver_type(tmp, mdp, 0); - Label not_flat; - test_non_flat_array_oop(array, tmp, not_flat); - set_mdp_flag_at(mdp, ArrayLoadData::flat_array_byte_constant()); + jmp(null_free_check); bind(not_flat); + increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_count_offset())); + bind(null_free_check); Label not_null_free; test_non_null_free_array_oop(array, tmp, not_null_free); diff --git a/src/hotspot/share/oops/methodData.cpp b/src/hotspot/share/oops/methodData.cpp index 7e46110b94e..44c92d33ea9 100644 --- a/src/hotspot/share/oops/methodData.cpp +++ b/src/hotspot/share/oops/methodData.cpp @@ -729,9 +729,11 @@ void ArrayStoreData::print_data_on(outputStream* st, const char* extra) const { void ArrayLoadData::print_data_on(outputStream* st, const char* extra) const { print_shared(st, "ArrayLoad", extra); + st->print(" not flat: %u", not_flat_count()); st->cr(); tab(st, true); st->print("array"); + tab(st); print_receiver_data_on(st); tab(st, true); st->print("element"); diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 699ba8b70e2..5340d4622c3 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -1985,8 +1985,21 @@ class ArrayLoadData : public ReceiverTypeData { null_free_array_flag = flat_array_flag + 1, }; + enum { + not_flat_count_off_in_extra_cells, + extra_cells_count + }; + SingleTypeEntry _element; + static int extra_cells_off() { + return ReceiverTypeData::static_cell_count() + SingleTypeEntry::static_cell_count(); + } + + static int not_flat_count_off() { + return extra_cells_off() + not_flat_count_off_in_extra_cells; + } + public: ArrayLoadData(DataLayout* layout) : ReceiverTypeData(layout), @@ -2002,7 +2015,7 @@ class ArrayLoadData : public ReceiverTypeData { virtual bool is_ArrayLoadData() const { return true; } static int static_cell_count() { - return SingleTypeEntry::static_cell_count() + ReceiverTypeData::static_cell_count(); + return extra_cells_off() + extra_cells_count; } virtual int cell_count() const { @@ -2015,6 +2028,10 @@ class ArrayLoadData : public ReceiverTypeData { void set_null_free_array() { set_flag_at(null_free_array_flag); } bool null_free_array() const { return flag_at(null_free_array_flag); } + int not_flat_count() const { + return uint_at(not_flat_count_off()); + } + // Code generation support static int flat_array_byte_constant() { return flag_number_to_constant(flat_array_flag); @@ -2028,6 +2045,10 @@ class ArrayLoadData : public ReceiverTypeData { return cell_offset(ReceiverTypeData::static_cell_count()); } + static ByteSize not_flat_count_offset() { + return cell_offset(not_flat_count_off()); + } + virtual void clean_weak_klass_links(bool always_clean) { ReceiverTypeData::clean_weak_klass_links(always_clean); _element.clean_weak_klass_links(always_clean); diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java new file mode 100644 index 00000000000..418103781d3 --- /dev/null +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2026 IBM Corporation. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @library /test/lib / + * @enablePreview + * @modules java.base/jdk.internal.value + * @requires (os.simpleArch == "x64" | os.simpleArch == "aarch64") + * @run main ${test.main.class} + */ + +package compiler.valhalla.inlinetypes; +import compiler.lib.ir_framework.*; +import jdk.internal.value.ValueClass; +public class TestArrayLoadProfiling { + public static void main(String[] args) { + TestFramework.runWithFlags("--enable-preview", "--add-exports", "java.base/jdk.internal.value=ALL-UNNAMED"); + } + + static MyValue1[] array1 = { new MyValue1(42) }; + static MyValue2[] array2 = { new MyValue2(42) }; + static MyValue1[] array3 = (MyValue1[])ValueClass.newNullRestrictedNonAtomicArray(MyValue1.class, 1, new MyValue1(42)); + static MyValue2[] array4 = (MyValue2[])ValueClass.newNullRestrictedNonAtomicArray(MyValue2.class, 1, new MyValue2(42)); + static Object[] array5 = { new Object() }; + + @Test + public static void test1(Object[] array) { + return array[0]; + } + + @Run(test = "test1") + public static void test1Runner() { + test1(array1); + } + + @Test + public static void test2(Object[] array) { + return array[0]; + } + + @Run(test = "test2") + public static void test2Runner() { + test2(array5); + } + + @Test + public static void test3(Object[] array) { + return array[0]; + } + + @Run(test = "test3") + public static void test3Runner() { + test3(array1); + test3(array2); + test3(array5); + } + + @Test + public static void test4(Object[] array) { + return array[0]; + } + + @Run(test = "test4") + public static void test3Runner() { + test4(array1); + test4(array2); + test4(array3); + test4(array4); + } + + interface I { + void m(); + } + + static value class MyValue1 implements I { + int intField; + + MyValue1(int intField) { + this.intField = intField; + } + + public void m() { + } + } + + static value class MyValue2 implements I { + int intField; + + MyValue2(int intField) { + this.intField = intField; + } + + public void m() { + } + } +} From 4f6cbe1e8e90774c0c966bb2f5bdd5845d7cfc9a Mon Sep 17 00:00:00 2001 From: rwestrel Date: Mon, 23 Mar 2026 16:16:48 +0100 Subject: [PATCH 05/49] more --- src/hotspot/share/c1/c1_LIRGenerator.cpp | 5 ++--- .../valhalla/inlinetypes/TestArrayLoadProfiling.java | 10 +++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp index ef57dd5b1ce..c3bb4d9e379 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.cpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp @@ -3017,9 +3017,8 @@ void LIRGenerator::profile_null_free_array(LIRItem array, ciMethodData* md, ciPr template void LIRGenerator::profile_array_type(AccessIndexed* x, ciMethodData*& md, ArrayData*& load_store) { assert(compilation()->profile_array_accesses(), "array access profiling is disabled"); LIR_Opr mdp = LIR_OprFact::illegalOpr; - ShouldNotReachHere(); - // profile_type(md, md->byte_offset_of_slot(load_store, ArrayData::array_offset()), 0, - // load_store->array()->type(), x->array(), mdp, true, nullptr, nullptr); + profile_type(md, md->byte_offset_of_slot(load_store, ArrayData::array_offset()), 0, + load_store->array()->type(), x->array(), mdp, true, nullptr, nullptr); } void LIRGenerator::profile_element_type(Value element, ciMethodData* md, ciArrayLoadData* load_data) { diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 418103781d3..42ea661c097 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -45,7 +45,7 @@ public static void main(String[] args) { static Object[] array5 = { new Object() }; @Test - public static void test1(Object[] array) { + public static Object test1(Object[] array) { return array[0]; } @@ -55,7 +55,7 @@ public static void test1Runner() { } @Test - public static void test2(Object[] array) { + public static Object test2(Object[] array) { return array[0]; } @@ -65,7 +65,7 @@ public static void test2Runner() { } @Test - public static void test3(Object[] array) { + public static Object test3(Object[] array) { return array[0]; } @@ -77,12 +77,12 @@ public static void test3Runner() { } @Test - public static void test4(Object[] array) { + public static Object test4(Object[] array) { return array[0]; } @Run(test = "test4") - public static void test3Runner() { + public static void test4Runner() { test4(array1); test4(array2); test4(array3); From a21a3064995d6f588361ee93f91f99a6efd80848 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Tue, 24 Mar 2026 16:50:50 +0100 Subject: [PATCH 06/49] c1 --- src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 25 +++++++++++++++ src/hotspot/cpu/x86/interp_masm_x86.cpp | 17 ++++------- src/hotspot/cpu/x86/interp_masm_x86.hpp | 2 +- src/hotspot/cpu/x86/templateTable_x86.cpp | 2 +- src/hotspot/share/c1/c1_LIR.cpp | 18 +++++++++++ src/hotspot/share/c1/c1_LIR.hpp | 34 +++++++++++++++++++++ src/hotspot/share/c1/c1_LIRAssembler.hpp | 1 + src/hotspot/share/c1/c1_LIRGenerator.cpp | 25 +++++++++------ src/hotspot/share/c1/c1_LIRGenerator.hpp | 5 +-- 9 files changed, 105 insertions(+), 24 deletions(-) diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index 01f4a8c1fb7..72349d8ed3d 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -3177,7 +3177,32 @@ void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) { __ bind(not_inline_type); } +void LIR_Assembler::emit_profile_multiple_array_types(LIR_OpProfileMultipleArrayTypes* op) { + Register array = op->array()->as_pointer_register(); + Register tmp1 = op->tmp1()->as_pointer_register(); + Register tmp2 = op->tmp2()->as_pointer_register(); + ciMethodData* md = op->md(); + Label not_flat, done; + __ test_non_flat_array_oop(array, tmp1, not_flat); + + Register klass = tmp1; + __ load_klass(klass, array, tmp2); + + Register mdo = tmp2; + __ mov_metadata(mdo, md->constant_encoding()); + + type_profile_helper(mdo, md, op->load(), tmp1); + + __ jmp(done); + __ bind(not_flat); + __ mov_metadata(mdo, md->constant_encoding()); + + Address counter_addr(mdo, md->byte_offset_of_slot(op->load(), ArrayLoadData::not_flat_count_offset())); + __ addptr(counter_addr, DataLayout::counter_increment); + + __ bind(done); +} void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no)); } diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index 9f2530f85b2..27f83ae481f 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -1614,9 +1614,9 @@ void InterpreterMacroAssembler::profile_switch_case(Register index, } } -template void InterpreterMacroAssembler::profile_array_type(Register mdp, - Register array, - Register tmp) { +void InterpreterMacroAssembler::profile_array_type(Register mdp, + Register array, + Register tmp) { if (ProfileInterpreter) { Label profile_continue; @@ -1624,19 +1624,19 @@ template void InterpreterMacroAssembler::profile_array_type(Re test_method_data_pointer(mdp, profile_continue); mov(tmp, array); - profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset()))); + profile_obj_type(tmp, Address(mdp, in_bytes(ArrayStoreData::array_offset()))); Label not_flat; test_non_flat_array_oop(array, tmp, not_flat); - set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant()); + set_mdp_flag_at(mdp, ArrayStoreData::flat_array_byte_constant()); bind(not_flat); Label not_null_free; test_non_null_free_array_oop(array, tmp, not_null_free); - set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant()); + set_mdp_flag_at(mdp, ArrayStoreData::null_free_array_byte_constant()); bind(not_null_free); @@ -1644,11 +1644,6 @@ template void InterpreterMacroAssembler::profile_array_type(Re } } -template void InterpreterMacroAssembler::profile_array_type(Register mdp, - Register array, - Register tmp); - - void InterpreterMacroAssembler::profile_multiple_array_types(Register mdp, Register array, Register tmp) { diff --git a/src/hotspot/cpu/x86/interp_masm_x86.hpp b/src/hotspot/cpu/x86/interp_masm_x86.hpp index 8c7a040d813..63b719badaf 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.hpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.hpp @@ -260,7 +260,7 @@ class InterpreterMacroAssembler: public MacroAssembler { void profile_switch_default(Register mdp); void profile_switch_case(Register index_in_scratch, Register mdp, Register scratch2); - template void profile_array_type(Register mdp, Register array, Register tmp); + void profile_array_type(Register mdp, Register array, Register tmp); void profile_multiple_array_types(Register mdp, Register array, Register tmp); void profile_multiple_element_types(Register mdp, Register element, Register tmp); diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp index 1371490a6bf..e5af853af23 100644 --- a/src/hotspot/cpu/x86/templateTable_x86.cpp +++ b/src/hotspot/cpu/x86/templateTable_x86.cpp @@ -1092,7 +1092,7 @@ void TemplateTable::aastore() { index_check_without_pop(rdx, rcx); // kills rbx - __ profile_array_type(rdi, rdx, rbx); + __ profile_array_type(rdi, rdx, rbx); __ profile_multiple_element_types(rdi, rax, rbx); __ testptr(rax, rax); diff --git a/src/hotspot/share/c1/c1_LIR.cpp b/src/hotspot/share/c1/c1_LIR.cpp index 02d22816529..ab254f6e366 100644 --- a/src/hotspot/share/c1/c1_LIR.cpp +++ b/src/hotspot/share/c1/c1_LIR.cpp @@ -980,6 +980,16 @@ void LIR_OpVisitState::visit(LIR_Op* op) { do_temp(opProfileInlineType->_tmp); break; } + + case lir_profile_multiple_array_types: { + assert(op->as_OpProfileMultipleArrayTypes() != nullptr, "must be"); + LIR_OpProfileMultipleArrayTypes* opProfileMultipleArrayTypes = (LIR_OpProfileMultipleArrayTypes*)op; + do_input(opProfileMultipleArrayTypes->_array); + do_temp(opProfileMultipleArrayTypes->_tmp1); + do_temp(opProfileMultipleArrayTypes->_tmp2); + + break; + } default: op->visit(this); } @@ -1203,6 +1213,10 @@ void LIR_OpProfileInlineType::emit_code(LIR_Assembler* masm) { masm->emit_profile_inline_type(this); } +void LIR_OpProfileMultipleArrayTypes::emit_code(LIR_Assembler* masm) { + masm->emit_profile_multiple_array_types(this); +} + // LIR_List LIR_List::LIR_List(Compilation* compilation, BlockBegin* block) : _operations(8) @@ -1924,6 +1938,7 @@ const char * LIR_Op::name() const { case lir_profile_type: s = "profile_type"; break; // LIR_OpProfileInlineType case lir_profile_inline_type: s = "profile_inline_type"; break; + case lir_profile_multiple_array_types: s = "profile_multiple_array_types"; break; // LIR_OpAssert #ifdef ASSERT case lir_assert: s = "assert"; break; @@ -2262,6 +2277,9 @@ void LIR_OpProfileInlineType::print_instr(outputStream* out) const { tmp()->print(out); out->print(" "); } +void LIR_OpProfileMultipleArrayTypes::print_instr(outputStream* out) const { +} + #endif // PRODUCT // Implementation of LIR_InsertionBuffer diff --git a/src/hotspot/share/c1/c1_LIR.hpp b/src/hotspot/share/c1/c1_LIR.hpp index 91a80bfdd12..371d6f5dd42 100644 --- a/src/hotspot/share/c1/c1_LIR.hpp +++ b/src/hotspot/share/c1/c1_LIR.hpp @@ -897,6 +897,7 @@ class LIR_OpLoadKlass; class LIR_OpProfileCall; class LIR_OpProfileType; class LIR_OpProfileInlineType; +class LIR_OpProfileMultipleArrayTypes; #ifdef ASSERT class LIR_OpAssert; #endif @@ -1012,6 +1013,7 @@ enum LIR_Code { , lir_profile_call , lir_profile_type , lir_profile_inline_type + , lir_profile_multiple_array_types , end_opMDOProfile , begin_opAssert , lir_assert @@ -1158,6 +1160,7 @@ class LIR_Op: public CompilationResourceObj { virtual LIR_OpProfileCall* as_OpProfileCall() { return nullptr; } virtual LIR_OpProfileType* as_OpProfileType() { return nullptr; } virtual LIR_OpProfileInlineType* as_OpProfileInlineType() { return nullptr; } + virtual LIR_OpProfileMultipleArrayTypes* as_OpProfileMultipleArrayTypes() { return nullptr; } #ifdef ASSERT virtual LIR_OpAssert* as_OpAssert() { return nullptr; } #endif @@ -2146,6 +2149,34 @@ class LIR_OpProfileInlineType : public LIR_Op { virtual void print_instr(outputStream* out) const PRODUCT_RETURN; }; +class LIR_OpProfileMultipleArrayTypes : public LIR_Op { + friend class LIR_OpVisitState; + ciMethodData* _md; + ciArrayLoadData* _load; + LIR_Opr _array; + LIR_Opr _tmp1; + LIR_Opr _tmp2; +public: + LIR_OpProfileMultipleArrayTypes(ciMethodData* md, ciArrayLoadData* load, LIR_Opr array, LIR_Opr tmp1, LIR_Opr tmp2) + : LIR_Op(lir_profile_multiple_array_types, LIR_OprFact::illegalOpr, nullptr) + , _md(md) + , _load(load) + , _array(array) + , _tmp1(tmp1) + , _tmp2(tmp2) { + + } + ciMethodData* md() const { return _md; } + ciArrayLoadData* load() const { return _load; } + LIR_Opr array() const { return _array; } + LIR_Opr tmp1() const { return _tmp1; } + LIR_Opr tmp2() const { return _tmp2; } + + virtual void emit_code(LIR_Assembler* masm); + virtual LIR_OpProfileMultipleArrayTypes* as_OpProfileMultipleArrayTypes() { return this; } + virtual void print_instr(outputStream* out) const PRODUCT_RETURN; +}; + class LIR_InsertionBuffer; //--------------------------------LIR_List--------------------------------------------------- @@ -2447,6 +2478,9 @@ class LIR_List: public CompilationResourceObj { void profile_inline_type(LIR_Address* mdp, LIR_Opr obj, int flag, LIR_Opr tmp, bool not_null) { append(new LIR_OpProfileInlineType(LIR_OprFact::address(mdp), obj, flag, tmp, not_null)); } + void profile_multiple_array_types(ciMethodData* md, ciArrayLoadData* load, LIR_Opr array, LIR_Opr tmp1, LIR_Opr tmp2) { + append(new LIR_OpProfileMultipleArrayTypes(md, load, array, tmp1, tmp2)); + } void xadd(LIR_Opr src, LIR_Opr add, LIR_Opr res, LIR_Opr tmp) { append(new LIR_Op2(lir_xadd, src, add, res, tmp)); } void xchg(LIR_Opr src, LIR_Opr set, LIR_Opr res, LIR_Opr tmp) { append(new LIR_Op2(lir_xchg, src, set, res, tmp)); } diff --git a/src/hotspot/share/c1/c1_LIRAssembler.hpp b/src/hotspot/share/c1/c1_LIRAssembler.hpp index 6b98a0b97dd..f68dcee3bd6 100644 --- a/src/hotspot/share/c1/c1_LIRAssembler.hpp +++ b/src/hotspot/share/c1/c1_LIRAssembler.hpp @@ -213,6 +213,7 @@ class LIR_Assembler: public CompilationResourceObj { void emit_profile_call(LIR_OpProfileCall* op); void emit_profile_type(LIR_OpProfileType* op); void emit_profile_inline_type(LIR_OpProfileInlineType* op); + void emit_profile_multiple_array_types(LIR_OpProfileMultipleArrayTypes* op); void emit_std_entries(); void emit_std_entry(CodeOffsets::Entries entry, const CompiledEntrySignature* ces); void add_scalarized_debug_info(int call_offset); diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp index c3bb4d9e379..0be71a3de0c 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.cpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp @@ -1968,7 +1968,7 @@ void LIRGenerator::do_StoreIndexed(StoreIndexed* x) { profile_array_type(x, md, store_data); assert(store_data->is_ArrayStoreData(), "incorrect profiling entry"); if (x->array()->maybe_null_free_array()) { - profile_null_free_array(array, md, data); + profile_null_free_array(array, md, store_data); } } } @@ -2344,7 +2344,7 @@ void LIRGenerator::do_LoadIndexed(LoadIndexed* x) { data = md->bci_to_data(bci); assert(data != nullptr && data->is_ArrayLoadData(), "incorrect profiling entry"); ciArrayLoadData* load_data = (ciArrayLoadData*)data; - profile_array_type(x, md, load_data); + profile_multiple_array_types(x, array, md, load_data); } } @@ -2377,7 +2377,9 @@ void LIRGenerator::do_LoadIndexed(LoadIndexed* x) { LoadFlattenedArrayStub* slow_path = nullptr; if (x->should_profile() && x->array()->maybe_null_free_array()) { - profile_null_free_array(array, md, data); + assert(data != nullptr && data->is_ArrayLoadData(), "incorrect profiling entry"); + ciArrayLoadData* load_data = (ciArrayLoadData*)data; + profile_null_free_array(array, md, load_data); } if (x->elt_type() == T_OBJECT && x->array()->maybe_flat_array()) { @@ -3005,20 +3007,25 @@ void LIRGenerator::profile_flags(ciMethodData* md, ciProfileData* data, int flag __ store(flags, addr); } -void LIRGenerator::profile_null_free_array(LIRItem array, ciMethodData* md, ciProfileData* data) { +template void LIRGenerator::profile_null_free_array(LIRItem array, ciMethodData* md, ArrayData* data) { assert(compilation()->profile_array_accesses(), "array access profiling is disabled"); - LabelObj* L_end = new LabelObj(); LIR_Opr tmp = new_register(T_METADATA); __ check_null_free_array(array.result(), tmp); - profile_flags(md, data, ArrayStoreData::null_free_array_byte_constant(), lir_cond_equal); + profile_flags(md, data, ArrayData::null_free_array_byte_constant(), lir_cond_equal); } -template void LIRGenerator::profile_array_type(AccessIndexed* x, ciMethodData*& md, ArrayData*& load_store) { +void LIRGenerator::profile_array_type(AccessIndexed* x, ciMethodData*& md, ciArrayStoreData* store) { assert(compilation()->profile_array_accesses(), "array access profiling is disabled"); LIR_Opr mdp = LIR_OprFact::illegalOpr; - profile_type(md, md->byte_offset_of_slot(load_store, ArrayData::array_offset()), 0, - load_store->array()->type(), x->array(), mdp, true, nullptr, nullptr); + profile_type(md, md->byte_offset_of_slot(store, ArrayStoreData::array_offset()), 0, + store->array()->type(), x->array(), mdp, true, nullptr, nullptr); +} + +void LIRGenerator::profile_multiple_array_types(AccessIndexed* x, LIRItem array, ciMethodData* md, ciArrayLoadData* load) { + assert(compilation()->profile_array_accesses(), "array access profiling is disabled"); + + __ profile_multiple_array_types(md, load, array.result(), new_pointer_register(), new_pointer_register()); } void LIRGenerator::profile_element_type(Value element, ciMethodData* md, ciArrayLoadData* load_data) { diff --git a/src/hotspot/share/c1/c1_LIRGenerator.hpp b/src/hotspot/share/c1/c1_LIRGenerator.hpp index 65c64d2262c..322d220d7bb 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.hpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.hpp @@ -493,8 +493,9 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure { void profile_parameters(Base* x); void profile_parameters_at_call(ProfileCall* x); void profile_flags(ciMethodData* md, ciProfileData* load_store, int flag, LIR_Condition condition = lir_cond_always); - void profile_null_free_array(LIRItem array, ciMethodData* md, ciProfileData* load_store); - template void profile_array_type(AccessIndexed* x, ciMethodData*& md, ArrayData*& load_store); + template void profile_null_free_array(LIRItem array, ciMethodData* md, ArrayData* load_store); + void profile_array_type(AccessIndexed* x, ciMethodData*& md, ciArrayStoreData* store); + void profile_multiple_array_types(AccessIndexed* x, LIRItem array, ciMethodData* md, ciArrayLoadData* load); void profile_element_type(Value element, ciMethodData* md, ciArrayLoadData* load_store); bool profile_inline_klass(ciMethodData* md, ciProfileData* data, Value value, int flag); LIR_Opr mask_boolean(LIR_Opr array, LIR_Opr value, CodeEmitInfo*& null_check_info); From 45ae0bdb34bb4f5e647a93884978edaf33cddead Mon Sep 17 00:00:00 2001 From: rwestrel Date: Tue, 24 Mar 2026 17:23:43 +0100 Subject: [PATCH 07/49] more --- .../inlinetypes/TestArrayLoadProfiling.java | 77 +++++++++++-------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 42ea661c097..3c4b4ef6653 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -42,52 +42,62 @@ public static void main(String[] args) { static MyValue2[] array2 = { new MyValue2(42) }; static MyValue1[] array3 = (MyValue1[])ValueClass.newNullRestrictedNonAtomicArray(MyValue1.class, 1, new MyValue1(42)); static MyValue2[] array4 = (MyValue2[])ValueClass.newNullRestrictedNonAtomicArray(MyValue2.class, 1, new MyValue2(42)); - static Object[] array5 = { new Object() }; + static A[] array5 = { new A() }; @Test - public static Object test1(Object[] array) { - return array[0]; + @IR(failOn = {IRNode.CALL_OF_METHOD}, count = { IRNode.IF, "3" }) + public static void test1(I[] array) { + return test1Inline(array[0]); } @Run(test = "test1") public static void test1Runner() { test1(array1); + test1Inline(array2); + test1Inline(array3); + test1Inline(array4); + test1Inline(array5); } - @Test - public static Object test2(Object[] array) { - return array[0]; + @Inline + void test1Inline(Interface i) { + i.m(); } - @Run(test = "test2") - public static void test2Runner() { - test2(array5); - } + // @Test + // public static Object test2(Object[] array) { + // return array[0]; + // } - @Test - public static Object test3(Object[] array) { - return array[0]; - } + // @Run(test = "test2") + // public static void test2Runner() { + // test2(array5); + // } - @Run(test = "test3") - public static void test3Runner() { - test3(array1); - test3(array2); - test3(array5); - } + // @Test + // public static Object test3(Object[] array) { + // return array[0]; + // } + + // @Run(test = "test3") + // public static void test3Runner() { + // test3(array1); + // test3(array2); + // test3(array5); + // } - @Test - public static Object test4(Object[] array) { - return array[0]; - } + // @Test + // public static Object test4(Object[] array) { + // return array[0]; + // } - @Run(test = "test4") - public static void test4Runner() { - test4(array1); - test4(array2); - test4(array3); - test4(array4); - } + // @Run(test = "test4") + // public static void test4Runner() { + // test4(array1); + // test4(array2); + // test4(array3); + // test4(array4); + // } interface I { void m(); @@ -114,4 +124,9 @@ static value class MyValue2 implements I { public void m() { } } + + static class A implements I { + public void m() { + } + } } From 8e8606cb68a4d14cb87156e3a8dc224c8191c676 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Tue, 24 Mar 2026 17:55:14 +0100 Subject: [PATCH 08/49] more --- src/hotspot/share/ci/ciMethod.cpp | 13 +++++++++++++ .../inlinetypes/TestArrayLoadProfiling.java | 16 ++++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp index 5ad75f69821..53b1ba912d3 100644 --- a/src/hotspot/share/ci/ciMethod.cpp +++ b/src/hotspot/share/ci/ciMethod.cpp @@ -444,6 +444,7 @@ const BitMap& ciMethod::bci_block_start() { int ciMethod::check_overflow(int c, Bytecodes::Code code) { switch (code) { case Bytecodes::_aastore: // fall-through + case Bytecodes::_aaload: // fall-through case Bytecodes::_checkcast: // fall-through case Bytecodes::_instanceof: { if (VM_Version::profile_all_receivers_at_type_check()) { @@ -678,12 +679,24 @@ bool ciMethod::array_access_profiled_type(int bci, ciKlass*& array_type, ciKlass ciProfileData* data = method_data()->bci_to_data(bci); if (data != nullptr) { if (data->is_ArrayLoadData()) { + ciCallProfile profile = call_profile_at_bci(bci); ciArrayLoadData* array_access = (ciArrayLoadData*) data->as_ArrayLoadData(); array_type = nullptr; //array_access->array()->valid_type(); element_type = array_access->element()->valid_type(); element_ptr = array_access->element()->ptr_kind(); flat_array = array_access->flat_array(); null_free_array = array_access->null_free_array(); + if (profile.morphism() == 1 && flat_array) { + array_type = profile.receiver(0); + } +#ifdef ASSERT + if (array_type != nullptr) { + bool flat = array_type->is_flat_array_klass(); + bool null_free = array_type->as_array_klass()->is_elem_null_free(); + assert(!flat || flat_array, "inconsistency"); + assert(!null_free || null_free_array, "inconsistency"); + } +#endif update_flags_from_type(array_type, flat_array, null_free_array); return true; } else if (data->is_ArrayStoreData()) { diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 3c4b4ef6653..1270c75f4cd 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -45,22 +45,22 @@ public static void main(String[] args) { static A[] array5 = { new A() }; @Test - @IR(failOn = {IRNode.CALL_OF_METHOD}, count = { IRNode.IF, "3" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.CALL, "4" }) public static void test1(I[] array) { - return test1Inline(array[0]); + test1Inline(array[0]); } @Run(test = "test1") public static void test1Runner() { test1(array1); - test1Inline(array2); - test1Inline(array3); - test1Inline(array4); - test1Inline(array5); + test1Inline(array2[0]); + test1Inline(array3[0]); + test1Inline(array4[0]); + test1Inline(array5[0]); } - @Inline - void test1Inline(Interface i) { + @ForceInline + static void test1Inline(I i) { i.m(); } From 51a974c0f652d77ac0a0383ad8dc5751d388c620 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 26 Mar 2026 12:17:40 +0100 Subject: [PATCH 09/49] more --- src/hotspot/share/ci/ciMethod.cpp | 12 ++-- src/hotspot/share/opto/parse2.cpp | 105 ++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp index 53b1ba912d3..61e83fc0518 100644 --- a/src/hotspot/share/ci/ciMethod.cpp +++ b/src/hotspot/share/ci/ciMethod.cpp @@ -444,7 +444,6 @@ const BitMap& ciMethod::bci_block_start() { int ciMethod::check_overflow(int c, Bytecodes::Code code) { switch (code) { case Bytecodes::_aastore: // fall-through - case Bytecodes::_aaload: // fall-through case Bytecodes::_checkcast: // fall-through case Bytecodes::_instanceof: { if (VM_Version::profile_all_receivers_at_type_check()) { @@ -453,7 +452,7 @@ int ciMethod::check_overflow(int c, Bytecodes::Code code) { return (c > 0 ? min_jint : c); // always non-positive } default: { - assert(Bytecodes::is_invoke(code), "%s", Bytecodes::name(code)); + assert(Bytecodes::is_invoke(code) || code == Bytecodes::_aaload, "%s", Bytecodes::name(code)); return (c < 0 ? max_jint : c); // always non-negative } } @@ -679,16 +678,17 @@ bool ciMethod::array_access_profiled_type(int bci, ciKlass*& array_type, ciKlass ciProfileData* data = method_data()->bci_to_data(bci); if (data != nullptr) { if (data->is_ArrayLoadData()) { - ciCallProfile profile = call_profile_at_bci(bci); + // ciCallProfile profile = call_profile_at_bci(bci); ciArrayLoadData* array_access = (ciArrayLoadData*) data->as_ArrayLoadData(); array_type = nullptr; //array_access->array()->valid_type(); element_type = array_access->element()->valid_type(); element_ptr = array_access->element()->ptr_kind(); flat_array = array_access->flat_array(); null_free_array = array_access->null_free_array(); - if (profile.morphism() == 1 && flat_array) { - array_type = profile.receiver(0); - } + array_type = nullptr; + // if (profile.morphism() == 1 && flat_array) { + // array_type = profile.receiver(0); + // } #ifdef ASSERT if (array_type != nullptr) { bool flat = array_type->is_flat_array_klass(); diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 1f9f648d3ae..823b1398151 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -29,6 +29,7 @@ #include "compiler/compileLog.hpp" #include "interpreter/linkResolver.hpp" #include "jvm_io.h" +#include "ci/ciFlatArrayKlass.hpp" #include "memory/resourceArea.hpp" #include "memory/universe.hpp" #include "oops/oop.inline.hpp" @@ -90,6 +91,110 @@ void Parse::array_load(BasicType bt) { // Cannot statically determine if array is a flat array, emit runtime check assert(UseArrayFlattening && is_reference_type(bt) && element_ptr->can_be_inline_type() && (!element_ptr->is_inlinetypeptr() || element_ptr->inline_klass()->maybe_flat_in_array()), "array can't be flat"); + + if (element_ptr->is_inlinetypeptr()) { + ciInlineKlass* vk = element_ptr->inline_klass(); + // Node* flat_array = cast_to_flat_array(array, vk); + Node* flat_array = array; + Node* vt = InlineTypeNode::make_from_flat_array(this, vk, flat_array, array_index); + Node* ld = _gvn.transform(vt); + push_node(bt, ld); + return; + } + ciMethodData* md = method()->method_data(); + if (md != nullptr && md->is_mature()) { + ciProfileData* data = md->bci_to_data(bci()); + if (data != nullptr && data->is_ArrayLoadData()) { + ciArrayLoadData* array_load = (ciArrayLoadData*) data->as_ArrayLoadData(); + int not_flat_count = array_load->not_flat_count(); + if (not_flat_count < 0) { + not_flat_count = max_jint; + } + ciCallProfile profile = method()->call_profile_at_bci(bci()); + int flat_count = profile.count(); + int flat_and_not_flat_count = saturated_add(flat_count, not_flat_count); + if (profile.morphism() > 0) { + bool not_flat_checked = false; + float prob = 1; + Node* region = new RegionNode(profile.morphism()+3); + Node* res_phi = new PhiNode(region, TypeOopPtr::BOTTOM); + Node* io_phi = new PhiNode(region, Type::ABIO); + Node* mem = reset_memory(); + Node* io = i_o(); + set_all_memory(mem); + Node* mem_phi = new PhiNode(region, Type::MEMORY, TypePtr::BOTTOM); + int j = 1; + for (int i = 0; i < profile.morphism() || !not_flat_checked; ) { + int count = i < profile.morphism() ? profile.receiver_count(i) : not_flat_count; + if (not_flat_count >= count && !not_flat_checked) { + not_flat_checked = true; + Node* test = flat_array_test(array, /* flat = */ false); + float p = ((float)not_flat_count) / ((float)flat_and_not_flat_count); + float this_prob = clamp(p / prob, PROB_MIN, PROB_MAX); + IfNode* iff = create_and_xform_if(control(), test, this_prob, COUNT_UNKNOWN); + set_control(_gvn.transform(new IfTrueNode(iff))); + assert(array_type->is_flat() || control()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found"); + const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); + DecoratorSet decorator_set = IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD; + if (needs_range_check(array_type->size(), array_index)) { + // We've emitted a RangeCheck but now insert an additional check between the range check and the actual load. + // We cannot pin the load to two separate nodes. Instead, we pin it conservatively here such that it cannot + // possibly float above the range check at any point. + decorator_set |= C2_UNKNOWN_CONTROL_LOAD; + } + Node* ld = access_load_at(array, adr, adr_type, element_ptr, bt, decorator_set); + if (element_ptr->is_inlinetypeptr()) { + ld = InlineTypeNode::make_from_oop(this, ld, element_ptr->inline_klass()); + } + res_phi->init_req(j, _gvn.transform(ld)); + region->init_req(j, control()); + io_phi->init_req(j, i_o()); + set_control(_gvn.transform(new IfFalseNode(iff))); + mem_phi->init_req(j, reset_memory()); + set_all_memory(mem); + set_i_o(io); + prob = 1 - p; + } else { + inc_sp(2); + float p = ((float)profile.receiver_count(i)) / ((float)flat_and_not_flat_count); + float this_prob = clamp(p / prob, PROB_MIN, PROB_MAX); + Node* casted_array = nullptr; + ciKlass* klass = profile.receiver(i); + Node* next_ctrl = type_check_receiver(array, klass, this_prob, &casted_array); + InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(this, klass->as_flat_array_klass()->element_klass()->as_inline_klass(), casted_array, array_index); + Node* ld = _gvn.transform(vt->buffer(this)); + res_phi->init_req(j, ld); + region->init_req(j, control()); + io_phi->init_req(j, i_o()); + set_control(next_ctrl); + mem_phi->init_req(j, reset_memory()); + set_all_memory(mem); + set_i_o(io); + prob = 1 - p; + i++; + dec_sp(2); + } + j++; + } + Node* unknown_inline_type = load_from_unknown_flat_array(array, array_index, element_ptr); + + region->init_req(j, control()); + res_phi->init_req(j, unknown_inline_type); + mem_phi->init_req(j, reset_memory()); + io_phi->init_req(j, i_o()); + set_control(_gvn.transform(region)); + set_all_memory(_gvn.transform(mem_phi)); + set_i_o(_gvn.transform(io_phi)); + Node* ld = _gvn.transform(res_phi); + ld = record_profile_for_speculation_at_array_load(ld); + push_node(bt, ld); + return; + } else { + ShouldNotReachHere(); + } + } + } + IdealKit ideal(this); IdealVariable res(ideal); ideal.declarations_done(); From 631b7fa0ea123387366441ba865a838ff27dcecb Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 26 Mar 2026 12:43:20 +0100 Subject: [PATCH 10/49] more --- src/hotspot/share/opto/parse2.cpp | 70 +++++++++++++++++-------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 823b1398151..f85424a567a 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -127,33 +127,39 @@ void Parse::array_load(BasicType bt) { for (int i = 0; i < profile.morphism() || !not_flat_checked; ) { int count = i < profile.morphism() ? profile.receiver_count(i) : not_flat_count; if (not_flat_count >= count && !not_flat_checked) { - not_flat_checked = true; - Node* test = flat_array_test(array, /* flat = */ false); - float p = ((float)not_flat_count) / ((float)flat_and_not_flat_count); - float this_prob = clamp(p / prob, PROB_MIN, PROB_MAX); - IfNode* iff = create_and_xform_if(control(), test, this_prob, COUNT_UNKNOWN); - set_control(_gvn.transform(new IfTrueNode(iff))); - assert(array_type->is_flat() || control()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found"); - const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); - DecoratorSet decorator_set = IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD; - if (needs_range_check(array_type->size(), array_index)) { - // We've emitted a RangeCheck but now insert an additional check between the range check and the actual load. - // We cannot pin the load to two separate nodes. Instead, we pin it conservatively here such that it cannot - // possibly float above the range check at any point. - decorator_set |= C2_UNKNOWN_CONTROL_LOAD; - } - Node* ld = access_load_at(array, adr, adr_type, element_ptr, bt, decorator_set); - if (element_ptr->is_inlinetypeptr()) { - ld = InlineTypeNode::make_from_oop(this, ld, element_ptr->inline_klass()); + if (not_flat_count == 0 && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + inc_sp(2); + uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + dec_sp(2); + } else { + not_flat_checked = true; + Node* test = flat_array_test(array, /* flat = */ false); + float p = ((float)not_flat_count) / ((float)flat_and_not_flat_count); + float this_prob = clamp(p / prob, PROB_MIN, PROB_MAX); + IfNode* iff = create_and_xform_if(control(), test, this_prob, COUNT_UNKNOWN); + set_control(_gvn.transform(new IfTrueNode(iff))); + assert(array_type->is_flat() || control()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found"); + const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); + DecoratorSet decorator_set = IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD; + if (needs_range_check(array_type->size(), array_index)) { + // We've emitted a RangeCheck but now insert an additional check between the range check and the actual load. + // We cannot pin the load to two separate nodes. Instead, we pin it conservatively here such that it cannot + // possibly float above the range check at any point. + decorator_set |= C2_UNKNOWN_CONTROL_LOAD; + } + Node* ld = access_load_at(array, adr, adr_type, element_ptr, bt, decorator_set); + if (element_ptr->is_inlinetypeptr()) { + ld = InlineTypeNode::make_from_oop(this, ld, element_ptr->inline_klass()); + } + res_phi->init_req(j, _gvn.transform(ld)); + region->init_req(j, control()); + io_phi->init_req(j, i_o()); + set_control(_gvn.transform(new IfFalseNode(iff))); + mem_phi->init_req(j, reset_memory()); + set_all_memory(mem); + set_i_o(io); + prob = 1 - p; } - res_phi->init_req(j, _gvn.transform(ld)); - region->init_req(j, control()); - io_phi->init_req(j, i_o()); - set_control(_gvn.transform(new IfFalseNode(iff))); - mem_phi->init_req(j, reset_memory()); - set_all_memory(mem); - set_i_o(io); - prob = 1 - p; } else { inc_sp(2); float p = ((float)profile.receiver_count(i)) / ((float)flat_and_not_flat_count); @@ -176,12 +182,14 @@ void Parse::array_load(BasicType bt) { } j++; } - Node* unknown_inline_type = load_from_unknown_flat_array(array, array_index, element_ptr); + if (!stop()) { + Node* unknown_inline_type = load_from_unknown_flat_array(array, array_index, element_ptr); - region->init_req(j, control()); - res_phi->init_req(j, unknown_inline_type); - mem_phi->init_req(j, reset_memory()); - io_phi->init_req(j, i_o()); + region->init_req(j, control()); + res_phi->init_req(j, unknown_inline_type); + mem_phi->init_req(j, reset_memory()); + io_phi->init_req(j, i_o()); + } set_control(_gvn.transform(region)); set_all_memory(_gvn.transform(mem_phi)); set_i_o(_gvn.transform(io_phi)); From 40fecc2e0ff4c895a48ed8602f06d75512d0ae7b Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 26 Mar 2026 16:48:13 +0100 Subject: [PATCH 11/49] more --- src/hotspot/share/opto/compile.cpp | 6 ++ src/hotspot/share/opto/parse2.cpp | 30 +++++-- .../inlinetypes/TestArrayLoadProfiling.java | 90 ++++++++++++++++++- 3 files changed, 116 insertions(+), 10 deletions(-) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 7c1fb53a55b..e556f660cbe 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -3195,6 +3195,7 @@ void Compile::Optimize() { // Loop transforms on the ideal graph. Range Check Elimination, // peeling, unrolling, etc. + bool split_if_progress = false;; // Set loop opts counter if((_loop_opts_cnt > 0) && (has_loops() || has_split_ifs())) { { @@ -3204,6 +3205,7 @@ void Compile::Optimize() { if (major_progress()) print_method(PHASE_PHASEIDEALLOOP1, 2); if (failing()) return; } + split_if_progress = major_progress() && !has_loops(); // Loop opts pass if partial peeling occurred in previous pass if(PartialPeelLoop && major_progress() && (_loop_opts_cnt > 0)) { TracePhase tp(_t_idealLoop); @@ -3248,6 +3250,10 @@ void Compile::Optimize() { if (failing()) return; + if (split_if_progress && !major_progress()) { + set_major_progress(); + } + // Loop transforms on the ideal graph. Range Check Elimination, // peeling, unrolling, etc. if (!optimize_loops(igvn, LoopOptsDefault)) { diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index f85424a567a..8ebca3bb5cd 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -116,7 +116,7 @@ void Parse::array_load(BasicType bt) { if (profile.morphism() > 0) { bool not_flat_checked = false; float prob = 1; - Node* region = new RegionNode(profile.morphism()+3); + Node* region = new RegionNode(profile.morphism() * 2 + 3); Node* res_phi = new PhiNode(region, TypeOopPtr::BOTTOM); Node* io_phi = new PhiNode(region, Type::ABIO); Node* mem = reset_memory(); @@ -127,12 +127,12 @@ void Parse::array_load(BasicType bt) { for (int i = 0; i < profile.morphism() || !not_flat_checked; ) { int count = i < profile.morphism() ? profile.receiver_count(i) : not_flat_count; if (not_flat_count >= count && !not_flat_checked) { + not_flat_checked = true; if (not_flat_count == 0 && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + PreserveJVMState pjvms(this); inc_sp(2); uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); - dec_sp(2); } else { - not_flat_checked = true; Node* test = flat_array_test(array, /* flat = */ false); float p = ((float)not_flat_count) / ((float)flat_and_not_flat_count); float this_prob = clamp(p / prob, PROB_MIN, PROB_MAX); @@ -168,6 +168,18 @@ void Parse::array_load(BasicType bt) { ciKlass* klass = profile.receiver(i); Node* next_ctrl = type_check_receiver(array, klass, this_prob, &casted_array); InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(this, klass->as_flat_array_klass()->element_klass()->as_inline_klass(), casted_array, array_index); + Node* null_ctl = top(); + + null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl); + + res_phi->init_req(j, zerocon(T_OBJECT)); + region->init_req(j, null_ctl); + io_phi->init_req(j, i_o()); + mem_phi->init_req(j, reset_memory()); + set_all_memory(mem); + + j++; + Node* ld = _gvn.transform(vt->buffer(this)); res_phi->init_req(j, ld); region->init_req(j, control()); @@ -182,7 +194,11 @@ void Parse::array_load(BasicType bt) { } j++; } - if (!stop()) { + if (not_flat_count != 0 && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + PreserveJVMState pjvms(this); + inc_sp(2); + uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } else if (too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { Node* unknown_inline_type = load_from_unknown_flat_array(array, array_index, element_ptr); region->init_req(j, control()); @@ -198,7 +214,7 @@ void Parse::array_load(BasicType bt) { push_node(bt, ld); return; } else { - ShouldNotReachHere(); + // ShouldNotReachHere(); } } } @@ -571,10 +587,10 @@ Node* Parse::create_speculative_inline_type_array_checks(Node* array, const Type // Even though the type does not tell us whether we have an inline type array or not, we can still check the profile data // whether we have a non-null-free or non-flat array. Speculating on a non-null-free array doesn't help aaload but could // be profitable for a subsequent aastore. - if (!array_type->is_null_free() && !array_type->is_not_null_free()) { + if (!array_type->is_null_free() && !array_type->is_not_null_free() && 0) { array = speculate_non_null_free_array(array, array_type); } - if (!array_type->is_flat() && !array_type->is_not_flat()) { + if (!array_type->is_flat() && !array_type->is_not_flat() && 0) { array = speculate_non_flat_array(array, array_type); } return array; diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 1270c75f4cd..8e66cff094f 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -27,15 +27,25 @@ * @enablePreview * @modules java.base/jdk.internal.value * @requires (os.simpleArch == "x64" | os.simpleArch == "aarch64") - * @run main ${test.main.class} + * @library /test/lib / + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI ${test.main.class} */ package compiler.valhalla.inlinetypes; + import compiler.lib.ir_framework.*; import jdk.internal.value.ValueClass; +import jdk.test.whitebox.WhiteBox; +import compiler.whitebox.CompilerWhiteBoxTest; +import java.lang.reflect.Method; + public class TestArrayLoadProfiling { + private final static WhiteBox WHITE_BOX = WhiteBox.getWhiteBox(); + public static void main(String[] args) { - TestFramework.runWithFlags("--enable-preview", "--add-exports", "java.base/jdk.internal.value=ALL-UNNAMED"); + TestFramework.runWithFlags("--enable-preview", "--add-exports", "java.base/jdk.internal.value=ALL-UNNAMED", "-Xbootclasspath/a:.", "-XX:+UnlockDiagnosticVMOptions", "-XX:+WhiteBoxAPI"); } static MyValue1[] array1 = { new MyValue1(42) }; @@ -45,7 +55,7 @@ public static void main(String[] args) { static A[] array5 = { new A() }; @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.CALL, "4" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.CALL, "4", IRNode.IF, "4" }) public static void test1(I[] array) { test1Inline(array[0]); } @@ -64,6 +74,80 @@ static void test1Inline(I i) { i.m(); } + // @Test + // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.CALL, "5", IRNode.IF, "5" }) + // @IR(failOn = { IRNode.CLASS_CHECK_TRAP }) + // public static void test2(I[] array) { + // test2Inline(array[0]); + // } + + // @Run(test = "test2") + // public static void test2Runner(RunInfo info) throws Exception { + // if (info.isWarmUp()) { + // test2(array1); + // test2Inline(array2[0]); + // test2Inline(array3[0]); + // test2Inline(array4[0]); + // test2Inline(array5[0]); + // } else { + // Method m = TestArrayLoadProfiling.class.getDeclaredMethod("test2", I[].class); + // if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) { + // throw new RuntimeException("should be compiled"); + // } + // int i = 0; + // do { + // test2(array2); + // i++; + // if (i > 10) { + // throw new RuntimeException("should not be compiled anymore"); + // } + // } while (WHITE_BOX.isMethodCompiled(m)); + // WHITE_BOX.enqueueMethodForCompilation(m, CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION); + // if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) { + // throw new RuntimeException("should be compiled"); + // } + // } + // } + + // @ForceInline + // static void test2Inline(I i) { + // i.m(); + // } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.CALL, "3", IRNode.IF, "4" }) + public static void test3(I[] array) { + test3Inline(array[0]); + } + + @Run(test = "test3") + public static void test3Runner() { + test3(array3); + test3(array4); + } + + @ForceInline + static void test3Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.CALL, "4", IRNode.IF, "6" }) + public static void test5(I[] array) { + test5Inline(array[0]); + } + + @Run(test = "test5") + public static void test5Runner() { + test5(array1); + test5(array2); + } + + @ForceInline + static void test5Inline(I i) { + i.m(); + } + // @Test // public static Object test2(Object[] array) { // return array[0]; From 5cf0e95f857a582eaf631688b7152cdb40c9fff5 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Mon, 30 Mar 2026 16:42:15 +0200 Subject: [PATCH 12/49] more --- src/hotspot/share/ci/ciCallProfile.hpp | 9 +- src/hotspot/share/opto/doCall.cpp | 2 +- src/hotspot/share/opto/parse.hpp | 1 + src/hotspot/share/opto/parse2.cpp | 569 ++++++++++++++---- .../compiler/lib/ir_framework/IRNode.java | 5 + .../inlinetypes/TestArrayLoadProfiling.java | 142 ++++- 6 files changed, 573 insertions(+), 155 deletions(-) diff --git a/src/hotspot/share/ci/ciCallProfile.hpp b/src/hotspot/share/ci/ciCallProfile.hpp index 76aca113169..43391aff1d0 100644 --- a/src/hotspot/share/ci/ciCallProfile.hpp +++ b/src/hotspot/share/ci/ciCallProfile.hpp @@ -61,18 +61,21 @@ class ciCallProfile : StackObj { int morphism() const { return _morphism; } int count() const { return _count; } - int receiver_count(int i) { + int receiver_count(int i) const { assert(i < _limit, "out of Call Profile MorphismLimit"); return _receiver_count[i]; } - float receiver_prob(int i) { + float receiver_prob(int i) const { assert(i < _limit, "out of Call Profile MorphismLimit"); return (float)_receiver_count[i]/(float)_count; } - ciKlass* receiver(int i) { + ciKlass* receiver(int i) const { assert(i < _limit, "out of Call Profile MorphismLimit"); return _receiver[i]; } + bool has_major_receiver() const { + return has_receiver(0) && (100.*receiver_prob(0) >= (float)TypeProfileMajorReceiverPercent); + } }; #endif // SHARE_CI_CICALLPROFILE_HPP diff --git a/src/hotspot/share/opto/doCall.cpp b/src/hotspot/share/opto/doCall.cpp index 4dc5404e598..fefff0f3354 100644 --- a/src/hotspot/share/opto/doCall.cpp +++ b/src/hotspot/share/opto/doCall.cpp @@ -234,7 +234,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool // Try using the type profile. if (call_does_dispatch && site_count > 0 && UseTypeProfile) { // The major receiver's count >= TypeProfileMajorReceiverPercent of site_count. - bool have_major_receiver = profile.has_receiver(0) && (100.*profile.receiver_prob(0) >= (float)TypeProfileMajorReceiverPercent); + bool have_major_receiver = profile.has_major_receiver(); ciMethod* receiver_method = nullptr; int morphism = profile.morphism(); diff --git a/src/hotspot/share/opto/parse.hpp b/src/hotspot/share/opto/parse.hpp index e86cf94a10e..3a9f4962447 100644 --- a/src/hotspot/share/opto/parse.hpp +++ b/src/hotspot/share/opto/parse.hpp @@ -146,6 +146,7 @@ class InlineTree : public AnyObj { //------------------------------Parse------------------------------------------ // Parse bytecodes, build a Graph class Parse : public GraphKit { + friend class ArrayLoad; public: // Per-block information needed by the parser: class Block { diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 8ebca3bb5cd..f56826dbd28 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -73,152 +73,485 @@ Node* Parse::record_profile_for_speculation_at_array_load(Node* ld) { return ld; } +class ArrayLoad { +private: + BasicType _bt; + Parse& _parse; + PhaseGVN& _gvn; + const Type* _elemtype; + Node* _array_index; + Node* _array; + + Node* _region; + Node* _res_phi; + Node* _io_phi; + Node* _mem_phi; + Node* _mem; + Node* _io; + + bool emit_null_and_range_checks() { + _array = _parse.null_check(_array, T_ARRAY); + // Compile-time detect of null-exception? + if (_parse.stopped()) return true; // _kit.top(); + + const TypeAryPtr* arytype = _gvn.type(_array)->is_aryptr(); + const TypeInt* sizetype = arytype->size(); + _elemtype = arytype->elem(); + + if (UseUniqueSubclasses) { + const Type* el = _elemtype->make_ptr(); + if (el && el->isa_instptr()) { + const TypeInstPtr* toop = el->is_instptr(); + if (toop->instance_klass()->unique_concrete_subklass()) { + // If we load from "AbstractClass[]" we must see "ConcreteSubClass". + const Type* subklass = Type::get_const_type(toop->instance_klass()); + _elemtype = subklass->join_speculative(el); + } + } + } -//---------------------------------array_load---------------------------------- -void Parse::array_load(BasicType bt) { - const Type* elemtype = Type::TOP; - Node* adr = array_addressing(bt, 0, elemtype); - if (stopped()) return; // guaranteed null or range check + if (!arytype->is_loaded()) { + // Only fails for some -Xcomp runs + // The class is unloaded. We have to run this bytecode in the interpreter. + ciKlass* klass = arytype->unloaded_klass(); - Node* array_index = pop(); - Node* array = pop(); + _parse.uncommon_trap(Deoptimization::Reason_unloaded, + Deoptimization::Action_reinterpret, + klass, "!loaded array"); + return true; // top(); + } - // Handle inline type arrays - const TypeOopPtr* element_ptr = elemtype->make_oopptr(); - const TypeAryPtr* array_type = _gvn.type(array)->is_aryptr(); + // ary = create_speculative_inline_type_array_checks(ary, arytype, elemtype); - if (!array_type->is_not_flat()) { - // Cannot statically determine if array is a flat array, emit runtime check - assert(UseArrayFlattening && is_reference_type(bt) && element_ptr->can_be_inline_type() && - (!element_ptr->is_inlinetypeptr() || element_ptr->inline_klass()->maybe_flat_in_array()), "array can't be flat"); + if (_parse.needs_range_check(sizetype, _array_index)) { + _parse.create_range_check(_array_index, _array, sizetype); + } else if (_parse.C->log() != nullptr) { + _parse.C->log()->elem("observe that='!need_range_check'"); + } + return false; + } + ciArrayLoadData* profile_data() const { + ciMethodData* md = _parse.method()->method_data(); + if (md == nullptr) { + return nullptr; + } + if (!md->is_mature()) { + return nullptr; + } + ciProfileData* data = md->bci_to_data(_parse.bci()); + if (data == nullptr) { + return nullptr; + } + if (!data->is_ArrayLoadData()) { + return nullptr; + } + return (ciArrayLoadData*) data->as_ArrayLoadData(); + } + + int profiled_not_flat_count() const { + if (profile_data()->not_flat_count() < 0) { + return max_jint; + } + return profile_data()->not_flat_count(); + } + +public: + ArrayLoad(BasicType bt, Parse &parse) : _bt(bt), _parse(parse), _gvn(parse.gvn()), _elemtype((nullptr)), + _array_index(nullptr), _array(nullptr), _region(nullptr), _res_phi(nullptr), + _io_phi(nullptr), _mem_phi(nullptr), _mem(nullptr), _io(nullptr) { + _array_index = _parse.peek(0); // Get from stack without popping + _array = _parse.peek(1); // in case of exception + } + + void pop_stack() { + Node* array_index = _parse.pop(); + Node* array = _parse.pop(); + assert(array_index == _array_index, ""); + assert(array == _array, ""); + } + + bool emit_load_if_known_flat_array(const TypeOopPtr* element_ptr) { if (element_ptr->is_inlinetypeptr()) { + pop_stack(); ciInlineKlass* vk = element_ptr->inline_klass(); // Node* flat_array = cast_to_flat_array(array, vk); - Node* flat_array = array; - Node* vt = InlineTypeNode::make_from_flat_array(this, vk, flat_array, array_index); + Node* flat_array = _array; + Node* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); Node* ld = _gvn.transform(vt); - push_node(bt, ld); - return; + _parse.push_node(_bt, ld); + return true; } - ciMethodData* md = method()->method_data(); - if (md != nullptr && md->is_mature()) { - ciProfileData* data = md->bci_to_data(bci()); - if (data != nullptr && data->is_ArrayLoadData()) { - ciArrayLoadData* array_load = (ciArrayLoadData*) data->as_ArrayLoadData(); - int not_flat_count = array_load->not_flat_count(); - if (not_flat_count < 0) { - not_flat_count = max_jint; - } - ciCallProfile profile = method()->call_profile_at_bci(bci()); + return false; + } + + Node* emit_plain_load(Node* array) { + const TypeAryPtr* array_type = _gvn.type(array)->is_aryptr(); + const TypeOopPtr* element_ptr = _elemtype->make_oopptr(); + const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(_bt); + DecoratorSet decorator_set = IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD; + if (_parse.needs_range_check(array_type->size(), _array_index)) { + // We've emitted a RangeCheck but now insert an additional check between the range check and the actual load. + // We cannot pin the load to two separate nodes. Instead, we pin it conservatively here such that it cannot + // possibly float above the range check at any point. + decorator_set |= C2_UNKNOWN_CONTROL_LOAD; + } + const TypeAryPtr* arytype = _gvn.type(array)->is_aryptr(); + const TypeInt* sizetype = arytype->size(); + Node* adr = _parse.array_element_address(array, _array_index, _bt, sizetype, _parse.control()); + assert(adr != _parse.top(), "top should go hand-in-hand with stopped"); + Node* ld = _parse.access_load_at(array, adr, adr_type, _elemtype, _bt, decorator_set); + if (element_ptr != nullptr && element_ptr->is_inlinetypeptr()) { + assert(!array_type->is_null_free() || !element_ptr->maybe_null(), "inline type array elements should never be null"); + ld = InlineTypeNode::make_from_oop(&_parse, ld, element_ptr->inline_klass()); + } + return _gvn.transform(ld); + } + + void test_non_flat_array_and_emit_reference_load(float p) { + const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr(); + Node* test = _parse.flat_array_test(_array, /* flat = */ false); + float this_prob = clamp(p, PROB_MIN, PROB_MAX); + IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, this_prob, COUNT_UNKNOWN); + _parse.set_control(_gvn.transform(new IfTrueNode(iff))); + assert(array_type->is_flat() || _parse.control()->in(0)->as_If()->is_flat_array_check(&_gvn), + "Should be found"); + Node* casted_array = _gvn.transform(new CheckCastPPNode(_parse.control(), _array, array_type->cast_to_not_flat())); + Node* ld = emit_plain_load(casted_array); + _res_phi->add_req(ld); + _region->add_req(_parse.control()); + _io_phi->add_req(_parse.i_o()); + _mem_phi->add_req(_parse.reset_memory()); + _parse.set_control(_gvn.transform(new IfFalseNode(iff))); + _parse.set_all_memory(_mem); + _parse.set_i_o(_io); + } + + void test_known_flat_array_and_emit_load_flat(ciKlass* klass, float p) { + float this_prob = clamp(p, PROB_MIN, PROB_MAX); + Node* casted_array = nullptr; + Node* next_ctrl = _parse.type_check_receiver(_array, klass, this_prob, &casted_array); + InlineTypeNode* vt = InlineTypeNode::make_from_flat_array( + &_parse, klass->as_flat_array_klass()->element_klass()->as_inline_klass(), casted_array, _array_index); + Node* null_ctl = _parse.top(); + + _parse.null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl); + + _res_phi->add_req(_parse.zerocon(T_OBJECT)); + _region->add_req(null_ctl); + _io_phi->add_req(_parse.i_o()); + _mem_phi->add_req(_parse.reset_memory()); + _parse.set_all_memory(_mem); + + Node* ld = _gvn.transform(vt->buffer(&_parse)); + _res_phi->add_req(ld); + _region->add_req(_parse.control()); + _io_phi->add_req(_parse.i_o()); + _mem_phi->add_req(_parse.reset_memory()); + _parse.set_control(next_ctrl); + _parse.set_all_memory(_mem); + _parse.set_i_o(_io); + } + + void load_from_unknown_flat_array(const TypeOopPtr* element_ptr) { + Node* unknown_inline_type = _parse.load_from_unknown_flat_array(_array, _array_index, element_ptr); + + _region->add_req(_parse.control()); + _res_phi->add_req(unknown_inline_type); + _mem_phi->add_req(_parse.reset_memory()); + _io_phi->add_req(_parse.i_o()); + } + + void create_merge_point() { + _region = new RegionNode(1); + _res_phi = new PhiNode(_region, TypeOopPtr::BOTTOM); + _io_phi = new PhiNode(_region, Type::ABIO); + _mem = _parse.reset_memory(); + _io = _parse.i_o(); + _parse.set_all_memory(_mem); + _mem_phi = new PhiNode(_region, Type::MEMORY, TypePtr::BOTTOM); + } + + bool emit() { + emit_null_and_range_checks(); + + // Check for always knowing you are throwing a range-check exception + if (_parse.stopped()) return true; //top(); + + const TypeOopPtr* element_ptr = _elemtype->make_oopptr(); + const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr(); + + if (array_type->is_flat()) { + if (_elemtype == TypeInt::BOOL) { + _bt = T_BOOLEAN; + } + Node* ld = emit_plain_load(_array); + pop_stack(); + _parse.push_node(_bt, ld); + return true; + } + + if (!array_type->is_not_flat()) { + // Cannot statically determine if array is a flat array, emit runtime check + assert(UseArrayFlattening && is_reference_type(_bt) && element_ptr->can_be_inline_type() && + (!element_ptr->is_inlinetypeptr() || element_ptr->inline_klass()->maybe_flat_in_array()), + "array can't be flat"); + + if (emit_load_if_known_flat_array(element_ptr)) { + return true; + } + + ciArrayLoadData* array_load = profile_data(); + if (array_load != nullptr) { + int not_flat_count = profiled_not_flat_count(); + ciCallProfile profile = _parse.method()->call_profile_at_bci(_parse.bci()); int flat_count = profile.count(); int flat_and_not_flat_count = saturated_add(flat_count, not_flat_count); - if (profile.morphism() > 0) { + if (profile.morphism() > 0 || profile.has_major_receiver()) { bool not_flat_checked = false; float prob = 1; - Node* region = new RegionNode(profile.morphism() * 2 + 3); - Node* res_phi = new PhiNode(region, TypeOopPtr::BOTTOM); - Node* io_phi = new PhiNode(region, Type::ABIO); - Node* mem = reset_memory(); - Node* io = i_o(); - set_all_memory(mem); - Node* mem_phi = new PhiNode(region, Type::MEMORY, TypePtr::BOTTOM); - int j = 1; - for (int i = 0; i < profile.morphism() || !not_flat_checked; ) { - int count = i < profile.morphism() ? profile.receiver_count(i) : not_flat_count; + create_merge_point(); + int limit = MAX2(profile.morphism(), 1); + bool done = false; + for (int i = 0; i < limit || !not_flat_checked;) { + int count = i < limit ? profile.receiver_count(i) : not_flat_count; if (not_flat_count >= count && !not_flat_checked) { not_flat_checked = true; - if (not_flat_count == 0 && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { - PreserveJVMState pjvms(this); - inc_sp(2); - uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); - } else { - Node* test = flat_array_test(array, /* flat = */ false); - float p = ((float)not_flat_count) / ((float)flat_and_not_flat_count); - float this_prob = clamp(p / prob, PROB_MIN, PROB_MAX); - IfNode* iff = create_and_xform_if(control(), test, this_prob, COUNT_UNKNOWN); - set_control(_gvn.transform(new IfTrueNode(iff))); - assert(array_type->is_flat() || control()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found"); - const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); - DecoratorSet decorator_set = IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD; - if (needs_range_check(array_type->size(), array_index)) { - // We've emitted a RangeCheck but now insert an additional check between the range check and the actual load. - // We cannot pin the load to two separate nodes. Instead, we pin it conservatively here such that it cannot - // possibly float above the range check at any point. - decorator_set |= C2_UNKNOWN_CONTROL_LOAD; - } - Node* ld = access_load_at(array, adr, adr_type, element_ptr, bt, decorator_set); - if (element_ptr->is_inlinetypeptr()) { - ld = InlineTypeNode::make_from_oop(this, ld, element_ptr->inline_klass()); + if (profile.morphism() > 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + PreserveJVMState pjvms(&_parse); + _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + done = true; + } else if (profile.morphism() <= 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr(); + Node* test = _parse.flat_array_test(_array, /* flat = */ false); + IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, PROB_MIN, COUNT_UNKNOWN); + _parse.set_control(_gvn.transform(new IfTrueNode(iff))); + assert(array_type->is_flat() || _parse.control()->in(0)->as_If()->is_flat_array_check(&_gvn), + "Should be found"); + { + PreserveJVMState pjvms(&_parse); + _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); } - res_phi->init_req(j, _gvn.transform(ld)); - region->init_req(j, control()); - io_phi->init_req(j, i_o()); - set_control(_gvn.transform(new IfFalseNode(iff))); - mem_phi->init_req(j, reset_memory()); - set_all_memory(mem); - set_i_o(io); + _parse.set_control(_gvn.transform(new IfFalseNode(iff))); + } else { + float p = ((float) not_flat_count) / ((float) flat_and_not_flat_count); + test_non_flat_array_and_emit_reference_load(p / prob); prob = 1 - p; } } else { - inc_sp(2); - float p = ((float)profile.receiver_count(i)) / ((float)flat_and_not_flat_count); - float this_prob = clamp(p / prob, PROB_MIN, PROB_MAX); - Node* casted_array = nullptr; + float p = ((float) profile.receiver_count(i)) / ((float) flat_and_not_flat_count); ciKlass* klass = profile.receiver(i); - Node* next_ctrl = type_check_receiver(array, klass, this_prob, &casted_array); - InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(this, klass->as_flat_array_klass()->element_klass()->as_inline_klass(), casted_array, array_index); - Node* null_ctl = top(); - - null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl); - - res_phi->init_req(j, zerocon(T_OBJECT)); - region->init_req(j, null_ctl); - io_phi->init_req(j, i_o()); - mem_phi->init_req(j, reset_memory()); - set_all_memory(mem); - - j++; - - Node* ld = _gvn.transform(vt->buffer(this)); - res_phi->init_req(j, ld); - region->init_req(j, control()); - io_phi->init_req(j, i_o()); - set_control(next_ctrl); - mem_phi->init_req(j, reset_memory()); - set_all_memory(mem); - set_i_o(io); - prob = 1 - p; + test_known_flat_array_and_emit_load_flat(klass, p / prob); i++; - dec_sp(2); + prob = 1 - p; + } + } + if (!done) { + if (profile.morphism() <= 0 || _parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + load_from_unknown_flat_array(element_ptr); + } else { + PreserveJVMState pjvms(&_parse); + _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); } - j++; } - if (not_flat_count != 0 && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { - PreserveJVMState pjvms(this); - inc_sp(2); - uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); - } else if (too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { - Node* unknown_inline_type = load_from_unknown_flat_array(array, array_index, element_ptr); - - region->init_req(j, control()); - res_phi->init_req(j, unknown_inline_type); - mem_phi->init_req(j, reset_memory()); - io_phi->init_req(j, i_o()); + // if (profile.morphism() > 0 && not_flat_count != 0 && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + // PreserveJVMState pjvms(&_parse); + // _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + // } else if (_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check) || profile.morphism() <= 0) { + // load_from_unknown_flat_array(element_ptr); + // } + _parse.set_control(_gvn.transform(_region)); + _parse.set_all_memory(_gvn.transform(_mem_phi)); + _parse.set_i_o(_gvn.transform(_io_phi)); + Node* ld = _gvn.transform(_res_phi); + ld = _parse.record_profile_for_speculation_at_array_load(ld); + pop_stack(); + _parse.push_node(_bt, ld); + return true; + } + if (flat_count == 0 && not_flat_count > 0 && !_parse.too_many_traps_or_recompiles( + Deoptimization::Reason_class_check)) { + { + // Deoptimize if flat array + BuildCutout unless(&_parse, _parse.flat_array_test(_array, /* flat = */ false), PROB_MAX); + _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); } - set_control(_gvn.transform(region)); - set_all_memory(_gvn.transform(mem_phi)); - set_i_o(_gvn.transform(io_phi)); - Node* ld = _gvn.transform(res_phi); - ld = record_profile_for_speculation_at_array_load(ld); - push_node(bt, ld); - return; - } else { - // ShouldNotReachHere(); + pop_stack(); + assert(!_parse.stopped(), "flat array should have been caught earlier"); + Node* casted_array = _gvn.transform( + new CheckCastPPNode(_parse.control(), _array, array_type->cast_to_not_flat())); + _parse.replace_in_map(_array, casted_array); + Node* ld = emit_plain_load(casted_array); + ld = _parse.record_profile_for_speculation_at_array_load(ld); + _parse.push_node(_bt, ld); + return true; } } } + return false; + } +}; + +//---------------------------------array_load---------------------------------- +void Parse::array_load(BasicType bt) { + ArrayLoad array_load(bt, *this); + if (array_load.emit()) { + return; + } + + const Type* elemtype = Type::TOP; + Node* adr = array_addressing(bt, 0, elemtype); + if (stopped()) return; // guaranteed null or range check + + Node* array_index = pop(); + Node* array = pop(); + + // Handle inline type arrays + const TypeOopPtr* element_ptr = elemtype->make_oopptr(); + const TypeAryPtr* array_type = _gvn.type(array)->is_aryptr(); + if (!array_type->is_not_flat()) { + // Cannot statically determine if array is a flat array, emit runtime check + assert(UseArrayFlattening && is_reference_type(bt) && element_ptr->can_be_inline_type() && + (!element_ptr->is_inlinetypeptr() || element_ptr->inline_klass()->maybe_flat_in_array()), "array can't be flat"); + // + // if (element_ptr->is_inlinetypeptr()) { + // ciInlineKlass* vk = element_ptr->inline_klass(); + // // Node* flat_array = cast_to_flat_array(array, vk); + // Node* flat_array = array; + // Node* vt = InlineTypeNode::make_from_flat_array(this, vk, flat_array, array_index); + // Node* ld = _gvn.transform(vt); + // push_node(bt, ld); + // return; + // } + // ciMethodData* md = method()->method_data(); + // if (md != nullptr && md->is_mature()) { + // ciProfileData* data = md->bci_to_data(bci()); + // if (data != nullptr && data->is_ArrayLoadData()) { + // ciArrayLoadData* array_load = (ciArrayLoadData*) data->as_ArrayLoadData(); + // int not_flat_count = array_load->not_flat_count(); + // if (not_flat_count < 0) { + // not_flat_count = max_jint; + // } + // ciCallProfile profile = method()->call_profile_at_bci(bci()); + // int flat_count = profile.count(); + // int flat_and_not_flat_count = saturated_add(flat_count, not_flat_count); + // if (profile.morphism() > 0) { + // bool not_flat_checked = false; + // float prob = 1; + // Node* region = new RegionNode(profile.morphism() * 2 + 3); + // Node* res_phi = new PhiNode(region, TypeOopPtr::BOTTOM); + // Node* io_phi = new PhiNode(region, Type::ABIO); + // Node* mem = reset_memory(); + // Node* io = i_o(); + // set_all_memory(mem); + // Node* mem_phi = new PhiNode(region, Type::MEMORY, TypePtr::BOTTOM); + // int j = 1; + // for (int i = 0; i < profile.morphism() || !not_flat_checked; ) { + // int count = i < profile.morphism() ? profile.receiver_count(i) : not_flat_count; + // if (not_flat_count >= count && !not_flat_checked) { + // not_flat_checked = true; + // if (not_flat_count == 0 && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + // PreserveJVMState pjvms(this); + // inc_sp(2); + // uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + // } else { + // Node* test = flat_array_test(array, /* flat = */ false); + // float p = ((float)not_flat_count) / ((float)flat_and_not_flat_count); + // float this_prob = clamp(p / prob, PROB_MIN, PROB_MAX); + // IfNode* iff = create_and_xform_if(control(), test, this_prob, COUNT_UNKNOWN); + // set_control(_gvn.transform(new IfTrueNode(iff))); + // assert(array_type->is_flat() || control()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found"); + // const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt); + // DecoratorSet decorator_set = IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD; + // if (needs_range_check(array_type->size(), array_index)) { + // // We've emitted a RangeCheck but now insert an additional check between the range check and the actual load. + // // We cannot pin the load to two separate nodes. Instead, we pin it conservatively here such that it cannot + // // possibly float above the range check at any point. + // decorator_set |= C2_UNKNOWN_CONTROL_LOAD; + // } + // Node* ld = access_load_at(array, adr, adr_type, element_ptr, bt, decorator_set); + // if (element_ptr->is_inlinetypeptr()) { + // ld = InlineTypeNode::make_from_oop(this, ld, element_ptr->inline_klass()); + // } + // res_phi->init_req(j, _gvn.transform(ld)); + // region->init_req(j, control()); + // io_phi->init_req(j, i_o()); + // set_control(_gvn.transform(new IfFalseNode(iff))); + // mem_phi->init_req(j, reset_memory()); + // set_all_memory(mem); + // set_i_o(io); + // prob = 1 - p; + // } + // } else { + // inc_sp(2); + // float p = ((float)profile.receiver_count(i)) / ((float)flat_and_not_flat_count); + // float this_prob = clamp(p / prob, PROB_MIN, PROB_MAX); + // Node* casted_array = nullptr; + // ciKlass* klass = profile.receiver(i); + // Node* next_ctrl = type_check_receiver(array, klass, this_prob, &casted_array); + // InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(this, klass->as_flat_array_klass()->element_klass()->as_inline_klass(), casted_array, array_index); + // Node* null_ctl = top(); + // + // null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl); + // + // res_phi->init_req(j, zerocon(T_OBJECT)); + // region->init_req(j, null_ctl); + // io_phi->init_req(j, i_o()); + // mem_phi->init_req(j, reset_memory()); + // set_all_memory(mem); + // + // j++; + // + // Node* ld = _gvn.transform(vt->buffer(this)); + // res_phi->init_req(j, ld); + // region->init_req(j, control()); + // io_phi->init_req(j, i_o()); + // set_control(next_ctrl); + // mem_phi->init_req(j, reset_memory()); + // set_all_memory(mem); + // set_i_o(io); + // prob = 1 - p; + // i++; + // dec_sp(2); + // } + // j++; + // } + // if (not_flat_count != 0 && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + // PreserveJVMState pjvms(this); + // inc_sp(2); + // uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + // } else if (too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + // Node* unknown_inline_type = load_from_unknown_flat_array(array, array_index, element_ptr); + // + // region->init_req(j, control()); + // res_phi->init_req(j, unknown_inline_type); + // mem_phi->init_req(j, reset_memory()); + // io_phi->init_req(j, i_o()); + // } + // set_control(_gvn.transform(region)); + // set_all_memory(_gvn.transform(mem_phi)); + // set_i_o(_gvn.transform(io_phi)); + // Node* ld = _gvn.transform(res_phi); + // ld = record_profile_for_speculation_at_array_load(ld); + // push_node(bt, ld); + // return; + // } + // if (flat_count == 0 && not_flat_count > 0 && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + // { // Deoptimize if flat array + // BuildCutout unless(this, flat_array_test(array, /* flat = */ false), PROB_MAX); + // inc_sp(2); + // uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + // } + // assert(!stopped(), "flat array should have been caught earlier"); + // Node* casted_array = _gvn.transform(new CheckCastPPNode(control(), array, array_type->cast_to_not_flat())); + // replace_in_map(array, casted_array); + // } else if (profile.has_major_receiver()) { + // + // } + // } + // } + // IdealKit ideal(this); IdealVariable res(ideal); ideal.declarations_done(); @@ -298,7 +631,7 @@ Node* Parse::load_from_unknown_flat_array(Node* array, Node* array_index, const PreserveReexecuteState preexecs(this); jvms()->set_bci(_bci); jvms()->set_should_reexecute(true); - inc_sp(2); + // inc_sp(2); kill_dead_locals(); call = make_runtime_call(RC_NO_LEAF | RC_NO_IO, OptoRuntime::load_unknown_inline_Type(), diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java index 950bec4c914..57adafcd15f 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java @@ -3291,6 +3291,11 @@ public static void anyStoreOfNodes(String irNodePlaceholder, String fieldHolder) beforeMatchingNameRegex(OPAQUE_CONSTANT_BOOL, "OpaqueConstantBool"); } + public static final String BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP = PREFIX + "BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP" + POSTFIX; + static { + trapNodes(BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "bimorphic_or_optimized_type_check"); + } + /* * Utility methods to set up IR_NODE_MAPPINGS. */ diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 8e66cff094f..fe190a6be30 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -53,9 +53,16 @@ public static void main(String[] args) { static MyValue1[] array3 = (MyValue1[])ValueClass.newNullRestrictedNonAtomicArray(MyValue1.class, 1, new MyValue1(42)); static MyValue2[] array4 = (MyValue2[])ValueClass.newNullRestrictedNonAtomicArray(MyValue2.class, 1, new MyValue2(42)); static A[] array5 = { new A() }; + static MyValue1[] array6 = (MyValue1[])ValueClass.newReferenceArray(MyValue1.class, 1); + static MyValue2[] array7 = (MyValue2[])ValueClass.newReferenceArray(MyValue2.class, 1); + { // doesn't run!? + array6[0] = new MyValue1(42); + array7[0] = new MyValue2(42); + } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.CALL, "4", IRNode.IF, "4" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "4" }) + @IR(failOn = IRNode.ALLOC) public static void test1(I[] array) { test1Inline(array[0]); } @@ -115,7 +122,8 @@ static void test1Inline(I i) { // } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.CALL, "3", IRNode.IF, "4" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "4" }) + @IR(failOn = IRNode.ALLOC) public static void test3(I[] array) { test3Inline(array[0]); } @@ -132,7 +140,8 @@ static void test3Inline(I i) { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.CALL, "4", IRNode.IF, "6" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "6" }) + @IR(failOn = IRNode.ALLOC) public static void test5(I[] array) { test5Inline(array[0]); } @@ -147,42 +156,109 @@ public static void test5Runner() { static void test5Inline(I i) { i.m(); } - - // @Test - // public static Object test2(Object[] array) { - // return array[0]; - // } - // @Run(test = "test2") - // public static void test2Runner() { - // test2(array5); - // } + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "6" }) + @IR(failOn = IRNode.ALLOC) + public static void test7(I[] array) { + test7Inline(array[0]); + } - // @Test - // public static Object test3(Object[] array) { - // return array[0]; - // } + @Run(test = "test7") + public static void test7Runner() { + test7Inline(array1[0]); + test7Inline(array2[0]); + test7Inline(array3[0]); + test7Inline(array4[0]); + test7(array5); + } + + @ForceInline + static void test7Inline(I i) { + i.m(); + } - // @Run(test = "test3") - // public static void test3Runner() { - // test3(array1); - // test3(array2); - // test3(array5); + // if (array == null) { + // trap1; // } - - // @Test - // public static Object test4(Object[] array) { - // return array[0]; + // if (0 not in range of array) { + // trap2; // } - - // @Run(test = "test4") - // public static void test4Runner() { - // test4(array1); - // test4(array2); - // test4(array3); - // test4(array4); + // if (array flat) { + // if (array.klass == MyValue1[]) { + // if (array[0] == null) { + // trap3; + // } + // // inlined call + // } else if (array.klass == MyValue2[]) { + // if (array[0] == null) { + // trap3; + // } + // // inlined call + // } else { + // trap4; + // } + // } else { + // if (array.klass == MyValue1[]) { + // if (array[0] == null) { + // trap3; + // } + // // inlined call + // } else if (array.klass == MyValue2[]) { + // if (array[0] == null) { + // trap3; + // } + // // inlined call + // } else { + // trap5; + // } // } - + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "11" }) + @IR(failOn = IRNode.ALLOC) + public static void test9(I[] array) { + test9Inline(array[0]); + } + + @Run(test = "test9") + public static void test9Runner() { + if (array6[0] == null) { + array6[0] = new MyValue1(42); + array7[0] = new MyValue2(42); + } + test9(array1); + test9(array2); + test9(array6); + test9(array7); + } + + @ForceInline + static void test9Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "11" }) + @IR(failOn = IRNode.ALLOC) + public static void test11(I[] array) { + test11Inline(array[0]); + } + + @Run(test = "test11") + public static void test11Runner() { + for (int i = 0; i < 50; i++) { + test11(array1); + } + test11(array2); + test11(array3); + test11(array4); + } + + @ForceInline + static void test11Inline(I i) { + i.m(); + } + interface I { void m(); } From 66513a657f071fcb9374df5f1b9c58458dfe8248 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Tue, 31 Mar 2026 12:53:46 +0200 Subject: [PATCH 13/49] more --- src/hotspot/share/opto/parse2.cpp | 10 +++-- .../inlinetypes/TestArrayLoadProfiling.java | 41 +++++++++++++++---- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index f56826dbd28..c7e5eb7bb9d 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -314,6 +314,7 @@ class ArrayLoad { int limit = MAX2(profile.morphism(), 1); bool done = false; for (int i = 0; i < limit || !not_flat_checked;) { + assert(!_parse.stopped(), ""); int count = i < limit ? profile.receiver_count(i) : not_flat_count; if (not_flat_count >= count && !not_flat_checked) { not_flat_checked = true; @@ -333,6 +334,8 @@ class ArrayLoad { _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); } _parse.set_control(_gvn.transform(new IfFalseNode(iff))); + load_from_unknown_flat_array(element_ptr); + done = true; } else { float p = ((float) not_flat_count) / ((float) flat_and_not_flat_count); test_non_flat_array_and_emit_reference_load(p / prob); @@ -347,12 +350,13 @@ class ArrayLoad { } } if (!done) { - if (profile.morphism() <= 0 || _parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { - load_from_unknown_flat_array(element_ptr); - } else { + if (!_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { PreserveJVMState pjvms(&_parse); _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } else { + load_from_unknown_flat_array(element_ptr); } + } // if (profile.morphism() > 0 && not_flat_count != 0 && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { // PreserveJVMState pjvms(&_parse); diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index fe190a6be30..c41cdc2f969 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -199,15 +199,12 @@ static void test7Inline(I i) { // trap4; // } // } else { - // if (array.klass == MyValue1[]) { - // if (array[0] == null) { + // if (array[0] == null) { // trap3; - // } + // } + // if (array[0].klass == MyValue1) { // // inlined call - // } else if (array.klass == MyValue2[]) { - // if (array[0] == null) { - // trap3; - // } + // } else if (array[0].klass == MyValue2) { // // inlined call // } else { // trap5; @@ -237,8 +234,36 @@ static void test9Inline(I i) { i.m(); } + // if (array == null) { + // trap1; + // } + // if (0 not in range of array) { + // trap2; + // } + // if (array.klass == MyValue1[]) { + // if (array[0] == null) { + // trap3; + // } + // // inlined call + // } else { + // if (array flat) { + // elt = load_unknown_inline(); + // if (elt == null) { + // trap3; + // } + // if (elt.klass == MyValue1) { + // // inlined call + // } else if (elt.klass == MyValue2) { + // // inlined call + // } else { + // trap4; + // } + // } else { + // trap5; + // } + // } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "11" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "6", IRNode.IF, "9" }) @IR(failOn = IRNode.ALLOC) public static void test11(I[] array) { test11Inline(array[0]); From 86fa856181e7c74330371cef0d9c48987604b03e Mon Sep 17 00:00:00 2001 From: rwestrel Date: Tue, 31 Mar 2026 16:00:18 +0200 Subject: [PATCH 14/49] more --- src/hotspot/share/opto/parse2.cpp | 84 +++++++++++----- .../inlinetypes/TestArrayLoadProfiling.java | 98 ++++++++++++++++++- 2 files changed, 159 insertions(+), 23 deletions(-) diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index c7e5eb7bb9d..7d8080c9cfb 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -254,10 +254,17 @@ class ArrayLoad { } void load_from_unknown_flat_array(const TypeOopPtr* element_ptr) { - Node* unknown_inline_type = _parse.load_from_unknown_flat_array(_array, _array_index, element_ptr); + Node* vt = nullptr; + if (element_ptr->is_inlinetypeptr()) { + ciInlineKlass* vk = element_ptr->inline_klass(); + Node* flat_array = _parse.cast_to_flat_array(_array, vk); + vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); + } else { + vt = _parse.load_from_unknown_flat_array(_array, _array_index, element_ptr); + } _region->add_req(_parse.control()); - _res_phi->add_req(unknown_inline_type); + _res_phi->add_req(vt); _mem_phi->add_req(_parse.reset_memory()); _io_phi->add_req(_parse.i_o()); } @@ -272,6 +279,16 @@ class ArrayLoad { _mem_phi = new PhiNode(_region, Type::MEMORY, TypePtr::BOTTOM); } + void finish_merge_point() { + _parse.set_control(_gvn.transform(_region)); + _parse.set_all_memory(_gvn.transform(_mem_phi)); + _parse.set_i_o(_gvn.transform(_io_phi)); + Node* ld = _gvn.transform(_res_phi); + ld = _parse.record_profile_for_speculation_at_array_load(ld); + pop_stack(); + _parse.push_node(_bt, ld); + } + bool emit() { emit_null_and_range_checks(); @@ -281,7 +298,7 @@ class ArrayLoad { const TypeOopPtr* element_ptr = _elemtype->make_oopptr(); const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr(); - if (array_type->is_flat()) { + if (array_type->is_not_flat()) { if (_elemtype == TypeInt::BOOL) { _bt = T_BOOLEAN; } @@ -291,15 +308,25 @@ class ArrayLoad { return true; } + if (array_type->is_flat()) { + pop_stack(); + ciInlineKlass* vk = element_ptr->inline_klass(); + Node* flat_array = _array; + Node* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); + Node* ld = _gvn.transform(vt); + _parse.push_node(_bt, ld); + return true; + } + if (!array_type->is_not_flat()) { // Cannot statically determine if array is a flat array, emit runtime check assert(UseArrayFlattening && is_reference_type(_bt) && element_ptr->can_be_inline_type() && (!element_ptr->is_inlinetypeptr() || element_ptr->inline_klass()->maybe_flat_in_array()), "array can't be flat"); - if (emit_load_if_known_flat_array(element_ptr)) { - return true; - } + // if (emit_load_if_known_flat_array(element_ptr)) { + // return true; + // } ciArrayLoadData* array_load = profile_data(); if (array_load != nullptr) { @@ -356,25 +383,11 @@ class ArrayLoad { } else { load_from_unknown_flat_array(element_ptr); } - } - // if (profile.morphism() > 0 && not_flat_count != 0 && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { - // PreserveJVMState pjvms(&_parse); - // _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); - // } else if (_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check) || profile.morphism() <= 0) { - // load_from_unknown_flat_array(element_ptr); - // } - _parse.set_control(_gvn.transform(_region)); - _parse.set_all_memory(_gvn.transform(_mem_phi)); - _parse.set_i_o(_gvn.transform(_io_phi)); - Node* ld = _gvn.transform(_res_phi); - ld = _parse.record_profile_for_speculation_at_array_load(ld); - pop_stack(); - _parse.push_node(_bt, ld); + finish_merge_point(); return true; } - if (flat_count == 0 && not_flat_count > 0 && !_parse.too_many_traps_or_recompiles( - Deoptimization::Reason_class_check)) { + if (flat_count == 0 && not_flat_count > 0 && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { { // Deoptimize if flat array BuildCutout unless(&_parse, _parse.flat_array_test(_array, /* flat = */ false), PROB_MAX); @@ -390,7 +403,33 @@ class ArrayLoad { _parse.push_node(_bt, ld); return true; } + if (flat_count != 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + { + // Deoptimize if not flat array + BuildCutout unless(&_parse, _parse.flat_array_test(_array, /* flat = */ true), PROB_MAX); + _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } + assert(!_parse.stopped(), "non flat array should have been caught earlier"); + Node* ld = _parse.load_from_unknown_flat_array(_array, _array_index, element_ptr); + pop_stack(); + ld = _parse.record_profile_for_speculation_at_array_load(ld); + _parse.push_node(_bt, ld); + return true; + } + if (flat_count != 0 && not_flat_count != 0) { + create_merge_point(); + float p = ((float) not_flat_count) / ((float) flat_and_not_flat_count); + test_non_flat_array_and_emit_reference_load(p); + load_from_unknown_flat_array(element_ptr); + finish_merge_point(); + return true; + } } + create_merge_point(); + test_non_flat_array_and_emit_reference_load(PROB_FAIR); + load_from_unknown_flat_array(element_ptr); + finish_merge_point(); + return true; } return false; } @@ -403,6 +442,7 @@ void Parse::array_load(BasicType bt) { return; } + ShouldNotReachHere(); const Type* elemtype = Type::TOP; Node* adr = array_addressing(bt, 0, elemtype); if (stopped()) return; // guaranteed null or range check diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index c41cdc2f969..7fb2f16f8be 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -48,7 +48,7 @@ public static void main(String[] args) { TestFramework.runWithFlags("--enable-preview", "--add-exports", "java.base/jdk.internal.value=ALL-UNNAMED", "-Xbootclasspath/a:.", "-XX:+UnlockDiagnosticVMOptions", "-XX:+WhiteBoxAPI"); } - static MyValue1[] array1 = { new MyValue1(42) }; + static final MyValue1[] array1 = { new MyValue1(42) }; static MyValue2[] array2 = { new MyValue2(42) }; static MyValue1[] array3 = (MyValue1[])ValueClass.newNullRestrictedNonAtomicArray(MyValue1.class, 1, new MyValue1(42)); static MyValue2[] array4 = (MyValue2[])ValueClass.newNullRestrictedNonAtomicArray(MyValue2.class, 1, new MyValue2(42)); @@ -284,6 +284,102 @@ static void test11Inline(I i) { i.m(); } + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "6", IRNode.IF, "7" }) + @IR(failOn = IRNode.ALLOC) + public static void test13(I[] array) { + test13Inline(array[0]); + } + + @Run(test = "test13") + public static void test13Runner() { + test13(array1); + test13(array2); + test13(array3); + test13(array4); + } + + @ForceInline + static void test13Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "5", IRNode.IF, "7" }) + @IR(failOn = IRNode.ALLOC) + public static void test15(I[] array) { + test15Inline(array[0]); + } + + @Run(test = "test15") + public static void test15Runner() { + if (array6[0] == null) { + array6[0] = new MyValue1(42); + array7[0] = new MyValue2(42); + } + test15(array1); + test15(array2); + test15(array3); + test15(array4); + test15(array6); + test15(array7); + } + + @ForceInline + static void test15Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "2", IRNode.CALL, "4", IRNode.IF, "4" }) + @IR(failOn = IRNode.ALLOC) + public static void test17(I[] array) { + test17Inline(array[0]); + } + + @Run(test = "test17") + @Warmup(0) + public static void test17Runner() { + test17(array1); + } + + @ForceInline + static void test17Inline(I i) { + i.m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.TRAP, "1", IRNode.CALL, "1", IRNode.IF, "1" }) + @IR(failOn = IRNode.ALLOC) + public static void test18() { + array1[0].m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "3" }) + @IR(failOn = IRNode.ALLOC) + public static void test19(A[] array) { + array[0].m(); + } + + @Run(test = "test19") + public static void test19Runner() { + test19(array5); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "8" }) + @IR(failOn = IRNode.ALLOC) + public static void test20(MyValue1[] array) { + array[0].m(); + } + + @Run(test = "test20") + @Warmup(0) + public static void test20Runner() { + test20(array1); + } + interface I { void m(); } From 130f6e16eb4d908a2d69260b7a91b128c240e263 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Wed, 1 Apr 2026 15:52:03 +0200 Subject: [PATCH 15/49] more --- src/hotspot/share/opto/parse2.cpp | 20 ++++- .../inlinetypes/TestArrayLoadProfiling.java | 78 ++++++++++++++----- 2 files changed, 76 insertions(+), 22 deletions(-) diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 7d8080c9cfb..e72e076bcd0 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -254,17 +254,29 @@ class ArrayLoad { } void load_from_unknown_flat_array(const TypeOopPtr* element_ptr) { - Node* vt = nullptr; + Node* ld = nullptr; if (element_ptr->is_inlinetypeptr()) { ciInlineKlass* vk = element_ptr->inline_klass(); Node* flat_array = _parse.cast_to_flat_array(_array, vk); - vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); + + InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); + ld = vt; + + Node* null_ctl = _parse.top(); + _parse.null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl); + + _res_phi->add_req(_parse.zerocon(T_OBJECT)); + _region->add_req(null_ctl); + _io_phi->add_req(_parse.i_o()); + _mem_phi->add_req(_parse.reset_memory()); + _parse.set_all_memory(_mem); + } else { - vt = _parse.load_from_unknown_flat_array(_array, _array_index, element_ptr); + ld = _parse.load_from_unknown_flat_array(_array, _array_index, element_ptr); } _region->add_req(_parse.control()); - _res_phi->add_req(vt); + _res_phi->add_req(ld); _mem_phi->add_req(_parse.reset_memory()); _io_phi->add_req(_parse.i_o()); } diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 7fb2f16f8be..114ac21e8f7 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -48,16 +48,18 @@ public static void main(String[] args) { TestFramework.runWithFlags("--enable-preview", "--add-exports", "java.base/jdk.internal.value=ALL-UNNAMED", "-Xbootclasspath/a:.", "-XX:+UnlockDiagnosticVMOptions", "-XX:+WhiteBoxAPI"); } - static final MyValue1[] array1 = { new MyValue1(42) }; - static MyValue2[] array2 = { new MyValue2(42) }; - static MyValue1[] array3 = (MyValue1[])ValueClass.newNullRestrictedNonAtomicArray(MyValue1.class, 1, new MyValue1(42)); - static MyValue2[] array4 = (MyValue2[])ValueClass.newNullRestrictedNonAtomicArray(MyValue2.class, 1, new MyValue2(42)); + static final MyValue1[] array1 = { new MyValue1((byte)42) }; + static MyValue2[] array2 = { new MyValue2((byte)42) }; + static MyValue1[] array3 = (MyValue1[])ValueClass.newNullRestrictedNonAtomicArray(MyValue1.class, 1, new MyValue1((byte)42)); + static MyValue2[] array4 = (MyValue2[])ValueClass.newNullRestrictedNonAtomicArray(MyValue2.class, 1, new MyValue2((byte)42)); static A[] array5 = { new A() }; static MyValue1[] array6 = (MyValue1[])ValueClass.newReferenceArray(MyValue1.class, 1); static MyValue2[] array7 = (MyValue2[])ValueClass.newReferenceArray(MyValue2.class, 1); - { // doesn't run!? - array6[0] = new MyValue1(42); - array7[0] = new MyValue2(42); + static MyValue1[] array8 = (MyValue1[])ValueClass.newNullRestrictedAtomicArray(MyValue1.class, 1, new MyValue1((byte)42)); + static { + array6[0] = new MyValue1((byte)42); + array7[0] = new MyValue2((byte)42); + array8[0] = new MyValue1((byte)42); } @Test @@ -219,10 +221,6 @@ public static void test9(I[] array) { @Run(test = "test9") public static void test9Runner() { - if (array6[0] == null) { - array6[0] = new MyValue1(42); - array7[0] = new MyValue2(42); - } test9(array1); test9(array2); test9(array6); @@ -313,10 +311,6 @@ public static void test15(I[] array) { @Run(test = "test15") public static void test15Runner() { - if (array6[0] == null) { - array6[0] = new MyValue1(42); - array7[0] = new MyValue2(42); - } test15(array1); test15(array2); test15(array3); @@ -380,15 +374,63 @@ public static void test20Runner() { test20(array1); } + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "8" }) + @IR(failOn = IRNode.ALLOC) + public static void test21() { + I[] array = array3; + test21Inline(array); + } + + @Run(test = "test21") + public static void test21Runner() { + test21(); + test21Inline(array2); + test21Inline(array4); + test21Inline(array5); + test21Inline(array7); + } + + @ForceInline + static void test21Inline(I[] array) { + array[0].m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "9" }) + @IR(failOn = IRNode.ALLOC) + public static void test22(I[] array) { + test22Inline(array[0]); + } + + @Run(test = "test22") + public static void test22Runner() { + test22(array1); + test22(array3); + test22(array6); + test22(array8); + test22Inline(array2[0]); + test22Inline(array4[0]); + test22Inline(array5[0]); + } + + @ForceInline + static void test22Inline(I i) { + i.m(); + } + interface I { void m(); } + static value class MyValue1 implements I { - int intField; + byte byteField; + byte byteField2; - MyValue1(int intField) { - this.intField = intField; + MyValue1(byte byteField) { + this.byteField = byteField; + this.byteField2 = byteField; } public void m() { From 14aef7a13b1b6c80c3f008f50ff8772e49d81583 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Tue, 7 Apr 2026 09:20:16 +0200 Subject: [PATCH 16/49] more --- src/hotspot/cpu/x86/interp_masm_x86.cpp | 2 - src/hotspot/share/c1/c1_Runtime1.cpp | 1 - src/hotspot/share/ci/ciMethod.cpp | 2 +- src/hotspot/share/oops/flatArrayKlass.cpp | 3 +- src/hotspot/share/oops/methodData.hpp | 9 +- src/hotspot/share/opto/parse2.cpp | 81 +++++++++----- src/hotspot/share/runtime/java.cpp | 10 ++ .../inlinetypes/TestArrayLoadProfiling.java | 102 ++++++++++++++++-- 8 files changed, 159 insertions(+), 51 deletions(-) diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index 27f83ae481f..e50173ffe40 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -1659,8 +1659,6 @@ void InterpreterMacroAssembler::profile_multiple_array_types(Register mdp, load_klass(tmp, array, rscratch1); profile_receiver_type(tmp, mdp, 0); - set_mdp_flag_at(mdp, ArrayLoadData::flat_array_byte_constant()); - jmp(null_free_check); bind(not_flat); increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_count_offset())); diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp index bcf55b454b6..423b09e9257 100644 --- a/src/hotspot/share/c1/c1_Runtime1.cpp +++ b/src/hotspot/share/c1/c1_Runtime1.cpp @@ -507,7 +507,6 @@ static void profile_flat_array(JavaThread* current, bool load, bool null_free) { if (data->is_ArrayLoadData()) { assert(load, "should be an array load"); ArrayLoadData* load_data = (ArrayLoadData*) data; - load_data->set_flat_array(); if (null_free) { load_data->set_null_free_array(); } diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp index 61e83fc0518..cc851262bba 100644 --- a/src/hotspot/share/ci/ciMethod.cpp +++ b/src/hotspot/share/ci/ciMethod.cpp @@ -683,7 +683,7 @@ bool ciMethod::array_access_profiled_type(int bci, ciKlass*& array_type, ciKlass array_type = nullptr; //array_access->array()->valid_type(); element_type = array_access->element()->valid_type(); element_ptr = array_access->element()->ptr_kind(); - flat_array = array_access->flat_array(); + flat_array = false; null_free_array = array_access->null_free_array(); array_type = nullptr; // if (profile.morphism() == 1 && flat_array) { diff --git a/src/hotspot/share/oops/flatArrayKlass.cpp b/src/hotspot/share/oops/flatArrayKlass.cpp index 815ba104bc2..e562a961470 100644 --- a/src/hotspot/share/oops/flatArrayKlass.cpp +++ b/src/hotspot/share/oops/flatArrayKlass.cpp @@ -392,7 +392,8 @@ void FlatArrayKlass::print_value_on(outputStream* st) const { assert(is_klass(), "must be klass"); element_klass()->print_value_on(st); - st->print("[]"); + st->print("[] "); + LayoutKindHelper::print_on(layout_kind(), st); } #ifndef PRODUCT diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 5340d4622c3..c9ddcd8b97e 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -1981,8 +1981,7 @@ class ArrayStoreData : public ReceiverTypeData { class ArrayLoadData : public ReceiverTypeData { private: enum { - flat_array_flag = BitData::last_bit_data_flag, - null_free_array_flag = flat_array_flag + 1, + null_free_array_flag = BitData::last_bit_data_flag + 1, }; enum { @@ -2022,9 +2021,6 @@ class ArrayLoadData : public ReceiverTypeData { return static_cell_count(); } - void set_flat_array() { set_flag_at(flat_array_flag); } - bool flat_array() const { return flag_at(flat_array_flag); } - void set_null_free_array() { set_flag_at(null_free_array_flag); } bool null_free_array() const { return flag_at(null_free_array_flag); } @@ -2033,9 +2029,6 @@ class ArrayLoadData : public ReceiverTypeData { } // Code generation support - static int flat_array_byte_constant() { - return flag_number_to_constant(flat_array_flag); - } static int null_free_array_byte_constant() { return flag_number_to_constant(null_free_array_flag); diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index e72e076bcd0..a1bb28d526f 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -185,12 +185,12 @@ class ArrayLoad { return false; } - Node* emit_plain_load(Node* array) { + Node* emit_plain_load(Node* array, bool pin_if_range_check = false) { const TypeAryPtr* array_type = _gvn.type(array)->is_aryptr(); const TypeOopPtr* element_ptr = _elemtype->make_oopptr(); const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(_bt); DecoratorSet decorator_set = IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD; - if (_parse.needs_range_check(array_type->size(), _array_index)) { + if (pin_if_range_check && _parse.needs_range_check(array_type->size(), _array_index)) { // We've emitted a RangeCheck but now insert an additional check between the range check and the actual load. // We cannot pin the load to two separate nodes. Instead, we pin it conservatively here such that it cannot // possibly float above the range check at any point. @@ -217,7 +217,7 @@ class ArrayLoad { assert(array_type->is_flat() || _parse.control()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found"); Node* casted_array = _gvn.transform(new CheckCastPPNode(_parse.control(), _array, array_type->cast_to_not_flat())); - Node* ld = emit_plain_load(casted_array); + Node* ld = emit_plain_load(casted_array, true); _res_phi->add_req(ld); _region->add_req(_parse.control()); _io_phi->add_req(_parse.i_o()); @@ -231,6 +231,7 @@ class ArrayLoad { float this_prob = clamp(p, PROB_MIN, PROB_MAX); Node* casted_array = nullptr; Node* next_ctrl = _parse.type_check_receiver(_array, klass, this_prob, &casted_array); + // TODO: pin load InlineTypeNode* vt = InlineTypeNode::make_from_flat_array( &_parse, klass->as_flat_array_klass()->element_klass()->as_inline_klass(), casted_array, _array_index); Node* null_ctl = _parse.top(); @@ -240,8 +241,10 @@ class ArrayLoad { _res_phi->add_req(_parse.zerocon(T_OBJECT)); _region->add_req(null_ctl); _io_phi->add_req(_parse.i_o()); - _mem_phi->add_req(_parse.reset_memory()); - _parse.set_all_memory(_mem); + Node* mem = _parse.reset_memory(); + _mem_phi->add_req(mem); + + _parse.set_all_memory(mem); Node* ld = _gvn.transform(vt->buffer(&_parse)); _res_phi->add_req(ld); @@ -253,32 +256,60 @@ class ArrayLoad { _parse.set_i_o(_io); } - void load_from_unknown_flat_array(const TypeOopPtr* element_ptr) { + Node* load_from_unknown_flat_array(const TypeOopPtr* element_ptr) { Node* ld = nullptr; + + Node* array = _array; + ciArrayLoadData* array_load = profile_data(); + if (array_load != nullptr && array_load->null_free_array() && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + Node* test = _parse.null_free_array_test(_array, false); + IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, PROB_MIN, COUNT_UNKNOWN); + _parse.set_control(_gvn.transform(new IfTrueNode(iff))); + { + PreserveJVMState pjvms(&_parse); + _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } + _parse.set_control(_gvn.transform(new IfFalseNode(iff))); + const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr()->cast_to_null_free(true); + array = _gvn.transform(new CheckCastPPNode(_parse.control(), array, array_type)); + } + if (element_ptr->is_inlinetypeptr()) { ciInlineKlass* vk = element_ptr->inline_klass(); - Node* flat_array = _parse.cast_to_flat_array(_array, vk); + Node* flat_array = _parse.cast_to_flat_array(array, vk); InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); ld = vt; - Node* null_ctl = _parse.top(); - _parse.null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl); - - _res_phi->add_req(_parse.zerocon(T_OBJECT)); - _region->add_req(null_ctl); - _io_phi->add_req(_parse.i_o()); - _mem_phi->add_req(_parse.reset_memory()); - _parse.set_all_memory(_mem); + if (_region != nullptr) { + Node* null_ctl = _parse.top(); + _parse.null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl); + _res_phi->add_req(_parse.zerocon(T_OBJECT)); + _region->add_req(null_ctl); + _io_phi->add_req(_parse.i_o()); + _mem_phi->add_req(_parse.reset_memory()); + _parse.set_all_memory(_mem); + } } else { - ld = _parse.load_from_unknown_flat_array(_array, _array_index, element_ptr); + ld = _parse.load_from_unknown_flat_array(array, _array_index, element_ptr); + const TypeAryPtr* array_type = _gvn.type(array)->is_aryptr(); + bool is_null_free = array_type->is_null_free() || + (!UseNullableAtomicValueFlattening && !UseNullableNonAtomicValueFlattening); + if (is_null_free) { + ld = _parse.cast_not_null(ld); + } } - _region->add_req(_parse.control()); - _res_phi->add_req(ld); - _mem_phi->add_req(_parse.reset_memory()); - _io_phi->add_req(_parse.i_o()); + if (_region != nullptr) { + _region->add_req(_parse.control()); + _res_phi->add_req(ld); + _mem_phi->add_req(_parse.reset_memory()); + _io_phi->add_req(_parse.i_o()); + return nullptr; + } else { + return ld; + } } void create_merge_point() { @@ -293,6 +324,7 @@ class ArrayLoad { void finish_merge_point() { _parse.set_control(_gvn.transform(_region)); + _parse.record_for_igvn(_parse.control()); _parse.set_all_memory(_gvn.transform(_mem_phi)); _parse.set_i_o(_gvn.transform(_io_phi)); Node* ld = _gvn.transform(_res_phi); @@ -336,10 +368,6 @@ class ArrayLoad { (!element_ptr->is_inlinetypeptr() || element_ptr->inline_klass()->maybe_flat_in_array()), "array can't be flat"); - // if (emit_load_if_known_flat_array(element_ptr)) { - // return true; - // } - ciArrayLoadData* array_load = profile_data(); if (array_load != nullptr) { int not_flat_count = profiled_not_flat_count(); @@ -410,7 +438,7 @@ class ArrayLoad { Node* casted_array = _gvn.transform( new CheckCastPPNode(_parse.control(), _array, array_type->cast_to_not_flat())); _parse.replace_in_map(_array, casted_array); - Node* ld = emit_plain_load(casted_array); + Node* ld = emit_plain_load(casted_array, true); ld = _parse.record_profile_for_speculation_at_array_load(ld); _parse.push_node(_bt, ld); return true; @@ -422,9 +450,8 @@ class ArrayLoad { _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); } assert(!_parse.stopped(), "non flat array should have been caught earlier"); - Node* ld = _parse.load_from_unknown_flat_array(_array, _array_index, element_ptr); + Node* ld = load_from_unknown_flat_array(element_ptr); pop_stack(); - ld = _parse.record_profile_for_speculation_at_array_load(ld); _parse.push_node(_bt, ld); return true; } diff --git a/src/hotspot/share/runtime/java.cpp b/src/hotspot/share/runtime/java.cpp index 776996ceb80..c229a69f48c 100644 --- a/src/hotspot/share/runtime/java.cpp +++ b/src/hotspot/share/runtime/java.cpp @@ -71,6 +71,8 @@ #include "runtime/init.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "runtime/java.hpp" + +#include "classfile/classPrinter.hpp" #include "runtime/javaThread.hpp" #include "runtime/os.hpp" #include "runtime/sharedRuntime.hpp" @@ -139,6 +141,14 @@ static void print_method_profiling_data() { if (count > 0) { for (int index = 0; index < count; index++) { Method* m = collected_profiled_methods->at(index); + { + ResourceMark rm; + stringStream ss; + m->print_short_name(&ss); + if (!strcmp(ss.as_string(), " compiler.valhalla.inlinetypes.TestArrayLoadProfiling::test22")) { + tty->print_cr("XXX"); + } + } // Instead of taking tty lock, we collect all lines into a string stream // and then print them all at once. diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 114ac21e8f7..ac6b78c1547 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -56,10 +56,13 @@ public static void main(String[] args) { static MyValue1[] array6 = (MyValue1[])ValueClass.newReferenceArray(MyValue1.class, 1); static MyValue2[] array7 = (MyValue2[])ValueClass.newReferenceArray(MyValue2.class, 1); static MyValue1[] array8 = (MyValue1[])ValueClass.newNullRestrictedAtomicArray(MyValue1.class, 1, new MyValue1((byte)42)); + static MyValue1[] array9 = (MyValue1[])ValueClass.newNullableAtomicArray(MyValue1.class, 1); + static MyValue3[] array10 = (MyValue3[])ValueClass.newNullRestrictedNonAtomicArray(MyValue3.class, 1, new MyValue3(42)); static { array6[0] = new MyValue1((byte)42); array7[0] = new MyValue2((byte)42); array8[0] = new MyValue1((byte)42); + array9[0] = new MyValue1((byte)42); } @Test @@ -397,27 +400,85 @@ static void test21Inline(I[] array) { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "9" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "7" }) @IR(failOn = IRNode.ALLOC) - public static void test22(I[] array) { - test22Inline(array[0]); + public static void test22() { + I[] array = array3; + test22Inline(array); } @Run(test = "test22") public static void test22Runner() { - test22(array1); - test22(array3); - test22(array6); - test22(array8); - test22Inline(array2[0]); - test22Inline(array4[0]); - test22Inline(array5[0]); + test22(); + test22Inline(array2); + test22Inline(array4); + } + + @ForceInline + static void test22Inline(I[] array) { + array[0].m(); + } + + // @Test + // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "9" }) + // @IR(failOn = IRNode.ALLOC) + // public static void test22(I[] array) { + // test22Inline(array[0]); + // } + + // @Run(test = "test22") + // public static void test22Runner() { + // test22(array1); + // test22(array3); + // test22(array6); + // test22(array8); + // test22(array9); + // test22Inline(array2[0]); + // test22Inline(array4[0]); + // test22Inline(array5[0]); + // } + + // @ForceInline + // static void test22Inline(I i) { + // i.m(); + // } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "9" }) + @IR(failOn = IRNode.ALLOC) + public static void test23(I[] array) { + test23Inline(array[0]); + } + + @Run(test = "test23") + public static void test23Runner() { + test23(array1); + test23(array3); + test23(array6); + test23(array8); + test23(array9); + test23Inline(array2[0]); + test23Inline(array4[0]); + test23Inline(array5[0]); } @ForceInline - static void test22Inline(I i) { + static void test23Inline(I i) { i.m(); } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "9" }) + @IR(failOn = IRNode.ALLOC) + public static void test24(I[] array) { + array[0].m(); + } + + @Run(test = "test24") + public static void test24Runner() { + test24(array3); + test24(array10); + } interface I { void m(); @@ -427,10 +488,12 @@ interface I { static value class MyValue1 implements I { byte byteField; byte byteField2; + int byteField3; MyValue1(byte byteField) { this.byteField = byteField; this.byteField2 = byteField; + this.byteField3 = byteField; } public void m() { @@ -448,6 +511,23 @@ public void m() { } } + static value class MyValue3 implements I { + int intField1; + int intField2; + int intField3; + int intField4; + + MyValue3(int intField) { + this.intField1 = intField; + this.intField2 = intField; + this.intField3 = intField; + this.intField4 = intField; + } + + public void m() { + } + } + static class A implements I { public void m() { } From 6c4a1e82067a70632e8d06bb635e7384bc4a8ae3 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 23 Apr 2026 09:35:08 +0200 Subject: [PATCH 17/49] more --- src/hotspot/share/opto/parse2.cpp | 24 +++++++++---------- .../inlinetypes/TestArrayLoadProfiling.java | 22 ++++++++--------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index a1bb28d526f..5e86400df7d 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -261,18 +261,18 @@ class ArrayLoad { Node* array = _array; ciArrayLoadData* array_load = profile_data(); - if (array_load != nullptr && array_load->null_free_array() && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { - Node* test = _parse.null_free_array_test(_array, false); - IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, PROB_MIN, COUNT_UNKNOWN); - _parse.set_control(_gvn.transform(new IfTrueNode(iff))); - { - PreserveJVMState pjvms(&_parse); - _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); - } - _parse.set_control(_gvn.transform(new IfFalseNode(iff))); - const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr()->cast_to_null_free(true); - array = _gvn.transform(new CheckCastPPNode(_parse.control(), array, array_type)); - } + // if (array_load != nullptr && array_load->null_free_array() && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + // Node* test = _parse.null_free_array_test(_array, false); + // IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, PROB_MIN, COUNT_UNKNOWN); + // _parse.set_control(_gvn.transform(new IfTrueNode(iff))); + // { + // PreserveJVMState pjvms(&_parse); + // _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + // } + // _parse.set_control(_gvn.transform(new IfFalseNode(iff))); + // const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr()->cast_to_null_free(true); + // array = _gvn.transform(new CheckCastPPNode(_parse.control(), array, array_type)); + // } if (element_ptr->is_inlinetypeptr()) { ciInlineKlass* vk = element_ptr->inline_klass(); diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index ac6b78c1547..42f386c3e07 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -467,18 +467,18 @@ static void test23Inline(I i) { i.m(); } - @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "9" }) - @IR(failOn = IRNode.ALLOC) - public static void test24(I[] array) { - array[0].m(); - } + // @Test + // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "9" }) + // @IR(failOn = IRNode.ALLOC) + // public static void test24(I[] array) { + // array[0].m(); + // } - @Run(test = "test24") - public static void test24Runner() { - test24(array3); - test24(array10); - } + // @Run(test = "test24") + // public static void test24Runner() { + // test24(array3); + // test24(array10); + // } interface I { void m(); From 1424efc230f2cfa3b8f789bbbd91486207f666e1 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 23 Apr 2026 17:42:16 +0200 Subject: [PATCH 18/49] more --- src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 18 ++++- src/hotspot/cpu/x86/interp_masm_x86.cpp | 12 +-- src/hotspot/share/c1/c1_LIR.cpp | 2 +- src/hotspot/share/c1/c1_LIRGenerator.cpp | 6 -- src/hotspot/share/c1/c1_Runtime1.cpp | 5 -- src/hotspot/share/ci/ciMethod.cpp | 2 +- src/hotspot/share/oops/methodData.cpp | 2 +- src/hotspot/share/oops/methodData.hpp | 77 +++++++++++++++---- src/hotspot/share/opto/parse2.cpp | 61 ++++++++++----- .../inlinetypes/TestArrayLoadProfiling.java | 27 +++++++ 10 files changed, 156 insertions(+), 56 deletions(-) diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index 72349d8ed3d..08ab322c326 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -3198,8 +3198,22 @@ void LIR_Assembler::emit_profile_multiple_array_types(LIR_OpProfileMultipleArray __ bind(not_flat); __ mov_metadata(mdo, md->constant_encoding()); - Address counter_addr(mdo, md->byte_offset_of_slot(op->load(), ArrayLoadData::not_flat_count_offset())); - __ addptr(counter_addr, DataLayout::counter_increment); + Label null_free; + + __ test_null_free_array_oop(array, tmp1, null_free); + + { + Address counter_addr(mdo, md->byte_offset_of_slot(op->load(), ArrayLoadData::not_flat_nullable_count_offset())); + __ addptr(counter_addr, DataLayout::counter_increment); + } + + __ jmp(done); + __ bind(null_free); + + { + Address counter_addr(mdo, md->byte_offset_of_slot(op->load(), ArrayLoadData::not_flat_null_free_count_offset())); + __ addptr(counter_addr, DataLayout::counter_increment); + } __ bind(done); } diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index e50173ffe40..12000ee8c7d 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -1653,24 +1653,26 @@ void InterpreterMacroAssembler::profile_multiple_array_types(Register mdp, // If no method data exists, go to profile_continue. test_method_data_pointer(mdp, profile_continue); - Label not_flat, null_free_check; + Label not_flat; test_non_flat_array_oop(array, tmp, not_flat); load_klass(tmp, array, rscratch1); profile_receiver_type(tmp, mdp, 0); - jmp(null_free_check); + jmp(profile_continue); bind(not_flat); - increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_count_offset())); - bind(null_free_check); Label not_null_free; test_non_null_free_array_oop(array, tmp, not_null_free); - set_mdp_flag_at(mdp, ArrayLoadData::null_free_array_byte_constant()); + increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_null_free_count_offset())); + + jmp(profile_continue); bind(not_null_free); + increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_nullable_count_offset())); + bind(profile_continue); } } diff --git a/src/hotspot/share/c1/c1_LIR.cpp b/src/hotspot/share/c1/c1_LIR.cpp index ab254f6e366..ad9dea81d7d 100644 --- a/src/hotspot/share/c1/c1_LIR.cpp +++ b/src/hotspot/share/c1/c1_LIR.cpp @@ -984,7 +984,7 @@ void LIR_OpVisitState::visit(LIR_Op* op) { case lir_profile_multiple_array_types: { assert(op->as_OpProfileMultipleArrayTypes() != nullptr, "must be"); LIR_OpProfileMultipleArrayTypes* opProfileMultipleArrayTypes = (LIR_OpProfileMultipleArrayTypes*)op; - do_input(opProfileMultipleArrayTypes->_array); + do_input(opProfileMultipleArrayTypes->_array); do_temp(opProfileMultipleArrayTypes->_array); do_temp(opProfileMultipleArrayTypes->_tmp1); do_temp(opProfileMultipleArrayTypes->_tmp2); diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp index 0be71a3de0c..68362353ca6 100644 --- a/src/hotspot/share/c1/c1_LIRGenerator.cpp +++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp @@ -2376,12 +2376,6 @@ void LIRGenerator::do_LoadIndexed(LoadIndexed* x) { LIR_Opr result = rlock_result(x, x->elt_type()); LoadFlattenedArrayStub* slow_path = nullptr; - if (x->should_profile() && x->array()->maybe_null_free_array()) { - assert(data != nullptr && data->is_ArrayLoadData(), "incorrect profiling entry"); - ciArrayLoadData* load_data = (ciArrayLoadData*)data; - profile_null_free_array(array, md, load_data); - } - if (x->elt_type() == T_OBJECT && x->array()->maybe_flat_array()) { assert(x->delayed() == nullptr, "Delayed LoadIndexed only apply to loaded_flat_arrays"); index.load_item(); diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp index 423b09e9257..97198c1cdc5 100644 --- a/src/hotspot/share/c1/c1_Runtime1.cpp +++ b/src/hotspot/share/c1/c1_Runtime1.cpp @@ -505,11 +505,6 @@ static void profile_flat_array(JavaThread* current, bool load, bool null_free) { ProfileData* data = md->bci_to_data(bci); assert(data != nullptr, "incorrect profiling entry"); if (data->is_ArrayLoadData()) { - assert(load, "should be an array load"); - ArrayLoadData* load_data = (ArrayLoadData*) data; - if (null_free) { - load_data->set_null_free_array(); - } } else { assert(data->is_ArrayStoreData(), ""); assert(!load, "should be an array store"); diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp index cc851262bba..4cc9a62ebe0 100644 --- a/src/hotspot/share/ci/ciMethod.cpp +++ b/src/hotspot/share/ci/ciMethod.cpp @@ -684,7 +684,7 @@ bool ciMethod::array_access_profiled_type(int bci, ciKlass*& array_type, ciKlass element_type = array_access->element()->valid_type(); element_ptr = array_access->element()->ptr_kind(); flat_array = false; - null_free_array = array_access->null_free_array(); + null_free_array = false; array_type = nullptr; // if (profile.morphism() == 1 && flat_array) { // array_type = profile.receiver(0); diff --git a/src/hotspot/share/oops/methodData.cpp b/src/hotspot/share/oops/methodData.cpp index 44c92d33ea9..348806a5e7f 100644 --- a/src/hotspot/share/oops/methodData.cpp +++ b/src/hotspot/share/oops/methodData.cpp @@ -729,7 +729,7 @@ void ArrayStoreData::print_data_on(outputStream* st, const char* extra) const { void ArrayLoadData::print_data_on(outputStream* st, const char* extra) const { print_shared(st, "ArrayLoad", extra); - st->print(" not flat: %u", not_flat_count()); + st->print("not flat %u (null free = %u, nullable = %u)", not_flat_count(), not_flat_null_free_count(), not_flat_nullable_count()); st->cr(); tab(st, true); st->print("array"); diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index c9ddcd8b97e..e85ec70f836 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -1981,11 +1981,11 @@ class ArrayStoreData : public ReceiverTypeData { class ArrayLoadData : public ReceiverTypeData { private: enum { - null_free_array_flag = BitData::last_bit_data_flag + 1, - }; - - enum { - not_flat_count_off_in_extra_cells, + not_flat_null_free_count_off_in_extra_cells, + not_flat_nullable_count_off_in_extra_cells, + flat_nullable_count_off_in_extra_cells, + flat_nullfree_atomic_count_off_in_extra_cells, + flat_nullfree_not_atomic_count_off_in_extra_cells, extra_cells_count }; @@ -1995,8 +1995,24 @@ class ArrayLoadData : public ReceiverTypeData { return ReceiverTypeData::static_cell_count() + SingleTypeEntry::static_cell_count(); } - static int not_flat_count_off() { - return extra_cells_off() + not_flat_count_off_in_extra_cells; + static int not_flat_null_free_count_off() { + return extra_cells_off() + not_flat_null_free_count_off_in_extra_cells; + } + + static int not_flat_nullable_count_off() { + return extra_cells_off() + not_flat_nullable_count_off_in_extra_cells; + } + + static int flat_nullable_count_off() { + return extra_cells_off() + flat_nullable_count_off_in_extra_cells; + } + + static int flat_nullfree_atomic_count_off() { + return extra_cells_off() + flat_nullfree_atomic_count_off_in_extra_cells; + } + + static int flat_nullfree_not_atomic_count_off() { + return extra_cells_off() + flat_nullfree_not_atomic_count_off_in_extra_cells; } public: @@ -2021,25 +2037,54 @@ class ArrayLoadData : public ReceiverTypeData { return static_cell_count(); } - void set_null_free_array() { set_flag_at(null_free_array_flag); } - bool null_free_array() const { return flag_at(null_free_array_flag); } - int not_flat_count() const { - return uint_at(not_flat_count_off()); + return not_flat_null_free_count() + not_flat_nullable_count(); } - // Code generation support + int not_flat_null_free_count() const { + return uint_at(not_flat_null_free_count_off()); + } - static int null_free_array_byte_constant() { - return flag_number_to_constant(null_free_array_flag); + int not_flat_nullable_count() const { + return uint_at(not_flat_nullable_count_off()); + } + + int flat_nullable_count() const { + return uint_at(flat_nullable_count_off()); + } + + int flat_nullfree_atomic_count() const { + return uint_at(flat_nullfree_atomic_count_off()); } + int flat_nullfree_not_atomic_count() const { + return uint_at(flat_nullfree_not_atomic_count_off()); + } + + // Code generation support + static ByteSize element_offset() { return cell_offset(ReceiverTypeData::static_cell_count()); } - static ByteSize not_flat_count_offset() { - return cell_offset(not_flat_count_off()); + static ByteSize not_flat_null_free_count_offset() { + return cell_offset(not_flat_null_free_count_off()); + } + + static ByteSize not_flat_nullable_count_offset() { + return cell_offset(not_flat_nullable_count_off()); + } + + static ByteSize flat_nullable_count_offset() { + return cell_offset(flat_nullable_count_off()); + } + + static ByteSize flat_nullfree_atomic_count_offset() { + return cell_offset(flat_nullfree_atomic_count_off()); + } + + static ByteSize flat_nullfree_not_atomic_count_offset() { + return cell_offset(flat_nullfree_not_atomic_count_off()); } virtual void clean_weak_klass_links(bool always_clean) { diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 5e86400df7d..0e239a6e07c 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -171,21 +171,21 @@ class ArrayLoad { assert(array == _array, ""); } - bool emit_load_if_known_flat_array(const TypeOopPtr* element_ptr) { - if (element_ptr->is_inlinetypeptr()) { - pop_stack(); - ciInlineKlass* vk = element_ptr->inline_klass(); - // Node* flat_array = cast_to_flat_array(array, vk); - Node* flat_array = _array; - Node* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); - Node* ld = _gvn.transform(vt); - _parse.push_node(_bt, ld); - return true; - } - return false; - } + // bool emit_load_if_known_flat_array(const TypeOopPtr* element_ptr) { + // if (element_ptr->is_inlinetypeptr()) { + // pop_stack(); + // ciInlineKlass* vk = element_ptr->inline_klass(); + // // Node* flat_array = cast_to_flat_array(array, vk); + // Node* flat_array = _array; + // Node* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); + // Node* ld = _gvn.transform(vt); + // _parse.push_node(_bt, ld); + // return true; + // } + // return false; + // } - Node* emit_plain_load(Node* array, bool pin_if_range_check = false) { + Node* emit_plain_load(Node* array, bool pin_if_range_check = false, bool safe_for_replace_in_map = false) { const TypeAryPtr* array_type = _gvn.type(array)->is_aryptr(); const TypeOopPtr* element_ptr = _elemtype->make_oopptr(); const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(_bt); @@ -196,8 +196,27 @@ class ArrayLoad { // possibly float above the range check at any point. decorator_set |= C2_UNKNOWN_CONTROL_LOAD; } - const TypeAryPtr* arytype = _gvn.type(array)->is_aryptr(); - const TypeInt* sizetype = arytype->size(); + const TypeInt* sizetype = array_type->size(); + if (element_ptr != nullptr && element_ptr->is_inlinetypeptr() && !array_type->is_null_free()) { + ciArrayLoadData* array_load = profile_data(); + if (array_load != nullptr && array_load->not_flat_nullable_count() == 0 && array_load->not_flat_null_free_count() > 0 && + !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + Node* test = _parse.null_free_array_test(_array, false); + IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, PROB_MIN, COUNT_UNKNOWN); + _parse.set_control(_gvn.transform(new IfTrueNode(iff))); + { + PreserveJVMState pjvms(&_parse); + _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } + _parse.set_control(_gvn.transform(new IfFalseNode(iff))); + const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr()->cast_to_null_free(true); + array = _gvn.transform(new CheckCastPPNode(_parse.control(), array, array_type)); + if (_parse.needs_range_check(array_type->size(), _array_index)) { + decorator_set |= C2_UNKNOWN_CONTROL_LOAD; + } + } + } + Node* adr = _parse.array_element_address(array, _array_index, _bt, sizetype, _parse.control()); assert(adr != _parse.top(), "top should go hand-in-hand with stopped"); Node* ld = _parse.access_load_at(array, adr, adr_type, _elemtype, _bt, decorator_set); @@ -205,6 +224,10 @@ class ArrayLoad { assert(!array_type->is_null_free() || !element_ptr->maybe_null(), "inline type array elements should never be null"); ld = InlineTypeNode::make_from_oop(&_parse, ld, element_ptr->inline_klass()); } + if (safe_for_replace_in_map) { + pop_stack(); + _parse.replace_in_map(_array, array); + } return _gvn.transform(ld); } @@ -260,7 +283,7 @@ class ArrayLoad { Node* ld = nullptr; Node* array = _array; - ciArrayLoadData* array_load = profile_data(); + // ciArrayLoadData* array_load = profile_data(); // if (array_load != nullptr && array_load->null_free_array() && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { // Node* test = _parse.null_free_array_test(_array, false); // IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, PROB_MIN, COUNT_UNKNOWN); @@ -346,8 +369,8 @@ class ArrayLoad { if (_elemtype == TypeInt::BOOL) { _bt = T_BOOLEAN; } - Node* ld = emit_plain_load(_array); - pop_stack(); + Node* ld = emit_plain_load(_array, false, true); + // pop_stack(); _parse.push_node(_bt, ld); return true; } diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 42f386c3e07..8e37d63e6b4 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -58,11 +58,14 @@ public static void main(String[] args) { static MyValue1[] array8 = (MyValue1[])ValueClass.newNullRestrictedAtomicArray(MyValue1.class, 1, new MyValue1((byte)42)); static MyValue1[] array9 = (MyValue1[])ValueClass.newNullableAtomicArray(MyValue1.class, 1); static MyValue3[] array10 = (MyValue3[])ValueClass.newNullRestrictedNonAtomicArray(MyValue3.class, 1, new MyValue3(42)); + static MyValue3[] array11 = (MyValue3[])ValueClass.newNullRestrictedAtomicArray(MyValue3.class, 1, new MyValue3(42)); + static MyValue3[] array12 = (MyValue3[])ValueClass.newReferenceArray(MyValue3.class, 1); static { array6[0] = new MyValue1((byte)42); array7[0] = new MyValue2((byte)42); array8[0] = new MyValue1((byte)42); array9[0] = new MyValue1((byte)42); + array12[0] = new MyValue3((byte)42); } @Test @@ -466,6 +469,30 @@ public static void test23Runner() { static void test23Inline(I i) { i.m(); } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "3", IRNode.CLASS_CHECK_TRAP, "1", IRNode.IF, "6" }) + @IR(failOn = IRNode.ALLOC) + public static int test24(MyValue3[] array, int i, int j, int k) { + return array[i].intField1 + array[j].intField1 + array[k].intField1; + } + + @Run(test = "test24") + public static void test24Runner() { + test24(array11, 0, 0, 0); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "4", IRNode.RANGE_CHECK_TRAP, "3", IRNode.IF, "7" }) + @IR(failOn = IRNode.ALLOC) + public static int test25(MyValue3[] array, int i, int j, int k) { + return array[i].intField1 + array[j].intField1 + array[k].intField1; + } + + @Run(test = "test25") + public static void test25Runner() { + test25(array12, 0, 0, 0); + } // @Test // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "9" }) From 96d7ef1875672c998c708aa2bf87ac0540eb0a22 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Fri, 24 Apr 2026 11:15:34 +0200 Subject: [PATCH 19/49] more --- src/hotspot/share/ci/ciMethodData.hpp | 5 +- src/hotspot/share/oops/methodData.cpp | 43 +++++--- src/hotspot/share/oops/methodData.hpp | 145 +++++++++++++++++++++----- 3 files changed, 154 insertions(+), 39 deletions(-) diff --git a/src/hotspot/share/ci/ciMethodData.hpp b/src/hotspot/share/ci/ciMethodData.hpp index 8b62d8779b1..cb6f06dfada 100644 --- a/src/hotspot/share/ci/ciMethodData.hpp +++ b/src/hotspot/share/ci/ciMethodData.hpp @@ -196,13 +196,12 @@ class ciReceiverTypeData : public ReceiverTypeData { void set_receiver(uint row, ciKlass* recv) { assert((uint)row < row_limit(), "oob"); - set_intptr_at(receiver0_offset + row * receiver_type_row_cell_count, - (intptr_t) recv); + set_intptr_at(_megamorphic_type_data.receiver_cell_index(row), (intptr_t) recv); } ciKlass* receiver(uint row) const { assert((uint)row < row_limit(), "oob"); - ciKlass* recv = (ciKlass*)intptr_at(receiver0_offset + row * receiver_type_row_cell_count); + ciKlass* recv = (ciKlass*)intptr_at(_megamorphic_type_data.receiver_cell_index(row)); assert(recv == nullptr || recv->is_klass(), "wrong type"); return recv; } diff --git a/src/hotspot/share/oops/methodData.cpp b/src/hotspot/share/oops/methodData.cpp index 348806a5e7f..3332a13ebc0 100644 --- a/src/hotspot/share/oops/methodData.cpp +++ b/src/hotspot/share/oops/methodData.cpp @@ -447,8 +447,8 @@ void VirtualCallTypeData::print_data_on(outputStream* st, const char* extra) con // that the check is reached, and a series of (Klass*, count) pairs // which are used to store a type profile for the receiver of the check. -void ReceiverTypeData::clean_weak_klass_links(bool always_clean) { - for (uint row = 0; row < row_limit(); row++) { +void MegamorphicTypeData::clean_weak_klass_links(bool always_clean) { + for (uint row = 0; row < row_limit(); row++) { Klass* p = receiver(row); if (p != nullptr) { if (!always_clean && p->is_instance_klass() && InstanceKlass::cast(p)->is_not_initialized()) { @@ -461,34 +461,53 @@ void ReceiverTypeData::clean_weak_klass_links(bool always_clean) { } } -void ReceiverTypeData::metaspace_pointers_do(MetaspaceClosure *it) { +void ReceiverTypeData::clean_weak_klass_links(bool always_clean) { + _megamorphic_type_data.clean_weak_klass_links(always_clean); +} + +void MegamorphicTypeData::metaspace_pointers_do(MetaspaceClosure *it) { for (uint row = 0; row < row_limit(); row++) { - Klass** recv = (Klass**)intptr_at_adr(receiver_cell_index(row)); + Klass** recv = (Klass**)_pd->intptr_at_adr(receiver_cell_index(row)); it->push(recv); } } -void ReceiverTypeData::print_receiver_data_on(outputStream* st) const { - uint row; +void ReceiverTypeData::metaspace_pointers_do(MetaspaceClosure *it) { + _megamorphic_type_data.metaspace_pointers_do(it); +} + +int MegamorphicTypeData::entries() const { int entries = 0; - for (row = 0; row < row_limit(); row++) { + for (uint row = 0; row < row_limit(); row++) { if (receiver(row) != nullptr) entries++; } - st->print_cr("count(%u) entries(%u)", count(), entries); - int total = count(); - for (row = 0; row < row_limit(); row++) { + return entries; +} + +int MegamorphicTypeData::count() const { + int total = 0; + for (uint row = 0; row < row_limit(); row++) { if (receiver(row) != nullptr) { total += receiver_count(row); } } - for (row = 0; row < row_limit(); row++) { + return total; +} + +void MegamorphicTypeData::print_receiver_data_on(outputStream* st, int total) const { + total += count(); + for (uint row = 0; row < row_limit(); row++) { if (receiver(row) != nullptr) { - tab(st); + _pd->tab(st); receiver(row)->print_value_on(st); st->print_cr("(%u %4.2f)", receiver_count(row), (float) receiver_count(row) / (float) total); } } } +void ReceiverTypeData::print_receiver_data_on(outputStream* st) const { + st->print_cr("count(%u) entries(%u)", count(), _megamorphic_type_data.entries()); + _megamorphic_type_data.print_receiver_data_on(st, count()); +} void ReceiverTypeData::print_data_on(outputStream* st, const char* extra) const { print_shared(st, "ReceiverTypeData", extra); print_receiver_data_on(st); diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index e85ec70f836..4f0701ea282 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -312,6 +312,7 @@ class ProfileData : public ResourceObj { friend class TypeEntries; friend class SingleTypeEntry; friend class TypeStackSlotEntries; + friend class MegamorphicTypeData; private: enum { tab_width_one = 16, @@ -1161,44 +1162,60 @@ class CallTypeData : public CounterData { // // Updated by platform-specific code, for example MacroAssembler::profile_receiver_type. // -class ReceiverTypeData : public CounterData { + +class MegamorphicTypeData { friend class VMStructs; protected: enum { - receiver0_offset = counter_cell_count, + receiver0_offset, count0_offset, receiver_type_row_cell_count = (count0_offset + 1) - receiver0_offset }; -public: - ReceiverTypeData(DataLayout* layout) : CounterData(layout) { - assert(layout->tag() == DataLayout::receiver_type_data_tag || - layout->tag() == DataLayout::virtual_call_data_tag || - layout->tag() == DataLayout::virtual_call_type_data_tag || - layout->tag() == DataLayout::array_store_data_tag || - layout->tag() == DataLayout::array_load_data_tag, "wrong type"); - } + ProfileData* _pd; + const int _base_off; + const int _type_width; - virtual bool is_ReceiverTypeData() const { return true; } + void set_intptr_at(int index, intptr_t value) { + _pd->set_intptr_at(index, value); + } - static int static_cell_count() { - return counter_cell_count + (uint) TypeProfileWidth * receiver_type_row_cell_count; + intptr_t intptr_at(int index) const { + return _pd->intptr_at(index); } - virtual int cell_count() const { - return static_cell_count(); + void set_uint_at(int index, uint value) { + _pd->set_uint_at(index, value); + } + uint uint_at(int index) const { + return _pd->uint_at(index); } +public: - // Direct accessors - static uint row_limit() { - return (uint) TypeProfileWidth; + MegamorphicTypeData(ProfileData* pd, int base_off, int type_width) + : _pd(pd), _base_off(base_off), _type_width(type_width) {} + + static int static_cell_count(int type_width) { + return type_width * receiver_type_row_cell_count; } - static int receiver_cell_index(uint row) { + int cell_count() const { + return _type_width * receiver_type_row_cell_count; + } + uint row_limit() const { + return (uint)_type_width; + } + static int static_receiver_cell_index(uint row) { return receiver0_offset + row * receiver_type_row_cell_count; } - static int receiver_count_cell_index(uint row) { + static int static_receiver_count_cell_index(uint row) { return count0_offset + row * receiver_type_row_cell_count; } + int receiver_cell_index(uint row) const { + return _base_off + receiver0_offset + row * receiver_type_row_cell_count; + } + int receiver_count_cell_index(uint row) const { + return _base_off + count0_offset + row * receiver_type_row_cell_count; + } Klass* receiver(uint row) const { assert(row < row_limit(), "oob"); @@ -1225,6 +1242,87 @@ class ReceiverTypeData : public CounterData { void clear_row(uint row) { assert(row < row_limit(), "oob"); + set_receiver(row, nullptr); + set_receiver_count(row, 0); + } + + // // Code generation support + // static ByteSize receiver_offset(uint row) { + // return cell_offset(receiver_cell_index(row)); + // } + // static ByteSize receiver_count_offset(uint row) { + // return cell_offset(receiver_count_cell_index(row)); + // } + // static ByteSize receiver_type_data_size() { + // return cell_offset(static_cell_count()); + // } + // + // GC support + void clean_weak_klass_links(bool always_clean); + + // CDS support + void metaspace_pointers_do(MetaspaceClosure* it); + + int entries() const; + int count() const; + void print_receiver_data_on(outputStream* st, int total) const; +}; + +class ReceiverTypeData : public CounterData { + friend class VMStructs; + friend class JVMCIVMStructs; +protected: + + MegamorphicTypeData _megamorphic_type_data; + +public: + ReceiverTypeData(DataLayout* layout) : CounterData(layout), + _megamorphic_type_data(this, counter_cell_count, TypeProfileWidth) { + assert(layout->tag() == DataLayout::receiver_type_data_tag || + layout->tag() == DataLayout::virtual_call_data_tag || + layout->tag() == DataLayout::virtual_call_type_data_tag || + layout->tag() == DataLayout::array_store_data_tag || + layout->tag() == DataLayout::array_load_data_tag, "wrong type"); + } + + virtual bool is_ReceiverTypeData() const { return true; } + + static int static_cell_count() { + return counter_cell_count + MegamorphicTypeData::static_cell_count(TypeProfileWidth); + } + + virtual int cell_count() const { + return static_cell_count(); + } + + // Direct accessors + static uint row_limit() { + return (uint) TypeProfileWidth; + } + int receiver_cell_index(uint row) const { + return _megamorphic_type_data.receiver_cell_index(row); + } + int receiver_count_cell_index(uint row) const { + return _megamorphic_type_data.receiver_count_cell_index(row); + } + + Klass* receiver(uint row) const { + return _megamorphic_type_data.receiver(row); + } + + void set_receiver(uint row, Klass* k) { + _megamorphic_type_data.set_receiver(row, k); + } + + uint receiver_count(uint row) const { + return _megamorphic_type_data.receiver_count(row); + } + + void set_receiver_count(uint row, uint count) { + _megamorphic_type_data.set_receiver_count(row, count); + } + + void clear_row(uint row) { // Clear total count - indicator of polymorphic call site. // The site may look like as monomorphic after that but // it allow to have more accurate profiling information because @@ -1242,16 +1340,15 @@ class ReceiverTypeData : public CounterData { // We do sorting a profiling info (ciCallProfile) for compilation. // set_count(0); - set_receiver(row, nullptr); - set_receiver_count(row, 0); + _megamorphic_type_data.clear_row(row); } // Code generation support static ByteSize receiver_offset(uint row) { - return cell_offset(receiver_cell_index(row)); + return cell_offset(MegamorphicTypeData::static_receiver_cell_index(row) + counter_cell_count); } static ByteSize receiver_count_offset(uint row) { - return cell_offset(receiver_count_cell_index(row)); + return cell_offset(MegamorphicTypeData::static_receiver_count_cell_index(row) + counter_cell_count); } static ByteSize receiver_type_data_size() { return cell_offset(static_cell_count()); From 4a2895efbdf7c266c06bb3933bcc1f665d2ed83a Mon Sep 17 00:00:00 2001 From: rwestrel Date: Mon, 27 Apr 2026 09:40:43 +0200 Subject: [PATCH 20/49] more --- src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 5 +- src/hotspot/cpu/x86/interp_masm_x86.cpp | 8 +- src/hotspot/cpu/x86/macroAssembler_x86.cpp | 65 +++++++-- src/hotspot/cpu/x86/macroAssembler_x86.hpp | 3 +- src/hotspot/share/ci/ciClassList.hpp | 1 + src/hotspot/share/ci/ciMethod.cpp | 142 +++++++++++--------- src/hotspot/share/ci/ciMethod.hpp | 4 +- src/hotspot/share/ci/ciMethodData.cpp | 43 +++--- src/hotspot/share/ci/ciMethodData.hpp | 47 +++++-- src/hotspot/share/oops/methodData.cpp | 6 +- src/hotspot/share/oops/methodData.hpp | 73 ++++++++-- src/hotspot/share/opto/parse2.cpp | 6 + 12 files changed, 278 insertions(+), 125 deletions(-) diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index 08ab322c326..5134f528181 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -1332,7 +1332,7 @@ void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data, Register recv) { int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0)); - __ profile_receiver_type(recv, mdo, mdp_offset); + __ profile_receiver_type(recv, mdo, mdp_offset); } void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) { @@ -3192,7 +3192,8 @@ void LIR_Assembler::emit_profile_multiple_array_types(LIR_OpProfileMultipleArray Register mdo = tmp2; __ mov_metadata(mdo, md->constant_encoding()); - type_profile_helper(mdo, md, op->load(), tmp1); + int mdp_offset = md->byte_offset_of_slot(op->load(), in_ByteSize(0)); + __ profile_receiver_type(tmp1, mdo, mdp_offset); __ jmp(done); __ bind(not_flat); diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index 12000ee8c7d..a35d2a9852c 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -1474,7 +1474,7 @@ void InterpreterMacroAssembler::profile_virtual_call(Register receiver, test_method_data_pointer(mdp, profile_continue); // Record the receiver type. - profile_receiver_type(receiver, mdp, 0); + profile_receiver_type(receiver, mdp, 0); // The method data pointer needs to be updated to reflect the new target. update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size())); @@ -1554,7 +1554,7 @@ void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass) mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); // Record the object type. - profile_receiver_type(klass, mdp, 0); + profile_receiver_type(klass, mdp, 0); } update_mdp_by_constant(mdp, mdp_delta); @@ -1657,7 +1657,7 @@ void InterpreterMacroAssembler::profile_multiple_array_types(Register mdp, test_non_flat_array_oop(array, tmp, not_flat); load_klass(tmp, array, rscratch1); - profile_receiver_type(tmp, mdp, 0); + profile_receiver_type(tmp, mdp, 0); jmp(profile_continue); bind(not_flat); @@ -1694,7 +1694,7 @@ void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Reg load_klass(tmp, element, rscratch1); // Record the object type. - profile_receiver_type(tmp, mdp, 0); + profile_receiver_type(tmp, mdp, 0); bind(done); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index ced747c6d11..a511f1e53e4 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -4946,12 +4946,12 @@ Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, // avoids grossly misrepresenting the profiles under concurrent updates. For speed, // counter updates are not atomic. // -void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { - int base_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(0)); - int end_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(ReceiverTypeData::row_limit())); - int poly_count_offset = in_bytes(CounterData::count_offset()); - int receiver_step = in_bytes(ReceiverTypeData::receiver_offset(1)) - base_receiver_offset; - int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset; +template void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { + int base_receiver_offset = in_bytes(ReceiverProfileData::receiver_offset(0)); + int end_receiver_offset = in_bytes(ReceiverProfileData::receiver_offset(ReceiverProfileData::row_limit())); + int poly_count_offset = in_bytes(ReceiverProfileData::count_offset()); + int receiver_step = in_bytes(ReceiverProfileData::receiver_offset(1)) - base_receiver_offset; + int receiver_to_count_step = in_bytes(ReceiverProfileData::receiver_count_offset(0)) - base_receiver_offset; // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset. assert(is_aligned(mdp_offset, BytesPerWord), "sanity"); @@ -4974,20 +4974,20 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ #ifdef ASSERT // We are about to walk the MDO slots without asking for offsets. // Check that our math hits all the right spots. - for (uint c = 0; c < ReceiverTypeData::row_limit(); c++) { - int real_recv_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_offset(c)); - int real_count_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_count_offset(c)); + for (uint c = 0; c < ReceiverProfileData::row_limit(); c++) { + int real_recv_offset = mdp_offset + in_bytes(ReceiverProfileData::receiver_offset(c)); + int real_count_offset = mdp_offset + in_bytes(ReceiverProfileData::receiver_count_offset(c)); int offset = base_receiver_offset + receiver_step*c; int count_offset = offset + receiver_to_count_step; assert((offset << LogBytesPerWord) == real_recv_offset, "receiver slot math"); assert((count_offset << LogBytesPerWord) == real_count_offset, "receiver count math"); } - int real_poly_count_offset = mdp_offset + in_bytes(CounterData::count_offset()); + int real_poly_count_offset = mdp_offset + in_bytes(ReceiverProfileData::count_offset()); assert(poly_count_offset << LogBytesPerWord == real_poly_count_offset, "poly counter math"); #endif // Corner case: no profile table. Increment poly counter and exit. - if (ReceiverTypeData::row_limit() == 0) { + if (ReceiverProfileData::row_limit() == 0) { addptr(Address(mdp, poly_count_offset, Address::times_ptr), DataLayout::counter_increment); return; } @@ -5121,7 +5121,8 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ // Increment polymorphic counter instead of receiver slot. bind(L_polymorphic); - movptr(offset, poly_count_offset); + profile_receiver_type_prolog(recv); + // movptr(offset, poly_count_offset); jmpb(L_count_update); // Found a receiver, convert its slot offset to corresponding count offset. @@ -5132,6 +5133,46 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ addptr(Address(mdp, offset, Address::times_ptr), DataLayout::counter_increment); } +template<> void MacroAssembler::profile_receiver_type_prolog(Register recv) { + int poly_count_offset = in_bytes(ReceiverTypeData::count_offset()); + Register offset = rscratch1; + movptr(offset, poly_count_offset); +} + +template<> void MacroAssembler::profile_receiver_type_prolog(Register recv) { + int flat_nullable_count_offset = in_bytes(ArrayLoadData::flat_nullable_count_offset()); + int flat_nullfree_atomic_count_offset = in_bytes(ArrayLoadData::flat_nullfree_atomic_count_offset()); + int flat_nullfree_not_atomic_count_offset = in_bytes(ArrayLoadData::flat_nullfree_not_atomic_count_offset()); + + int layout_kind_offset = in_bytes(FlatArrayKlass::layout_kind_offset()); + Label null_free_non_atomic, null_free_atomic, nullable_atomic_flat, done, failure; + Register offset = rscratch1; + cmpl(Address(recv, layout_kind_offset), (int)LayoutKind::NULL_FREE_ATOMIC_FLAT); + jccb(Assembler::notEqual, null_free_non_atomic); + movptr(offset, flat_nullfree_atomic_count_offset); + + jmpb(done); + bind(null_free_non_atomic); + cmpl(Address(recv, layout_kind_offset), (int)LayoutKind::NULL_FREE_NON_ATOMIC_FLAT); + jccb(Assembler::notEqual, nullable_atomic_flat); + movptr(offset, flat_nullfree_not_atomic_count_offset); + + jmpb(done); + bind(nullable_atomic_flat); + cmpl(Address(recv, layout_kind_offset), (int)LayoutKind::NULLABLE_ATOMIC_FLAT); + jccb(Assembler::notEqual, failure); + movptr(offset, flat_nullable_count_offset); + + jmpb(done); + bind(failure); + stop("unexpected flat array"); + bind(done); +} + + +template void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset); +template void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset); + void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) { if (!VerifyOops || VerifyAdapterSharing) { // Below address of the code string confuses VerifyAdapterSharing diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp index 3dc883d27f8..c36d51e3604 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp @@ -708,7 +708,8 @@ class MacroAssembler: public Assembler { // method handles (JSR 292) Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0); - void profile_receiver_type(Register recv, Register mdp, int mdp_offset); + template void profile_receiver_type(Register recv, Register mdp, int mdp_offset); + template void profile_receiver_type_prolog(Register recv); // Debugging diff --git a/src/hotspot/share/ci/ciClassList.hpp b/src/hotspot/share/ci/ciClassList.hpp index 04b13d5cae9..e5e964539c2 100644 --- a/src/hotspot/share/ci/ciClassList.hpp +++ b/src/hotspot/share/ci/ciClassList.hpp @@ -100,6 +100,7 @@ friend class ciExceptionHandlerStream; \ friend class ciObject; \ friend class ciNullObject; \ friend class ciInstance; \ +friend class ciMegamorphicTypeData; \ friend class ciMemberName; \ friend class ciMethod; \ friend class ciMethodData; \ diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp index 4cc9a62ebe0..589f0f3323b 100644 --- a/src/hotspot/share/ci/ciMethod.cpp +++ b/src/hotspot/share/ci/ciMethod.cpp @@ -459,6 +459,68 @@ int ciMethod::check_overflow(int c, Bytecodes::Code code) { } +ciCallProfile ciMethod::megamorphic_profile_at_bci(int bci, int count, ciMegamorphicTypeData* megamorphic_type_data) { + ciCallProfile result; + + // In addition, virtual call sites have receiver type information + int receivers_count_total = 0; + int morphism = 0; + // Precompute morphism for the possible fixup + for (uint i = 0; i < megamorphic_type_data->row_limit(); i++) { + ciKlass* receiver = megamorphic_type_data->receiver(i); + if (receiver == nullptr) continue; + morphism++; + } + int epsilon = 0; + // For a call, it is assumed that either the type of the receiver(s) + // is recorded or an associated counter is incremented, but not both. With + // tiered compilation, however, both can happen due to the interpreter and + // C1 profiling invocations differently. Address that inconsistency here. + if (morphism == 1 && count > 0) { + epsilon = count; + count = 0; + } + for (uint i = 0; i < megamorphic_type_data->row_limit(); i++) { + ciKlass* receiver = megamorphic_type_data->receiver(i); + if (receiver == nullptr) continue; + int rcount = saturated_add(megamorphic_type_data->receiver_count(i), epsilon); + if (rcount == 0) rcount = 1; // Should be valid value + receivers_count_total = saturated_add(receivers_count_total, rcount); + // Add the receiver to result data. + result.add_receiver(receiver, rcount); + // If we extend profiling to record methods, + // we will set result._method also. + } + // Determine call site's morphism. + // The call site count is 0 with known morphism (only 1 or 2 receivers) + // or < 0 in the case of a type check failure for checkcast, aastore, instanceof. + // The call site count is > 0 in the case of a polymorphic virtual call. + if (morphism > 0 && morphism == result._limit) { + // The morphism <= MorphismLimit. + if ((morphism < ciCallProfile::MorphismLimit) || + (morphism == ciCallProfile::MorphismLimit && count == 0)) { +#ifdef ASSERT + if (count > 0) { + this->print_short_name(tty); + tty->print_cr(" @ bci:%d", bci); + this->print_codes(); + assert(false, "this call site should not be polymorphic"); + } +#endif + result._morphism = morphism; + } + } + // Make the count consistent if this is a call profile. If count is + // zero or less, presume that this is a typecheck profile and + // do nothing. Otherwise, increase count to be the sum of all + // receiver's counts. + if (count >= 0) { + count = saturated_add(count, receivers_count_total); + } + result._count = count; + return result; +} + // ------------------------------------------------------------------ // ciMethod::call_profile_at_bci // @@ -469,71 +531,25 @@ ciCallProfile ciMethod::call_profile_at_bci(int bci) { ciCallProfile result; if (method_data() != nullptr && method_data()->is_mature()) { ciProfileData* data = method_data()->bci_to_data(bci); - if (data != nullptr && data->is_CounterData()) { - // Every profiled call site has a counter. - int count = check_overflow(data->as_CounterData()->count(), java_code_at_bci(bci)); - - if (!data->is_ReceiverTypeData()) { - result._receiver_count[0] = 0; // that's a definite zero - } else { // ReceiverTypeData is a subclass of CounterData - ciReceiverTypeData* call = (ciReceiverTypeData*)data->as_ReceiverTypeData(); - // In addition, virtual call sites have receiver type information - int receivers_count_total = 0; - int morphism = 0; - // Precompute morphism for the possible fixup - for (uint i = 0; i < call->row_limit(); i++) { - ciKlass* receiver = call->receiver(i); - if (receiver == nullptr) continue; - morphism++; - } - int epsilon = 0; - // For a call, it is assumed that either the type of the receiver(s) - // is recorded or an associated counter is incremented, but not both. With - // tiered compilation, however, both can happen due to the interpreter and - // C1 profiling invocations differently. Address that inconsistency here. - if (morphism == 1 && count > 0) { - epsilon = count; - count = 0; - } - for (uint i = 0; i < call->row_limit(); i++) { - ciKlass* receiver = call->receiver(i); - if (receiver == nullptr) continue; - int rcount = saturated_add(call->receiver_count(i), epsilon); - if (rcount == 0) rcount = 1; // Should be valid value - receivers_count_total = saturated_add(receivers_count_total, rcount); - // Add the receiver to result data. - result.add_receiver(receiver, rcount); - // If we extend profiling to record methods, - // we will set result._method also. - } - // Determine call site's morphism. - // The call site count is 0 with known morphism (only 1 or 2 receivers) - // or < 0 in the case of a type check failure for checkcast, aastore, instanceof. - // The call site count is > 0 in the case of a polymorphic virtual call. - if (morphism > 0 && morphism == result._limit) { - // The morphism <= MorphismLimit. - if ((morphism < ciCallProfile::MorphismLimit) || - (morphism == ciCallProfile::MorphismLimit && count == 0)) { -#ifdef ASSERT - if (count > 0) { - this->print_short_name(tty); - tty->print_cr(" @ bci:%d", bci); - this->print_codes(); - assert(false, "this call site should not be polymorphic"); - } -#endif - result._morphism = morphism; - } - } - // Make the count consistent if this is a call profile. If count is - // zero or less, presume that this is a typecheck profile and - // do nothing. Otherwise, increase count to be the sum of all - // receiver's counts. - if (count >= 0) { - count = saturated_add(count, receivers_count_total); + if (data != nullptr) { + if (data->is_CounterData()) { + // Every profiled call site has a counter. + int count = check_overflow(data->as_CounterData()->count(), java_code_at_bci(bci)); + + if (!data->is_ReceiverTypeData()) { + result._receiver_count[0] = 0; // that's a definite zero + } else { // ReceiverTypeData is a subclass of CounterData + ciReceiverTypeData* call = (ciReceiverTypeData*) data->as_ReceiverTypeData(); + ciMegamorphicTypeData* megamorphic_type_data = call->megamorphic_type_data(); + return megamorphic_profile_at_bci(bci, count, megamorphic_type_data); } + result._count = count; + } else if (data->is_ArrayLoadData()) { + ciArrayLoadData* load = (ciArrayLoadData*) data->as_ArrayLoadData(); + int count = saturated_add(load->flat_nullable_count(), load->flat_nullfree_atomic_count()) + load->flat_nullfree_not_atomic_count(); + ciMegamorphicTypeData* megamorphic_type_data = load->megamorphic_type_data(); + return megamorphic_profile_at_bci(bci, count, megamorphic_type_data); } - result._count = count; } } return result; diff --git a/src/hotspot/share/ci/ciMethod.hpp b/src/hotspot/share/ci/ciMethod.hpp index 70f75c02c16..a91ec8e8017 100644 --- a/src/hotspot/share/ci/ciMethod.hpp +++ b/src/hotspot/share/ci/ciMethod.hpp @@ -37,6 +37,7 @@ #include "utilities/vmEnums.hpp" class ciMethodBlocks; +class ciMegamorphicTypeData; class MethodLiveness; class Arena; class BCEscapeAnalyzer; @@ -135,7 +136,7 @@ class ciMethod : public ciMetadata { // Check and update the profile counter in case of overflow static int check_overflow(int c, Bytecodes::Code code); - public: +public: void check_is_loaded() const { assert(is_loaded(), "not loaded"); } // Basic method information. @@ -266,6 +267,7 @@ class ciMethod : public ciMetadata { ciTypeFlow* get_flow_analysis(); ciTypeFlow* get_osr_flow_analysis(int osr_bci); // alternate entry point ciCallProfile call_profile_at_bci(int bci); + ciCallProfile megamorphic_profile_at_bci(int bci, int count, ciMegamorphicTypeData* megamorphic_type_data); // Does type profiling provide any useful information at this point? bool argument_profiled_type(int bci, int i, ciKlass*& type, ProfilePtrKind& ptr_kind); diff --git a/src/hotspot/share/ci/ciMethodData.cpp b/src/hotspot/share/ci/ciMethodData.cpp index b6c8598c146..a439415c2cd 100644 --- a/src/hotspot/share/ci/ciMethodData.cpp +++ b/src/hotspot/share/ci/ciMethodData.cpp @@ -308,9 +308,10 @@ bool ciMethodData::load_data() { return true; } -void ciReceiverTypeData::translate_receiver_data_from(const ProfileData* data) { +bool ciMegamorphicTypeData::translate_type_data_from(const MegamorphicTypeData* data) { + bool cleared_row = false; for (uint row = 0; row < row_limit(); row++) { - Klass* k = data->as_ReceiverTypeData()->receiver(row); + Klass* k = data->receiver(row); if (k != nullptr && k->class_loader_data() != nullptr && is_klass_loaded(k)) { if (k->is_loader_alive()) { ciKlass* klass = CURRENT_ENV->get_klass(k); @@ -318,11 +319,24 @@ void ciReceiverTypeData::translate_receiver_data_from(const ProfileData* data) { } else { // With concurrent class unloading, the MDO could have stale metadata; override it clear_row(row); + cleared_row = true; } } else { set_receiver(row, nullptr); } } + return cleared_row; +} + +void ciMegamorphicTypeData::print_receiver_data_on(outputStream* st, int total) const { + total += count(); + for (uint row = 0; row < row_limit(); row++) { + if (receiver(row) != nullptr) { + _pd->tab(st); + receiver(row)->print_name_on(st); + st->print_cr("(%u %4.2f)", receiver_count(row), (float) receiver_count(row) / (float) total); + } + } } void ciTypeStackSlotEntries::translate_type_data_from(const TypeStackSlotEntries* entries) { @@ -950,19 +964,8 @@ void ciCallTypeData::print_data_on(outputStream* st, const char* extra) const { } void ciReceiverTypeData::print_receiver_data_on(outputStream* st) const { - uint row; - int entries = 0; - for (row = 0; row < row_limit(); row++) { - if (receiver(row) != nullptr) entries++; - } - st->print_cr("count(%u) entries(%u)", count(), entries); - for (row = 0; row < row_limit(); row++) { - if (receiver(row) != nullptr) { - tab(st); - receiver(row)->print_name_on(st); - st->print_cr("(%u)", receiver_count(row)); - } - } + st->print_cr("count(%u) entries(%u)", count(), _megamorphic_type_data.entries()); + megamorphic_type_data()->print_receiver_data_on(st, count()); } void ciReceiverTypeData::print_data_on(outputStream* st, const char* extra) const { @@ -1014,11 +1017,17 @@ void ciArrayStoreData::print_data_on(outputStream* st, const char* extra) const } void ciArrayLoadData::print_data_on(outputStream* st, const char* extra) const { - print_shared(st, "ciArrayLoadData", extra); + print_shared(st, "ArrayLoad", extra); + st->print("not flat %u (null free = %u, nullable = %u)", not_flat_count(), not_flat_null_free_count(), not_flat_nullable_count()); + st->cr(); + tab(st); + st->print("flat %u (null free atomic = %u, null free not atomic = %u, nullable = %u)", flat_count(), flat_nullfree_atomic_count(), flat_nullfree_not_atomic_count(), flat_nullable_count()); st->cr(); tab(st, true); st->print("array"); - rtd_super()->print_receiver_data_on(st); + tab(st); + megamorphic_type_data()->print_receiver_data_on(st, 0); + st->cr(); tab(st, true); st->print("element"); element()->print_data_on(st); diff --git a/src/hotspot/share/ci/ciMethodData.hpp b/src/hotspot/share/ci/ciMethodData.hpp index cb6f06dfada..e88bcab0201 100644 --- a/src/hotspot/share/ci/ciMethodData.hpp +++ b/src/hotspot/share/ci/ciMethodData.hpp @@ -141,6 +141,28 @@ class ciSingleTypeEntry : public SingleTypeEntry, ciTypeEntries { #endif }; +class ciMegamorphicTypeData : public MegamorphicTypeData { +public: + bool translate_type_data_from(const MegamorphicTypeData* ret); + + void set_receiver(uint row, ciKlass* recv) { + assert((uint)row < row_limit(), "oob"); + set_intptr_at(receiver_cell_index(row), (intptr_t) recv); + } + + ciKlass* receiver(uint row) const { + assert((uint)row < row_limit(), "oob"); + ciKlass* recv = (ciKlass*)intptr_at(receiver_cell_index(row)); + assert(recv == nullptr || recv->is_klass(), "wrong type"); + return recv; + } + +#ifndef PRODUCT + void print_receiver_data_on(outputStream* st, int total) const; +#endif +}; + + class ciCallTypeData : public CallTypeData { public: ciCallTypeData(DataLayout* layout) : CallTypeData(layout) {} @@ -193,6 +215,7 @@ class ciCallTypeData : public CallTypeData { class ciReceiverTypeData : public ReceiverTypeData { public: ciReceiverTypeData(DataLayout* layout) : ReceiverTypeData(layout) {}; + ciMegamorphicTypeData* megamorphic_type_data() const { return (ciMegamorphicTypeData*)ReceiverTypeData::megamorphic_type_data(); } void set_receiver(uint row, ciKlass* recv) { assert((uint)row < row_limit(), "oob"); @@ -200,17 +223,18 @@ class ciReceiverTypeData : public ReceiverTypeData { } ciKlass* receiver(uint row) const { - assert((uint)row < row_limit(), "oob"); - ciKlass* recv = (ciKlass*)intptr_at(_megamorphic_type_data.receiver_cell_index(row)); - assert(recv == nullptr || recv->is_klass(), "wrong type"); - return recv; + return megamorphic_type_data()->receiver(row); } // Copy & translate from oop based ReceiverTypeData virtual void translate_from(const ProfileData* data) { translate_receiver_data_from(data); } - void translate_receiver_data_from(const ProfileData* data); + void translate_receiver_data_from(const ProfileData* data) { + if (megamorphic_type_data()->translate_type_data_from(data->as_ReceiverTypeData()->megamorphic_type_data())) { + set_count(0); + } + } #ifndef PRODUCT void print_data_on(outputStream* st, const char* extra = nullptr) const; void print_receiver_data_on(outputStream* st) const; @@ -384,25 +408,28 @@ class ciArrayStoreData : public ArrayStoreData { }; class ciArrayLoadData : public ArrayLoadData { - // Fake multiple inheritance... It's a ciReceiverTypeData also. - ciReceiverTypeData* rtd_super() const { return (ciReceiverTypeData*) this; } public: ciArrayLoadData(DataLayout* layout) : ArrayLoadData(layout) {} // ciSingleTypeEntry* array() const { return (ciSingleTypeEntry*)ArrayLoadData::array(); } ciSingleTypeEntry* element() const { return (ciSingleTypeEntry*)ArrayLoadData::element(); } + ciMegamorphicTypeData* megamorphic_type_data() const { return (ciMegamorphicTypeData*)ArrayLoadData::megamorphic_type_data(); } virtual void translate_from(const ProfileData* data) { - rtd_super()->translate_receiver_data_from(data); + if (megamorphic_type_data()->translate_type_data_from(data->as_ArrayLoadData()->megamorphic_type_data())) { + set_flat_nullable_count(0); + set_flat_nullfree_atomic_count(0); + set_flat_nullfree_not_atomic_count(0); + } element()->translate_type_data_from(data->as_ArrayLoadData()->element()); } ciKlass* receiver(uint row) { - return rtd_super()->receiver(row); + return megamorphic_type_data()->receiver(row); } - #ifndef PRODUCT +#ifndef PRODUCT void print_data_on(outputStream* st, const char* extra = nullptr) const; #endif }; diff --git a/src/hotspot/share/oops/methodData.cpp b/src/hotspot/share/oops/methodData.cpp index 3332a13ebc0..c7ce56403d8 100644 --- a/src/hotspot/share/oops/methodData.cpp +++ b/src/hotspot/share/oops/methodData.cpp @@ -750,10 +750,14 @@ void ArrayLoadData::print_data_on(outputStream* st, const char* extra) const { print_shared(st, "ArrayLoad", extra); st->print("not flat %u (null free = %u, nullable = %u)", not_flat_count(), not_flat_null_free_count(), not_flat_nullable_count()); st->cr(); + tab(st); + st->print("flat %u (null free atomic = %u, null free not atomic = %u, nullable = %u)", flat_count(), flat_nullfree_atomic_count(), flat_nullfree_not_atomic_count(), flat_nullable_count()); + st->cr(); tab(st, true); st->print("array"); tab(st); - print_receiver_data_on(st); + _megamorphic_type_data.print_receiver_data_on(st, 0); + st->cr(); tab(st, true); st->print("element"); _element.print_data_on(st); diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 4f0701ea282..41cd29c2666 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -1172,10 +1172,18 @@ class MegamorphicTypeData { receiver_type_row_cell_count = (count0_offset + 1) - receiver0_offset }; - ProfileData* _pd; const int _base_off; const int _type_width; + void set_uint_at(int index, uint value) { + _pd->set_uint_at(index, value); + } + uint uint_at(int index) const { + return _pd->uint_at(index); + } + +protected: + void set_intptr_at(int index, intptr_t value) { _pd->set_intptr_at(index, value); } @@ -1184,12 +1192,6 @@ class MegamorphicTypeData { return _pd->intptr_at(index); } - void set_uint_at(int index, uint value) { - _pd->set_uint_at(index, value); - } - uint uint_at(int index) const { - return _pd->uint_at(index); - } public: MegamorphicTypeData(ProfileData* pd, int base_off, int type_width) @@ -1285,6 +1287,9 @@ class ReceiverTypeData : public CounterData { layout->tag() == DataLayout::array_load_data_tag, "wrong type"); } + const MegamorphicTypeData* megamorphic_type_data() const { + return &_megamorphic_type_data; + } virtual bool is_ReceiverTypeData() const { return true; } static int static_cell_count() { @@ -2075,7 +2080,7 @@ class ArrayStoreData : public ReceiverTypeData { virtual void print_data_on(outputStream* st, const char* extra = nullptr) const; }; -class ArrayLoadData : public ReceiverTypeData { +class ArrayLoadData : public ProfileData { private: enum { not_flat_null_free_count_off_in_extra_cells, @@ -2087,9 +2092,10 @@ class ArrayLoadData : public ReceiverTypeData { }; SingleTypeEntry _element; + MegamorphicTypeData _megamorphic_type_data; static int extra_cells_off() { - return ReceiverTypeData::static_cell_count() + SingleTypeEntry::static_cell_count(); + return SingleTypeEntry::static_cell_count() + MegamorphicTypeData::static_cell_count(TypeProfileWidth); } static int not_flat_null_free_count_off() { @@ -2114,8 +2120,9 @@ class ArrayLoadData : public ReceiverTypeData { public: ArrayLoadData(DataLayout* layout) : - ReceiverTypeData(layout), - _element(ReceiverTypeData::static_cell_count()) { + ProfileData(layout), + _element(0), + _megamorphic_type_data(this, SingleTypeEntry::static_cell_count(), TypeProfileWidth) { assert(layout->tag() == DataLayout::array_load_data_tag, "wrong type"); _element.set_profile_data(this); } @@ -2124,6 +2131,10 @@ class ArrayLoadData : public ReceiverTypeData { return &_element; } + const MegamorphicTypeData* megamorphic_type_data() const { + return &_megamorphic_type_data; + } + virtual bool is_ArrayLoadData() const { return true; } static int static_cell_count() { @@ -2138,6 +2149,10 @@ class ArrayLoadData : public ReceiverTypeData { return not_flat_null_free_count() + not_flat_nullable_count(); } + int flat_count() const { + return flat_nullfree_atomic_count() + flat_nullfree_not_atomic_count() + flat_nullable_count(); + } + int not_flat_null_free_count() const { return uint_at(not_flat_null_free_count_off()); } @@ -2158,10 +2173,26 @@ class ArrayLoadData : public ReceiverTypeData { return uint_at(flat_nullfree_not_atomic_count_off()); } + void set_flat_nullable_count(uint count) { + set_uint_at(flat_nullable_count_off(), count); + } + + void set_flat_nullfree_atomic_count(uint count) { + set_uint_at(flat_nullfree_atomic_count_off(), count); + } + + void set_flat_nullfree_not_atomic_count(uint count) { + set_uint_at(flat_nullfree_not_atomic_count_off(), count); + } + + static uint row_limit() { + return (uint) TypeProfileWidth; + } + // Code generation support static ByteSize element_offset() { - return cell_offset(ReceiverTypeData::static_cell_count()); + return cell_offset(0); } static ByteSize not_flat_null_free_count_offset() { @@ -2172,6 +2203,10 @@ class ArrayLoadData : public ReceiverTypeData { return cell_offset(not_flat_nullable_count_off()); } + static ByteSize count_offset() { + return cell_offset(flat_nullable_count_off()); + } + static ByteSize flat_nullable_count_offset() { return cell_offset(flat_nullable_count_off()); } @@ -2184,13 +2219,23 @@ class ArrayLoadData : public ReceiverTypeData { return cell_offset(flat_nullfree_not_atomic_count_off()); } + static ByteSize receiver_offset(uint row) { + return cell_offset(MegamorphicTypeData::static_receiver_cell_index(row) + SingleTypeEntry::static_cell_count()); + } + static ByteSize receiver_count_offset(uint row) { + return cell_offset(MegamorphicTypeData::static_receiver_count_cell_index(row) + SingleTypeEntry::static_cell_count()); + } + // static ByteSize receiver_type_data_size() { + // return cell_offset(static_cell_count()); + // } + virtual void clean_weak_klass_links(bool always_clean) { - ReceiverTypeData::clean_weak_klass_links(always_clean); + _megamorphic_type_data.clean_weak_klass_links(always_clean); _element.clean_weak_klass_links(always_clean); } virtual void metaspace_pointers_do(MetaspaceClosure* it) { - ReceiverTypeData::metaspace_pointers_do(it); + _megamorphic_type_data.metaspace_pointers_do(it); _element.metaspace_pointers_do(it); } diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 0e239a6e07c..500fbc8c14e 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -254,6 +254,12 @@ class ArrayLoad { float this_prob = clamp(p, PROB_MIN, PROB_MAX); Node* casted_array = nullptr; Node* next_ctrl = _parse.type_check_receiver(_array, klass, this_prob, &casted_array); + if (_parse.stopped()) { + _parse.set_control(next_ctrl); + _parse.set_all_memory(_mem); + _parse.set_i_o(_io); + return; + } // TODO: pin load InlineTypeNode* vt = InlineTypeNode::make_from_flat_array( &_parse, klass->as_flat_array_klass()->element_klass()->as_inline_klass(), casted_array, _array_index); From d716e2c44047bbfd0526ae2b25a27c5d8ee6ae44 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Wed, 29 Apr 2026 08:56:48 +0200 Subject: [PATCH 21/49] more --- src/hotspot/cpu/x86/macroAssembler_x86.cpp | 141 +++++++++++---------- src/hotspot/cpu/x86/macroAssembler_x86.hpp | 3 + src/hotspot/share/oops/methodData.hpp | 30 +++-- 3 files changed, 99 insertions(+), 75 deletions(-) diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index a511f1e53e4..2fa6f2a8e03 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -4935,67 +4935,16 @@ Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, return Address(rsp, scale_reg, scale_factor, offset); } -// Handle the receiver type profile update given the "recv" klass. -// -// Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset". -// If there are no matching or claimable receiver entries in RD, updates -// the polymorphic counter. -// -// This code expected to run by either the interpreter or JIT-ed code, without -// extra synchronization. For safety, receiver cells are claimed atomically, which -// avoids grossly misrepresenting the profiles under concurrent updates. For speed, -// counter updates are not atomic. -// -template void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { - int base_receiver_offset = in_bytes(ReceiverProfileData::receiver_offset(0)); - int end_receiver_offset = in_bytes(ReceiverProfileData::receiver_offset(ReceiverProfileData::row_limit())); - int poly_count_offset = in_bytes(ReceiverProfileData::count_offset()); - int receiver_step = in_bytes(ReceiverProfileData::receiver_offset(1)) - base_receiver_offset; - int receiver_to_count_step = in_bytes(ReceiverProfileData::receiver_count_offset(0)) - base_receiver_offset; - - // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset. - assert(is_aligned(mdp_offset, BytesPerWord), "sanity"); - base_receiver_offset += mdp_offset; - end_receiver_offset += mdp_offset; - poly_count_offset += mdp_offset; - - // Scale down to optimize encoding. Slots are pointer-sized. - assert(is_aligned(base_receiver_offset, BytesPerWord), "sanity"); - assert(is_aligned(end_receiver_offset, BytesPerWord), "sanity"); - assert(is_aligned(poly_count_offset, BytesPerWord), "sanity"); - assert(is_aligned(receiver_step, BytesPerWord), "sanity"); - assert(is_aligned(receiver_to_count_step, BytesPerWord), "sanity"); - base_receiver_offset >>= LogBytesPerWord; - end_receiver_offset >>= LogBytesPerWord; - poly_count_offset >>= LogBytesPerWord; - receiver_step >>= LogBytesPerWord; - receiver_to_count_step >>= LogBytesPerWord; +bool MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, Register &offset, Label &L_found_recv, int base, uint row_limit) { + int base_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, 0)); + int end_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, row_limit)); + int receiver_step = in_bytes(MegamorphicTypeData::receiver_offset(base, 1)) - base_receiver_offset; + int receiver_to_count_step = in_bytes(MegamorphicTypeData::receiver_count_offset(base, 0)) - base_receiver_offset; -#ifdef ASSERT - // We are about to walk the MDO slots without asking for offsets. - // Check that our math hits all the right spots. - for (uint c = 0; c < ReceiverProfileData::row_limit(); c++) { - int real_recv_offset = mdp_offset + in_bytes(ReceiverProfileData::receiver_offset(c)); - int real_count_offset = mdp_offset + in_bytes(ReceiverProfileData::receiver_count_offset(c)); - int offset = base_receiver_offset + receiver_step*c; - int count_offset = offset + receiver_to_count_step; - assert((offset << LogBytesPerWord) == real_recv_offset, "receiver slot math"); - assert((count_offset << LogBytesPerWord) == real_count_offset, "receiver count math"); - } - int real_poly_count_offset = mdp_offset + in_bytes(ReceiverProfileData::count_offset()); - assert(poly_count_offset << LogBytesPerWord == real_poly_count_offset, "poly counter math"); -#endif - - // Corner case: no profile table. Increment poly counter and exit. - if (ReceiverProfileData::row_limit() == 0) { - addptr(Address(mdp, poly_count_offset, Address::times_ptr), DataLayout::counter_increment); - return; - } - - Register offset = rscratch1; + offset = rscratch1; Label L_loop_search_receiver, L_loop_search_empty; - Label L_restart, L_found_recv, L_found_empty, L_polymorphic, L_count_update; + Label L_restart, L_found_empty, L_polymorphic; // The code here recognizes three major cases: // A. Fastest: receiver found in the table @@ -5049,8 +4998,8 @@ template void MacroAssembler::profile_receiver_type( // Fastest: receiver is already installed movptr(offset, base_receiver_offset); bind(L_loop_search_receiver); - cmpptr(recv, Address(mdp, offset, Address::times_ptr)); - jccb(Assembler::equal, L_found_recv); + cmpptr(recv, Address(mdp, offset, Address::times_ptr)); + jccb(Assembler::equal, L_found_recv); addptr(offset, receiver_step); cmpptr(offset, end_receiver_offset); jccb(Assembler::notEqual, L_loop_search_receiver); @@ -5058,8 +5007,8 @@ template void MacroAssembler::profile_receiver_type( // Fast: no receiver, but profile is full movptr(offset, base_receiver_offset); bind(L_loop_search_empty); - cmpptr(Address(mdp, offset, Address::times_ptr), NULL_WORD); - jccb(Assembler::equal, L_found_empty); + cmpptr(Address(mdp, offset, Address::times_ptr), NULL_WORD); + jccb(Assembler::equal, L_found_empty); addptr(offset, receiver_step); cmpptr(offset, end_receiver_offset); jccb(Assembler::notEqual, L_loop_search_empty); @@ -5080,8 +5029,8 @@ template void MacroAssembler::profile_receiver_type( Register shifted_recv = recv; if (recv == rax || mdp == rax) { spare_reg = (recv != rbx && mdp != rbx) ? rbx : - (recv != rcx && mdp != rcx) ? rcx : - rdx; + (recv != rcx && mdp != rcx) ? rcx : + rdx; assert_different_registers(mdp, recv, offset, spare_reg); push(spare_reg); @@ -5121,6 +5070,70 @@ template void MacroAssembler::profile_receiver_type( // Increment polymorphic counter instead of receiver slot. bind(L_polymorphic); + return false; +} + +// Handle the receiver type profile update given the "recv" klass. +// +// Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset". +// If there are no matching or claimable receiver entries in RD, updates +// the polymorphic counter. +// +// This code expected to run by either the interpreter or JIT-ed code, without +// extra synchronization. For safety, receiver cells are claimed atomically, which +// avoids grossly misrepresenting the profiles under concurrent updates. For speed, +// counter updates are not atomic. +// +template void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { + int base_receiver_offset = in_bytes(ReceiverProfileData::receiver_offset(0)); + int end_receiver_offset = in_bytes(ReceiverProfileData::receiver_offset(ReceiverProfileData::row_limit())); + int poly_count_offset = in_bytes(ReceiverProfileData::count_offset()); + int receiver_step = in_bytes(ReceiverProfileData::receiver_offset(1)) - base_receiver_offset; + int receiver_to_count_step = in_bytes(ReceiverProfileData::receiver_count_offset(0)) - base_receiver_offset; + + // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset. + assert(is_aligned(mdp_offset, BytesPerWord), "sanity"); + base_receiver_offset += mdp_offset; + end_receiver_offset += mdp_offset; + poly_count_offset += mdp_offset; + + // Scale down to optimize encoding. Slots are pointer-sized. + assert(is_aligned(base_receiver_offset, BytesPerWord), "sanity"); + assert(is_aligned(end_receiver_offset, BytesPerWord), "sanity"); + assert(is_aligned(poly_count_offset, BytesPerWord), "sanity"); + assert(is_aligned(receiver_step, BytesPerWord), "sanity"); + assert(is_aligned(receiver_to_count_step, BytesPerWord), "sanity"); + base_receiver_offset >>= LogBytesPerWord; + end_receiver_offset >>= LogBytesPerWord; + poly_count_offset >>= LogBytesPerWord; + receiver_step >>= LogBytesPerWord; + receiver_to_count_step >>= LogBytesPerWord; + +#ifdef ASSERT + // We are about to walk the MDO slots without asking for offsets. + // Check that our math hits all the right spots. + for (uint c = 0; c < ReceiverProfileData::row_limit(); c++) { + int real_recv_offset = mdp_offset + in_bytes(ReceiverProfileData::receiver_offset(c)); + int real_count_offset = mdp_offset + in_bytes(ReceiverProfileData::receiver_count_offset(c)); + int offset = base_receiver_offset + receiver_step*c; + int count_offset = offset + receiver_to_count_step; + assert((offset << LogBytesPerWord) == real_recv_offset, "receiver slot math"); + assert((count_offset << LogBytesPerWord) == real_count_offset, "receiver count math"); + } + int real_poly_count_offset = mdp_offset + in_bytes(ReceiverProfileData::count_offset()); + assert(poly_count_offset << LogBytesPerWord == real_poly_count_offset, "poly counter math"); +#endif + + // Corner case: no profile table. Increment poly counter and exit. + if (ReceiverProfileData::row_limit() == 0) { + addptr(Address(mdp, poly_count_offset, Address::times_ptr), DataLayout::counter_increment); + return; + } + + Register offset; + Label L_found_recv; + Label L_count_update; + profile_receiver_type_helper(recv, mdp, offset, L_found_recv, ReceiverProfileData::base_of_megamorphic_type_data(), ReceiverProfileData::row_limit()); profile_receiver_type_prolog(recv); // movptr(offset, poly_count_offset); jmpb(L_count_update); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp index c36d51e3604..6040995eb46 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp @@ -708,6 +708,9 @@ class MacroAssembler: public Assembler { // method handles (JSR 292) Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0); + bool profile_receiver_type_helper(Register recv, Register mdp, + Register &offset, Label &L_found_recv, int base, uint row_limit); + template void profile_receiver_type(Register recv, Register mdp, int mdp_offset); template void profile_receiver_type_prolog(Register recv); diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 41cd29c2666..462be5e342a 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -1248,13 +1248,13 @@ class MegamorphicTypeData { set_receiver_count(row, 0); } - // // Code generation support - // static ByteSize receiver_offset(uint row) { - // return cell_offset(receiver_cell_index(row)); - // } - // static ByteSize receiver_count_offset(uint row) { - // return cell_offset(receiver_count_cell_index(row)); - // } + // Code generation support + static ByteSize receiver_offset(int base, uint row) { + return ProfileData::cell_offset(base + static_receiver_cell_index(row)); + } + static ByteSize receiver_count_offset(int base, uint row) { + return ProfileData::cell_offset(base + static_receiver_count_cell_index(row)); + } // static ByteSize receiver_type_data_size() { // return cell_offset(static_cell_count()); // } @@ -1287,6 +1287,10 @@ class ReceiverTypeData : public CounterData { layout->tag() == DataLayout::array_load_data_tag, "wrong type"); } + static int base_of_megamorphic_type_data() { + return counter_cell_count; + } + const MegamorphicTypeData* megamorphic_type_data() const { return &_megamorphic_type_data; } @@ -1350,10 +1354,10 @@ class ReceiverTypeData : public CounterData { // Code generation support static ByteSize receiver_offset(uint row) { - return cell_offset(MegamorphicTypeData::static_receiver_cell_index(row) + counter_cell_count); + return MegamorphicTypeData::receiver_offset(counter_cell_count, row); } static ByteSize receiver_count_offset(uint row) { - return cell_offset(MegamorphicTypeData::static_receiver_count_cell_index(row) + counter_cell_count); + return MegamorphicTypeData::receiver_count_offset(counter_cell_count, row); } static ByteSize receiver_type_data_size() { return cell_offset(static_cell_count()); @@ -2127,6 +2131,10 @@ class ArrayLoadData : public ProfileData { _element.set_profile_data(this); } + static int base_of_megamorphic_type_data() { + return SingleTypeEntry::static_cell_count(); + } + const SingleTypeEntry* element() const { return &_element; } @@ -2220,10 +2228,10 @@ class ArrayLoadData : public ProfileData { } static ByteSize receiver_offset(uint row) { - return cell_offset(MegamorphicTypeData::static_receiver_cell_index(row) + SingleTypeEntry::static_cell_count()); + return MegamorphicTypeData::receiver_offset(SingleTypeEntry::static_cell_count(), row); } static ByteSize receiver_count_offset(uint row) { - return cell_offset(MegamorphicTypeData::static_receiver_count_cell_index(row) + SingleTypeEntry::static_cell_count()); + return MegamorphicTypeData::receiver_count_offset(SingleTypeEntry::static_cell_count(), row); } // static ByteSize receiver_type_data_size() { // return cell_offset(static_cell_count()); From 17d4c8a23f8659bf29aa2df1e19dd89fb11bc7b8 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Wed, 29 Apr 2026 14:03:08 +0200 Subject: [PATCH 22/49] more --- src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 4 +- src/hotspot/cpu/x86/interp_masm_x86.cpp | 8 +- src/hotspot/cpu/x86/macroAssembler_x86.cpp | 134 +++++++++++++------- src/hotspot/cpu/x86/macroAssembler_x86.hpp | 6 +- src/hotspot/share/oops/methodData.hpp | 17 ++- 5 files changed, 104 insertions(+), 65 deletions(-) diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp index 5134f528181..d97395ab1de 100644 --- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp @@ -1332,7 +1332,7 @@ void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data, Register recv) { int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0)); - __ profile_receiver_type(recv, mdo, mdp_offset); + __ profile_receiver_type(recv, mdo, mdp_offset); } void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) { @@ -3193,7 +3193,7 @@ void LIR_Assembler::emit_profile_multiple_array_types(LIR_OpProfileMultipleArray __ mov_metadata(mdo, md->constant_encoding()); int mdp_offset = md->byte_offset_of_slot(op->load(), in_ByteSize(0)); - __ profile_receiver_type(tmp1, mdo, mdp_offset); + __ profile_array_type_at_load(tmp1, mdo, mdp_offset); __ jmp(done); __ bind(not_flat); diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp index a35d2a9852c..b8b2e780f83 100644 --- a/src/hotspot/cpu/x86/interp_masm_x86.cpp +++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp @@ -1474,7 +1474,7 @@ void InterpreterMacroAssembler::profile_virtual_call(Register receiver, test_method_data_pointer(mdp, profile_continue); // Record the receiver type. - profile_receiver_type(receiver, mdp, 0); + profile_receiver_type(receiver, mdp, 0); // The method data pointer needs to be updated to reflect the new target. update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size())); @@ -1554,7 +1554,7 @@ void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass) mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size()); // Record the object type. - profile_receiver_type(klass, mdp, 0); + profile_receiver_type(klass, mdp, 0); } update_mdp_by_constant(mdp, mdp_delta); @@ -1657,7 +1657,7 @@ void InterpreterMacroAssembler::profile_multiple_array_types(Register mdp, test_non_flat_array_oop(array, tmp, not_flat); load_klass(tmp, array, rscratch1); - profile_receiver_type(tmp, mdp, 0); + profile_array_type_at_load(tmp, mdp, 0); jmp(profile_continue); bind(not_flat); @@ -1694,7 +1694,7 @@ void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Reg load_klass(tmp, element, rscratch1); // Record the object type. - profile_receiver_type(tmp, mdp, 0); + profile_receiver_type(tmp, mdp, 0); bind(done); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 2fa6f2a8e03..ebffe6dcf68 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -4935,13 +4935,39 @@ Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, return Address(rsp, scale_reg, scale_factor, offset); } -bool MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, Register &offset, Label &L_found_recv, int base, uint row_limit) { - int base_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, 0)); - int end_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, row_limit)); - int receiver_step = in_bytes(MegamorphicTypeData::receiver_offset(base, 1)) - base_receiver_offset; - int receiver_to_count_step = in_bytes(MegamorphicTypeData::receiver_count_offset(base, 0)) - base_receiver_offset; +bool MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, Label &L_found_recv, int mdp_offset, uint row_limit) { + int base_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(0)); + int end_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(row_limit)); + int receiver_step = in_bytes(MegamorphicTypeData::receiver_offset(1)) - base_receiver_offset; + int receiver_to_count_step = in_bytes(MegamorphicTypeData::receiver_count_offset(0)) - base_receiver_offset; - offset = rscratch1; + assert(is_aligned(mdp_offset, BytesPerWord), "sanity"); + base_receiver_offset += mdp_offset; + end_receiver_offset += mdp_offset; + + assert(is_aligned(base_receiver_offset, BytesPerWord), "sanity"); + assert(is_aligned(end_receiver_offset, BytesPerWord), "sanity"); + assert(is_aligned(receiver_step, BytesPerWord), "sanity"); + assert(is_aligned(receiver_to_count_step, BytesPerWord), "sanity"); + base_receiver_offset >>= LogBytesPerWord; + end_receiver_offset >>= LogBytesPerWord; + receiver_step >>= LogBytesPerWord; + receiver_to_count_step >>= LogBytesPerWord; + +#ifdef ASSERT + // We are about to walk the MDO slots without asking for offsets. + // Check that our math hits all the right spots. + for (uint c = 0; c < ReceiverTypeData::row_limit(); c++) { + int real_recv_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_offset(c)); + int real_count_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_count_offset(c)); + int offset = base_receiver_offset + receiver_step*c; + int count_offset = offset + receiver_to_count_step; + assert((offset << LogBytesPerWord) == real_recv_offset, "receiver slot math"); + assert((count_offset << LogBytesPerWord) == real_count_offset, "receiver count math"); + } +#endif + + Register offset = rscratch1; Label L_loop_search_receiver, L_loop_search_empty; Label L_restart, L_found_empty, L_polymorphic; @@ -5084,58 +5110,37 @@ bool MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, R // avoids grossly misrepresenting the profiles under concurrent updates. For speed, // counter updates are not atomic. // -template void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { - int base_receiver_offset = in_bytes(ReceiverProfileData::receiver_offset(0)); - int end_receiver_offset = in_bytes(ReceiverProfileData::receiver_offset(ReceiverProfileData::row_limit())); - int poly_count_offset = in_bytes(ReceiverProfileData::count_offset()); - int receiver_step = in_bytes(ReceiverProfileData::receiver_offset(1)) - base_receiver_offset; - int receiver_to_count_step = in_bytes(ReceiverProfileData::receiver_count_offset(0)) - base_receiver_offset; +void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { + int poly_count_offset = in_bytes(ReceiverTypeData::count_offset()); + int base_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(0)); + int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset; // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset. assert(is_aligned(mdp_offset, BytesPerWord), "sanity"); - base_receiver_offset += mdp_offset; - end_receiver_offset += mdp_offset; poly_count_offset += mdp_offset; // Scale down to optimize encoding. Slots are pointer-sized. - assert(is_aligned(base_receiver_offset, BytesPerWord), "sanity"); - assert(is_aligned(end_receiver_offset, BytesPerWord), "sanity"); assert(is_aligned(poly_count_offset, BytesPerWord), "sanity"); - assert(is_aligned(receiver_step, BytesPerWord), "sanity"); assert(is_aligned(receiver_to_count_step, BytesPerWord), "sanity"); - base_receiver_offset >>= LogBytesPerWord; - end_receiver_offset >>= LogBytesPerWord; poly_count_offset >>= LogBytesPerWord; - receiver_step >>= LogBytesPerWord; receiver_to_count_step >>= LogBytesPerWord; #ifdef ASSERT - // We are about to walk the MDO slots without asking for offsets. - // Check that our math hits all the right spots. - for (uint c = 0; c < ReceiverProfileData::row_limit(); c++) { - int real_recv_offset = mdp_offset + in_bytes(ReceiverProfileData::receiver_offset(c)); - int real_count_offset = mdp_offset + in_bytes(ReceiverProfileData::receiver_count_offset(c)); - int offset = base_receiver_offset + receiver_step*c; - int count_offset = offset + receiver_to_count_step; - assert((offset << LogBytesPerWord) == real_recv_offset, "receiver slot math"); - assert((count_offset << LogBytesPerWord) == real_count_offset, "receiver count math"); - } - int real_poly_count_offset = mdp_offset + in_bytes(ReceiverProfileData::count_offset()); + int real_poly_count_offset = mdp_offset + in_bytes(ReceiverTypeData::count_offset()); assert(poly_count_offset << LogBytesPerWord == real_poly_count_offset, "poly counter math"); #endif // Corner case: no profile table. Increment poly counter and exit. - if (ReceiverProfileData::row_limit() == 0) { + if (ReceiverTypeData::row_limit() == 0) { addptr(Address(mdp, poly_count_offset, Address::times_ptr), DataLayout::counter_increment); return; } - Register offset; Label L_found_recv; + profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset + ReceiverTypeData::base_of_megamorphic_type_data(), ReceiverTypeData::row_limit()); + Register offset = rscratch1; + movptr(offset, poly_count_offset); Label L_count_update; - profile_receiver_type_helper(recv, mdp, offset, L_found_recv, ReceiverProfileData::base_of_megamorphic_type_data(), ReceiverProfileData::row_limit()); - profile_receiver_type_prolog(recv); - // movptr(offset, poly_count_offset); jmpb(L_count_update); // Found a receiver, convert its slot offset to corresponding count offset. @@ -5146,20 +5151,49 @@ template void MacroAssembler::profile_receiver_type( addptr(Address(mdp, offset, Address::times_ptr), DataLayout::counter_increment); } -template<> void MacroAssembler::profile_receiver_type_prolog(Register recv) { - int poly_count_offset = in_bytes(ReceiverTypeData::count_offset()); - Register offset = rscratch1; - movptr(offset, poly_count_offset); -} - -template<> void MacroAssembler::profile_receiver_type_prolog(Register recv) { +void MacroAssembler::profile_array_type_at_load(Register recv, Register mdp, int mdp_offset) { + int base_receiver_offset = in_bytes(ArrayLoadData::receiver_offset(0)); + int receiver_to_count_step = in_bytes(ArrayLoadData::receiver_count_offset(0)) - base_receiver_offset; int flat_nullable_count_offset = in_bytes(ArrayLoadData::flat_nullable_count_offset()); int flat_nullfree_atomic_count_offset = in_bytes(ArrayLoadData::flat_nullfree_atomic_count_offset()); int flat_nullfree_not_atomic_count_offset = in_bytes(ArrayLoadData::flat_nullfree_not_atomic_count_offset()); + // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset. + assert(is_aligned(mdp_offset, BytesPerWord), "sanity"); + flat_nullable_count_offset += mdp_offset; + flat_nullfree_atomic_count_offset += mdp_offset; + flat_nullfree_not_atomic_count_offset += mdp_offset; + + // Scale down to optimize encoding. Slots are pointer-sized. + assert(is_aligned(flat_nullable_count_offset, BytesPerWord), "sanity"); + assert(is_aligned(flat_nullfree_atomic_count_offset, BytesPerWord), "sanity"); + assert(is_aligned(flat_nullfree_not_atomic_count_offset, BytesPerWord), "sanity"); + assert(is_aligned(receiver_to_count_step, BytesPerWord), "sanity"); + flat_nullable_count_offset >>= LogBytesPerWord; + flat_nullfree_atomic_count_offset >>= LogBytesPerWord; + flat_nullfree_not_atomic_count_offset >>= LogBytesPerWord; + receiver_to_count_step >>= LogBytesPerWord; + +#ifdef ASSERT + int real_flat_nullable_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullable_count_offset()); + assert(flat_nullable_count_offset << LogBytesPerWord == real_flat_nullable_count_offset, "poly counter math"); + int real_flat_nullfree_atomic_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullfree_atomic_count_offset()); + assert(flat_nullfree_atomic_count_offset << LogBytesPerWord == real_flat_nullfree_atomic_count_offset, "poly counter math"); + int real_flat_nullfree_not_atomic_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullfree_not_atomic_count_offset()); + assert(flat_nullfree_not_atomic_count_offset << LogBytesPerWord == real_flat_nullfree_not_atomic_count_offset, "poly counter math"); +#endif + + // Corner case: no profile table. Increment poly counter and exit. + if (ReceiverTypeData::row_limit() == 0) { + ShouldNotReachHere(); + return; + } + + Label L_found_recv; + profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset + ReceiverTypeData::base_of_megamorphic_type_data(), ReceiverTypeData::row_limit()); + Register offset = rscratch1; int layout_kind_offset = in_bytes(FlatArrayKlass::layout_kind_offset()); Label null_free_non_atomic, null_free_atomic, nullable_atomic_flat, done, failure; - Register offset = rscratch1; cmpl(Address(recv, layout_kind_offset), (int)LayoutKind::NULL_FREE_ATOMIC_FLAT); jccb(Assembler::notEqual, null_free_non_atomic); movptr(offset, flat_nullfree_atomic_count_offset); @@ -5180,11 +5214,17 @@ template<> void MacroAssembler::profile_receiver_type_prolog(Regi bind(failure); stop("unexpected flat array"); bind(done); -} + Label L_count_update; + jmpb(L_count_update); -template void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset); -template void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset); + // Found a receiver, convert its slot offset to corresponding count offset. + bind(L_found_recv); + addptr(offset, receiver_to_count_step); + + bind(L_count_update); + addptr(Address(mdp, offset, Address::times_ptr), DataLayout::counter_increment); +} void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) { if (!VerifyOops || VerifyAdapterSharing) { diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp index 6040995eb46..d46fe9f4a91 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp @@ -709,10 +709,10 @@ class MacroAssembler: public Assembler { Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0); bool profile_receiver_type_helper(Register recv, Register mdp, - Register &offset, Label &L_found_recv, int base, uint row_limit); + Label &L_found_recv, int base, uint row_limit); - template void profile_receiver_type(Register recv, Register mdp, int mdp_offset); - template void profile_receiver_type_prolog(Register recv); + void profile_receiver_type(Register recv, Register mdp, int mdp_offset); + void profile_array_type_at_load(Register recv, Register mdp, int mdp_offset); // Debugging diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 462be5e342a..5ec8e9117ee 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -25,7 +25,6 @@ #ifndef SHARE_OOPS_METHODDATA_HPP #define SHARE_OOPS_METHODDATA_HPP -#include "methodData.hpp" #include "interpreter/bytecodes.hpp" #include "interpreter/invocationCounter.hpp" #include "oops/metadata.hpp" @@ -1249,11 +1248,11 @@ class MegamorphicTypeData { } // Code generation support - static ByteSize receiver_offset(int base, uint row) { - return ProfileData::cell_offset(base + static_receiver_cell_index(row)); + static ByteSize receiver_offset(uint row) { + return ProfileData::cell_offset(static_receiver_cell_index(row)); } - static ByteSize receiver_count_offset(int base, uint row) { - return ProfileData::cell_offset(base + static_receiver_count_cell_index(row)); + static ByteSize receiver_count_offset(uint row) { + return ProfileData::cell_offset(static_receiver_count_cell_index(row)); } // static ByteSize receiver_type_data_size() { // return cell_offset(static_cell_count()); @@ -1354,10 +1353,10 @@ class ReceiverTypeData : public CounterData { // Code generation support static ByteSize receiver_offset(uint row) { - return MegamorphicTypeData::receiver_offset(counter_cell_count, row); + return cell_offset(MegamorphicTypeData::static_receiver_cell_index(row) + counter_cell_count); } static ByteSize receiver_count_offset(uint row) { - return MegamorphicTypeData::receiver_count_offset(counter_cell_count, row); + return cell_offset(MegamorphicTypeData::static_receiver_count_cell_index(row) + counter_cell_count); } static ByteSize receiver_type_data_size() { return cell_offset(static_cell_count()); @@ -2228,10 +2227,10 @@ class ArrayLoadData : public ProfileData { } static ByteSize receiver_offset(uint row) { - return MegamorphicTypeData::receiver_offset(SingleTypeEntry::static_cell_count(), row); + return cell_offset(MegamorphicTypeData::static_receiver_cell_index(row) + SingleTypeEntry::static_cell_count()); } static ByteSize receiver_count_offset(uint row) { - return MegamorphicTypeData::receiver_count_offset(SingleTypeEntry::static_cell_count(), row); + return cell_offset(MegamorphicTypeData::static_receiver_count_cell_index(row) + SingleTypeEntry::static_cell_count()); } // static ByteSize receiver_type_data_size() { // return cell_offset(static_cell_count()); From 6d1ea801937825bca17cfc7e17d3b8ed24847900 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Wed, 29 Apr 2026 17:16:50 +0200 Subject: [PATCH 23/49] more --- src/hotspot/cpu/x86/macroAssembler_x86.cpp | 38 +++++++++---------- src/hotspot/cpu/x86/macroAssembler_x86.hpp | 2 +- src/hotspot/share/oops/methodData.hpp | 33 ++++++++-------- .../inlinetypes/TestArrayLoadProfiling.java | 35 +++++++++++++++++ 4 files changed, 71 insertions(+), 37 deletions(-) diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index ebffe6dcf68..f1b73250e5c 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -4935,11 +4935,11 @@ Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, return Address(rsp, scale_reg, scale_factor, offset); } -bool MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, Label &L_found_recv, int mdp_offset, uint row_limit) { - int base_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(0)); - int end_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(row_limit)); - int receiver_step = in_bytes(MegamorphicTypeData::receiver_offset(1)) - base_receiver_offset; - int receiver_to_count_step = in_bytes(MegamorphicTypeData::receiver_count_offset(0)) - base_receiver_offset; +bool MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, Label &L_found_recv, int mdp_offset, int base, uint row_limit) { + int base_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, 0)); + int end_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, row_limit)); + int receiver_step = in_bytes(MegamorphicTypeData::receiver_offset(base, 1)) - base_receiver_offset; + int receiver_to_count_step = in_bytes(MegamorphicTypeData::receiver_count_offset(base, 0)) - base_receiver_offset; assert(is_aligned(mdp_offset, BytesPerWord), "sanity"); base_receiver_offset += mdp_offset; @@ -4958,8 +4958,8 @@ bool MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, L // We are about to walk the MDO slots without asking for offsets. // Check that our math hits all the right spots. for (uint c = 0; c < ReceiverTypeData::row_limit(); c++) { - int real_recv_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_offset(c)); - int real_count_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_count_offset(c)); + int real_recv_offset = mdp_offset + in_bytes(MegamorphicTypeData::receiver_offset(base, c)); + int real_count_offset = mdp_offset + in_bytes(MegamorphicTypeData::receiver_count_offset(base, c)); int offset = base_receiver_offset + receiver_step*c; int count_offset = offset + receiver_to_count_step; assert((offset << LogBytesPerWord) == real_recv_offset, "receiver slot math"); @@ -5025,7 +5025,7 @@ bool MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, L movptr(offset, base_receiver_offset); bind(L_loop_search_receiver); cmpptr(recv, Address(mdp, offset, Address::times_ptr)); - jccb(Assembler::equal, L_found_recv); + jcc(Assembler::equal, L_found_recv); addptr(offset, receiver_step); cmpptr(offset, end_receiver_offset); jccb(Assembler::notEqual, L_loop_search_receiver); @@ -5111,19 +5111,19 @@ bool MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, L // counter updates are not atomic. // void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { - int poly_count_offset = in_bytes(ReceiverTypeData::count_offset()); - int base_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(0)); - int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset; + int poly_count_offset = in_bytes(ReceiverTypeData::count_offset()); + int base_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(0)); + int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset; // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset. assert(is_aligned(mdp_offset, BytesPerWord), "sanity"); - poly_count_offset += mdp_offset; + poly_count_offset += mdp_offset; // Scale down to optimize encoding. Slots are pointer-sized. - assert(is_aligned(poly_count_offset, BytesPerWord), "sanity"); - assert(is_aligned(receiver_to_count_step, BytesPerWord), "sanity"); - poly_count_offset >>= LogBytesPerWord; - receiver_to_count_step >>= LogBytesPerWord; + assert(is_aligned(poly_count_offset, BytesPerWord), "sanity"); + assert(is_aligned(receiver_to_count_step, BytesPerWord), "sanity"); + poly_count_offset >>= LogBytesPerWord; + receiver_to_count_step >>= LogBytesPerWord; #ifdef ASSERT int real_poly_count_offset = mdp_offset + in_bytes(ReceiverTypeData::count_offset()); @@ -5137,7 +5137,7 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ } Label L_found_recv; - profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset + ReceiverTypeData::base_of_megamorphic_type_data(), ReceiverTypeData::row_limit()); + profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ReceiverTypeData::base_of_megamorphic_type_data(), ReceiverTypeData::row_limit()); Register offset = rscratch1; movptr(offset, poly_count_offset); Label L_count_update; @@ -5172,7 +5172,7 @@ void MacroAssembler::profile_array_type_at_load(Register recv, Register mdp, int flat_nullable_count_offset >>= LogBytesPerWord; flat_nullfree_atomic_count_offset >>= LogBytesPerWord; flat_nullfree_not_atomic_count_offset >>= LogBytesPerWord; - receiver_to_count_step >>= LogBytesPerWord; + receiver_to_count_step >>= LogBytesPerWord; #ifdef ASSERT int real_flat_nullable_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullable_count_offset()); @@ -5190,7 +5190,7 @@ void MacroAssembler::profile_array_type_at_load(Register recv, Register mdp, int } Label L_found_recv; - profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset + ReceiverTypeData::base_of_megamorphic_type_data(), ReceiverTypeData::row_limit()); + profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ArrayLoadData::base_of_megamorphic_type_data(), ArrayLoadData::row_limit()); Register offset = rscratch1; int layout_kind_offset = in_bytes(FlatArrayKlass::layout_kind_offset()); Label null_free_non_atomic, null_free_atomic, nullable_atomic_flat, done, failure; diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp index d46fe9f4a91..a063bb14b62 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp @@ -709,7 +709,7 @@ class MacroAssembler: public Assembler { Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0); bool profile_receiver_type_helper(Register recv, Register mdp, - Label &L_found_recv, int base, uint row_limit); + Label &L_found_recv, int mdp_offset, int base, uint row_limit); void profile_receiver_type(Register recv, Register mdp, int mdp_offset); void profile_array_type_at_load(Register recv, Register mdp, int mdp_offset); diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 5ec8e9117ee..ca13217c4d7 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -1205,17 +1205,17 @@ class MegamorphicTypeData { uint row_limit() const { return (uint)_type_width; } - static int static_receiver_cell_index(uint row) { - return receiver0_offset + row * receiver_type_row_cell_count; + static int static_receiver_cell_index(int base, uint row) { + return base + receiver0_offset + row * receiver_type_row_cell_count; } - static int static_receiver_count_cell_index(uint row) { - return count0_offset + row * receiver_type_row_cell_count; + static int static_receiver_count_cell_index(int base, uint row) { + return base + count0_offset + row * receiver_type_row_cell_count; } int receiver_cell_index(uint row) const { - return _base_off + receiver0_offset + row * receiver_type_row_cell_count; + return static_receiver_cell_index(_base_off, row); } int receiver_count_cell_index(uint row) const { - return _base_off + count0_offset + row * receiver_type_row_cell_count; + return static_receiver_count_cell_index(_base_off, row); } Klass* receiver(uint row) const { @@ -1248,11 +1248,11 @@ class MegamorphicTypeData { } // Code generation support - static ByteSize receiver_offset(uint row) { - return ProfileData::cell_offset(static_receiver_cell_index(row)); + static ByteSize receiver_offset(int base, uint row) { + return ProfileData::cell_offset(static_receiver_cell_index(base, row)); } - static ByteSize receiver_count_offset(uint row) { - return ProfileData::cell_offset(static_receiver_count_cell_index(row)); + static ByteSize receiver_count_offset(int base, uint row) { + return ProfileData::cell_offset(static_receiver_count_cell_index(base, row)); } // static ByteSize receiver_type_data_size() { // return cell_offset(static_cell_count()); @@ -1278,7 +1278,7 @@ class ReceiverTypeData : public CounterData { public: ReceiverTypeData(DataLayout* layout) : CounterData(layout), - _megamorphic_type_data(this, counter_cell_count, TypeProfileWidth) { + _megamorphic_type_data(this, base_of_megamorphic_type_data(), TypeProfileWidth) { assert(layout->tag() == DataLayout::receiver_type_data_tag || layout->tag() == DataLayout::virtual_call_data_tag || layout->tag() == DataLayout::virtual_call_type_data_tag || @@ -1353,10 +1353,10 @@ class ReceiverTypeData : public CounterData { // Code generation support static ByteSize receiver_offset(uint row) { - return cell_offset(MegamorphicTypeData::static_receiver_cell_index(row) + counter_cell_count); + return cell_offset(MegamorphicTypeData::static_receiver_cell_index(base_of_megamorphic_type_data(), row)); } static ByteSize receiver_count_offset(uint row) { - return cell_offset(MegamorphicTypeData::static_receiver_count_cell_index(row) + counter_cell_count); + return cell_offset(MegamorphicTypeData::static_receiver_count_cell_index(base_of_megamorphic_type_data(), row)); } static ByteSize receiver_type_data_size() { return cell_offset(static_cell_count()); @@ -2125,7 +2125,7 @@ class ArrayLoadData : public ProfileData { ArrayLoadData(DataLayout* layout) : ProfileData(layout), _element(0), - _megamorphic_type_data(this, SingleTypeEntry::static_cell_count(), TypeProfileWidth) { + _megamorphic_type_data(this, base_of_megamorphic_type_data(), TypeProfileWidth) { assert(layout->tag() == DataLayout::array_load_data_tag, "wrong type"); _element.set_profile_data(this); } @@ -2197,7 +2197,6 @@ class ArrayLoadData : public ProfileData { } // Code generation support - static ByteSize element_offset() { return cell_offset(0); } @@ -2227,10 +2226,10 @@ class ArrayLoadData : public ProfileData { } static ByteSize receiver_offset(uint row) { - return cell_offset(MegamorphicTypeData::static_receiver_cell_index(row) + SingleTypeEntry::static_cell_count()); + return cell_offset(MegamorphicTypeData::static_receiver_cell_index(base_of_megamorphic_type_data(), row)); } static ByteSize receiver_count_offset(uint row) { - return cell_offset(MegamorphicTypeData::static_receiver_count_cell_index(row) + SingleTypeEntry::static_cell_count()); + return cell_offset(MegamorphicTypeData::static_receiver_count_cell_index(base_of_megamorphic_type_data(), row)); } // static ByteSize receiver_type_data_size() { // return cell_offset(static_cell_count()); diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 8e37d63e6b4..40ef7f87598 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -60,12 +60,17 @@ public static void main(String[] args) { static MyValue3[] array10 = (MyValue3[])ValueClass.newNullRestrictedNonAtomicArray(MyValue3.class, 1, new MyValue3(42)); static MyValue3[] array11 = (MyValue3[])ValueClass.newNullRestrictedAtomicArray(MyValue3.class, 1, new MyValue3(42)); static MyValue3[] array12 = (MyValue3[])ValueClass.newReferenceArray(MyValue3.class, 1); + static MyValue4[] array13 = (MyValue4[])ValueClass.newNullRestrictedNonAtomicArray(MyValue4.class, 1, new MyValue4(42)); + static MyValue4[] array14 = (MyValue4[])ValueClass.newNullRestrictedAtomicArray(MyValue4.class, 1, new MyValue4(42)); + static MyValue4[] array15 = (MyValue4[])ValueClass.newNullableAtomicArray(MyValue4.class, 1); + static MyValue4[] array16 = { new MyValue4((byte)42) }; static { array6[0] = new MyValue1((byte)42); array7[0] = new MyValue2((byte)42); array8[0] = new MyValue1((byte)42); array9[0] = new MyValue1((byte)42); array12[0] = new MyValue3((byte)42); + array15[0] = new MyValue4((byte)42); } @Test @@ -494,6 +499,25 @@ public static void test25Runner() { test25(array12, 0, 0, 0); } + @Test + // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "4" }) + // @IR(failOn = IRNode.ALLOC) + public static I test26(I[] array) { + return array[0]; + } + + @Run(test = "test26") + public static void test26Runner() { + // for (int i = 0; i < 10; i++) { + // test26(array1); + // test26(array2); + // } + test26(array13); + test26(array14); + // test26(array15); + // test26(array16); + } + // @Test // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "9" }) // @IR(failOn = IRNode.ALLOC) @@ -555,6 +579,17 @@ public void m() { } } + static value class MyValue4 implements I { + int intField; + + MyValue4(int intField) { + this.intField = intField; + } + + public void m() { + } + } + static class A implements I { public void m() { } From 182b9dda5b392711f2f2e13458a9598656f81f04 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 30 Apr 2026 09:41:51 +0200 Subject: [PATCH 24/49] more --- src/hotspot/share/ci/ciMethod.cpp | 2 +- src/hotspot/share/oops/methodData.hpp | 4 -- .../inlinetypes/TestArrayLoadProfiling.java | 46 +++++++++++++------ 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp index 589f0f3323b..ef2b256a144 100644 --- a/src/hotspot/share/ci/ciMethod.cpp +++ b/src/hotspot/share/ci/ciMethod.cpp @@ -546,7 +546,7 @@ ciCallProfile ciMethod::call_profile_at_bci(int bci) { result._count = count; } else if (data->is_ArrayLoadData()) { ciArrayLoadData* load = (ciArrayLoadData*) data->as_ArrayLoadData(); - int count = saturated_add(load->flat_nullable_count(), load->flat_nullfree_atomic_count()) + load->flat_nullfree_not_atomic_count(); + int count = load->flat_count(); ciMegamorphicTypeData* megamorphic_type_data = load->megamorphic_type_data(); return megamorphic_profile_at_bci(bci, count, megamorphic_type_data); } diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index ca13217c4d7..1386e4d509d 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -2209,10 +2209,6 @@ class ArrayLoadData : public ProfileData { return cell_offset(not_flat_nullable_count_off()); } - static ByteSize count_offset() { - return cell_offset(flat_nullable_count_off()); - } - static ByteSize flat_nullable_count_offset() { return cell_offset(flat_nullable_count_off()); } diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 40ef7f87598..969b259c901 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -60,10 +60,14 @@ public static void main(String[] args) { static MyValue3[] array10 = (MyValue3[])ValueClass.newNullRestrictedNonAtomicArray(MyValue3.class, 1, new MyValue3(42)); static MyValue3[] array11 = (MyValue3[])ValueClass.newNullRestrictedAtomicArray(MyValue3.class, 1, new MyValue3(42)); static MyValue3[] array12 = (MyValue3[])ValueClass.newReferenceArray(MyValue3.class, 1); - static MyValue4[] array13 = (MyValue4[])ValueClass.newNullRestrictedNonAtomicArray(MyValue4.class, 1, new MyValue4(42)); - static MyValue4[] array14 = (MyValue4[])ValueClass.newNullRestrictedAtomicArray(MyValue4.class, 1, new MyValue4(42)); + static MyValue4[] array13 = (MyValue4[])ValueClass.newNullRestrictedNonAtomicArray(MyValue4.class, 1, new MyValue4((byte)42)); // atomic + static MyValue4[] array14 = (MyValue4[])ValueClass.newNullRestrictedAtomicArray(MyValue4.class, 1, new MyValue4((byte)42)); // atomic static MyValue4[] array15 = (MyValue4[])ValueClass.newNullableAtomicArray(MyValue4.class, 1); static MyValue4[] array16 = { new MyValue4((byte)42) }; + static MyValue5[] array17 = (MyValue5[])ValueClass.newNullRestrictedNonAtomicArray(MyValue5.class, 1, new MyValue5((byte)42)); // non atomic + static MyValue5[] array18 = (MyValue5[])ValueClass.newNullRestrictedAtomicArray(MyValue5.class, 1, new MyValue5((byte)42)); // non atomic + static MyValue5[] array19 = (MyValue5[])ValueClass.newNullableAtomicArray(MyValue5.class, 1); + static MyValue5[] array20 = { new MyValue5((byte)42) }; static { array6[0] = new MyValue1((byte)42); array7[0] = new MyValue2((byte)42); @@ -71,6 +75,7 @@ public static void main(String[] args) { array9[0] = new MyValue1((byte)42); array12[0] = new MyValue3((byte)42); array15[0] = new MyValue4((byte)42); + array19[0] = new MyValue5((byte)42); } @Test @@ -508,14 +513,16 @@ public static I test26(I[] array) { @Run(test = "test26") public static void test26Runner() { - // for (int i = 0; i < 10; i++) { - // test26(array1); - // test26(array2); - // } - test26(array13); - test26(array14); - // test26(array15); - // test26(array16); + for (int i = 0; i < 10; i++) { + test26(array1); + test26(array2); + } + // test26(array13); + // test26(array14); + // test26(array17); + // test26(array18); + test26(array15); + test26(array16); } // @Test @@ -580,10 +587,23 @@ public void m() { } static value class MyValue4 implements I { - int intField; + byte byteField; + byte byteField2; - MyValue4(int intField) { - this.intField = intField; + MyValue4(byte byteField) { + this.byteField = byteField; + this.byteField2 = byteField; + } + + public void m() { + } + } + + static value class MyValue5 implements I { + byte byteField; + + MyValue5(byte byteField) { + this.byteField = byteField; } public void m() { From 0f75a701991284f58848cec43cf2dda0227bfbc9 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 30 Apr 2026 15:21:30 +0200 Subject: [PATCH 25/49] exp --- .../valhalla/inlinetypes/TestArrayLoadProfiling.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 969b259c901..d8acb643dd1 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -513,6 +513,18 @@ public static I test26(I[] array) { @Run(test = "test26") public static void test26Runner() { + // if (ValueClass.isAtomicArray(array13)) { + // throw new RuntimeException("atomic"); + // } + if (!ValueClass.isAtomicArray(array14)) { + throw new RuntimeException("not atomic"); + } + // if (ValueClass.isAtomicArray(array17)) { + // throw new RuntimeException("atomic"); + // } + if (!ValueClass.isAtomicArray(array18)) { + throw new RuntimeException("not atomic"); + } for (int i = 0; i < 10; i++) { test26(array1); test26(array2); From 50cb4c5d282cbdf1d37108d1b7590da44a5ce75b Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 30 Apr 2026 15:52:51 +0200 Subject: [PATCH 26/49] exp --- .../inlinetypes/TestArrayLoadProfiling.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index d8acb643dd1..3ca30d31dc7 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -513,18 +513,10 @@ public static I test26(I[] array) { @Run(test = "test26") public static void test26Runner() { - // if (ValueClass.isAtomicArray(array13)) { - // throw new RuntimeException("atomic"); - // } - if (!ValueClass.isAtomicArray(array14)) { - throw new RuntimeException("not atomic"); - } - // if (ValueClass.isAtomicArray(array17)) { - // throw new RuntimeException("atomic"); - // } - if (!ValueClass.isAtomicArray(array18)) { - throw new RuntimeException("not atomic"); - } + System.out.println("array13 is " + (ValueClass.isFlatArray(array13) ? "flat" : " non flat") + " " + (ValueClass.isAtomicArray(array13) ? "atomic" : " non atomic")); + System.out.println("array14 is " + (ValueClass.isFlatArray(array13) ? "flat" : " non flat") + " " + (ValueClass.isAtomicArray(array14) ? "atomic" : " non atomic")); + System.out.println("array17 is " + (ValueClass.isFlatArray(array13) ? "flat" : " non flat") + " " + (ValueClass.isAtomicArray(array17) ? "atomic" : " non atomic")); + System.out.println("array18 is " + (ValueClass.isFlatArray(array13) ? "flat" : " non flat") + " " + (ValueClass.isAtomicArray(array18) ? "atomic" : " non atomic")); for (int i = 0; i < 10; i++) { test26(array1); test26(array2); From 189380502bff5d3b31d78d38ec73faf753f2d9b8 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 30 Apr 2026 16:06:23 +0200 Subject: [PATCH 27/49] more --- .../compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 3ca30d31dc7..c9bbb1bc1f3 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -26,6 +26,7 @@ * @library /test/lib / * @enablePreview * @modules java.base/jdk.internal.value + * @modules java.base/jdk.internal.vm.annotation * @requires (os.simpleArch == "x64" | os.simpleArch == "aarch64") * @library /test/lib / * @build jdk.test.whitebox.WhiteBox @@ -37,6 +38,7 @@ import compiler.lib.ir_framework.*; import jdk.internal.value.ValueClass; +import jdk.internal.vm.annotation.LooselyConsistentValue; import jdk.test.whitebox.WhiteBox; import compiler.whitebox.CompilerWhiteBoxTest; import java.lang.reflect.Method; @@ -590,6 +592,7 @@ public void m() { } } + @LooselyConsistentValue static value class MyValue4 implements I { byte byteField; byte byteField2; From aad15cfd85325dcb246d84a287d89f02ab5dce62 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 7 May 2026 10:08:44 +0200 Subject: [PATCH 28/49] more --- src/hotspot/share/oops/methodData.hpp | 4 +- src/hotspot/share/opto/inlinetypenode.cpp | 93 ++++++++++++------- src/hotspot/share/opto/inlinetypenode.hpp | 2 +- src/hotspot/share/opto/parse2.cpp | 43 ++++++++- .../inlinetypes/TestArrayLoadProfiling.java | 27 +++--- 5 files changed, 116 insertions(+), 53 deletions(-) diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 1386e4d509d..da15fed8833 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -2153,11 +2153,11 @@ class ArrayLoadData : public ProfileData { } int not_flat_count() const { - return not_flat_null_free_count() + not_flat_nullable_count(); + return saturated_add(not_flat_null_free_count(), not_flat_nullable_count()); } int flat_count() const { - return flat_nullfree_atomic_count() + flat_nullfree_not_atomic_count() + flat_nullable_count(); + return saturated_add(saturated_add(flat_nullfree_atomic_count(), flat_nullfree_not_atomic_count()), flat_nullable_count()); } int not_flat_null_free_count() const { diff --git a/src/hotspot/share/opto/inlinetypenode.cpp b/src/hotspot/share/opto/inlinetypenode.cpp index 0a5939ba7bf..4d5eb33f32a 100644 --- a/src/hotspot/share/opto/inlinetypenode.cpp +++ b/src/hotspot/share/opto/inlinetypenode.cpp @@ -1397,7 +1397,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_impl(GraphKit* kit, ciInlineKlass return LoadFlatNode::load(kit, vk, base, ptr, null_free, trust_null_free_oop, decorators); } -InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx) { +InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx, float null_free_prob,float null_free_atomic_prob) { assert(vk->maybe_flat_in_array(), "element type %s cannot be flat in array", vk->name()->as_utf8()); PhaseGVN& gvn = kit->gvn(); // The flat field loads are dependent on both the array layout checks as well as the range check. @@ -1406,6 +1406,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas InlineTypeNode* vt_nullable = nullptr; InlineTypeNode* vt_null_free = nullptr; InlineTypeNode* vt_non_atomic = nullptr; + Node *top = kit->C->top(); RegionNode* region = new RegionNode(4); gvn.set_type(region, Type::CONTROL); @@ -1423,20 +1424,25 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas kit->record_for_igvn(io); Node* bol_null_free = kit->null_free_array_test(base); // Argument evaluation order is undefined in C++ and since this sets control, it needs to come first - IfNode* iff_null_free = kit->create_and_map_if(kit->control(), bol_null_free, PROB_FAIR, COUNT_UNKNOWN); + IfNode* iff_null_free = kit->create_and_map_if(kit->control(), bol_null_free, clamp(null_free_prob, PROB_MIN, PROB_MAX), COUNT_UNKNOWN); // Nullable kit->set_control(kit->IfFalse(iff_null_free)); if (!kit->stopped()) { assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8()); kit->set_all_memory(input_memory_state); - Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true); - Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); - vt_nullable = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, false, decorators); + if (null_free_prob == 0 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + PreserveJVMState pjvms(kit); + kit->uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } else { + Node* cast = kit->cast_to_flat_array_exact(base, vk, false, true); + Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); + vt_nullable = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, false, decorators); - region->init_req(1, kit->control()); - mem->set_req(1, kit->reset_memory()); - io->set_req(1, kit->i_o()); + region->init_req(1, kit->control()); + mem->set_req(1, kit->reset_memory()); + io->set_req(1, kit->i_o()); + } } // Null-free @@ -1444,35 +1450,52 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas if (!kit->stopped()) { kit->set_all_memory(input_memory_state); - Node* bol_atomic = kit->null_free_atomic_array_test(base, vk); - IfNode* iff_atomic = kit->create_and_map_if(kit->control(), bol_atomic, PROB_FAIR, COUNT_UNKNOWN); - - // Atomic - kit->set_control(kit->IfTrue(iff_atomic)); - if (!kit->stopped()) { - assert(vk->has_null_free_atomic_layout(), "element type %s does not have a null-free atomic flat layout", vk->name()->as_utf8()); - kit->set_all_memory(input_memory_state); - Node* cast = kit->cast_to_flat_array_exact(base, vk, true, true); - Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); - vt_null_free = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, true, decorators); - - region->init_req(2, kit->control()); - mem->set_req(2, kit->reset_memory()); - io->set_req(2, kit->i_o()); - } + if (null_free_prob == 1 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + PreserveJVMState pjvms(kit); + kit->uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } else { + Node *bol_atomic = kit->null_free_atomic_array_test(base, vk); + IfNode *iff_atomic = kit->create_and_map_if(kit->control(), bol_atomic, + clamp(null_free_atomic_prob, PROB_MIN, PROB_MAX), COUNT_UNKNOWN); - // Non-Atomic - kit->set_control(kit->IfFalse(iff_atomic)); - if (!kit->stopped()) { - assert(vk->has_null_free_non_atomic_layout(), "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8()); - kit->set_all_memory(input_memory_state); - Node* cast = kit->cast_to_flat_array_exact(base, vk, true, false); - Node* ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); - vt_non_atomic = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, false, false, true, decorators); + // Atomic + kit->set_control(kit->IfTrue(iff_atomic)); + if (!kit->stopped()) { + assert(vk->has_null_free_atomic_layout(), "element type %s does not have a null-free atomic flat layout", + vk->name()->as_utf8()); + kit->set_all_memory(input_memory_state); + if (null_free_atomic_prob == 0 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + PreserveJVMState pjvms(kit); + kit->uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } else { + Node *cast = kit->cast_to_flat_array_exact(base, vk, true, true); + Node *ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); + vt_null_free = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, true, false, true, decorators); + region->init_req(2, kit->control()); + mem->set_req(2, kit->reset_memory()); + io->set_req(2, kit->i_o()); + } + } - region->init_req(3, kit->control()); - mem->set_req(3, kit->reset_memory()); - io->set_req(3, kit->i_o()); + // Non-Atomic + kit->set_control(kit->IfFalse(iff_atomic)); + if (!kit->stopped()) { + if (null_free_atomic_prob == 1 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + PreserveJVMState pjvms(kit); + kit->uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + } else { + assert(vk->has_null_free_non_atomic_layout(), + "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8()); + kit->set_all_memory(input_memory_state); + Node *cast = kit->cast_to_flat_array_exact(base, vk, true, false); + Node *ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); + vt_non_atomic = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, false, false, true, decorators); + + region->init_req(3, kit->control()); + mem->set_req(3, kit->reset_memory()); + io->set_req(3, kit->i_o()); + } + } } } diff --git a/src/hotspot/share/opto/inlinetypenode.hpp b/src/hotspot/share/opto/inlinetypenode.hpp index 44eba6fbc72..4d48d7f08e6 100644 --- a/src/hotspot/share/opto/inlinetypenode.hpp +++ b/src/hotspot/share/opto/inlinetypenode.hpp @@ -88,7 +88,7 @@ class InlineTypeNode : public TypeNode { // Create and initialize by loading the field values from a flat field or array static InlineTypeNode* make_from_flat(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* ptr, bool atomic, bool immutable_memory, bool null_free, DecoratorSet decorators); - static InlineTypeNode* make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx); + static InlineTypeNode* make_from_flat_array(GraphKit* kit, ciInlineKlass* vk, Node* base, Node* idx, float null_free_prob = PROB_FAIR, float null_free_atomic_prob = PROB_FAIR); // Create and initialize with the inputs or outputs of a MultiNode (method entry or call) static InlineTypeNode* make_from_multi(GraphKit* kit, MultiNode* multi, ciInlineKlass* vk, uint& base_input, bool in, bool null_free = true); // Create with null field values diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 500fbc8c14e..46715b507b1 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -304,10 +304,51 @@ class ArrayLoad { // } if (element_ptr->is_inlinetypeptr()) { + ciArrayLoadData* array_load = profile_data(); + float null_free_prob = PROB_FAIR; + float null_free_atomic_prob = PROB_FAIR; + if (array_load != nullptr && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + int flat_nullable_count = array_load->flat_nullable_count(); + int flat_nullfree_atomic_count = array_load->flat_nullfree_atomic_count(); + int flat_nullfree_not_atomic_count = array_load->flat_nullfree_not_atomic_count(); + ciMegamorphicTypeData* megamorphic_type_data = array_load->megamorphic_type_data(); + for (uint i = 0; i < megamorphic_type_data->row_limit(); ++i) { + ciKlass* receiver = megamorphic_type_data->receiver(i); + if (receiver != nullptr && receiver->is_subtype_of(element_ptr->is_instptr()->instance_klass())) { + ciFlatArrayKlass* flat_array_klass = receiver->as_flat_array_klass();; + if (!flat_array_klass->is_elem_null_free()) { + saturated_add(flat_nullable_count, megamorphic_type_data->receiver_count(i)); + } else { + if (flat_array_klass->is_elem_atomic()) { + saturated_add(flat_nullfree_atomic_count, megamorphic_type_data->receiver_count(i)); + } else { + saturated_add(flat_nullfree_not_atomic_count, megamorphic_type_data->receiver_count(i)); + } + } + } + } + float count = saturated_add(flat_nullable_count, saturated_add(flat_nullfree_atomic_count, flat_nullfree_not_atomic_count)); + if (flat_nullable_count == 0) { + null_free_prob = 1; + } else if (flat_nullable_count == count) { + null_free_prob = 0; + } else { + null_free_prob = clamp(1 - (float)flat_nullable_count / count, PROB_MIN, PROB_MAX); + } + float null_free_count = saturated_add(flat_nullfree_atomic_count, flat_nullfree_not_atomic_count); + if (flat_nullfree_atomic_count == 0) { + null_free_atomic_prob = 0; + } else if (flat_nullfree_atomic_count == null_free_count) { + null_free_atomic_prob = 1; + } else { + null_free_atomic_prob = clamp((float) flat_nullfree_atomic_count / null_free_count, PROB_MIN, PROB_MAX); + } + } + ciInlineKlass* vk = element_ptr->inline_klass(); Node* flat_array = _parse.cast_to_flat_array(array, vk); - InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); + InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index, null_free_prob, null_free_atomic_prob); ld = vt; if (_region != nullptr) { diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index c9bbb1bc1f3..6a5b024f1fb 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -70,6 +70,7 @@ public static void main(String[] args) { static MyValue5[] array18 = (MyValue5[])ValueClass.newNullRestrictedAtomicArray(MyValue5.class, 1, new MyValue5((byte)42)); // non atomic static MyValue5[] array19 = (MyValue5[])ValueClass.newNullableAtomicArray(MyValue5.class, 1); static MyValue5[] array20 = { new MyValue5((byte)42) }; + static MyValue4[] array21 = (MyValue4[])ValueClass.newReferenceArray(MyValue4.class, 1); static { array6[0] = new MyValue1((byte)42); array7[0] = new MyValue2((byte)42); @@ -78,6 +79,7 @@ public static void main(String[] args) { array12[0] = new MyValue3((byte)42); array15[0] = new MyValue4((byte)42); array19[0] = new MyValue5((byte)42); + array21[0] = new MyValue4((byte)42); } @Test @@ -396,7 +398,7 @@ public static void test20Runner() { @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "8" }) @IR(failOn = IRNode.ALLOC) public static void test21() { - I[] array = array3; + I[] array = array16; test21Inline(array); } @@ -404,9 +406,9 @@ public static void test21() { public static void test21Runner() { test21(); test21Inline(array2); - test21Inline(array4); + test21Inline(array20); test21Inline(array5); - test21Inline(array7); + test21Inline(array1); } @ForceInline @@ -515,20 +517,17 @@ public static I test26(I[] array) { @Run(test = "test26") public static void test26Runner() { - System.out.println("array13 is " + (ValueClass.isFlatArray(array13) ? "flat" : " non flat") + " " + (ValueClass.isAtomicArray(array13) ? "atomic" : " non atomic")); - System.out.println("array14 is " + (ValueClass.isFlatArray(array13) ? "flat" : " non flat") + " " + (ValueClass.isAtomicArray(array14) ? "atomic" : " non atomic")); - System.out.println("array17 is " + (ValueClass.isFlatArray(array13) ? "flat" : " non flat") + " " + (ValueClass.isAtomicArray(array17) ? "atomic" : " non atomic")); - System.out.println("array18 is " + (ValueClass.isFlatArray(array13) ? "flat" : " non flat") + " " + (ValueClass.isAtomicArray(array18) ? "atomic" : " non atomic")); - for (int i = 0; i < 10; i++) { - test26(array1); - test26(array2); - } - // test26(array13); + // test26(array21); // not flat, nullable + // test26(array20); // flat, nullable, atomic + // test26(array19); // flat, nullable, atomic + // test26(array14); // flat, null restricted atomic + test26(array13); // flat, null restricted, non atomic + // test26(array14); // test26(array17); // test26(array18); - test26(array15); - test26(array16); + // test26(array15); + // test26(array16); } // @Test From 1ab5fb17814d5148cfde8af21bf59db72b7fcb10 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 7 May 2026 17:25:42 +0200 Subject: [PATCH 29/49] more --- src/hotspot/share/opto/inlinetypenode.cpp | 4 +- .../inlinetypes/TestArrayLoadProfiling.java | 110 +++++++++++++----- 2 files changed, 81 insertions(+), 33 deletions(-) diff --git a/src/hotspot/share/opto/inlinetypenode.cpp b/src/hotspot/share/opto/inlinetypenode.cpp index 4d5eb33f32a..cfb1adb6d02 100644 --- a/src/hotspot/share/opto/inlinetypenode.cpp +++ b/src/hotspot/share/opto/inlinetypenode.cpp @@ -1431,7 +1431,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas if (!kit->stopped()) { assert(vk->has_nullable_atomic_layout(), "element type %s does not have a nullable flat layout", vk->name()->as_utf8()); kit->set_all_memory(input_memory_state); - if (null_free_prob == 0 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + if (null_free_prob == 1 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { PreserveJVMState pjvms(kit); kit->uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); } else { @@ -1450,7 +1450,7 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas if (!kit->stopped()) { kit->set_all_memory(input_memory_state); - if (null_free_prob == 1 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + if (null_free_prob == 0 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { PreserveJVMState pjvms(kit); kit->uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); } else { diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 6a5b024f1fb..dad68aedee9 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -71,6 +71,7 @@ public static void main(String[] args) { static MyValue5[] array19 = (MyValue5[])ValueClass.newNullableAtomicArray(MyValue5.class, 1); static MyValue5[] array20 = { new MyValue5((byte)42) }; static MyValue4[] array21 = (MyValue4[])ValueClass.newReferenceArray(MyValue4.class, 1); + static MyValue2[] array22 = (MyValue2[])ValueClass.newNullRestrictedAtomicArray(MyValue2.class, 1, new MyValue2((byte)42)); static { array6[0] = new MyValue1((byte)42); array7[0] = new MyValue2((byte)42); @@ -382,7 +383,7 @@ public static void test19Runner() { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "8" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "9" }) @IR(failOn = IRNode.ALLOC) public static void test20(MyValue1[] array) { array[0].m(); @@ -395,7 +396,7 @@ public static void test20Runner() { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "8" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "8" }) @IR(failOn = IRNode.ALLOC) public static void test21() { I[] array = array16; @@ -405,10 +406,10 @@ public static void test21() { @Run(test = "test21") public static void test21Runner() { test21(); - test21Inline(array2); - test21Inline(array20); - test21Inline(array5); - test21Inline(array1); + test21Inline(array2); // flat, nullable + test21Inline(array20); // flat, nullable + test21Inline(array5); // not flat + test21Inline(array1); // flat, nullable } @ForceInline @@ -417,18 +418,19 @@ static void test21Inline(I[] array) { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "7" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "7" }) @IR(failOn = IRNode.ALLOC) public static void test22() { - I[] array = array3; + I[] array = array16; test22Inline(array); } @Run(test = "test22") public static void test22Runner() { test22(); - test22Inline(array2); - test22Inline(array4); + test22Inline(array2); // flat, nullable + test22Inline(array20); // flat, nullable + test22Inline(array1); // flat, nullable } @ForceInline @@ -461,7 +463,7 @@ static void test22Inline(I[] array) { // } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "9" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "9" }) @IR(failOn = IRNode.ALLOC) public static void test23(I[] array) { test23Inline(array[0]); @@ -469,11 +471,11 @@ public static void test23(I[] array) { @Run(test = "test23") public static void test23Runner() { - test23(array1); - test23(array3); - test23(array6); - test23(array8); - test23(array9); + test23(array1); // flat nullable + //test23(array3); // flat null free non atomic + test23(array6); // not flat + test23(array8); // flat null free atomic + test23(array9); // flat nullable atomic test23Inline(array2[0]); test23Inline(array4[0]); test23Inline(array5[0]); @@ -507,29 +509,71 @@ public static int test25(MyValue3[] array, int i, int j, int k) { public static void test25Runner() { test25(array12, 0, 0, 0); } - + @Test - // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "4" }) - // @IR(failOn = IRNode.ALLOC) - public static I test26(I[] array) { - return array[0]; + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "3", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "7" }) + @IR(failOn = IRNode.ALLOC) + public static void test26() { + I[] array = array14; + test26Inline(array); } @Run(test = "test26") public static void test26Runner() { - // test26(array21); // not flat, nullable - // test26(array20); // flat, nullable, atomic - // test26(array19); // flat, nullable, atomic - // test26(array14); // flat, null restricted atomic - test26(array13); // flat, null restricted, non atomic + test26(); + test26Inline(array8); // flat, null free atomic + test26Inline(array22); // flat, null free atomic + test26Inline(array18); // flat, null free atomic + } + + @ForceInline + static void test26Inline(I[] array) { + array[0].m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "3", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "7" }) + @IR(failOn = IRNode.ALLOC) + public static void test27() { + I[] array = array13; + test27Inline(array); + } - // test26(array14); - // test26(array17); - // test26(array18); - // test26(array15); - // test26(array16); + @Run(test = "test27") + public static void test27Runner() { + test27(); + test27Inline(array3); // flat, null free non atomic + test27Inline(array10); // flat, null free non atomic + test27Inline(array17); // flat, null free non atomic + } + + @ForceInline + static void test27Inline(I[] array) { + array[0].m(); } + // @Test + // // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "4" }) + // // @IR(failOn = IRNode.ALLOC) + // public static I test26(I[] array) { + // return array[0]; + // } + + // @Run(test = "test26") + // public static void test26Runner() { + // // test26(array21); // not flat, nullable + // // test26(array20); // flat, nullable, atomic + // // test26(array19); // flat, nullable, atomic + // // test26(array14); // flat, null restricted atomic + // test26(array13); // flat, null restricted, non atomic + + // // test26(array14); + // // test26(array17); + // // test26(array18); + // // test26(array15); + // // test26(array16); + // } + // @Test // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "9" }) // @IR(failOn = IRNode.ALLOC) @@ -548,6 +592,7 @@ interface I { } + @LooselyConsistentValue static value class MyValue1 implements I { byte byteField; byte byteField2; @@ -605,11 +650,14 @@ public void m() { } } + @LooselyConsistentValue static value class MyValue5 implements I { byte byteField; + byte byteField2; MyValue5(byte byteField) { this.byteField = byteField; + this.byteField2 = byteField; } public void m() { From 74c8fb0a6bdd33255d71401de1fc43cef01f5cc7 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Mon, 11 May 2026 11:03:34 +0200 Subject: [PATCH 30/49] more --- src/hotspot/share/opto/inlinetypenode.cpp | 6 +-- .../inlinetypes/TestArrayLoadProfiling.java | 42 ++++++++++++++++--- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/opto/inlinetypenode.cpp b/src/hotspot/share/opto/inlinetypenode.cpp index cfb1adb6d02..94c8f96455b 100644 --- a/src/hotspot/share/opto/inlinetypenode.cpp +++ b/src/hotspot/share/opto/inlinetypenode.cpp @@ -1480,13 +1480,13 @@ InlineTypeNode* InlineTypeNode::make_from_flat_array(GraphKit* kit, ciInlineKlas // Non-Atomic kit->set_control(kit->IfFalse(iff_atomic)); if (!kit->stopped()) { + assert(vk->has_null_free_non_atomic_layout(), + "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8()); + kit->set_all_memory(input_memory_state); if (null_free_atomic_prob == 1 && !kit->too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { PreserveJVMState pjvms(kit); kit->uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); } else { - assert(vk->has_null_free_non_atomic_layout(), - "element type %s does not have a null-free non-atomic flat layout", vk->name()->as_utf8()); - kit->set_all_memory(input_memory_state); Node *cast = kit->cast_to_flat_array_exact(base, vk, true, false); Node *ptr = kit->array_element_address(cast, idx, T_FLAT_ELEMENT); vt_non_atomic = InlineTypeNode::make_from_flat(kit, vk, cast, ptr, false, false, true, decorators); diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index dad68aedee9..b3728c5da9b 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -47,6 +47,8 @@ public class TestArrayLoadProfiling { private final static WhiteBox WHITE_BOX = WhiteBox.getWhiteBox(); public static void main(String[] args) { + TestFramework.runWithFlags("--enable-preview", "--add-exports", "java.base/jdk.internal.value=ALL-UNNAMED", "-Xbootclasspath/a:.", "-XX:+UnlockDiagnosticVMOptions", "-XX:+WhiteBoxAPI", "-XX:-TieredCompilation"); + TestFramework.runWithFlags("--enable-preview", "--add-exports", "java.base/jdk.internal.value=ALL-UNNAMED", "-Xbootclasspath/a:.", "-XX:+UnlockDiagnosticVMOptions", "-XX:+WhiteBoxAPI", "-XX:-ProfileInterpreter"); TestFramework.runWithFlags("--enable-preview", "--add-exports", "java.base/jdk.internal.value=ALL-UNNAMED", "-Xbootclasspath/a:.", "-XX:+UnlockDiagnosticVMOptions", "-XX:+WhiteBoxAPI"); } @@ -82,6 +84,8 @@ public static void main(String[] args) { array19[0] = new MyValue5((byte)42); array21[0] = new MyValue4((byte)42); } + + static I staticField; @Test @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "4" }) @@ -91,6 +95,7 @@ public static void test1(I[] array) { } @Run(test = "test1") + @Warmup(10_000) public static void test1Runner() { test1(array1); test1Inline(array2[0]); @@ -152,6 +157,7 @@ public static void test3(I[] array) { } @Run(test = "test3") + @Warmup(10_000) public static void test3Runner() { test3(array3); test3(array4); @@ -170,6 +176,7 @@ public static void test5(I[] array) { } @Run(test = "test5") + @Warmup(10_000) public static void test5Runner() { test5(array1); test5(array2); @@ -188,6 +195,7 @@ public static void test7(I[] array) { } @Run(test = "test7") + @Warmup(10_000) public static void test7Runner() { test7Inline(array1[0]); test7Inline(array2[0]); @@ -241,6 +249,7 @@ public static void test9(I[] array) { } @Run(test = "test9") + @Warmup(10_000) public static void test9Runner() { test9(array1); test9(array2); @@ -289,6 +298,7 @@ public static void test11(I[] array) { } @Run(test = "test11") + @Warmup(10_000) public static void test11Runner() { for (int i = 0; i < 50; i++) { test11(array1); @@ -311,6 +321,7 @@ public static void test13(I[] array) { } @Run(test = "test13") + @Warmup(10_000) public static void test13Runner() { test13(array1); test13(array2); @@ -331,6 +342,7 @@ public static void test15(I[] array) { } @Run(test = "test15") + @Warmup(10_000) public static void test15Runner() { test15(array1); test15(array2); @@ -378,6 +390,7 @@ public static void test19(A[] array) { } @Run(test = "test19") + @Warmup(10_000) public static void test19Runner() { test19(array5); } @@ -404,6 +417,7 @@ public static void test21() { } @Run(test = "test21") + @Warmup(10_000) public static void test21Runner() { test21(); test21Inline(array2); // flat, nullable @@ -426,6 +440,7 @@ public static void test22() { } @Run(test = "test22") + @Warmup(10_000) public static void test22Runner() { test22(); test22Inline(array2); // flat, nullable @@ -463,13 +478,14 @@ static void test22Inline(I[] array) { // } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "9" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "9" }) @IR(failOn = IRNode.ALLOC) public static void test23(I[] array) { test23Inline(array[0]); } @Run(test = "test23") + @Warmup(10_000) public static void test23Runner() { test23(array1); // flat nullable //test23(array3); // flat null free non atomic @@ -494,6 +510,7 @@ public static int test24(MyValue3[] array, int i, int j, int k) { } @Run(test = "test24") + @Warmup(10_000) public static void test24Runner() { test24(array11, 0, 0, 0); } @@ -506,6 +523,7 @@ public static int test25(MyValue3[] array, int i, int j, int k) { } @Run(test = "test25") + @Warmup(10_000) public static void test25Runner() { test25(array12, 0, 0, 0); } @@ -519,11 +537,11 @@ public static void test26() { } @Run(test = "test26") + @Warmup(10_000) public static void test26Runner() { test26(); - test26Inline(array8); // flat, null free atomic - test26Inline(array22); // flat, null free atomic - test26Inline(array18); // flat, null free atomic + test26Inline(array8); // flat, null free atomic + test26Inline(array18); // flat, null free atomic } @ForceInline @@ -540,10 +558,10 @@ public static void test27() { } @Run(test = "test27") + @Warmup(10_000) public static void test27Runner() { test27(); test27Inline(array3); // flat, null free non atomic - test27Inline(array10); // flat, null free non atomic test27Inline(array17); // flat, null free non atomic } @@ -552,6 +570,20 @@ static void test27Inline(I[] array) { array[0].m(); } + // @Test + // @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "3" }) + // @IR(failOn = IRNode.ALLOC) + // public static void test28(I[] array) { + // array[0].m(); + // array[0].m(); + // } + + // @Run(test = "test28") + // public static void test28Runner() { + // test28(array5); + // } + + // @Test // // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "4" }) // // @IR(failOn = IRNode.ALLOC) From 3ac97b34e46dcccf12d0085d492f19ca30c30213 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Mon, 11 May 2026 15:42:07 +0200 Subject: [PATCH 31/49] more --- .../inlinetypes/TestArrayLoadProfiling.java | 249 +++++++++++++++--- 1 file changed, 212 insertions(+), 37 deletions(-) diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index b3728c5da9b..d65e933d4de 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -109,45 +109,51 @@ static void test1Inline(I i) { i.m(); } - // @Test - // @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.CALL, "5", IRNode.IF, "5" }) - // @IR(failOn = { IRNode.CLASS_CHECK_TRAP }) - // public static void test2(I[] array) { - // test2Inline(array[0]); - // } + static void trapAndRecompile(String name, Runnable causesTrap) throws Exception { + Method m = TestArrayLoadProfiling.class.getDeclaredMethod(name, I[].class); + if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) { + throw new RuntimeException("should be compiled"); + } + int i = 0; + do { + causesTrap.run(); + i++; + if (i > 10) { + throw new RuntimeException("should not be compiled anymore"); + } + } while (WHITE_BOX.isMethodCompiled(m)); + WHITE_BOX.enqueueMethodForCompilation(m, CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION); + if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) { + throw new RuntimeException("should be compiled"); + } + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.DYNAMIC_CALL_OF_METHOD, "m", "1", IRNode.UNHANDLED_TRAP, "2", IRNode.CALL, "6", IRNode.ALLOC, "2" }, + phase = { CompilePhase.BEFORE_MACRO_EXPANSION }, applyIf = { "ProfileInterpreter", "true"}) + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test2(I[] array) { + test2Inline(array[0]); + } - // @Run(test = "test2") - // public static void test2Runner(RunInfo info) throws Exception { - // if (info.isWarmUp()) { - // test2(array1); - // test2Inline(array2[0]); - // test2Inline(array3[0]); - // test2Inline(array4[0]); - // test2Inline(array5[0]); - // } else { - // Method m = TestArrayLoadProfiling.class.getDeclaredMethod("test2", I[].class); - // if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) { - // throw new RuntimeException("should be compiled"); - // } - // int i = 0; - // do { - // test2(array2); - // i++; - // if (i > 10) { - // throw new RuntimeException("should not be compiled anymore"); - // } - // } while (WHITE_BOX.isMethodCompiled(m)); - // WHITE_BOX.enqueueMethodForCompilation(m, CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION); - // if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) { - // throw new RuntimeException("should be compiled"); - // } - // } - // } + @Run(test = "test2") + @Warmup(10_000) + public static void test2Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test2(array1); + test2Inline(array2[0]); + test2Inline(array3[0]); + test2Inline(array4[0]); + test2Inline(array5[0]); + } else { + trapAndRecompile("test2", () -> { test2(array2); }); + } + } - // @ForceInline - // static void test2Inline(I i) { - // i.m(); - // } + @ForceInline + static void test2Inline(I i) { + i.m(); + } @Test @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "4" }) @@ -168,6 +174,28 @@ static void test3Inline(I i) { i.m(); } + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test4(I[] array) { + test4Inline(array[0]); + } + + @Run(test = "test4") + @Warmup(10_000) + public static void test4Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test4(array3); + test4(array4); + } else { + trapAndRecompile("test4", () -> { test4(array1); }); + } + } + + @ForceInline + static void test4Inline(I i) { + i.m(); + } + @Test @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "6" }) @IR(failOn = IRNode.ALLOC) @@ -187,6 +215,28 @@ static void test5Inline(I i) { i.m(); } + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test6(I[] array) { + test6Inline(array[0]); + } + + @Run(test = "test6") + @Warmup(10_000) + public static void test6Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test6(array1); + test6(array2); + } else { + trapAndRecompile("test6", () -> { test6(array3); }); + } + } + + @ForceInline + static void test6Inline(I i) { + i.m(); + } + @Test @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "6" }) @IR(failOn = IRNode.ALLOC) @@ -209,6 +259,31 @@ static void test7Inline(I i) { i.m(); } + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test8(I[] array) { + test8Inline(array[0]); + } + + @Run(test = "test8") + @Warmup(10_000) + public static void test8Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test8Inline(array1[0]); + test8Inline(array2[0]); + test8Inline(array3[0]); + test8Inline(array4[0]); + test8(array5); + } else { + trapAndRecompile("test8", () -> { test8(array1); }); + } + } + + @ForceInline + static void test8Inline(I i) { + i.m(); + } + // if (array == null) { // trap1; // } @@ -262,6 +337,30 @@ static void test9Inline(I i) { i.m(); } + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test10(I[] array) { + test10Inline(array[0]); + } + + @Run(test = "test10") + @Warmup(10_000) + public static void test10Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test10(array1); + test10(array2); + test10(array6); + test10(array7); + } else { + trapAndRecompile("test10", () -> { test10(array3); }); + } + } + + @ForceInline + static void test10Inline(I i) { + i.m(); + } + // if (array == null) { // trap1; // } @@ -313,6 +412,32 @@ static void test11Inline(I i) { i.m(); } + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test12(I[] array) { + test12Inline(array[0]); + } + + @Run(test = "test12") + @Warmup(10_000) + public static void test12Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + for (int i = 0; i < 50; i++) { + test12(array1); + } + test12(array2); + test12(array3); + test12(array4); + } else { + trapAndRecompile("test12", () -> { test12(array5); }); + } + } + + @ForceInline + static void test12Inline(I i) { + i.m(); + } + @Test @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "6", IRNode.IF, "7" }) @IR(failOn = IRNode.ALLOC) @@ -334,6 +459,30 @@ static void test13Inline(I i) { i.m(); } + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) + public static void test14(I[] array) { + test14Inline(array[0]); + } + + @Run(test = "test14") + @Warmup(10_000) + public static void test14Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test14(array1); + test14(array2); + test14(array3); + test14(array4); + } else { + trapAndRecompile("test14", () -> { test14(array5); }); + } + } + + @ForceInline + static void test14Inline(I i) { + i.m(); + } + @Test @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "5", IRNode.IF, "7" }) @IR(failOn = IRNode.ALLOC) @@ -357,6 +506,32 @@ static void test15Inline(I i) { i.m(); } + @Test + @IR(failOn = { IRNode.CLASS_CHECK_TRAP, IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP } ) + public static void test16(I[] array) { + test16Inline(array[0]); + } + + @Run(test = "test16") + @Warmup(10_000) + public static void test16Runner(RunInfo info) throws Exception { + if (info.isWarmUp()) { + test16(array1); + test16(array2); + test16(array3); + test16(array4); + test16(array6); + test16(array7); + } else { + trapAndRecompile("test16", () -> { test16(array5); }); + } + } + + @ForceInline + static void test16Inline(I i) { + i.m(); + } + @Test @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "2", IRNode.CALL, "4", IRNode.IF, "4" }) @IR(failOn = IRNode.ALLOC) From 2d250f23c083a4749ef083aa205aa22ad4aa2f73 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Tue, 12 May 2026 09:32:53 +0200 Subject: [PATCH 32/49] exps --- src/hotspot/share/opto/parse2.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 46715b507b1..c44e2d9dde0 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -351,7 +351,7 @@ class ArrayLoad { InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index, null_free_prob, null_free_atomic_prob); ld = vt; - if (_region != nullptr) { + if (_region != nullptr && 0) { Node* null_ctl = _parse.top(); _parse.null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl); @@ -439,7 +439,7 @@ class ArrayLoad { "array can't be flat"); ciArrayLoadData* array_load = profile_data(); - if (array_load != nullptr) { + if (array_load != nullptr && 0) { int not_flat_count = profiled_not_flat_count(); ciCallProfile profile = _parse.method()->call_profile_at_bci(_parse.bci()); int flat_count = profile.count(); @@ -546,12 +546,12 @@ class ArrayLoad { //---------------------------------array_load---------------------------------- void Parse::array_load(BasicType bt) { - ArrayLoad array_load(bt, *this); - if (array_load.emit()) { - return; - } - - ShouldNotReachHere(); + // ArrayLoad array_load(bt, *this); + // if (array_load.emit()) { + // return; + // } + // + // ShouldNotReachHere(); const Type* elemtype = Type::TOP; Node* adr = array_addressing(bt, 0, elemtype); if (stopped()) return; // guaranteed null or range check From b67e9c2460282493993aa86e739756bb2d043a09 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Wed, 13 May 2026 09:30:03 +0200 Subject: [PATCH 33/49] exps --- src/hotspot/share/opto/parse2.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index c44e2d9dde0..10f69108b29 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -348,7 +348,7 @@ class ArrayLoad { ciInlineKlass* vk = element_ptr->inline_klass(); Node* flat_array = _parse.cast_to_flat_array(array, vk); - InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index, null_free_prob, null_free_atomic_prob); + InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); ld = vt; if (_region != nullptr && 0) { @@ -384,7 +384,7 @@ class ArrayLoad { void create_merge_point() { _region = new RegionNode(1); - _res_phi = new PhiNode(_region, TypeOopPtr::BOTTOM); + _res_phi = new PhiNode(_region, _elemtype->make_oopptr()); _io_phi = new PhiNode(_region, Type::ABIO); _mem = _parse.reset_memory(); _io = _parse.i_o(); @@ -546,12 +546,12 @@ class ArrayLoad { //---------------------------------array_load---------------------------------- void Parse::array_load(BasicType bt) { - // ArrayLoad array_load(bt, *this); - // if (array_load.emit()) { - // return; - // } - // - // ShouldNotReachHere(); + ArrayLoad array_load(bt, *this); + if (array_load.emit()) { + return; + } + + ShouldNotReachHere(); const Type* elemtype = Type::TOP; Node* adr = array_addressing(bt, 0, elemtype); if (stopped()) return; // guaranteed null or range check From b1d39701dfe7f61cf1456c2ce6e680039eb7d0ec Mon Sep 17 00:00:00 2001 From: rwestrel Date: Wed, 13 May 2026 10:43:11 +0200 Subject: [PATCH 34/49] more --- src/hotspot/share/opto/parse2.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 10f69108b29..fa886f41d8e 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -348,7 +348,7 @@ class ArrayLoad { ciInlineKlass* vk = element_ptr->inline_klass(); Node* flat_array = _parse.cast_to_flat_array(array, vk); - InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index); + InlineTypeNode* vt = InlineTypeNode::make_from_flat_array(&_parse, vk, flat_array, _array_index, null_free_prob, null_free_atomic_prob); ld = vt; if (_region != nullptr && 0) { @@ -439,7 +439,7 @@ class ArrayLoad { "array can't be flat"); ciArrayLoadData* array_load = profile_data(); - if (array_load != nullptr && 0) { + if (array_load != nullptr) { int not_flat_count = profiled_not_flat_count(); ciCallProfile profile = _parse.method()->call_profile_at_bci(_parse.bci()); int flat_count = profile.count(); From a91b22fb4e17d9e2c9bb89a0e709b3e0657ee253 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Tue, 19 May 2026 16:52:08 +0200 Subject: [PATCH 35/49] fix --- src/hotspot/share/oops/methodData.hpp | 8 ++ src/hotspot/share/opto/graphKit.cpp | 4 +- src/hotspot/share/opto/parse2.cpp | 133 +++++++++++++++--- .../valhalla/inlinetypes/TestLWorld.java | 14 ++ .../inlinetypes/TestLWorldProfiling.java | 14 +- 5 files changed, 153 insertions(+), 20 deletions(-) diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index da15fed8833..45c78751bbd 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -2160,6 +2160,14 @@ class ArrayLoadData : public ProfileData { return saturated_add(saturated_add(flat_nullfree_atomic_count(), flat_nullfree_not_atomic_count()), flat_nullable_count()); } + int nullable_count() const { + return saturated_add(flat_nullable_count(), not_flat_nullable_count()); + } + + int null_free_count() const { + return saturated_add(saturated_add(flat_nullfree_atomic_count(), flat_nullfree_not_atomic_count()), not_flat_null_free_count()); + } + int not_flat_null_free_count() const { return uint_at(not_flat_null_free_count_off()); } diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index b5d0540858f..a42484e714e 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -3876,8 +3876,8 @@ Node* GraphKit::gen_checkcast(Node* obj, Node* superklass, Node** failure_contro } else if (obj->is_Phi()) { Node* region = obj->in(0); // TODO make this more robust (see JDK-8231346) - if (region->req() == 3 && region->in(2) != nullptr && region->in(2)->in(0) != nullptr) { - IfNode* iff = region->in(2)->in(0)->isa_If(); + if (region->req() == 3 && region->in(1) != nullptr && region->in(1)->in(0) != nullptr) { + IfNode* iff = region->in(1)->in(0)->isa_If(); if (iff != nullptr) { iff->is_flat_array_check(&_gvn, &array); } diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index fa886f41d8e..a0b5b67f49b 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -132,6 +132,9 @@ class ArrayLoad { } ciArrayLoadData* profile_data() const { + if (!UseArrayLoadStoreProfile) { + return nullptr; + } ciMethodData* md = _parse.method()->method_data(); if (md == nullptr) { return nullptr; @@ -239,12 +242,15 @@ class ArrayLoad { _parse.set_control(_gvn.transform(new IfTrueNode(iff))); assert(array_type->is_flat() || _parse.control()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found"); - Node* casted_array = _gvn.transform(new CheckCastPPNode(_parse.control(), _array, array_type->cast_to_not_flat())); - Node* ld = emit_plain_load(casted_array, true); - _res_phi->add_req(ld); - _region->add_req(_parse.control()); - _io_phi->add_req(_parse.i_o()); - _mem_phi->add_req(_parse.reset_memory()); + if (!_parse.stopped()) { + Node *casted_array = _gvn. + transform(new CheckCastPPNode(_parse.control(), _array, array_type->cast_to_not_flat())); + Node *ld = emit_plain_load(casted_array, true); + _res_phi->add_req(ld); + _region->add_req(_parse.control()); + _io_phi->add_req(_parse.i_o()); + _mem_phi->add_req(_parse.reset_memory()); + } _parse.set_control(_gvn.transform(new IfFalseNode(iff))); _parse.set_all_memory(_mem); _parse.set_i_o(_io); @@ -392,15 +398,99 @@ class ArrayLoad { _mem_phi = new PhiNode(_region, Type::MEMORY, TypePtr::BOTTOM); } + void record_array_profile_for_speculation() { + ciArrayLoadData* array_load = profile_data(); + if (UseTypeSpeculation && array_load != nullptr && array_load->nullable_count() > 0 && array_load->null_free_count() == 0) { + const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr(); + if (!array_type->is_not_null_free() && array_type->speculative() == nullptr) { + const TypeAryPtr* spec_array_type = TypeAryPtr::BOTTOM->cast_to_not_null_free(); + const TypeOopPtr* spec_type = TypeOopPtr::make(TypePtr::BotPTR, Type::Offset::bottom, TypeOopPtr::InstanceBot, spec_array_type); + Node* cast = new CheckCastPPNode(_parse.control(), _array, array_type->remove_speculative()->join_speculative(spec_type)); + cast = _gvn.transform(cast); + _parse.replace_in_map(_array, cast); + } + } + } + void finish_merge_point() { + _parse.record_for_igvn(_region); _parse.set_control(_gvn.transform(_region)); - _parse.record_for_igvn(_parse.control()); - _parse.set_all_memory(_gvn.transform(_mem_phi)); + Node* base = nullptr; + for (uint i = 1; i < _mem_phi->req(); ++i) { + Node* in = _mem_phi->in(i); + Node* new_base = in; + if (in->is_MergeMem()) { + new_base = in->as_MergeMem()->base_memory(); + } + if (base == nullptr) { + base = new_base; + } else if (base != new_base) { + base = NodeSentinel; + } + } + assert(base != nullptr, ""); + if (base != NodeSentinel) { + MergeMemNode* mm = MergeMemNode::make(base); + for (uint i = 1; i < _mem_phi->req(); ++i) { + MergeMemNode* in = _mem_phi->in(i)->isa_MergeMem(); + if (in != nullptr) { + for (MergeMemStream mms(in); mms.next_non_empty(); ) { + if (mms.at_base_memory()) { + continue; + } + Node* mem = mms.memory(); + if (mm->memory_at(mms.alias_idx()) == base) { + Node* phi = PhiNode::make(_region, base, Type::MEMORY, mms.adr_type()); + _gvn.set_type(phi, phi->bottom_type()); + _parse.record_for_igvn(phi); + mm->set_memory_at(mms.alias_idx(), phi); + } + PhiNode* phi = mm->memory_at(mms.alias_idx())->as_Phi(); + phi->set_req(i, mem); + } + } + } + _parse.set_all_memory(_gvn.transform(mm)); + } else { + _parse.record_for_igvn(_mem_phi); + _parse.set_all_memory(_gvn.transform(_mem_phi)); + } _parse.set_i_o(_gvn.transform(_io_phi)); Node* ld = _gvn.transform(_res_phi); ld = _parse.record_profile_for_speculation_at_array_load(ld); pop_stack(); _parse.push_node(_bt, ld); + + // record_array_profile_for_speculation(); + } + + void cast_to_speculative_array_type(const TypeAryPtr *&array_type) { + if (!array_type->is_flat() && !array_type->is_not_flat()) { + // For arrays that might be flat, speculate that the array has the exact type reported in the profile data such that + // we can rely on a fixed memory layout (i.e. either a flat layout or not). + Deoptimization::DeoptReason reason = Deoptimization::Reason_speculate_class_check; + ciKlass* speculative_array_type = array_type->speculative_type(); + if (speculative_array_type != nullptr && !_parse.too_many_traps_or_recompiles(reason)) { + // Speculate that this array has the exact type reported by profile data + Node* casted_array = nullptr; + DEBUG_ONLY(Node* old_control = _parse.control();) + Node* slow_ctl = _parse.type_check_receiver(_array, speculative_array_type, 1.0, &casted_array); + if (_parse.stopped()) { + // The check always fails and therefore profile information is incorrect. Don't use it. + assert(old_control == slow_ctl, "type check should have been removed"); + _parse.set_control(slow_ctl); + } else if (!slow_ctl->is_top()) { + { PreserveJVMState pjvms(&_parse); + _parse.set_control(slow_ctl); + _parse.uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile); + } + _parse.replace_in_map(_array, casted_array); + array_type = _gvn.type(casted_array)->is_aryptr(); + _elemtype = array_type->elem(); + _array = casted_array; + } + } + } } bool emit() { @@ -409,9 +499,12 @@ class ArrayLoad { // Check for always knowing you are throwing a range-check exception if (_parse.stopped()) return true; //top(); - const TypeOopPtr* element_ptr = _elemtype->make_oopptr(); const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr(); + cast_to_speculative_array_type(array_type); + + const TypeOopPtr* element_ptr = _elemtype->make_oopptr(); + if (array_type->is_not_flat()) { if (_elemtype == TypeInt::BOOL) { _bt = T_BOOLEAN; @@ -419,10 +512,11 @@ class ArrayLoad { Node* ld = emit_plain_load(_array, false, true); // pop_stack(); _parse.push_node(_bt, ld); + // record_array_profile_for_speculation(); return true; } - if (array_type->is_flat()) { + if (array_type->is_flat() && element_ptr->is_inlinetypeptr()) { pop_stack(); ciInlineKlass* vk = element_ptr->inline_klass(); Node* flat_array = _array; @@ -508,9 +602,11 @@ class ArrayLoad { Node* casted_array = _gvn.transform( new CheckCastPPNode(_parse.control(), _array, array_type->cast_to_not_flat())); _parse.replace_in_map(_array, casted_array); + _array = casted_array; Node* ld = emit_plain_load(casted_array, true); ld = _parse.record_profile_for_speculation_at_array_load(ld); _parse.push_node(_bt, ld); + record_array_profile_for_speculation(); return true; } if (flat_count != 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { @@ -523,6 +619,7 @@ class ArrayLoad { Node* ld = load_from_unknown_flat_array(element_ptr); pop_stack(); _parse.push_node(_bt, ld); + // record_array_profile_for_speculation(); return true; } if (flat_count != 0 && not_flat_count != 0) { @@ -546,12 +643,14 @@ class ArrayLoad { //---------------------------------array_load---------------------------------- void Parse::array_load(BasicType bt) { - ArrayLoad array_load(bt, *this); - if (array_load.emit()) { - return; - } + if (!UseNewCode) { + ArrayLoad array_load(bt, *this); + if (array_load.emit()) { + return; + } - ShouldNotReachHere(); + ShouldNotReachHere(); + } const Type* elemtype = Type::TOP; Node* adr = array_addressing(bt, 0, elemtype); if (stopped()) return; // guaranteed null or range check @@ -1073,10 +1172,10 @@ Node* Parse::create_speculative_inline_type_array_checks(Node* array, const Type // Even though the type does not tell us whether we have an inline type array or not, we can still check the profile data // whether we have a non-null-free or non-flat array. Speculating on a non-null-free array doesn't help aaload but could // be profitable for a subsequent aastore. - if (!array_type->is_null_free() && !array_type->is_not_null_free() && 0) { + if (!array_type->is_null_free() && !array_type->is_not_null_free()) { array = speculate_non_null_free_array(array, array_type); } - if (!array_type->is_flat() && !array_type->is_not_flat() && 0) { + if (!array_type->is_flat() && !array_type->is_not_flat()) { array = speculate_non_flat_array(array, array_type); } return array; diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java index 17924e32947..fdee749e15f 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorld.java @@ -206,6 +206,8 @@ private static void call() {} private static final MyValue1 testValue1 = MyValue1.createWithFieldsInline(rI, rL); @NullRestricted private static final MyValue2 testValue2 = MyValue2.createWithFieldsInline(rI, rD); + @NullRestricted + private static final MyValue3 testValue3 = MyValue3.create(); protected long hash() { return testValue1.hash(); @@ -789,6 +791,13 @@ public void test20_verifier() { } } + private static final MyValue3[] testValue3Array = (MyValue3[])ValueClass.newNullRestrictedNonAtomicArray(MyValue3.class, 3, MyValue3.DEFAULT); + static { + for (int i = 0; i < 3; ++i) { + testValue3Array[i] = testValue3; + } + } + private static final NonValueClass[] testNonValueArray = new NonValueClass[42]; // Test load from (flattened) inline type array disguised as object array @@ -1315,6 +1324,9 @@ public Object[] test38(Object[] oa, Object o, int i1, int i2, int num) { case 6: result = testValue1Array2; break; + case 7: + result = testValue3Array; + break; } result[i1] = result[i2]; result[i2] = o; @@ -1354,6 +1366,8 @@ public void test38_verifier() { } result = test38(null, testValue1Array, index, index, 6); Asserts.assertEQ(testValue1, ((MyValue1[][])result)[index][index]); + result = test38(null, testValue3, index, index, 7); + Asserts.assertEQ(testValue3, result[index]); } @ForceInline diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorldProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorldProfiling.java index 8810c9a1722..ad52d36138a 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorldProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestLWorldProfiling.java @@ -113,6 +113,8 @@ public static void main(String[] args) { private static final MyValue1 testValue1 = MyValue1.createWithFieldsInline(rI, rL); @NullRestricted private static final MyValue2 testValue2 = MyValue2.createWithFieldsInline(rI, rD); + @NullRestricted + private static final MyValue3 testValue3 = MyValue3.create(); private static final MyValue1[] testValue1Array = (MyValue1[])ValueClass.newNullRestrictedNonAtomicArray(MyValue1.class, 1, MyValue1.DEFAULT); static { testValue1Array[0] = testValue1; @@ -121,6 +123,10 @@ public static void main(String[] args) { static { testValue2Array[0] = testValue2; } + private static final MyValue3[] testValue3Array = (MyValue3[])ValueClass.newNullRestrictedNonAtomicArray(MyValue3.class, 1, MyValue3.DEFAULT); + static { + testValue3Array[0] = testValue3; + } // Some non-value classes static class MyInteger extends Number { @@ -258,6 +264,8 @@ public void test3_verifier() { Asserts.assertEQ(testValue1, o); o = test3(testValue2Array); Asserts.assertEQ(testValue2, o); + o = test3(testValue3Array); + Asserts.assertEQ(testValue3, o); } @Test @@ -296,6 +304,10 @@ public void test5_verifier() { Asserts.assertEQ(testValue1, o); o = test5(testValue1NotFlatArray); Asserts.assertEQ(testValue1, o); + o = test5(testValue2Array); + Asserts.assertEQ(testValue2, o); + o = test5(testValue3Array); + Asserts.assertEQ(testValue3, o); } // Check that profile data that's useless at the aaload is @@ -378,7 +390,7 @@ public void test8_helper(Object arg) { @Test @IR(applyIf = {"UseArrayLoadStoreProfile", "true"}, - counts = {STATIC_CALL, "= 5", CLASS_CHECK_TRAP, "= 1", NULL_CHECK_TRAP, "= 2", + counts = {STATIC_CALL, "= 7", CLASS_CHECK_TRAP, "= 2", NULL_CHECK_TRAP, "= 2", RANGE_CHECK_TRAP, "= 1"}) @IR(applyIf = {"UseArrayLoadStoreProfile", "false"}, counts = {STATIC_CALL, "= 5", RANGE_CHECK_TRAP, "= 1", NULL_CHECK_TRAP, "= 2"}) From 9222a3eb16239f1a0c8566382d4eb93dcf9a7bc9 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Wed, 20 May 2026 14:17:53 +0200 Subject: [PATCH 36/49] product build fix --- src/hotspot/share/ci/ciMethodData.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hotspot/share/ci/ciMethodData.cpp b/src/hotspot/share/ci/ciMethodData.cpp index a439415c2cd..44f39d6dbb4 100644 --- a/src/hotspot/share/ci/ciMethodData.cpp +++ b/src/hotspot/share/ci/ciMethodData.cpp @@ -328,6 +328,7 @@ bool ciMegamorphicTypeData::translate_type_data_from(const MegamorphicTypeData* return cleared_row; } +#ifndef PRODUCT void ciMegamorphicTypeData::print_receiver_data_on(outputStream* st, int total) const { total += count(); for (uint row = 0; row < row_limit(); row++) { @@ -338,6 +339,7 @@ void ciMegamorphicTypeData::print_receiver_data_on(outputStream* st, int total) } } } +#endif void ciTypeStackSlotEntries::translate_type_data_from(const TypeStackSlotEntries* entries) { for (int i = 0; i < number_of_entries(); i++) { From 78ce406f5da481904671869f38159255460b309a Mon Sep 17 00:00:00 2001 From: rwestrel Date: Wed, 20 May 2026 14:33:13 +0200 Subject: [PATCH 37/49] build fix --- src/hotspot/share/ci/ciCallProfile.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hotspot/share/ci/ciCallProfile.hpp b/src/hotspot/share/ci/ciCallProfile.hpp index 43391aff1d0..a0810eab2ff 100644 --- a/src/hotspot/share/ci/ciCallProfile.hpp +++ b/src/hotspot/share/ci/ciCallProfile.hpp @@ -27,6 +27,7 @@ #include "ci/ciClassList.hpp" #include "memory/allocation.hpp" +#include "opto/c2_globals.hpp" // ciCallProfile // From 56c50bc8e23ef5a2d44d98b6d3b0a9510db743b2 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Wed, 20 May 2026 16:12:48 +0200 Subject: [PATCH 38/49] whitespaces --- src/hotspot/share/oops/methodData.hpp | 2 +- .../inlinetypes/TestArrayLoadProfiling.java | 152 +++++++++--------- 2 files changed, 77 insertions(+), 77 deletions(-) diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index 45c78751bbd..d6e95806ecd 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -2100,7 +2100,7 @@ class ArrayLoadData : public ProfileData { static int extra_cells_off() { return SingleTypeEntry::static_cell_count() + MegamorphicTypeData::static_cell_count(TypeProfileWidth); } - + static int not_flat_null_free_count_off() { return extra_cells_off() + not_flat_null_free_count_off_in_extra_cells; } diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index d65e933d4de..d06f6b9bf5e 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -86,7 +86,7 @@ public static void main(String[] args) { } static I staticField; - + @Test @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "4" }) @IR(failOn = IRNode.ALLOC) @@ -110,27 +110,27 @@ static void test1Inline(I i) { } static void trapAndRecompile(String name, Runnable causesTrap) throws Exception { - Method m = TestArrayLoadProfiling.class.getDeclaredMethod(name, I[].class); - if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) { - throw new RuntimeException("should be compiled"); - } - int i = 0; - do { - causesTrap.run(); - i++; - if (i > 10) { - throw new RuntimeException("should not be compiled anymore"); - } - } while (WHITE_BOX.isMethodCompiled(m)); - WHITE_BOX.enqueueMethodForCompilation(m, CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION); - if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) { - throw new RuntimeException("should be compiled"); - } - } - + Method m = TestArrayLoadProfiling.class.getDeclaredMethod(name, I[].class); + if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) { + throw new RuntimeException("should be compiled"); + } + int i = 0; + do { + causesTrap.run(); + i++; + if (i > 10) { + throw new RuntimeException("should not be compiled anymore"); + } + } while (WHITE_BOX.isMethodCompiled(m)); + WHITE_BOX.enqueueMethodForCompilation(m, CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION); + if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) { + throw new RuntimeException("should be compiled"); + } + } + @Test @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.DYNAMIC_CALL_OF_METHOD, "m", "1", IRNode.UNHANDLED_TRAP, "2", IRNode.CALL, "6", IRNode.ALLOC, "2" }, - phase = { CompilePhase.BEFORE_MACRO_EXPANSION }, applyIf = { "ProfileInterpreter", "true"}) + phase = { CompilePhase.BEFORE_MACRO_EXPANSION }, applyIf = { "ProfileInterpreter", "true"}) @IR(failOn = { IRNode.CLASS_CHECK_TRAP } , applyIf = { "ProfileInterpreter", "true" }) public static void test2(I[] array) { test2Inline(array[0]); @@ -146,7 +146,7 @@ public static void test2Runner(RunInfo info) throws Exception { test2Inline(array4[0]); test2Inline(array5[0]); } else { - trapAndRecompile("test2", () -> { test2(array2); }); + trapAndRecompile("test2", () -> { test2(array2); }); } } @@ -184,10 +184,10 @@ public static void test4(I[] array) { @Warmup(10_000) public static void test4Runner(RunInfo info) throws Exception { if (info.isWarmUp()) { - test4(array3); - test4(array4); + test4(array3); + test4(array4); } else { - trapAndRecompile("test4", () -> { test4(array1); }); + trapAndRecompile("test4", () -> { test4(array1); }); } } @@ -225,11 +225,11 @@ public static void test6(I[] array) { @Warmup(10_000) public static void test6Runner(RunInfo info) throws Exception { if (info.isWarmUp()) { - test6(array1); - test6(array2); - } else { - trapAndRecompile("test6", () -> { test6(array3); }); - } + test6(array1); + test6(array2); + } else { + trapAndRecompile("test6", () -> { test6(array3); }); + } } @ForceInline @@ -269,14 +269,14 @@ public static void test8(I[] array) { @Warmup(10_000) public static void test8Runner(RunInfo info) throws Exception { if (info.isWarmUp()) { - test8Inline(array1[0]); - test8Inline(array2[0]); - test8Inline(array3[0]); - test8Inline(array4[0]); - test8(array5); - } else { - trapAndRecompile("test8", () -> { test8(array1); }); - } + test8Inline(array1[0]); + test8Inline(array2[0]); + test8Inline(array3[0]); + test8Inline(array4[0]); + test8(array5); + } else { + trapAndRecompile("test8", () -> { test8(array1); }); + } } @ForceInline @@ -347,13 +347,13 @@ public static void test10(I[] array) { @Warmup(10_000) public static void test10Runner(RunInfo info) throws Exception { if (info.isWarmUp()) { - test10(array1); - test10(array2); - test10(array6); - test10(array7); - } else { - trapAndRecompile("test10", () -> { test10(array3); }); - } + test10(array1); + test10(array2); + test10(array6); + test10(array7); + } else { + trapAndRecompile("test10", () -> { test10(array3); }); + } } @ForceInline @@ -370,14 +370,14 @@ static void test10Inline(I i) { // if (array.klass == MyValue1[]) { // if (array[0] == null) { // trap3; - // } + // } // // inlined call // } else { // if (array flat) { // elt = load_unknown_inline(); // if (elt == null) { // trap3; - // } + // } // if (elt.klass == MyValue1) { // // inlined call // } else if (elt.klass == MyValue2) { @@ -422,15 +422,15 @@ public static void test12(I[] array) { @Warmup(10_000) public static void test12Runner(RunInfo info) throws Exception { if (info.isWarmUp()) { - for (int i = 0; i < 50; i++) { - test12(array1); - } - test12(array2); - test12(array3); - test12(array4); - } else { - trapAndRecompile("test12", () -> { test12(array5); }); - } + for (int i = 0; i < 50; i++) { + test12(array1); + } + test12(array2); + test12(array3); + test12(array4); + } else { + trapAndRecompile("test12", () -> { test12(array5); }); + } } @ForceInline @@ -469,13 +469,13 @@ public static void test14(I[] array) { @Warmup(10_000) public static void test14Runner(RunInfo info) throws Exception { if (info.isWarmUp()) { - test14(array1); - test14(array2); - test14(array3); - test14(array4); - } else { - trapAndRecompile("test14", () -> { test14(array5); }); - } + test14(array1); + test14(array2); + test14(array3); + test14(array4); + } else { + trapAndRecompile("test14", () -> { test14(array5); }); + } } @ForceInline @@ -516,15 +516,15 @@ public static void test16(I[] array) { @Warmup(10_000) public static void test16Runner(RunInfo info) throws Exception { if (info.isWarmUp()) { - test16(array1); - test16(array2); - test16(array3); - test16(array4); - test16(array6); - test16(array7); - } else { - trapAndRecompile("test16", () -> { test16(array5); }); - } + test16(array1); + test16(array2); + test16(array3); + test16(array4); + test16(array6); + test16(array7); + } else { + trapAndRecompile("test16", () -> { test16(array5); }); + } } @ForceInline @@ -715,15 +715,15 @@ public static void test26() { @Warmup(10_000) public static void test26Runner() { test26(); - test26Inline(array8); // flat, null free atomic - test26Inline(array18); // flat, null free atomic + test26Inline(array8); // flat, null free atomic + test26Inline(array18); // flat, null free atomic } @ForceInline static void test26Inline(I[] array) { array[0].m(); } - + @Test @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "3", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "7" }) @IR(failOn = IRNode.ALLOC) @@ -797,7 +797,7 @@ static void test27Inline(I[] array) { interface I { void m(); } - + @LooselyConsistentValue static value class MyValue1 implements I { @@ -810,7 +810,7 @@ static value class MyValue1 implements I { this.byteField2 = byteField; this.byteField3 = byteField; } - + public void m() { } } From efafdcfe12936a21980d732e7530113c625896e4 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 21 May 2026 09:32:05 +0200 Subject: [PATCH 39/49] profile_receiver_type_helper fixes --- src/hotspot/cpu/x86/macroAssembler_x86.cpp | 14 +++++--------- src/hotspot/cpu/x86/macroAssembler_x86.hpp | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index f1b73250e5c..24f52d4c9d3 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -4935,7 +4935,7 @@ Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, return Address(rsp, scale_reg, scale_factor, offset); } -bool MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, Label &L_found_recv, int mdp_offset, int base, uint row_limit) { +void MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, Label &L_found_recv, int mdp_offset, int base, uint row_limit) { int base_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, 0)); int end_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, row_limit)); int receiver_step = in_bytes(MegamorphicTypeData::receiver_offset(base, 1)) - base_receiver_offset; @@ -5193,30 +5193,26 @@ void MacroAssembler::profile_array_type_at_load(Register recv, Register mdp, int profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ArrayLoadData::base_of_megamorphic_type_data(), ArrayLoadData::row_limit()); Register offset = rscratch1; int layout_kind_offset = in_bytes(FlatArrayKlass::layout_kind_offset()); - Label null_free_non_atomic, null_free_atomic, nullable_atomic_flat, done, failure; + Label null_free_non_atomic, null_free_atomic, nullable_atomic_flat, failure, L_count_update; cmpl(Address(recv, layout_kind_offset), (int)LayoutKind::NULL_FREE_ATOMIC_FLAT); jccb(Assembler::notEqual, null_free_non_atomic); movptr(offset, flat_nullfree_atomic_count_offset); - jmpb(done); + jmpb(L_count_update); bind(null_free_non_atomic); cmpl(Address(recv, layout_kind_offset), (int)LayoutKind::NULL_FREE_NON_ATOMIC_FLAT); jccb(Assembler::notEqual, nullable_atomic_flat); movptr(offset, flat_nullfree_not_atomic_count_offset); - jmpb(done); + jmpb(L_count_update); bind(nullable_atomic_flat); cmpl(Address(recv, layout_kind_offset), (int)LayoutKind::NULLABLE_ATOMIC_FLAT); jccb(Assembler::notEqual, failure); movptr(offset, flat_nullable_count_offset); - jmpb(done); + jmpb(L_count_update); bind(failure); stop("unexpected flat array"); - bind(done); - - Label L_count_update; - jmpb(L_count_update); // Found a receiver, convert its slot offset to corresponding count offset. bind(L_found_recv); diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp index a063bb14b62..ab73d21be56 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp @@ -708,7 +708,7 @@ class MacroAssembler: public Assembler { // method handles (JSR 292) Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0); - bool profile_receiver_type_helper(Register recv, Register mdp, + void profile_receiver_type_helper(Register recv, Register mdp, Label &L_found_recv, int mdp_offset, int base, uint row_limit); void profile_receiver_type(Register recv, Register mdp, int mdp_offset); From 67f9a31b30de1e788b39afbed44a162aca398a9a Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 21 May 2026 09:47:30 +0200 Subject: [PATCH 40/49] profile_receiver_type_helper fixes --- src/hotspot/cpu/x86/macroAssembler_x86.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 24f52d4c9d3..a84abc1dbd6 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -5096,7 +5096,6 @@ void MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, L // Increment polymorphic counter instead of receiver slot. bind(L_polymorphic); - return false; } // Handle the receiver type profile update given the "recv" klass. From e58cec75737d50129563074c4e8845607a38a82d Mon Sep 17 00:00:00 2001 From: rwestrel Date: Mon, 1 Jun 2026 15:48:31 +0200 Subject: [PATCH 41/49] merge --- src/hotspot/share/oops/methodData.hpp | 1 + .../compiler/lib/ir_framework/IRNode.java | 4 ++-- .../inlinetypes/TestArrayLoadProfiling.java | 22 +++++++++---------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/oops/methodData.hpp b/src/hotspot/share/oops/methodData.hpp index d6e95806ecd..52143fadad3 100644 --- a/src/hotspot/share/oops/methodData.hpp +++ b/src/hotspot/share/oops/methodData.hpp @@ -1171,6 +1171,7 @@ class MegamorphicTypeData { receiver_type_row_cell_count = (count0_offset + 1) - receiver0_offset }; + ProfileData* _pd; const int _base_off; const int _type_width; diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java index 57adafcd15f..2e197a6e8b3 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java @@ -3291,9 +3291,9 @@ public static void anyStoreOfNodes(String irNodePlaceholder, String fieldHolder) beforeMatchingNameRegex(OPAQUE_CONSTANT_BOOL, "OpaqueConstantBool"); } - public static final String BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP = PREFIX + "BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP" + POSTFIX; + public static final String BIMORPHIC_TRAP = PREFIX + "BIMORPHIC_TRAP" + POSTFIX; static { - trapNodes(BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "bimorphic_or_optimized_type_check"); + trapNodes(BIMORPHIC_TRAP, "bimorphic"); } /* diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index d06f6b9bf5e..9c69401ee41 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -317,7 +317,7 @@ static void test8Inline(I i) { // } // } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "11" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "11" }) @IR(failOn = IRNode.ALLOC) public static void test9(I[] array) { test9Inline(array[0]); @@ -390,7 +390,7 @@ static void test10Inline(I i) { // } // } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "6", IRNode.IF, "9" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "6", IRNode.IF, "9" }) @IR(failOn = IRNode.ALLOC) public static void test11(I[] array) { test11Inline(array[0]); @@ -439,7 +439,7 @@ static void test12Inline(I i) { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "6", IRNode.IF, "7" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.BIMORPHIC_TRAP, "1", IRNode.TRAP, "5", IRNode.CALL, "6", IRNode.IF, "7" }) @IR(failOn = IRNode.ALLOC) public static void test13(I[] array) { test13Inline(array[0]); @@ -484,7 +484,7 @@ static void test14Inline(I i) { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "5", IRNode.IF, "7" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.BIMORPHIC_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "5", IRNode.IF, "7" }) @IR(failOn = IRNode.ALLOC) public static void test15(I[] array) { test15Inline(array[0]); @@ -507,7 +507,7 @@ static void test15Inline(I i) { } @Test - @IR(failOn = { IRNode.CLASS_CHECK_TRAP, IRNode.BIMORPHIC_OR_OPTIMIZED_TYPE_CHECK_TRAP } ) + @IR(failOn = { IRNode.CLASS_CHECK_TRAP, IRNode.BIMORPHIC_TRAP } ) public static void test16(I[] array) { test16Inline(array[0]); } @@ -571,7 +571,7 @@ public static void test19Runner() { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "9" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "8" }) @IR(failOn = IRNode.ALLOC) public static void test20(MyValue1[] array) { array[0].m(); @@ -584,7 +584,7 @@ public static void test20Runner() { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "8" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "4", IRNode.CALL, "4", IRNode.IF, "7" }) @IR(failOn = IRNode.ALLOC) public static void test21() { I[] array = array16; @@ -607,7 +607,7 @@ static void test21Inline(I[] array) { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "7" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "2", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "2", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "6" }) @IR(failOn = IRNode.ALLOC) public static void test22() { I[] array = array16; @@ -678,7 +678,7 @@ static void test23Inline(I i) { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "3", IRNode.CLASS_CHECK_TRAP, "1", IRNode.IF, "6" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "3", IRNode.CLASS_CHECK_TRAP, "1", IRNode.IF, "5" }) @IR(failOn = IRNode.ALLOC) public static int test24(MyValue3[] array, int i, int j, int k) { return array[i].intField1 + array[j].intField1 + array[k].intField1; @@ -704,7 +704,7 @@ public static void test25Runner() { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "3", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "7" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "3", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "6" }) @IR(failOn = IRNode.ALLOC) public static void test26() { I[] array = array14; @@ -725,7 +725,7 @@ static void test26Inline(I[] array) { } @Test - @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "3", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "7" }) + @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "3", IRNode.TRAP, "5", IRNode.CALL, "5", IRNode.IF, "6" }) @IR(failOn = IRNode.ALLOC) public static void test27() { I[] array = array13; From c9f28150c05778b78facf146ee201b9b3fc19756 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Mon, 1 Jun 2026 16:33:34 +0200 Subject: [PATCH 42/49] more --- src/hotspot/cpu/x86/macroAssembler_x86.cpp | 4 ++-- src/hotspot/share/opto/parse2.cpp | 6 ++++++ .../valhalla/inlinetypes/TestArrayLoadProfiling.java | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index a84abc1dbd6..6e411179eb4 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -5197,13 +5197,13 @@ void MacroAssembler::profile_array_type_at_load(Register recv, Register mdp, int jccb(Assembler::notEqual, null_free_non_atomic); movptr(offset, flat_nullfree_atomic_count_offset); - jmpb(L_count_update); + jmp(L_count_update); bind(null_free_non_atomic); cmpl(Address(recv, layout_kind_offset), (int)LayoutKind::NULL_FREE_NON_ATOMIC_FLAT); jccb(Assembler::notEqual, nullable_atomic_flat); movptr(offset, flat_nullfree_not_atomic_count_offset); - jmpb(L_count_update); + jmp(L_count_update); bind(nullable_atomic_flat); cmpl(Address(recv, layout_kind_offset), (int)LayoutKind::NULLABLE_ATOMIC_FLAT); jccb(Assembler::notEqual, failure); diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index a0b5b67f49b..3ebf8d2687a 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -212,6 +212,12 @@ class ArrayLoad { _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); } _parse.set_control(_gvn.transform(new IfFalseNode(iff))); + if (_parse.stopped()) { + if (safe_for_replace_in_map) { + pop_stack(); + } + return _parse.C->top(); + } const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr()->cast_to_null_free(true); array = _gvn.transform(new CheckCastPPNode(_parse.control(), array, array_type)); if (_parse.needs_range_check(array_type->size(), _array_index)) { diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index 9c69401ee41..ea164e36b4a 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -24,6 +24,7 @@ /** * @test * @library /test/lib / + * @requires vm.flagless * @enablePreview * @modules java.base/jdk.internal.value * @modules java.base/jdk.internal.vm.annotation From 937cab982cb3d401f9d6a09e511760135ba83bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Thu, 21 May 2026 11:27:03 +0200 Subject: [PATCH 43/49] Port changes to aarch64 --- .../cpu/aarch64/c1_LIRAssembler_aarch64.cpp | 42 +++++ .../cpu/aarch64/interp_masm_aarch64.cpp | 52 ++++-- .../cpu/aarch64/interp_masm_aarch64.hpp | 5 +- .../cpu/aarch64/macroAssembler_aarch64.cpp | 153 +++++++++++++----- .../cpu/aarch64/macroAssembler_aarch64.hpp | 4 + .../cpu/aarch64/templateTable_aarch64.cpp | 6 +- 6 files changed, 206 insertions(+), 56 deletions(-) diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp index c74ff16ae26..11d48e52b4f 100644 --- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp @@ -2971,6 +2971,48 @@ void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) { __ bind(not_inline_type); } +void LIR_Assembler::emit_profile_multiple_array_types(LIR_OpProfileMultipleArrayTypes* op) { + Register array = op->array()->as_pointer_register(); + Register tmp1 = op->tmp1()->as_pointer_register(); + Register tmp2 = op->tmp2()->as_pointer_register(); + ciMethodData* md = op->md(); + + Label not_flat, done; + __ test_non_flat_array_oop (array, tmp1, not_flat); + + Register klass = tmp1; + __ load_klass(klass, array); + + Register mdo = tmp2; + __ mov_metadata(mdo, md->constant_encoding()); + + int mdp_offset = md->byte_offset_of_slot(op->load(), in_ByteSize(0)); + __ profile_array_type_at_load(tmp1, mdo, mdp_offset); + + __ b(done); + __ bind(not_flat); + __ mov_metadata(mdo, md->constant_encoding()); + + Label null_free; + + __ test_null_free_array_oop(array, tmp1, null_free); + + { + Address counter_addr(mdo, md->byte_offset_of_slot(op->load(), ArrayLoadData::not_flat_nullable_count_offset())); + __ addptr(counter_addr, DataLayout::counter_increment); + } + + __ b(done); + __ bind(null_free); + + { + Address counter_addr(mdo, md->byte_offset_of_slot(op->load(), ArrayLoadData::not_flat_null_free_count_offset())); + __ addptr(counter_addr, DataLayout::counter_increment); + } + + __ bind(done); +} + void LIR_Assembler::align_backward_branch_target() { } diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp index 1b16ec25ca7..57048e1937e 100644 --- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp @@ -1202,9 +1202,9 @@ void InterpreterMacroAssembler::profile_switch_case(Register index, } } -template void InterpreterMacroAssembler::profile_array_type(Register mdp, - Register array, - Register tmp) { +void InterpreterMacroAssembler::profile_array_type(Register mdp, + Register array, + Register tmp) { if (ProfileInterpreter) { Label profile_continue; @@ -1212,19 +1212,19 @@ template void InterpreterMacroAssembler::profile_array_type(Re test_method_data_pointer(mdp, profile_continue); mov(tmp, array); - profile_obj_type(tmp, Address(mdp, in_bytes(ArrayData::array_offset()))); + profile_obj_type(tmp, Address(mdp, in_bytes(ArrayStoreData::array_offset()))); Label not_flat; test_non_flat_array_oop(array, tmp, not_flat); - set_mdp_flag_at(mdp, ArrayData::flat_array_byte_constant()); + set_mdp_flag_at(mdp, ArrayStoreData::flat_array_byte_constant()); bind(not_flat); Label not_null_free; test_non_null_free_array_oop(array, tmp, not_null_free); - set_mdp_flag_at(mdp, ArrayData::null_free_array_byte_constant()); + set_mdp_flag_at(mdp, ArrayStoreData::null_free_array_byte_constant()); bind(not_null_free); @@ -1232,14 +1232,40 @@ template void InterpreterMacroAssembler::profile_array_type(Re } } -template void InterpreterMacroAssembler::profile_array_type(Register mdp, - Register array, - Register tmp); -template void InterpreterMacroAssembler::profile_array_type(Register mdp, - Register array, - Register tmp); +void InterpreterMacroAssembler::profile_multiple_array_types(Register mdp, + Register array, + Register tmp) { + if (ProfileInterpreter) { + Label profile_continue; + + // If no method data exists, go to profile_continue. + test_method_data_pointer(mdp, profile_continue); + + Label not_flat; + test_non_flat_array_oop(array, tmp, not_flat); + + load_klass(tmp, array); + profile_array_type_at_load(tmp, mdp, 0); + + b(profile_continue); + bind(not_flat); + + Label not_null_free; + test_non_null_free_array_oop(array, tmp, not_null_free); + + increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_null_free_count_offset())); + + b(profile_continue); + + bind(not_null_free); + + increment_mdp_data_at(mdp, in_bytes(ArrayLoadData::not_flat_nullable_count_offset())); + + bind(profile_continue); + } +} -void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp, const Register tmp2) { +void InterpreterMacroAssembler::profile_multiple_element_types(Register mdp, Register element, Register tmp) { if (ProfileInterpreter) { Label profile_continue; diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp index d72137a0944..12f359de284 100644 --- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp @@ -303,8 +303,9 @@ class InterpreterMacroAssembler: public MacroAssembler { void profile_switch_default(Register mdp); void profile_switch_case(Register index_in_scratch, Register mdp, Register scratch2); - template void profile_array_type(Register mdp, Register array, Register tmp); - void profile_multiple_element_types(Register mdp, Register element, Register tmp, Register tmp2); + void profile_array_type(Register mdp, Register array, Register tmp); + void profile_multiple_array_types(Register mdp, Register array, Register tmp); + void profile_multiple_element_types(Register mdp, Register element, Register tmp); void profile_element_type(Register mdp, Register element, Register tmp); void profile_acmp(Register mdp, Register left, Register right, Register tmp); diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index f9451cbbf42..52be0cdc5fe 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -2112,56 +2112,32 @@ Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, } } -// Handle the receiver type profile update given the "recv" klass. -// -// Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset". -// If there are no matching or claimable receiver entries in RD, updates -// the polymorphic counter. -// -// This code expected to run by either the interpreter or JIT-ed code, without -// extra synchronization. For safety, receiver cells are claimed atomically, which -// avoids grossly misrepresenting the profiles under concurrent updates. For speed, -// counter updates are not atomic. -// -void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { - assert_different_registers(recv, mdp, rscratch1, rscratch2); - - int base_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(0)); - int end_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(ReceiverTypeData::row_limit())); - int poly_count_offset = in_bytes(CounterData::count_offset()); - int receiver_step = in_bytes(ReceiverTypeData::receiver_offset(1)) - base_receiver_offset; - int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset; +void MacroAssembler::profile_receiver_type_helper(Register recv, Register mdp, Label &L_found_recv, int mdp_offset, int base, uint row_limit) { + int base_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, 0)); + int end_receiver_offset = in_bytes(MegamorphicTypeData::receiver_offset(base, row_limit)); + int receiver_step = in_bytes(MegamorphicTypeData::receiver_offset(base, 1)) - base_receiver_offset; + int receiver_to_count_step = in_bytes(MegamorphicTypeData::receiver_count_offset(base, 0)) - base_receiver_offset; - // Adjust for MDP offsets. base_receiver_offset += mdp_offset; end_receiver_offset += mdp_offset; - poly_count_offset += mdp_offset; #ifdef ASSERT // We are about to walk the MDO slots without asking for offsets. // Check that our math hits all the right spots. for (uint c = 0; c < ReceiverTypeData::row_limit(); c++) { - int real_recv_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_offset(c)); - int real_count_offset = mdp_offset + in_bytes(ReceiverTypeData::receiver_count_offset(c)); + int real_recv_offset = mdp_offset + in_bytes(MegamorphicTypeData::receiver_offset(base, c)); + int real_count_offset = mdp_offset + in_bytes(MegamorphicTypeData::receiver_count_offset(base, c)); int offset = base_receiver_offset + receiver_step*c; int count_offset = offset + receiver_to_count_step; - assert(offset == real_recv_offset, "receiver slot math"); + assert(offset== real_recv_offset, "receiver slot math"); assert(count_offset == real_count_offset, "receiver count math"); } - int real_poly_count_offset = mdp_offset + in_bytes(CounterData::count_offset()); - assert(poly_count_offset == real_poly_count_offset, "poly counter math"); #endif - // Corner case: no profile table. Increment poly counter and exit. - if (ReceiverTypeData::row_limit() == 0) { - increment(Address(mdp, poly_count_offset), DataLayout::counter_increment); - return; - } - Register offset = rscratch2; Label L_loop_search_receiver, L_loop_search_empty; - Label L_restart, L_found_recv, L_found_empty, L_polymorphic, L_count_update; + Label L_restart, L_found_empty, L_polymorphic; // The code here recognizes three major cases: // A. Fastest: receiver found in the table @@ -2215,9 +2191,9 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ // Fastest: receiver is already installed mov(offset, base_receiver_offset); bind(L_loop_search_receiver); - ldr(rscratch1, Address(mdp, offset)); - cmp(rscratch1, recv); - br(Assembler::EQ, L_found_recv); + ldr(rscratch1, Address(mdp, offset)); + cmp(rscratch1, recv); + br(Assembler::EQ, L_found_recv); add(offset, offset, receiver_step); sub(rscratch1, offset, end_receiver_offset); cbnz(rscratch1, L_loop_search_receiver); @@ -2225,8 +2201,8 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ // Fast: no receiver, but profile is full mov(offset, base_receiver_offset); bind(L_loop_search_empty); - ldr(rscratch1, Address(mdp, offset)); - cbz(rscratch1, L_found_empty); + ldr(rscratch1, Address(mdp, offset)); + cbz(rscratch1, L_found_empty); add(offset, offset, receiver_step); sub(rscratch1, offset, end_receiver_offset); cbnz(rscratch1, L_loop_search_empty); @@ -2255,7 +2231,46 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ // Increment polymorphic counter instead of receiver slot. bind(L_polymorphic); +} + +// Handle the receiver type profile update given the "recv" klass. +// +// Normally updates the ReceiverData (RD) that starts at "mdp" + "mdp_offset". +// If there are no matching or claimable receiver entries in RD, updates +// the polymorphic counter. +// +// This code expected to run by either the interpreter or JIT-ed code, without +// extra synchronization. For safety, receiver cells are claimed atomically, which +// avoids grossly misrepresenting the profiles under concurrent updates. For speed, +// counter updates are not atomic. +// +void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_offset) { + assert_different_registers(recv, mdp, rscratch1, rscratch2); + + int poly_count_offset = in_bytes(ReceiverTypeData::count_offset()); + int base_receiver_offset = in_bytes(ReceiverTypeData::receiver_offset(0)); + int receiver_to_count_step = in_bytes(ReceiverTypeData::receiver_count_offset(0)) - base_receiver_offset; + + // Adjust for MDP offsets. + poly_count_offset += mdp_offset; + +#ifdef ASSERT + int real_poly_count_offset = mdp_offset + in_bytes(ReceiverTypeData::count_offset()); + assert(poly_count_offset == real_poly_count_offset, "poly counter math"); +#endif + + // Corner case: no profile table. Increment poly counter and exit. + if (ReceiverTypeData::row_limit() == 0) { + increment(Address(mdp, poly_count_offset), DataLayout::counter_increment); + return; + } + + Label L_found_recv; + profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ReceiverTypeData::base_of_megamorphic_type_data(), ReceiverTypeData::row_limit()); + // Increment polymorphic counter instead of receiver slot. + Register offset = rscratch2; mov(offset, poly_count_offset); + Label L_count_update; b(L_count_update); // Found a receiver, convert its slot offset to corresponding count offset. @@ -2266,6 +2281,68 @@ void MacroAssembler::profile_receiver_type(Register recv, Register mdp, int mdp_ increment(Address(mdp, offset), DataLayout::counter_increment); } +void MacroAssembler::profile_array_type_at_load(Register recv, Register mdp, int mdp_offset) { + int base_receiver_offset = in_bytes(ArrayLoadData::receiver_offset(0)); + int receiver_to_count_step = in_bytes(ArrayLoadData::receiver_count_offset(0)) - base_receiver_offset; + int flat_nullable_count_offset = in_bytes(ArrayLoadData::flat_nullable_count_offset()); + int flat_nullfree_atomic_count_offset = in_bytes(ArrayLoadData::flat_nullfree_atomic_count_offset()); + int flat_nullfree_not_atomic_count_offset = in_bytes(ArrayLoadData::flat_nullfree_not_atomic_count_offset()); + + // Adjust for MDP offsets. Slots are pointer-sized, so is the global offset. + flat_nullable_count_offset += mdp_offset; + flat_nullfree_atomic_count_offset += mdp_offset; + flat_nullfree_not_atomic_count_offset += mdp_offset; + +#ifdef ASSERT + int real_flat_nullable_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullable_count_offset()); + assert(flat_nullable_count_offset == real_flat_nullable_count_offset, "poly counter math"); + int real_flat_nullfree_atomic_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullfree_atomic_count_offset()); + assert(flat_nullfree_atomic_count_offset == real_flat_nullfree_atomic_count_offset, "poly counter math"); + int real_flat_nullfree_not_atomic_count_offset = mdp_offset + in_bytes(ArrayLoadData::flat_nullfree_not_atomic_count_offset()); + assert(flat_nullfree_not_atomic_count_offset == real_flat_nullfree_not_atomic_count_offset, "poly counter math"); +#endif + + // Corner case: no profile table. Increment poly counter and exit. + if (ReceiverTypeData::row_limit() == 0) { + ShouldNotReachHere(); + return; + } + + Label L_found_recv; + profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ArrayLoadData::base_of_megamorphic_type_data(), ArrayLoadData::row_limit()); + Register offset = rscratch2; + int layout_kind_offset = in_bytes(FlatArrayKlass::layout_kind_offset()); + Label null_free_non_atomic, null_free_atomic, nullable_atomic_flat, failure, L_count_update; + ldrw(rscratch1, Address(recv, layout_kind_offset)); + cmpw(rscratch1, (int)LayoutKind::NULL_FREE_ATOMIC_FLAT); + br(NE, null_free_non_atomic); + mov(offset, flat_nullfree_atomic_count_offset); + + b(L_count_update); + bind(null_free_non_atomic); + ldrw(rscratch1, Address(recv, layout_kind_offset)); + cmpw(rscratch1, (int)LayoutKind::NULL_FREE_NON_ATOMIC_FLAT); + br(NE, nullable_atomic_flat); + mov(offset, flat_nullfree_not_atomic_count_offset); + + b(L_count_update); + bind(nullable_atomic_flat); + ldrw(rscratch1, Address(recv, layout_kind_offset)); + cmpw(rscratch1, (int)LayoutKind::NULLABLE_ATOMIC_FLAT); + br(NE, failure); + mov(offset, flat_nullable_count_offset); + + b(L_count_update); + bind(failure); + stop("unexpected flat array"); + + // Found a receiver, convert its slot offset to corresponding count offset. + bind(L_found_recv); + add(offset, offset, receiver_to_count_step); + + bind(L_count_update); + increment(Address(mdp, offset), DataLayout::counter_increment); +} void MacroAssembler::call_VM_leaf_base(address entry_point, int number_of_arguments, diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp index de2aa31dabe..5ae8617be0c 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp @@ -1158,7 +1158,11 @@ class MacroAssembler: public Assembler { Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0); + void profile_receiver_type_helper(Register recv, Register mdp, + Label &L_found_recv, int mdp_offset, int base, uint row_limit); + void profile_receiver_type(Register recv, Register mdp, int mdp_offset); + void profile_array_type_at_load(Register recv, Register mdp, int mdp_offset); void verify_sve_vector_length(Register tmp = rscratch1); void reinitialize_ptrue() { diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp index 698ca7fd67a..e40170ba533 100644 --- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp @@ -818,7 +818,7 @@ void TemplateTable::aaload() // r0: array // r1: index index_check(r0, r1); // leaves index in r1, kills rscratch1 - __ profile_array_type(r2, r0, r4); + __ profile_multiple_array_types(r2, r0, r4); if (UseArrayFlattening) { Label is_flat_array, done; @@ -1130,8 +1130,8 @@ void TemplateTable::aastore() { index_check(r3, r2); // kills r1 - __ profile_array_type(r4, r3, r5); - __ profile_multiple_element_types(r4, r0, r5, r6); + __ profile_array_type(r4, r3, r5); + __ profile_multiple_element_types(r4, r0, r5); __ add(r4, r2, arrayOopDesc::base_offset_in_bytes(T_OBJECT) >> LogBytesPerHeapOop); Address element_address(r3, r4, Address::uxtw(LogBytesPerHeapOop)); From bc7497d5a5a7c94821a71cc723786a9bef979ed0 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Wed, 3 Jun 2026 17:56:04 +0200 Subject: [PATCH 44/49] more --- src/hotspot/share/opto/parse2.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 3ebf8d2687a..6a1ba2248b2 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -279,13 +279,15 @@ class ArrayLoad { _parse.null_check_common(vt->get_null_marker(), T_INT, false, &null_ctl); - _res_phi->add_req(_parse.zerocon(T_OBJECT)); - _region->add_req(null_ctl); - _io_phi->add_req(_parse.i_o()); Node* mem = _parse.reset_memory(); - _mem_phi->add_req(mem); - _parse.set_all_memory(mem); + if (!null_ctl->is_top()) { + _res_phi->add_req(_parse.zerocon(T_OBJECT)); + _region->add_req(null_ctl); + _io_phi->add_req(_parse.i_o()); + _mem_phi->add_req(mem); + } + Node* ld = _gvn.transform(vt->buffer(&_parse)); _res_phi->add_req(ld); From 3a649680e8d5216805a293fe8776be4c40b97619 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 4 Jun 2026 16:21:36 +0200 Subject: [PATCH 45/49] more --- src/hotspot/share/opto/parse2.cpp | 58 +++++-- .../inlinetypes/TestArrayLoadProfiling.java | 158 ++++++++++++++++++ 2 files changed, 206 insertions(+), 10 deletions(-) diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index 6a1ba2248b2..b308bf504f2 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -200,16 +200,25 @@ class ArrayLoad { decorator_set |= C2_UNKNOWN_CONTROL_LOAD; } const TypeInt* sizetype = array_type->size(); - if (element_ptr != nullptr && element_ptr->is_inlinetypeptr() && !array_type->is_null_free()) { + if (element_ptr != nullptr && element_ptr->can_be_inline_type() && !array_type->is_null_free()) { ciArrayLoadData* array_load = profile_data(); - if (array_load != nullptr && array_load->not_flat_nullable_count() == 0 && array_load->not_flat_null_free_count() > 0 && - !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + Deoptimization::DeoptReason reason = Deoptimization::Reason_none; + if (array_type->speculative() != nullptr && + !array_type->speculative()->isa_aryptr()->is_not_null_free() && + !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) { + reason = Deoptimization::Reason_speculate_class_check; + } else if (array_load != nullptr && array_load->not_flat_nullable_count() == 0 && array_load-> + not_flat_null_free_count() > 0 && + !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + reason = Deoptimization::Reason_class_check; + } + if (reason != Deoptimization::Reason_none) { Node* test = _parse.null_free_array_test(_array, false); IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, PROB_MIN, COUNT_UNKNOWN); _parse.set_control(_gvn.transform(new IfTrueNode(iff))); { PreserveJVMState pjvms(&_parse); - _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + _parse.uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile); } _parse.set_control(_gvn.transform(new IfFalseNode(iff))); if (_parse.stopped()) { @@ -499,6 +508,24 @@ class ArrayLoad { } } } + if (!array_type->is_flat() && !array_type->is_not_flat()) { + const TypePtr* speculative = array_type->speculative(); + Deoptimization::DeoptReason reason = Deoptimization::Reason_speculate_class_check; + if (array_type->speculative() != nullptr && + array_type->speculative()->is_aryptr()->is_not_flat() && + !_parse.too_many_traps_or_recompiles(reason)) { + { // Deoptimize if flat array + BuildCutout unless(&_parse, _parse.flat_array_test(_array, /* flat = */ false), PROB_MAX); + _parse.uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile); + } + assert(!_parse.stopped(), "flat array should have been caught earlier"); + Node* casted_array = _gvn.transform(new CheckCastPPNode(_parse.control(), _array, array_type->cast_to_not_flat())); + _parse.replace_in_map(_array, casted_array); + array_type = _gvn.type(casted_array)->is_aryptr(); + _elemtype = array_type->elem(); + _array = casted_array; + } + } } bool emit() { @@ -520,6 +547,7 @@ class ArrayLoad { Node* ld = emit_plain_load(_array, false, true); // pop_stack(); _parse.push_node(_bt, ld); + ld = _parse.record_profile_for_speculation_at_array_load(ld); // record_array_profile_for_speculation(); return true; } @@ -543,6 +571,16 @@ class ArrayLoad { ciArrayLoadData* array_load = profile_data(); if (array_load != nullptr) { int not_flat_count = profiled_not_flat_count(); + Deoptimization::DeoptReason not_flat_reason = Deoptimization::Reason_none; + if (not_flat_count != 0) { + if (array_type->speculative() != nullptr && + array_type->speculative()->is_aryptr()->is_flat()) { + not_flat_count = 0; + not_flat_reason = Deoptimization::Reason_speculate_class_check; + } + } else { + not_flat_reason = Deoptimization::Reason_class_check; + } ciCallProfile profile = _parse.method()->call_profile_at_bci(_parse.bci()); int flat_count = profile.count(); int flat_and_not_flat_count = saturated_add(flat_count, not_flat_count); @@ -557,11 +595,11 @@ class ArrayLoad { int count = i < limit ? profile.receiver_count(i) : not_flat_count; if (not_flat_count >= count && !not_flat_checked) { not_flat_checked = true; - if (profile.morphism() > 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + if (profile.morphism() > 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(not_flat_reason)) { PreserveJVMState pjvms(&_parse); - _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + _parse.uncommon_trap_exact(not_flat_reason, Deoptimization::Action_maybe_recompile); done = true; - } else if (profile.morphism() <= 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + } else if (profile.morphism() <= 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(not_flat_reason)) { const TypeAryPtr* array_type = _gvn.type(_array)->is_aryptr(); Node* test = _parse.flat_array_test(_array, /* flat = */ false); IfNode* iff = _parse.create_and_xform_if(_parse.control(), test, PROB_MIN, COUNT_UNKNOWN); @@ -570,7 +608,7 @@ class ArrayLoad { "Should be found"); { PreserveJVMState pjvms(&_parse); - _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + _parse.uncommon_trap_exact(not_flat_reason, Deoptimization::Action_maybe_recompile); } _parse.set_control(_gvn.transform(new IfFalseNode(iff))); load_from_unknown_flat_array(element_ptr); @@ -617,11 +655,11 @@ class ArrayLoad { record_array_profile_for_speculation(); return true; } - if (flat_count != 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) { + if (flat_count != 0 && not_flat_count == 0 && !_parse.too_many_traps_or_recompiles(not_flat_reason)) { { // Deoptimize if not flat array BuildCutout unless(&_parse, _parse.flat_array_test(_array, /* flat = */ true), PROB_MAX); - _parse.uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile); + _parse.uncommon_trap_exact(not_flat_reason, Deoptimization::Action_maybe_recompile); } assert(!_parse.stopped(), "non flat array should have been caught earlier"); Node* ld = load_from_unknown_flat_array(element_ptr); diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index ea164e36b4a..fe08e898170 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -75,6 +75,12 @@ public static void main(String[] args) { static MyValue5[] array20 = { new MyValue5((byte)42) }; static MyValue4[] array21 = (MyValue4[])ValueClass.newReferenceArray(MyValue4.class, 1); static MyValue2[] array22 = (MyValue2[])ValueClass.newNullRestrictedAtomicArray(MyValue2.class, 1, new MyValue2((byte)42)); + static A[][] array23 = {{ new A() }}; + static MyValue1[][] array24 = {{ new MyValue1((byte)42) }}; + static B[][] array25 = {{ new B() }}; + static MyValue2[][] array26 = {{ new MyValue2((byte)42) }}; + static MyValue3[][] array27 = { (MyValue3[])ValueClass.newNullRestrictedAtomicArray(MyValue3.class, 1, new MyValue3(42)) }; + static MyValue6[][] array28 = { (MyValue6[])ValueClass.newNullRestrictedAtomicArray(MyValue6.class, 1, new MyValue6(42)) }; static { array6[0] = new MyValue1((byte)42); array7[0] = new MyValue2((byte)42); @@ -746,6 +752,136 @@ static void test27Inline(I[] array) { array[0].m(); } + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "3", IRNode.RANGE_CHECK_TRAP, "2", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "6", IRNode.CALL, "6", IRNode.IF, "6" }) + @IR(failOn = IRNode.ALLOC) + public static void test28(I[][] arrayOfArray) { + I[] array = arrayOfArray[0]; + test28Inline(array); + } + + @Run(test = "test28") + @Warmup(10_000) + public static void test28Runner() { + test28(array23); + test28Inline(array1); + test28Inline(array2); + test28Inline(array5); + } + + @ForceInline + static void test28Inline(I[] array) { + array[0].m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "3", IRNode.RANGE_CHECK_TRAP, "2", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "6", IRNode.CALL, "6", IRNode.IF, "6" }) + @IR(failOn = IRNode.ALLOC) + public static void test29(I[][] arrayOfArray) { + I[] array = arrayOfArray[0]; + test29Inline(array); + } + + @Run(test = "test29") + @Warmup(10_000) + public static void test29Runner() { + test29(array24); + test29Inline(array1); + test29Inline(array2); + test29Inline(array5); + } + + @ForceInline + static void test29Inline(I[] array) { + array[0].m(); + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "3", IRNode.RANGE_CHECK_TRAP, "3", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "7", IRNode.CALL, "7", IRNode.IF, "9" }) + @IR(failOn = IRNode.ALLOC) + public static Object test30(boolean flag, I[][] arrayOfArray1, I[][] arrayOfArray2) { + I[] array = null; + if (flag) { + array = arrayOfArray1[0]; + } else { + array = arrayOfArray2[0]; + } + return test30Inline(array); + } + + @Run(test = "test30") + @Warmup(10_000) + public static void test30Runner() { + test30(true, array23, array25); + test30(false, array23, array25); + test30Inline(array1); + test30Inline(array2); + test30Inline(array5); + } + + @ForceInline + static Object test30Inline(I[] array) { + return array[0]; + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "4", IRNode.RANGE_CHECK_TRAP, "3", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "8", IRNode.CALL, "8", IRNode.IF, "11" }) + @IR(failOn = IRNode.ALLOC) + public static void test31(boolean flag, I[][] arrayOfArray1, I[][] arrayOfArray2) { + I[] array = null; + if (flag) { + array = arrayOfArray1[0]; + } else { + array = arrayOfArray2[0]; + } + I i = test31Inline(array); + i.m(); + } + + @Run(test = "test31") + @Warmup(10_000) + public static void test31Runner() { + test31(true, array24, array26); + test31(false, array24, array26); + test31Inline(array1); + test31Inline(array2); + test31Inline(array5); + } + + @ForceInline + static I test31Inline(I[] array) { + return array[0]; + } + + @Test + @IR(counts = { IRNode.NULL_CHECK_TRAP, "3", IRNode.RANGE_CHECK_TRAP, "3", IRNode.CLASS_CHECK_TRAP, "2", IRNode.BIMORPHIC_TRAP, "1", IRNode.TRAP, "9", IRNode.CALL, "9", IRNode.IF, "12" }) + @IR(failOn = IRNode.ALLOC) + public static void test32(boolean flag, I[][] arrayOfArray1, I[][] arrayOfArray2) { + I[] array = null; + if (flag) { + array = arrayOfArray1[0]; + } else { + array = arrayOfArray2[0]; + } + I i = test32Inline(array); + i.m(); + } + + @Run(test = "test32") + @Warmup(10_000) + public static void test32Runner() { + test32(true, array27, array28); + test32(false, array27, array28); + test32Inline(array1); + test32Inline(array2); + test32Inline(array5); + } + + @ForceInline + static I test32Inline(I[] array) { + return array[0]; + } + // @Test // @IR(counts = { IRNode.NULL_CHECK_TRAP, "1", IRNode.RANGE_CHECK_TRAP, "1", IRNode.CLASS_CHECK_TRAP, "1", IRNode.TRAP, "3", IRNode.CALL, "3", IRNode.IF, "3" }) // @IR(failOn = IRNode.ALLOC) @@ -876,4 +1012,26 @@ static class A implements I { public void m() { } } + + static class B implements I { + public void m() { + } + } + + static value class MyValue6 implements I { + int intField1; + int intField2; + int intField3; + int intField4; + + MyValue6(int intField) { + this.intField1 = intField; + this.intField2 = intField; + this.intField3 = intField; + this.intField4 = intField; + } + + public void m() { + } + } } From 3c6c80a9a37357fc9655d8891a41a566df9d72d1 Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 4 Jun 2026 16:53:54 +0200 Subject: [PATCH 46/49] more --- src/hotspot/share/opto/type.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/type.cpp b/src/hotspot/share/opto/type.cpp index 6c27311b35b..66caf23688a 100644 --- a/src/hotspot/share/opto/type.cpp +++ b/src/hotspot/share/opto/type.cpp @@ -5151,9 +5151,15 @@ const TypeAryPtr* TypeAryPtr::cast_to_null_free(bool null_free) const { } else { new_elem = new_elem->meet_speculative(TypePtr::NULL_PTR); } + const TypePtr* speculative = _speculative; + if (speculative != nullptr && speculative->isa_aryptr()) { + if (null_free && speculative->is_aryptr()->is_not_null_free()) { + speculative = nullptr; + } + } new_elem = elem->isa_narrowoop() ? new_elem->make_narrowoop() : new_elem; const TypeAry* new_ary = TypeAry::make(new_elem, size(), is_stable(), is_flat(), is_not_flat(), is_not_null_free(), is_atomic()); - const TypeAryPtr* res = make(ptr(), const_oop(), new_ary, klass(), klass_is_exact(), _offset, _field_offset, _instance_id, _speculative, _inline_depth, _is_autobox_cache); + const TypeAryPtr* res = make(ptr(), const_oop(), new_ary, klass(), klass_is_exact(), _offset, _field_offset, _instance_id, speculative, _inline_depth, _is_autobox_cache); if (res->speculative() == res->remove_speculative()) { return res->remove_speculative(); } From e13665d70a7d0f55e462e9b33ada782afba2a5ac Mon Sep 17 00:00:00 2001 From: rwestrel Date: Thu, 4 Jun 2026 17:13:43 +0200 Subject: [PATCH 47/49] more --- src/hotspot/cpu/x86/macroAssembler_x86.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp index 6e411179eb4..e3337d5522f 100644 --- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp @@ -5182,14 +5182,11 @@ void MacroAssembler::profile_array_type_at_load(Register recv, Register mdp, int assert(flat_nullfree_not_atomic_count_offset << LogBytesPerWord == real_flat_nullfree_not_atomic_count_offset, "poly counter math"); #endif + Label L_found_recv; // Corner case: no profile table. Increment poly counter and exit. - if (ReceiverTypeData::row_limit() == 0) { - ShouldNotReachHere(); - return; + if (ReceiverTypeData::row_limit() != 0) { + profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ArrayLoadData::base_of_megamorphic_type_data(), ArrayLoadData::row_limit()); } - - Label L_found_recv; - profile_receiver_type_helper(recv, mdp, L_found_recv, mdp_offset, ArrayLoadData::base_of_megamorphic_type_data(), ArrayLoadData::row_limit()); Register offset = rscratch1; int layout_kind_offset = in_bytes(FlatArrayKlass::layout_kind_offset()); Label null_free_non_atomic, null_free_atomic, nullable_atomic_flat, failure, L_count_update; @@ -5213,9 +5210,11 @@ void MacroAssembler::profile_array_type_at_load(Register recv, Register mdp, int bind(failure); stop("unexpected flat array"); - // Found a receiver, convert its slot offset to corresponding count offset. - bind(L_found_recv); - addptr(offset, receiver_to_count_step); + if (ReceiverTypeData::row_limit() != 0) { + // Found a receiver, convert its slot offset to corresponding count offset. + bind(L_found_recv); + addptr(offset, receiver_to_count_step); + } bind(L_count_update); addptr(Address(mdp, offset, Address::times_ptr), DataLayout::counter_increment); From 8cac664a2edbf269778e19cb87dd92a3da7b891d Mon Sep 17 00:00:00 2001 From: rwestrel Date: Fri, 5 Jun 2026 09:30:52 +0200 Subject: [PATCH 48/49] more --- .../valhalla/inlinetypes/TestArrayLoadProfiling.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java index fe08e898170..56f41121bc9 100644 --- a/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java +++ b/test/hotspot/jtreg/compiler/valhalla/inlinetypes/TestArrayLoadProfiling.java @@ -118,7 +118,7 @@ static void test1Inline(I i) { static void trapAndRecompile(String name, Runnable causesTrap) throws Exception { Method m = TestArrayLoadProfiling.class.getDeclaredMethod(name, I[].class); - if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) { + if (TestFramework.isStableDeopt(m, CompLevel.C2) && (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION)) { throw new RuntimeException("should be compiled"); } int i = 0; @@ -126,11 +126,15 @@ static void trapAndRecompile(String name, Runnable causesTrap) throws Exception causesTrap.run(); i++; if (i > 10) { - throw new RuntimeException("should not be compiled anymore"); + if (TestFramework.isStableDeopt(m, CompLevel.C2)) { + throw new RuntimeException("should not be compiled anymore"); + } else { + break; + } } } while (WHITE_BOX.isMethodCompiled(m)); WHITE_BOX.enqueueMethodForCompilation(m, CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION); - if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) { + if (TestFramework.isStableDeopt(m, CompLevel.C2) && (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION)) { throw new RuntimeException("should be compiled"); } } From 354f66fb4c6d7ea8907067fb58a826961b15858d Mon Sep 17 00:00:00 2001 From: rwestrel Date: Fri, 5 Jun 2026 09:45:31 +0200 Subject: [PATCH 49/49] more --- src/hotspot/share/opto/parse2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/parse2.cpp b/src/hotspot/share/opto/parse2.cpp index b308bf504f2..d3a419ea74f 100644 --- a/src/hotspot/share/opto/parse2.cpp +++ b/src/hotspot/share/opto/parse2.cpp @@ -246,7 +246,7 @@ class ArrayLoad { pop_stack(); _parse.replace_in_map(_array, array); } - return _gvn.transform(ld); + return ld; } void test_non_flat_array_and_emit_reference_load(float p) {