代码拉取完成，页面将自动刷新
同步操作将从 jiangfeilong/openjdk-11 强制同步，此操作会覆盖自 Fork 仓库以来所做的任何修改，且无法恢复！！！
确定后同步将在后台操作，完成时将刷新页面，请耐心等待。
diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad
index af3e593..05b36e2 100644
--- a/src/hotspot/cpu/aarch64/aarch64.ad
+++ b/src/hotspot/cpu/aarch64/aarch64.ad
@@ -1131,6 +1131,7 @@ definitions %{
source_hpp %{
#include "gc/z/c2/zBarrierSetC2.hpp"
+#include "gc/z/zThreadLocalData.hpp"
%}
@@ -2501,17 +2502,7 @@ void Compile::reshape_address(AddPNode* addp) {
__ INSN(REG, as_Register(BASE)); \
}
-typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
-typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
-typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
- MacroAssembler::SIMD_RegVariant T, const Address &adr);
-
- // Used for all non-volatile memory accesses. The use of
- // $mem->opcode() to discover whether this pattern uses sign-extended
- // offsets is something of a kludge.
- static void loadStore(MacroAssembler masm, mem_insn insn,
- Register reg, int opcode,
- Register base, int index, int size, int disp)
+static Address mem2address(int opcode, Register base, int index, int size, int disp)
{
Address::extend scale;
@@ -2530,16 +2521,33 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
}
if (index == -1) {
- (masm.*insn)(reg, Address(base, disp));
+ return Address(base, disp);
} else {
assert(disp == 0, "unsupported address mode: disp = %d", disp);
- (masm.*insn)(reg, Address(base, as_Register(index), scale));
+ return Address(base, as_Register(index), scale);
}
}
+typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
+typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
+typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
+typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
+ MacroAssembler::SIMD_RegVariant T, const Address &adr);
+
+ // Used for all non-volatile memory accesses. The use of
+ // $mem->opcode() to discover whether this pattern uses sign-extended
+ // offsets is something of a kludge.
+ static void loadStore(MacroAssembler masm, mem_insn insn,
+ Register reg, int opcode,
+ Register base, int index, int size, int disp)
+ {
+ Address addr = mem2address(opcode, base, index, size, disp);
+ (masm.*insn)(reg, addr);
+ }
+
static void loadStore(MacroAssembler masm, mem_float_insn insn,
- FloatRegister reg, int opcode,
- Register base, int index, int size, int disp)
+ FloatRegister reg, int opcode,
+ Register base, int index, int size, int disp)
{
Address::extend scale;
@@ -2561,8 +2569,8 @@ typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
}
static void loadStore(MacroAssembler masm, mem_vector_insn insn,
- FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
- int opcode, Register base, int index, int size, int disp)
+ FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
+ int opcode, Register base, int index, int size, int disp)
{
if (index == -1) {
(masm.*insn)(reg, T, Address(base, disp));
@@ -3797,7 +3805,7 @@ frame %{
static const int hi[Op_RegL + 1] = { // enum name
0, // Op_Node
0, // Op_Set
- OptoReg::Bad, // Op_RegN
+ OptoReg::Bad, // Op_RegN
OptoReg::Bad, // Op_RegI
R0_H_num, // Op_RegP
OptoReg::Bad, // Op_RegF
@@ -6929,7 +6937,7 @@ instruct loadRange(iRegINoSp dst, memory mem)
instruct loadP(iRegPNoSp dst, memory mem)
%{
match(Set dst (LoadP mem));
- predicate(!needs_acquiring_load(n));
+ predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
ins_cost(4 * INSN_COST);
format %{ "ldr $dst, $mem\t# ptr" %}
@@ -7622,6 +7630,7 @@ instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
%{
match(Set dst (LoadP mem));
+ predicate(n->as_Load()->barrier_data() == 0);
ins_cost(VOLATILE_REF_COST);
format %{ "ldar $dst, $mem\t# ptr" %}
@@ -8506,6 +8515,7 @@ instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoS
instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ predicate(n->as_LoadStore()->barrier_data() == 0);
ins_cost(2 * VOLATILE_REF_COST);
effect(KILL cr);
@@ -8619,7 +8629,7 @@ instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegL
instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
- predicate(needs_acquiring_load_exclusive(n));
+ predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
@@ -8750,6 +8760,7 @@ instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN ne
%}
instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr);
@@ -8849,7 +8860,7 @@ instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN
%}
instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
- predicate(needs_acquiring_load_exclusive(n));
+ predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
ins_cost(VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr);
@@ -8950,6 +8961,7 @@ instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN ne
%}
instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
ins_cost(2 * VOLATILE_REF_COST);
effect(KILL cr);
@@ -9057,8 +9069,8 @@ instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN
%}
instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
- predicate(needs_acquiring_load_exclusive(n));
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+ predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
ins_cost(VOLATILE_REF_COST);
effect(KILL cr);
format %{
@@ -9108,6 +9120,7 @@ instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
%}
instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set prev (GetAndSetP mem newv));
ins_cost(2 * VOLATILE_REF_COST);
format %{ "atomic_xchg $prev, $newv, [$mem]" %}
@@ -9151,7 +9164,7 @@ instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
%}
instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
- predicate(needs_acquiring_load_exclusive(n));
+ predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
match(Set prev (GetAndSetP mem newv));
ins_cost(VOLATILE_REF_COST);
format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
@@ -17477,145 +17490,238 @@ instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
source %{
-#include "gc/z/zBarrierSetAssembler.hpp"
+static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) {
+ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak);
+ __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
+ __ andr(tmp, tmp, ref);
+ __ cbnz(tmp, *stub->entry());
+ __ bind(*stub->continuation());
+}
-static void z_load_barrier_slow_reg(MacroAssembler& _masm, Register dst,
- Register base, int index, int scale,
- int disp, bool weak) {
- const address stub = weak ? ZBarrierSet::assembler()->load_barrier_weak_slow_stub(dst)
- : ZBarrierSet::assembler()->load_barrier_slow_stub(dst);
+static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
+ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */);
+ __ b(*stub->entry());
+ __ bind(*stub->continuation());
+}
- if (index == -1) {
- if (disp != 0) {
- __ lea(dst, Address(base, disp));
- } else {
- __ mov(dst, base);
- }
- } else {
- Register index_reg = as_Register(index);
- if (disp == 0) {
- __ lea(dst, Address(base, index_reg, Address::lsl(scale)));
- } else {
- __ lea(dst, Address(base, disp));
- __ lea(dst, Address(dst, index_reg, Address::lsl(scale)));
+%}
+
+// Load Pointer
+instruct zLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr)
+%{
+ match(Set dst (LoadP mem));
+ predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierStrong));
+ effect(TEMP dst, KILL cr);
+
+ ins_cost(4 * INSN_COST);
+
+ format %{ "ldr $dst, $mem" %}
+
+ ins_encode %{
+ const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
+ __ ldr($dst$$Register, ref_addr);
+ if (barrier_data() != ZLoadBarrierElided) {
+ z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, false /* weak */);
}
- }
+ %}
- __ far_call(RuntimeAddress(stub));
-}
+ ins_pipe(iload_reg_mem);
+%}
+
+// Load Weak Pointer
+instruct zLoadWeakP(iRegPNoSp dst, memory mem, rFlagsReg cr)
+%{
+ match(Set dst (LoadP mem));
+ predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierWeak));
+ effect(TEMP dst, KILL cr);
+
+ ins_cost(4 * INSN_COST);
+ format %{ "ldr $dst, $mem" %}
+
+ ins_encode %{
+ const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
+ __ ldr($dst$$Register, ref_addr);
+ z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, true /* weak */);
+ %}
+
+ ins_pipe(iload_reg_mem);
%}
-//
-// Execute ZGC load barrier (strong) slow path
-//
-instruct loadBarrierSlowReg(iRegP dst, memory mem, rFlagsReg cr,
- vRegD_V0 v0, vRegD_V1 v1, vRegD_V2 v2, vRegD_V3 v3, vRegD_V4 v4,
- vRegD_V5 v5, vRegD_V6 v6, vRegD_V7 v7, vRegD_V8 v8, vRegD_V9 v9,
- vRegD_V10 v10, vRegD_V11 v11, vRegD_V12 v12, vRegD_V13 v13, vRegD_V14 v14,
- vRegD_V15 v15, vRegD_V16 v16, vRegD_V17 v17, vRegD_V18 v18, vRegD_V19 v19,
- vRegD_V20 v20, vRegD_V21 v21, vRegD_V22 v22, vRegD_V23 v23, vRegD_V24 v24,
- vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29,
- vRegD_V30 v30, vRegD_V31 v31) %{
- match(Set dst (LoadBarrierSlowReg mem));
- predicate(!n->as_LoadBarrierSlowReg()->is_weak());
-
- effect(DEF dst, KILL cr,
- KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7,
- KILL v8, KILL v9, KILL v10, KILL v11, KILL v12, KILL v13, KILL v14,
- KILL v15, KILL v16, KILL v17, KILL v18, KILL v19, KILL v20, KILL v21,
- KILL v22, KILL v23, KILL v24, KILL v25, KILL v26, KILL v27, KILL v28,
- KILL v29, KILL v30, KILL v31);
-
- format %{"LoadBarrierSlowReg $dst, $mem" %}
- ins_encode %{
- z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register,
- $mem$$index, $mem$$scale, $mem$$disp, false);
+// Load Pointer Volatile
+instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr)
+%{
+ match(Set dst (LoadP mem));
+ predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() == ZLoadBarrierStrong);
+ effect(TEMP dst, KILL cr);
+
+ ins_cost(VOLATILE_REF_COST);
+
+ format %{ "ldar $dst, $mem\t" %}
+
+ ins_encode %{
+ __ ldar($dst$$Register, $mem$$Register);
+ if (barrier_data() != ZLoadBarrierElided) {
+ z_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, false /* weak */);
+ }
%}
- ins_pipe(pipe_slow);
+
+ ins_pipe(pipe_serial);
%}
-//
-// Execute ZGC load barrier (weak) slow path
-//
-instruct loadBarrierWeakSlowReg(iRegP dst, memory mem, rFlagsReg cr,
- vRegD_V0 v0, vRegD_V1 v1, vRegD_V2 v2, vRegD_V3 v3, vRegD_V4 v4,
- vRegD_V5 v5, vRegD_V6 v6, vRegD_V7 v7, vRegD_V8 v8, vRegD_V9 v9,
- vRegD_V10 v10, vRegD_V11 v11, vRegD_V12 v12, vRegD_V13 v13, vRegD_V14 v14,
- vRegD_V15 v15, vRegD_V16 v16, vRegD_V17 v17, vRegD_V18 v18, vRegD_V19 v19,
- vRegD_V20 v20, vRegD_V21 v21, vRegD_V22 v22, vRegD_V23 v23, vRegD_V24 v24,
- vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29,
- vRegD_V30 v30, vRegD_V31 v31) %{
- match(Set dst (LoadBarrierSlowReg mem));
- predicate(n->as_LoadBarrierSlowReg()->is_weak());
-
- effect(DEF dst, KILL cr,
- KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7,
- KILL v8, KILL v9, KILL v10, KILL v11, KILL v12, KILL v13, KILL v14,
- KILL v15, KILL v16, KILL v17, KILL v18, KILL v19, KILL v20, KILL v21,
- KILL v22, KILL v23, KILL v24, KILL v25, KILL v26, KILL v27, KILL v28,
- KILL v29, KILL v30, KILL v31);
-
- format %{"LoadBarrierWeakSlowReg $dst, $mem" %}
- ins_encode %{
- z_load_barrier_slow_reg(_masm, $dst$$Register, $mem$$base$$Register,
- $mem$$index, $mem$$scale, $mem$$disp, true);
+instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+ match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
+ effect(KILL cr, TEMP_DEF res);
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{ "cmpxchg $mem, $oldval, $newval\n\t"
+ "cset $res, EQ" %}
+
+ ins_encode %{
+ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ false /* acquire */, true /* release */, false /* weak */, rscratch2);
+ __ cset($res$$Register, Assembler::EQ);
+ if (barrier_data() != ZLoadBarrierElided) {
+ Label good;
+ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
+ __ andr(rscratch1, rscratch1, rscratch2);
+ __ cbz(rscratch1, good);
+ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ false /* acquire */, true /* release */, false /* weak */, rscratch2);
+ __ cset($res$$Register, Assembler::EQ);
+ __ bind(good);
+ }
%}
+
ins_pipe(pipe_slow);
%}
-// Specialized versions of compareAndExchangeP that adds a keepalive that is consumed
-// but doesn't affect output.
+instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+ match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+ predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong));
+ effect(KILL cr, TEMP_DEF res);
-instruct z_compareAndExchangeP(iRegPNoSp res, indirect mem,
- iRegP oldval, iRegP newval, iRegP keepalive,
- rFlagsReg cr) %{
- match(Set res (ZCompareAndExchangeP (Binary mem keepalive) (Binary oldval newval)));
ins_cost(2 * VOLATILE_REF_COST);
- effect(TEMP_DEF res, KILL cr);
- format %{
- "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
- %}
+
+ format %{ "cmpxchg $mem, $oldval, $newval\n\t"
+ "cset $res, EQ" %}
+
ins_encode %{
- __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
- Assembler::xword, /*acquire*/ false, /*release*/ true,
- /*weak*/ false, $res$$Register);
+ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ true /* acquire */, true /* release */, false /* weak */, rscratch2);
+ __ cset($res$$Register, Assembler::EQ);
+ if (barrier_data() != ZLoadBarrierElided) {
+ Label good;
+ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
+ __ andr(rscratch1, rscratch1, rscratch2);
+ __ cbz(rscratch1, good);
+ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ );
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ true /* acquire */, true /* release */, false /* weak */, rscratch2);
+ __ cset($res$$Register, Assembler::EQ);
+ __ bind(good);
+ }
%}
+
ins_pipe(pipe_slow);
%}
-instruct z_compareAndSwapP(iRegINoSp res,
- indirect mem,
- iRegP oldval, iRegP newval, iRegP keepalive,
- rFlagsReg cr) %{
-
- match(Set res (ZCompareAndSwapP (Binary mem keepalive) (Binary oldval newval)));
- match(Set res (ZWeakCompareAndSwapP (Binary mem keepalive) (Binary oldval newval)));
+instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+ match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
+ effect(TEMP_DEF res, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
- effect(KILL cr);
+ format %{ "cmpxchg $res = $mem, $oldval, $newval" %}
- format %{
- "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
- "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
- %}
-
- ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
- aarch64_enc_cset_eq(res));
+ ins_encode %{
+ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ false /* acquire */, true /* release */, false /* weak */, $res$$Register);
+ if (barrier_data() != ZLoadBarrierElided) {
+ Label good;
+ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
+ __ andr(rscratch1, rscratch1, $res$$Register);
+ __ cbz(rscratch1, good);
+ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ false /* acquire */, true /* release */, false /* weak */, $res$$Register);
+ __ bind(good);
+ }
+ %}
ins_pipe(pipe_slow);
%}
+instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+ match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+ predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
+ effect(TEMP_DEF res, KILL cr);
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{ "cmpxchg $res = $mem, $oldval, $newval" %}
-instruct z_get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev,
- iRegP keepalive) %{
- match(Set prev (ZGetAndSetP mem (Binary newv keepalive)));
+ ins_encode %{
+ guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ true /* acquire */, true /* release */, false /* weak */, $res$$Register);
+ if (barrier_data() != ZLoadBarrierElided) {
+ Label good;
+ __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
+ __ andr(rscratch1, rscratch1, $res$$Register);
+ __ cbz(rscratch1, good);
+ z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */);
+ __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+ true /* acquire */, true /* release */, false /* weak */, $res$$Register);
+ __ bind(good);
+ }
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+ match(Set prev (GetAndSetP mem newv));
+ predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
+ effect(TEMP_DEF prev, KILL cr);
ins_cost(2 * VOLATILE_REF_COST);
+
format %{ "atomic_xchg $prev, $newv, [$mem]" %}
+
ins_encode %{
- __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
+ __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register);
+ if (barrier_data() != ZLoadBarrierElided) {
+ z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */);
+ }
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
+instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+ match(Set prev (GetAndSetP mem newv));
+ predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong));
+ effect(TEMP_DEF prev, KILL cr);
+
+ ins_cost(VOLATILE_REF_COST);
+
+ format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
+
+ ins_encode %{
+ __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register);
+ if (barrier_data() != ZLoadBarrierElided) {
+ z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */);
+ }
%}
ins_pipe(pipe_serial);
%}
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
index 8e169ac..787c0c1 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
@@ -24,22 +24,23 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/codeBlob.hpp"
+#include "code/vmreg.inline.hpp"
#include "gc/z/zBarrier.inline.hpp"
#include "gc/z/zBarrierSet.hpp"
#include "gc/z/zBarrierSetAssembler.hpp"
#include "gc/z/zBarrierSetRuntime.hpp"
+#include "gc/z/zThreadLocalData.hpp"
#include "memory/resourceArea.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "utilities/macros.hpp"
#ifdef COMPILER1
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "gc/z/c1/zBarrierSetC1.hpp"
#endif // COMPILER1
-
-#include "gc/z/zThreadLocalData.hpp"
-
-ZBarrierSetAssembler::ZBarrierSetAssembler() :
- _load_barrier_slow_stub(),
- _load_barrier_weak_slow_stub() {}
+#ifdef COMPILER2
+#include "gc/z/c2/zBarrierSetC2.hpp"
+#endif // COMPILER2
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
@@ -63,27 +64,25 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
return;
}
- // rscratch1 can be passed as src or dst, so don't use it.
- RegSet savedRegs = RegSet::of(rscratch2, rheapbase);
+ assert_different_registers(rscratch1, rscratch2, src.base());
+ assert_different_registers(rscratch1, rscratch2, dst);
- Label done;
- assert_different_registers(rheapbase, rscratch2, dst);
- assert_different_registers(rheapbase, rscratch2, src.base());
+ RegSet savedRegs = RegSet::range(r0, r28) - RegSet::of(dst, rscratch1, rscratch2);
- __ push(savedRegs, sp);
+ Label done;
// Load bad mask into scratch register.
- __ ldr(rheapbase, address_bad_mask_from_thread(rthread));
+ __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
__ lea(rscratch2, src);
__ ldr(dst, src);
// Test reference against bad mask. If mask bad, then we need to fix it up.
- __ tst(dst, rheapbase);
+ __ tst(dst, rscratch1);
__ br(Assembler::EQ, done);
__ enter();
- __ push(RegSet::range(r0,r28) - RegSet::of(dst), sp);
+ __ push(savedRegs, sp);
if (c_rarg0 != dst) {
__ mov(c_rarg0, dst);
@@ -91,13 +90,15 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
__ mov(c_rarg1, rscratch2);
int step = 4 * wordSize;
- __ mov(rscratch1, -step);
+ __ mov(rscratch2, -step);
__ sub(sp, sp, step);
for (int i = 28; i >= 4; i -= 4) {
__ st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
- as_FloatRegister(i+3), __ T1D, Address(__ post(sp, rscratch1)));
+ as_FloatRegister(i+3), __ T1D, Address(__ post(sp, rscratch2)));
}
+ __ st1(as_FloatRegister(0), as_FloatRegister(1), as_FloatRegister(2),
+ as_FloatRegister(3), __ T1D, Address(sp));
__ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
@@ -111,13 +112,10 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
__ mov(dst, r0);
}
- __ pop(RegSet::range(r0,r28) - RegSet::of(dst), sp);
+ __ pop(savedRegs, sp);
__ leave();
__ bind(done);
-
- // Restore tmps
- __ pop(savedRegs, sp);
}
#ifdef ASSERT
@@ -209,7 +207,7 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
// The Address offset is too large to direct load - -784. Our range is +127, -128.
__ mov(tmp, (long int)(in_bytes(ZThreadLocalData::address_bad_mask_offset()) -
- in_bytes(JavaThread::jni_environment_offset())));
+ in_bytes(JavaThread::jni_environment_offset())));
// Load address bad mask
__ add(tmp, jni_env, tmp);
__ ldr(tmp, Address(tmp));
@@ -297,111 +295,173 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler*
__ prologue("zgc_load_barrier stub", false);
// We don't use push/pop_clobbered_registers() - we need to pull out the result from r0.
- for (int i = 0; i < 32; i +=2) {
- __ stpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ pre(sp,-16)));
+ for (int i = 0; i < 32; i += 2) {
+ __ stpd(as_FloatRegister(i), as_FloatRegister(i + 1), Address(__ pre(sp,-16)));
}
- RegSet saveRegs = RegSet::range(r0,r28) - RegSet::of(r0);
- __ push(saveRegs, sp);
-
+ const RegSet save_regs = RegSet::range(r1, r28);
+ __ push(save_regs, sp);
// Setup arguments
__ load_parameter(0, c_rarg0);
__ load_parameter(1, c_rarg1);
__ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
- __ pop(saveRegs, sp);
+ __ pop(save_regs, sp);
- for (int i = 30; i >0; i -=2) {
- __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1), Address(__ post(sp, 16)));
+ for (int i = 30; i >0; i -= 2) {
+ __ ldpd(as_FloatRegister(i), as_FloatRegister(i + 1), Address(__ post(sp, 16)));
}
__ epilogue();
}
#endif // COMPILER1
-#undef __
-#define __ cgen->assembler()->
-
-// Generates a register specific stub for calling
-// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or
-// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded().
-//
-// The raddr register serves as both input and output for this stub. When the stub is
-// called the raddr register contains the object field address (oop*) where the bad oop
-// was loaded from, which caused the slow path to be taken. On return from the stub the
-// raddr register contains the good/healed oop returned from
-// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or
-// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded().
-static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) {
- // Don't generate stub for invalid registers
- if (raddr == zr || raddr == r29 || raddr == r30) {
- return NULL;
+#ifdef COMPILER2
+
+OptoReg::Name ZBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
+ if (!OptoReg::is_reg(opto_reg)) {
+ return OptoReg::Bad;
}
- // Create stub name
- char name[64];
- const bool weak = (decorators & ON_WEAK_OOP_REF) != 0;
- os::snprintf(name, sizeof(name), "zgc_load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name());
+ const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+ if (vm_reg->is_FloatRegister()) {
+ return opto_reg & ~1;
+ }
- __ align(CodeEntryAlignment);
- StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode));
- address start = __ pc();
+ return opto_reg;
+}
- // Save live registers
- RegSet savedRegs = RegSet::range(r0,r18) - RegSet::of(raddr);
+#undef __
+#define __ _masm->
+
+class ZSaveLiveRegisters {
+private:
+ MacroAssembler* const _masm;
+ RegSet _gp_regs;
+ RegSet _fp_regs;
+
+public:
+ void initialize(ZLoadBarrierStubC2* stub) {
+ // Create mask of live registers
+ RegMask live = stub->live();
+
+ // Record registers that needs to be saved/restored
+ while (live.is_NotEmpty()) {
+ const OptoReg::Name opto_reg = live.find_first_elem();
+ live.Remove(opto_reg);
+ if (OptoReg::is_reg(opto_reg)) {
+ const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+ if (vm_reg->is_Register()) {
+ _gp_regs += RegSet::of(vm_reg->as_Register());
+ } else if (vm_reg->is_FloatRegister()) {
+ _fp_regs += RegSet::of((Register)vm_reg->as_FloatRegister());
+ } else {
+ fatal("Unknown register type");
+ }
+ }
+ }
- __ enter();
- __ push(savedRegs, sp);
+ // Remove C-ABI SOE registers, scratch regs and _ref register that will be updated
+ _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->ref());
+ }
- // Setup arguments
- if (raddr != c_rarg1) {
- __ mov(c_rarg1, raddr);
- }
+ ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
+ _masm(masm),
+ _gp_regs(),
+ _fp_regs() {
- __ ldr(c_rarg0, Address(raddr));
+ // Figure out what registers to save/restore
+ initialize(stub);
- // Call barrier function
- __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1);
+ // Save registers
+ __ push(_gp_regs, sp);
+ __ push_fp(_fp_regs, sp);
+ }
- // Move result returned in r0 to raddr, if needed
- if (raddr != r0) {
- __ mov(raddr, r0);
+ ~ZSaveLiveRegisters() {
+ // Restore registers
+ __ pop_fp(_fp_regs, sp);
+ __ pop(_gp_regs, sp);
}
+};
- __ pop(savedRegs, sp);
- __ leave();
- __ ret(lr);
+#undef __
+#define __ _masm->
+
+class ZSetupArguments {
+private:
+ MacroAssembler* const _masm;
+ const Register _ref;
+ const Address _ref_addr;
+
+public:
+ ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
+ _masm(masm),
+ _ref(stub->ref()),
+ _ref_addr(stub->ref_addr()) {
+
+ // Setup arguments
+ if (_ref_addr.base() == noreg) {
+ // No self healing
+ if (_ref != c_rarg0) {
+ __ mov(c_rarg0, _ref);
+ }
+ __ mov(c_rarg1, 0);
+ } else {
+ // Self healing
+ if (_ref == c_rarg0) {
+ // _ref is already at correct place
+ __ lea(c_rarg1, _ref_addr);
+ } else if (_ref != c_rarg1) {
+ // _ref is in wrong place, but not in c_rarg1, so fix it first
+ __ lea(c_rarg1, _ref_addr);
+ __ mov(c_rarg0, _ref);
+ } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) {
+ assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0");
+ __ mov(c_rarg0, _ref);
+ __ lea(c_rarg1, _ref_addr);
+ } else {
+ assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0");
+ if (_ref_addr.base() == c_rarg0 || _ref_addr.index() == c_rarg0) {
+ __ mov(rscratch2, c_rarg1);
+ __ lea(c_rarg1, _ref_addr);
+ __ mov(c_rarg0, rscratch2);
+ } else {
+ ShouldNotReachHere();
+ }
+ }
+ }
+ }
- return start;
-}
+ ~ZSetupArguments() {
+ // Transfer result
+ if (_ref != r0) {
+ __ mov(_ref, r0);
+ }
+ }
+};
#undef __
+#define __ masm->
-static void barrier_stubs_init_inner(const char* label, const DecoratorSet decorators, address* stub) {
- const int nregs = 28; // Exclude FP, XZR, SP from calculation.
- const int code_size = nregs * 254; // Rough estimate of code size
-
- ResourceMark rm;
+void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const {
+ BLOCK_COMMENT("ZLoadBarrierStubC2");
- CodeBuffer buf(BufferBlob::create(label, code_size));
- StubCodeGenerator cgen(&buf);
+ // Stub entry
+ __ bind(*stub->entry());
- for (int i = 0; i < nregs; i++) {
- const Register reg = as_Register(i);
- stub[i] = generate_load_barrier_stub(&cgen, reg, decorators);
+ {
+ ZSaveLiveRegisters save_live_registers(masm, stub);
+ ZSetupArguments setup_arguments(masm, stub);
+ __ mov(rscratch1, stub->slow_path());
+ __ blr(rscratch1);
}
-}
-void ZBarrierSetAssembler::barrier_stubs_init() {
- barrier_stubs_init_inner("zgc_load_barrier_stubs", ON_STRONG_OOP_REF, _load_barrier_slow_stub);
- barrier_stubs_init_inner("zgc_load_barrier_weak_stubs", ON_WEAK_OOP_REF, _load_barrier_weak_slow_stub);
+ // Stub exit
+ __ b(*stub->continuation());
}
-address ZBarrierSetAssembler::load_barrier_slow_stub(Register reg) {
- return _load_barrier_slow_stub[reg->encoding()];
-}
+#undef __
-address ZBarrierSetAssembler::load_barrier_weak_slow_stub(Register reg) {
- return _load_barrier_weak_slow_stub[reg->encoding()];
-}
+#endif // COMPILER2
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
index 7e8be01..cca8738 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
@@ -24,6 +24,12 @@
#ifndef CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP
#define CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP
+#include "code/vmreg.hpp"
+#include "oops/accessDecorators.hpp"
+#ifdef COMPILER2
+#include "opto/optoreg.hpp"
+#endif // COMPILER2
+
#ifdef COMPILER1
class LIR_Assembler;
class LIR_OprDesc;
@@ -32,14 +38,13 @@ class StubAssembler;
class ZLoadBarrierStubC1;
#endif // COMPILER1
-class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
-private:
- address _load_barrier_slow_stub[RegisterImpl::number_of_registers];
- address _load_barrier_weak_slow_stub[RegisterImpl::number_of_registers];
+#ifdef COMPILER2
+class Node;
+class ZLoadBarrierStubC2;
+#endif // COMPILER2
+class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
public:
- ZBarrierSetAssembler();
-
virtual void load_at(MacroAssembler* masm,
DecoratorSet decorators,
BasicType type,
@@ -83,10 +88,13 @@ public:
DecoratorSet decorators) const;
#endif // COMPILER1
- virtual void barrier_stubs_init();
+#ifdef COMPILER2
+ OptoReg::Name refine_register(const Node* node,
+ OptoReg::Name opto_reg);
- address load_barrier_slow_stub(Register reg);
- address load_barrier_weak_slow_stub(Register reg);
+ void generate_c2_load_barrier_stub(MacroAssembler* masm,
+ ZLoadBarrierStubC2* stub) const;
+#endif // COMPILER2
};
#endif // CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index 279ff79..f9b2b3b 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -2096,6 +2096,65 @@ int MacroAssembler::pop(unsigned int bitset, Register stack) {
return count;
}
+
+// Push lots of registers in the bit set supplied. Don't push sp.
+// Return the number of words pushed
+int MacroAssembler::push_fp(unsigned int bitset, Register stack) {
+ int words_pushed = 0;
+
+ // Scan bitset to accumulate register pairs
+ unsigned char regs[32];
+ int count = 0;
+ for (int reg = 0; reg <= 31; reg++) {
+ if (1 & bitset)
+ regs[count++] = reg;
+ bitset >>= 1;
+ }
+ regs[count++] = zr->encoding_nocheck();
+ count &= ~1; // Only push an even number of regs
+
+ // Always pushing full 128 bit registers.
+ if (count) {
+ stpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(pre(stack, -count * wordSize * 2)));
+ words_pushed += 2;
+ }
+ for (int i = 2; i < count; i += 2) {
+ stpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2));
+ words_pushed += 2;
+ }
+
+ assert(words_pushed == count, "oops, pushed != count");
+ return count;
+}
+
+int MacroAssembler::pop_fp(unsigned int bitset, Register stack) {
+ int words_pushed = 0;
+
+ // Scan bitset to accumulate register pairs
+ unsigned char regs[32];
+ int count = 0;
+ for (int reg = 0; reg <= 31; reg++) {
+ if (1 & bitset)
+ regs[count++] = reg;
+ bitset >>= 1;
+ }
+ regs[count++] = zr->encoding_nocheck();
+ count &= ~1;
+
+ for (int i = 2; i < count; i += 2) {
+ ldpq(as_FloatRegister(regs[i]), as_FloatRegister(regs[i+1]), Address(stack, i * wordSize * 2));
+ words_pushed += 2;
+ }
+ if (count) {
+ ldpq(as_FloatRegister(regs[0]), as_FloatRegister(regs[1]), Address(post(stack, count * wordSize * 2)));
+ words_pushed += 2;
+ }
+
+ assert(words_pushed == count, "oops, pushed != count");
+
+ return count;
+}
+
#ifdef ASSERT
void MacroAssembler::verify_heapbase(const char* msg) {
#if 0
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index af8056b..44497ea 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -440,12 +440,18 @@ private:
int push(unsigned int bitset, Register stack);
int pop(unsigned int bitset, Register stack);
+ int push_fp(unsigned int bitset, Register stack);
+ int pop_fp(unsigned int bitset, Register stack);
+
void mov(Register dst, Address a);
public:
void push(RegSet regs, Register stack) { if (regs.bits()) push(regs.bits(), stack); }
void pop(RegSet regs, Register stack) { if (regs.bits()) pop(regs.bits(), stack); }
+ void push_fp(RegSet regs, Register stack) { if (regs.bits()) push_fp(regs.bits(), stack); }
+ void pop_fp(RegSet regs, Register stack) { if (regs.bits()) pop_fp(regs.bits(), stack); }
+
// Push and pop everything that might be clobbered by a native
// runtime call except rscratch1 and rscratch2. (They are always
// scratch, so we don't have to protect them.) Only save the lower
diff --git a/src/hotspot/cpu/aarch64/register_aarch64.hpp b/src/hotspot/cpu/aarch64/register_aarch64.hpp
index 8cda52a..5f7662c 100644
--- a/src/hotspot/cpu/aarch64/register_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/register_aarch64.hpp
@@ -230,6 +230,11 @@ public:
return *this;
}
+ RegSet &operator-=(const RegSet aSet) {
+ *this = *this - aSet;
+ return *this;
+ }
+
static RegSet of(Register r1) {
return RegSet(r1);
}
diff --git a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
index c162024..6e4eb1a 100644
--- a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
@@ -880,8 +880,8 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
} else
#endif
{
- __ load_mirror(rscratch1, rmethod);
- __ stp(rscratch1, zr, Address(sp, 4 * wordSize));
+ __ load_mirror(r10, rmethod);
+ __ stp(r10, zr, Address(sp, 4 * wordSize));
}
__ ldr(rcpool, Address(rmethod, Method::const_offset()));
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
index 381211e..d88ecf7 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
@@ -27,16 +27,16 @@
#include "gc/z/zBarrierSet.hpp"
#include "gc/z/zBarrierSetAssembler.hpp"
#include "gc/z/zBarrierSetRuntime.hpp"
-#include "runtime/stubCodeGenerator.hpp"
+#include "runtime/sharedRuntime.hpp"
#include "utilities/macros.hpp"
#ifdef COMPILER1
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "gc/z/c1/zBarrierSetC1.hpp"
#endif // COMPILER1
-
-#undef __
-#define __ masm->
+#ifdef COMPILER2
+#include "gc/z/c2/zBarrierSetC2.hpp"
+#endif // COMPILER2
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
@@ -44,6 +44,9 @@
#define BLOCK_COMMENT(str) __ block_comment(str)
#endif
+#undef __
+#define __ masm->
+
static void call_vm(MacroAssembler* masm,
address entry_point,
Register arg0,
@@ -333,126 +336,326 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler*
#endif // COMPILER1
-#undef __
-#define __ cgen->assembler()->
-
-// Generates a register specific stub for calling
-// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or
-// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded().
-//
-// The raddr register serves as both input and output for this stub. When the stub is
-// called the raddr register contains the object field address (oop*) where the bad oop
-// was loaded from, which caused the slow path to be taken. On return from the stub the
-// raddr register contains the good/healed oop returned from
-// ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or
-// ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded().
-static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) {
- // Don't generate stub for invalid registers
- if (raddr == rsp || raddr == r12 || raddr == r15) {
- return NULL;
+#ifdef COMPILER2
+
+OptoReg::Name ZBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
+ if (!OptoReg::is_reg(opto_reg)) {
+ return OptoReg::Bad;
+ }
+
+ const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+ if (vm_reg->is_XMMRegister()) {
+ opto_reg &= ~15;
+ switch (node->ideal_reg()) {
+ case Op_VecX:
+ opto_reg |= 2;
+ break;
+ case Op_VecY:
+ opto_reg |= 4;
+ break;
+ case Op_VecZ:
+ opto_reg |= 8;
+ break;
+ default:
+ opto_reg |= 1;
+ break;
+ }
}
- // Create stub name
- char name[64];
- const bool weak = (decorators & ON_WEAK_OOP_REF) != 0;
- os::snprintf(name, sizeof(name), "load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name());
+ return opto_reg;
+}
+
+// We use the vec_spill_helper from the x86.ad file to avoid reinventing this wheel
+extern int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
+ int stack_offset, int reg, uint ireg, outputStream* st);
+#undef __
+#define __ _masm->
- __ align(CodeEntryAlignment);
- StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode));
- address start = __ pc();
+class ZSaveLiveRegisters {
+private:
+ struct XMMRegisterData {
+ XMMRegister _reg;
+ int _size;
- // Save live registers
- if (raddr != rax) {
- __ push(rax);
- }
- if (raddr != rcx) {
- __ push(rcx);
- }
- if (raddr != rdx) {
- __ push(rdx);
+ // Used by GrowableArray::find()
+ bool operator == (const XMMRegisterData& other) {
+ return _reg == other._reg;
+ }
+ };
+
+ MacroAssembler* const _masm;
+ GrowableArray<Register> _gp_registers;
+ GrowableArray<XMMRegisterData> _xmm_registers;
+ int _spill_size;
+ int _spill_offset;
+
+ static int xmm_compare_register_size(XMMRegisterData* left, XMMRegisterData* right) {
+ if (left->_size == right->_size) {
+ return 0;
+ }
+
+ return (left->_size < right->_size) ? -1 : 1;
}
- if (raddr != rsi) {
- __ push(rsi);
+
+ static int xmm_slot_size(OptoReg::Name opto_reg) {
+ // The low order 4 bytes denote what size of the XMM register is live
+ return (opto_reg & 15) << 3;
}
- if (raddr != rdi) {
- __ push(rdi);
+
+ static uint xmm_ideal_reg_for_size(int reg_size) {
+ switch (reg_size) {
+ case 8:
+ return Op_VecD;
+ case 16:
+ return Op_VecX;
+ case 32:
+ return Op_VecY;
+ case 64:
+ return Op_VecZ;
+ default:
+ fatal("Invalid register size %d", reg_size);
+ return 0;
+ }
}
- if (raddr != r8) {
- __ push(r8);
+
+ bool xmm_needs_vzeroupper() const {
+ return _xmm_registers.is_nonempty() && _xmm_registers.at(0)._size > 16;
}
- if (raddr != r9) {
- __ push(r9);
+
+ void xmm_register_save(const XMMRegisterData& reg_data) {
+ const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
+ const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
+ _spill_offset -= reg_data._size;
+ vec_spill_helper(__ code(), false /* do_size */, false /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
}
- if (raddr != r10) {
- __ push(r10);
+
+ void xmm_register_restore(const XMMRegisterData& reg_data) {
+ const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
+ const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
+ vec_spill_helper(__ code(), false /* do_size */, true /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
+ _spill_offset += reg_data._size;
}
- if (raddr != r11) {
- __ push(r11);
+
+ void gp_register_save(Register reg) {
+ _spill_offset -= 8;
+ __ movq(Address(rsp, _spill_offset), reg);
}
- // Setup arguments
- if (c_rarg1 != raddr) {
- __ movq(c_rarg1, raddr);
+ void gp_register_restore(Register reg) {
+ __ movq(reg, Address(rsp, _spill_offset));
+ _spill_offset += 8;
}
- __ movq(c_rarg0, Address(raddr, 0));
- // Call barrier function
- __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1);
+ void initialize(ZLoadBarrierStubC2* stub) {
+ // Create mask of caller saved registers that need to
+ // be saved/restored if live
+ RegMask caller_saved;
+ caller_saved.Insert(OptoReg::as_OptoReg(rax->as_VMReg()));
+ caller_saved.Insert(OptoReg::as_OptoReg(rcx->as_VMReg()));
+ caller_saved.Insert(OptoReg::as_OptoReg(rdx->as_VMReg()));
+ caller_saved.Insert(OptoReg::as_OptoReg(rsi->as_VMReg()));
+ caller_saved.Insert(OptoReg::as_OptoReg(rdi->as_VMReg()));
+ caller_saved.Insert(OptoReg::as_OptoReg(r8->as_VMReg()));
+ caller_saved.Insert(OptoReg::as_OptoReg(r9->as_VMReg()));
+ caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg()));
+ caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg()));
+ caller_saved.Remove(OptoReg::as_OptoReg(stub->ref()->as_VMReg()));
- // Move result returned in rax to raddr, if needed
- if (raddr != rax) {
- __ movq(raddr, rax);
- }
+ // Create mask of live registers
+ RegMask live = stub->live();
+ if (stub->tmp() != noreg) {
+ live.Insert(OptoReg::as_OptoReg(stub->tmp()->as_VMReg()));
+ }
- // Restore saved registers
- if (raddr != r11) {
- __ pop(r11);
- }
- if (raddr != r10) {
- __ pop(r10);
- }
- if (raddr != r9) {
- __ pop(r9);
- }
- if (raddr != r8) {
- __ pop(r8);
- }
- if (raddr != rdi) {
- __ pop(rdi);
- }
- if (raddr != rsi) {
- __ pop(rsi);
- }
- if (raddr != rdx) {
- __ pop(rdx);
+ int gp_spill_size = 0;
+ int xmm_spill_size = 0;
+
+ // Record registers that needs to be saved/restored
+ while (live.is_NotEmpty()) {
+ const OptoReg::Name opto_reg = live.find_first_elem();
+ const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+
+ live.Remove(opto_reg);
+
+ if (vm_reg->is_Register()) {
+ if (caller_saved.Member(opto_reg)) {
+ _gp_registers.append(vm_reg->as_Register());
+ gp_spill_size += 8;
+ }
+ } else if (vm_reg->is_XMMRegister()) {
+ // We encode in the low order 4 bits of the opto_reg, how large part of the register is live
+ const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~15);
+ const int reg_size = xmm_slot_size(opto_reg);
+ const XMMRegisterData reg_data = { vm_reg_base->as_XMMRegister(), reg_size };
+ const int reg_index = _xmm_registers.find(reg_data);
+ if (reg_index == -1) {
+ // Not previously appended
+ _xmm_registers.append(reg_data);
+ xmm_spill_size += reg_size;
+ } else {
+ // Previously appended, update size
+ const int reg_size_prev = _xmm_registers.at(reg_index)._size;
+ if (reg_size > reg_size_prev) {
+ _xmm_registers.at_put(reg_index, reg_data);
+ xmm_spill_size += reg_size - reg_size_prev;
+ }
+ }
+ } else {
+ fatal("Unexpected register type");
+ }
+ }
+
+ // Sort by size, largest first
+ _xmm_registers.sort(xmm_compare_register_size);
+
+ // Stack pointer must be 16 bytes aligned for the call
+ _spill_offset = _spill_size = align_up(xmm_spill_size + gp_spill_size, 16);
+ }
+
+public:
+ ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
+ _masm(masm),
+ _gp_registers(),
+ _xmm_registers(),
+ _spill_size(0),
+ _spill_offset(0) {
+
+ //
+ // Stack layout after registers have been spilled:
+ //
+ // | ... | original rsp, 16 bytes aligned
+ // ------------------
+ // | zmm0 high |
+ // | ... |
+ // | zmm0 low | 16 bytes aligned
+ // | ... |
+ // | ymm1 high |
+ // | ... |
+ // | ymm1 low | 16 bytes aligned
+ // | ... |
+ // | xmmN high |
+ // | ... |
+ // | xmmN low | 8 bytes aligned
+ // | reg0 | 8 bytes aligned
+ // | reg1 |
+ // | ... |
+ // | regN | new rsp, if 16 bytes aligned
+ // | <padding> | else new rsp, 16 bytes aligned
+ // ------------------
+ //
+
+ // Figure out what registers to save/restore
+ initialize(stub);
+
+ // Allocate stack space
+ if (_spill_size > 0) {
+ __ subptr(rsp, _spill_size);
+ }
+
+ // Save XMM/YMM/ZMM registers
+ for (int i = 0; i < _xmm_registers.length(); i++) {
+ xmm_register_save(_xmm_registers.at(i));
+ }
+
+ if (xmm_needs_vzeroupper()) {
+ __ vzeroupper();
+ }
+
+ // Save general purpose registers
+ for (int i = 0; i < _gp_registers.length(); i++) {
+ gp_register_save(_gp_registers.at(i));
+ }
}
- if (raddr != rcx) {
- __ pop(rcx);
+
+ ~ZSaveLiveRegisters() {
+ // Restore general purpose registers
+ for (int i = _gp_registers.length() - 1; i >= 0; i--) {
+ gp_register_restore(_gp_registers.at(i));
+ }
+
+ __ vzeroupper();
+
+ // Restore XMM/YMM/ZMM registers
+ for (int i = _xmm_registers.length() - 1; i >= 0; i--) {
+ xmm_register_restore(_xmm_registers.at(i));
+ }
+
+ // Free stack space
+ if (_spill_size > 0) {
+ __ addptr(rsp, _spill_size);
+ }
}
- if (raddr != rax) {
- __ pop(rax);
+};
+
+class ZSetupArguments {
+private:
+ MacroAssembler* const _masm;
+ const Register _ref;
+ const Address _ref_addr;
+
+public:
+ ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
+ _masm(masm),
+ _ref(stub->ref()),
+ _ref_addr(stub->ref_addr()) {
+
+ // Setup arguments
+ if (_ref_addr.base() == noreg) {
+ // No self healing
+ if (_ref != c_rarg0) {
+ __ movq(c_rarg0, _ref);
+ }
+ __ xorq(c_rarg1, c_rarg1);
+ } else {
+ // Self healing
+ if (_ref == c_rarg0) {
+ __ lea(c_rarg1, _ref_addr);
+ } else if (_ref != c_rarg1) {
+ __ lea(c_rarg1, _ref_addr);
+ __ movq(c_rarg0, _ref);
+ } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) {
+ __ movq(c_rarg0, _ref);
+ __ lea(c_rarg1, _ref_addr);
+ } else {
+ __ xchgq(c_rarg0, c_rarg1);
+ if (_ref_addr.base() == c_rarg0) {
+ __ lea(c_rarg1, Address(c_rarg1, _ref_addr.index(), _ref_addr.scale(), _ref_addr.disp()));
+ } else if (_ref_addr.index() == c_rarg0) {
+ __ lea(c_rarg1, Address(_ref_addr.base(), c_rarg1, _ref_addr.scale(), _ref_addr.disp()));
+ } else {
+ ShouldNotReachHere();
+ }
+ }
+ }
}
- __ ret(0);
-
- return start;
-}
+ ~ZSetupArguments() {
+ // Transfer result
+ if (_ref != rax) {
+ __ movq(_ref, rax);
+ }
+ }
+};
#undef __
+#define __ masm->
-void ZBarrierSetAssembler::barrier_stubs_init() {
- // Load barrier stubs
- int stub_code_size = 256 * 16; // Rough estimate of code size
+void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const {
+ BLOCK_COMMENT("ZLoadBarrierStubC2");
- ResourceMark rm;
- BufferBlob* bb = BufferBlob::create("zgc_load_barrier_stubs", stub_code_size);
- CodeBuffer buf(bb);
- StubCodeGenerator cgen(&buf);
+ // Stub entry
+ __ bind(*stub->entry());
- Register rr = as_Register(0);
- for (int i = 0; i < RegisterImpl::number_of_registers; i++) {
- _load_barrier_slow_stub[i] = generate_load_barrier_stub(&cgen, rr, ON_STRONG_OOP_REF);
- _load_barrier_weak_slow_stub[i] = generate_load_barrier_stub(&cgen, rr, ON_WEAK_OOP_REF);
- rr = rr->successor();
+ {
+ ZSaveLiveRegisters save_live_registers(masm, stub);
+ ZSetupArguments setup_arguments(masm, stub);
+ __ call(RuntimeAddress(stub->slow_path()));
}
+
+ // Stub exit
+ __ jmp(*stub->continuation());
}
+
+#undef __
+
+#endif // COMPILER2
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
index 3687754..e433882 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
@@ -24,6 +24,14 @@
#ifndef CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP
#define CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP
+#include "code/vmreg.hpp"
+#include "oops/accessDecorators.hpp"
+#ifdef COMPILER2
+#include "opto/optoreg.hpp"
+#endif // COMPILER2
+
+class MacroAssembler;
+
#ifdef COMPILER1
class LIR_Assembler;
class LIR_OprDesc;
@@ -32,18 +40,13 @@ class StubAssembler;
class ZLoadBarrierStubC1;
#endif // COMPILER1
-class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
- address _load_barrier_slow_stub[RegisterImpl::number_of_registers];
- address _load_barrier_weak_slow_stub[RegisterImpl::number_of_registers];
+#ifdef COMPILER2
+class Node;
+class ZLoadBarrierStubC2;
+#endif // COMPILER2
+class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
public:
- ZBarrierSetAssembler() :
- _load_barrier_slow_stub(),
- _load_barrier_weak_slow_stub() {}
-
- address load_barrier_slow_stub(Register reg) { return _load_barrier_slow_stub[reg->encoding()]; }
- address load_barrier_weak_slow_stub(Register reg) { return _load_barrier_weak_slow_stub[reg->encoding()]; }
-
virtual void load_at(MacroAssembler* masm,
DecoratorSet decorators,
BasicType type,
@@ -86,7 +89,12 @@ public:
DecoratorSet decorators) const;
#endif // COMPILER1
- virtual void barrier_stubs_init();
+#ifdef COMPILER2
+ OptoReg::Name refine_register(const Node* node,
+ OptoReg::Name opto_reg);
+ void generate_c2_load_barrier_stub(MacroAssembler* masm,
+ ZLoadBarrierStubC2* stub) const;
+#endif // COMPILER2
};
#endif // CPU_X86_GC_Z_ZBARRIERSETASSEMBLER_X86_HPP
diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad
index 6eefe33..8fb9a3e 100644
--- a/src/hotspot/cpu/x86/x86.ad
+++ b/src/hotspot/cpu/x86/x86.ad
@@ -1097,138 +1097,6 @@ reg_class vectorz_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0
reg_class_dynamic vectorz_reg(vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() %} );
reg_class_dynamic vectorz_reg_vl(vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
-reg_class xmm0_reg(XMM0, XMM0b, XMM0c, XMM0d);
-reg_class ymm0_reg(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h);
-reg_class zmm0_reg(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p);
-
-reg_class xmm1_reg(XMM1, XMM1b, XMM1c, XMM1d);
-reg_class ymm1_reg(XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h);
-reg_class zmm1_reg(XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p);
-
-reg_class xmm2_reg(XMM2, XMM2b, XMM2c, XMM2d);
-reg_class ymm2_reg(XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h);
-reg_class zmm2_reg(XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p);
-
-reg_class xmm3_reg(XMM3, XMM3b, XMM3c, XMM3d);
-reg_class ymm3_reg(XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h);
-reg_class zmm3_reg(XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p);
-
-reg_class xmm4_reg(XMM4, XMM4b, XMM4c, XMM4d);
-reg_class ymm4_reg(XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h);
-reg_class zmm4_reg(XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p);
-
-reg_class xmm5_reg(XMM5, XMM5b, XMM5c, XMM5d);
-reg_class ymm5_reg(XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h);
-reg_class zmm5_reg(XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p);
-
-reg_class xmm6_reg(XMM6, XMM6b, XMM6c, XMM6d);
-reg_class ymm6_reg(XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h);
-reg_class zmm6_reg(XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p);
-
-reg_class xmm7_reg(XMM7, XMM7b, XMM7c, XMM7d);
-reg_class ymm7_reg(XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h);
-reg_class zmm7_reg(XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p);
-
-#ifdef _LP64
-
-reg_class xmm8_reg(XMM8, XMM8b, XMM8c, XMM8d);
-reg_class ymm8_reg(XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h);
-reg_class zmm8_reg(XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p);
-
-reg_class xmm9_reg(XMM9, XMM9b, XMM9c, XMM9d);
-reg_class ymm9_reg(XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h);
-reg_class zmm9_reg(XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p);
-
-reg_class xmm10_reg(XMM10, XMM10b, XMM10c, XMM10d);
-reg_class ymm10_reg(XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h);
-reg_class zmm10_reg(XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p);
-
-reg_class xmm11_reg(XMM11, XMM11b, XMM11c, XMM11d);
-reg_class ymm11_reg(XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h);
-reg_class zmm11_reg(XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p);
-
-reg_class xmm12_reg(XMM12, XMM12b, XMM12c, XMM12d);
-reg_class ymm12_reg(XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h);
-reg_class zmm12_reg(XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p);
-
-reg_class xmm13_reg(XMM13, XMM13b, XMM13c, XMM13d);
-reg_class ymm13_reg(XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h);
-reg_class zmm13_reg(XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p);
-
-reg_class xmm14_reg(XMM14, XMM14b, XMM14c, XMM14d);
-reg_class ymm14_reg(XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h);
-reg_class zmm14_reg(XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p);
-
-reg_class xmm15_reg(XMM15, XMM15b, XMM15c, XMM15d);
-reg_class ymm15_reg(XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h);
-reg_class zmm15_reg(XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p);
-
-reg_class xmm16_reg(XMM16, XMM16b, XMM16c, XMM16d);
-reg_class ymm16_reg(XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h);
-reg_class zmm16_reg(XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p);
-
-reg_class xmm17_reg(XMM17, XMM17b, XMM17c, XMM17d);
-reg_class ymm17_reg(XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h);
-reg_class zmm17_reg(XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p);
-
-reg_class xmm18_reg(XMM18, XMM18b, XMM18c, XMM18d);
-reg_class ymm18_reg(XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h);
-reg_class zmm18_reg(XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p);
-
-reg_class xmm19_reg(XMM19, XMM19b, XMM19c, XMM19d);
-reg_class ymm19_reg(XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h);
-reg_class zmm19_reg(XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p);
-
-reg_class xmm20_reg(XMM20, XMM20b, XMM20c, XMM20d);
-reg_class ymm20_reg(XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h);
-reg_class zmm20_reg(XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p);
-
-reg_class xmm21_reg(XMM21, XMM21b, XMM21c, XMM21d);
-reg_class ymm21_reg(XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h);
-reg_class zmm21_reg(XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p);
-
-reg_class xmm22_reg(XMM22, XMM22b, XMM22c, XMM22d);
-reg_class ymm22_reg(XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h);
-reg_class zmm22_reg(XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p);
-
-reg_class xmm23_reg(XMM23, XMM23b, XMM23c, XMM23d);
-reg_class ymm23_reg(XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h);
-reg_class zmm23_reg(XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p);
-
-reg_class xmm24_reg(XMM24, XMM24b, XMM24c, XMM24d);
-reg_class ymm24_reg(XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h);
-reg_class zmm24_reg(XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p);
-
-reg_class xmm25_reg(XMM25, XMM25b, XMM25c, XMM25d);
-reg_class ymm25_reg(XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h);
-reg_class zmm25_reg(XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p);
-
-reg_class xmm26_reg(XMM26, XMM26b, XMM26c, XMM26d);
-reg_class ymm26_reg(XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h);
-reg_class zmm26_reg(XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p);
-
-reg_class xmm27_reg(XMM27, XMM27b, XMM27c, XMM27d);
-reg_class ymm27_reg(XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h);
-reg_class zmm27_reg(XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p);
-
-reg_class xmm28_reg(XMM28, XMM28b, XMM28c, XMM28d);
-reg_class ymm28_reg(XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h);
-reg_class zmm28_reg(XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p);
-
-reg_class xmm29_reg(XMM29, XMM29b, XMM29c, XMM29d);
-reg_class ymm29_reg(XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h);
-reg_class zmm29_reg(XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p);
-
-reg_class xmm30_reg(XMM30, XMM30b, XMM30c, XMM30d);
-reg_class ymm30_reg(XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h);
-reg_class zmm30_reg(XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p);
-
-reg_class xmm31_reg(XMM31, XMM31b, XMM31c, XMM31d);
-reg_class ymm31_reg(XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h);
-reg_class zmm31_reg(XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p);
-
-#endif
-
%}
@@ -1775,8 +1643,8 @@ static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo
return (UseAVX > 2) ? 6 : 4;
}
-static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
- int stack_offset, int reg, uint ireg, outputStream* st) {
+int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
+ int stack_offset, int reg, uint ireg, outputStream* st) {
// In 64-bit VM size calculation is very complex. Emitting instructions
// into scratch buffer is used to get size in 64-bit VM.
LP64_ONLY( assert(!do_size, "this method calculates size only for 32-bit VM"); )
diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad
index e256e22..d127732 100644
--- a/src/hotspot/cpu/x86/x86_64.ad
+++ b/src/hotspot/cpu/x86/x86_64.ad
@@ -541,6 +541,7 @@ reg_class int_rdi_reg(RDI);
source_hpp %{
#include "gc/z/c2/zBarrierSetC2.hpp"
+#include "gc/z/zThreadLocalData.hpp"
%}
@@ -1088,8 +1089,8 @@ static enum RC rc_class(OptoReg::Name reg)
static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
int src_hi, int dst_hi, uint ireg, outputStream* st);
-static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
- int stack_offset, int reg, uint ireg, outputStream* st);
+int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
+ int stack_offset, int reg, uint ireg, outputStream* st);
static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
int dst_offset, uint ireg, outputStream* st) {
@@ -1800,6 +1801,19 @@ const RegMask Matcher::method_handle_invoke_SP_save_mask() {
return NO_REG_mask();
}
+static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) {
+ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak);
+ __ testptr(ref, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset()));
+ __ jcc(Assembler::notZero, *stub->entry());
+ __ bind(*stub->continuation());
+}
+
+static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
+ ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */);
+ __ jmp(*stub->entry());
+ __ bind(*stub->continuation());
+}
+
%}
//----------ENCODING BLOCK-----------------------------------------------------
@@ -4284,136 +4298,6 @@ operand cmpOpUCF2() %{
%}
%}
-// Operands for bound floating pointer register arguments
-operand rxmm0() %{
- constraint(ALLOC_IN_RC(xmm0_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX<= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm1() %{
- constraint(ALLOC_IN_RC(xmm1_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm2() %{
- constraint(ALLOC_IN_RC(xmm2_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm3() %{
- constraint(ALLOC_IN_RC(xmm3_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm4() %{
- constraint(ALLOC_IN_RC(xmm4_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm5() %{
- constraint(ALLOC_IN_RC(xmm5_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm6() %{
- constraint(ALLOC_IN_RC(xmm6_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm7() %{
- constraint(ALLOC_IN_RC(xmm7_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm8() %{
- constraint(ALLOC_IN_RC(xmm8_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm9() %{
- constraint(ALLOC_IN_RC(xmm9_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm10() %{
- constraint(ALLOC_IN_RC(xmm10_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm11() %{
- constraint(ALLOC_IN_RC(xmm11_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm12() %{
- constraint(ALLOC_IN_RC(xmm12_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm13() %{
- constraint(ALLOC_IN_RC(xmm13_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm14() %{
- constraint(ALLOC_IN_RC(xmm14_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm15() %{
- constraint(ALLOC_IN_RC(xmm15_reg)); match(VecX);
- predicate((UseSSE > 0) && (UseAVX <= 2)); format%{%} interface(REG_INTER);
-%}
-operand rxmm16() %{
- constraint(ALLOC_IN_RC(xmm16_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm17() %{
- constraint(ALLOC_IN_RC(xmm17_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm18() %{
- constraint(ALLOC_IN_RC(xmm18_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm19() %{
- constraint(ALLOC_IN_RC(xmm19_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm20() %{
- constraint(ALLOC_IN_RC(xmm20_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm21() %{
- constraint(ALLOC_IN_RC(xmm21_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm22() %{
- constraint(ALLOC_IN_RC(xmm22_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm23() %{
- constraint(ALLOC_IN_RC(xmm23_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm24() %{
- constraint(ALLOC_IN_RC(xmm24_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm25() %{
- constraint(ALLOC_IN_RC(xmm25_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm26() %{
- constraint(ALLOC_IN_RC(xmm26_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm27() %{
- constraint(ALLOC_IN_RC(xmm27_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm28() %{
- constraint(ALLOC_IN_RC(xmm28_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm29() %{
- constraint(ALLOC_IN_RC(xmm29_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm30() %{
- constraint(ALLOC_IN_RC(xmm30_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-operand rxmm31() %{
- constraint(ALLOC_IN_RC(xmm31_reg)); match(VecX);
- predicate(UseAVX == 3); format%{%} interface(REG_INTER);
-%}
-
//----------OPERAND CLASSES----------------------------------------------------
// Operand Classes are groups of operands that are used as to simplify
// instruction definitions by not requiring the AD writer to specify separate
@@ -5306,6 +5190,7 @@ instruct loadRange(rRegI dst, memory mem)
instruct loadP(rRegP dst, memory mem)
%{
match(Set dst (LoadP mem));
+ predicate(n->as_Load()->barrier_data() == 0);
ins_cost(125); // XXX
format %{ "movq $dst, $mem\t# ptr" %}
@@ -7515,6 +7400,7 @@ instruct storePConditional(memory heap_top_ptr,
rax_RegP oldval, rRegP newval,
rFlagsReg cr)
%{
+ predicate(n->as_LoadStore()->barrier_data() == 0);
match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
@@ -7566,7 +7452,7 @@ instruct compareAndSwapP(rRegI res,
rax_RegP oldval, rRegP newval,
rFlagsReg cr)
%{
- predicate(VM_Version::supports_cx8());
+ predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
effect(KILL cr, KILL oldval);
@@ -7808,7 +7694,7 @@ instruct compareAndExchangeP(
rax_RegP oldval, rRegP newval,
rFlagsReg cr)
%{
- predicate(VM_Version::supports_cx8());
+ predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
effect(KILL cr);
@@ -7953,6 +7839,7 @@ instruct xchgL( memory mem, rRegL newval) %{
instruct xchgP( memory mem, rRegP newval) %{
match(Set newval (GetAndSetP mem newval));
+ predicate(n->as_LoadStore()->barrier_data() == 0);
format %{ "XCHGQ $newval,[$mem]" %}
ins_encode %{
__ xchgq($newval$$Register, $mem$$Address);
@@ -11649,6 +11536,7 @@ instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
%{
match(Set cr (CmpP op1 (LoadP op2)));
+ predicate(n->in(2)->as_Load()->barrier_data() == 0);
ins_cost(500); // XXX
format %{ "cmpq $op1, $op2\t# ptr" %}
@@ -11674,7 +11562,8 @@ instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
// and raw pointers have no anti-dependencies.
instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
%{
- predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
+ predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
+ n->in(2)->as_Load()->barrier_data() == 0);
match(Set cr (CmpP op1 (LoadP op2)));
format %{ "cmpq $op1, $op2\t# raw ptr" %}
@@ -11699,7 +11588,8 @@ instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
// any compare to a zero should be eq/neq.
instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
%{
- predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
+ predicate((!UseCompressedOops || (Universe::narrow_oop_base() != NULL)) &&
+ n->in(1)->as_Load()->barrier_data() == 0);
match(Set cr (CmpP (LoadP op) zero));
ins_cost(500); // XXX
@@ -11712,7 +11602,9 @@ instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
%{
- predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
+ predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) &&
+ (Universe::narrow_klass_base() == NULL) &&
+ n->in(1)->as_Load()->barrier_data() == 0);
match(Set cr (CmpP (LoadP mem) zero));
format %{ "cmpq R12, $mem\t# ptr (R12_heapbase==0)" %}
@@ -12667,274 +12559,126 @@ instruct RethrowException()
// Execute ZGC load barrier (strong) slow path
//
-// When running without XMM regs
-instruct loadBarrierSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{
+// Load Pointer
+instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr)
+%{
+ predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierStrong);
+ match(Set dst (LoadP mem));
+ effect(KILL cr, TEMP dst);
- match(Set dst (LoadBarrierSlowReg mem));
- predicate(MaxVectorSize < 16 && !n->as_LoadBarrierSlowReg()->is_weak());
+ ins_cost(125);
- effect(DEF dst, KILL cr);
+ format %{ "movq $dst, $mem" %}
- format %{"LoadBarrierSlowRegNoVec $dst, $mem" %}
ins_encode %{
-#if INCLUDE_ZGC
- Register d = $dst$$Register;
- ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
-
- assert(d != r12, "Can't be R12!");
- assert(d != r15, "Can't be R15!");
- assert(d != rsp, "Can't be RSP!");
-
- __ lea(d, $mem$$Address);
- __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
-#else
- ShouldNotReachHere();
-#endif
+ __ movptr($dst$$Register, $mem$$Address);
+ if (barrier_data() != ZLoadBarrierElided) {
+ z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, false /* weak */);
+ }
%}
- ins_pipe(pipe_slow);
-%}
-// For XMM and YMM enabled processors
-instruct loadBarrierSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr,
- rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
- rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
- rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
- rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
+ ins_pipe(ialu_reg_mem);
+%}
- match(Set dst (LoadBarrierSlowReg mem));
- predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16) && !n->as_LoadBarrierSlowReg()->is_weak());
+// Load Weak Pointer
+instruct zLoadWeakP(rRegP dst, memory mem, rFlagsReg cr)
+%{
+ predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierWeak);
+ match(Set dst (LoadP mem));
+ effect(KILL cr, TEMP dst);
- effect(DEF dst, KILL cr,
- KILL x0, KILL x1, KILL x2, KILL x3,
- KILL x4, KILL x5, KILL x6, KILL x7,
- KILL x8, KILL x9, KILL x10, KILL x11,
- KILL x12, KILL x13, KILL x14, KILL x15);
+ ins_cost(125);
- format %{"LoadBarrierSlowRegXmm $dst, $mem" %}
+ format %{ "movq $dst, $mem" %}
ins_encode %{
-#if INCLUDE_ZGC
- Register d = $dst$$Register;
- ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
-
- assert(d != r12, "Can't be R12!");
- assert(d != r15, "Can't be R15!");
- assert(d != rsp, "Can't be RSP!");
-
- __ lea(d, $mem$$Address);
- __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
-#else
- ShouldNotReachHere();
-#endif
+ __ movptr($dst$$Register, $mem$$Address);
+ z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, true /* weak */);
%}
- ins_pipe(pipe_slow);
-%}
-// For ZMM enabled processors
-instruct loadBarrierSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr,
- rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
- rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
- rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
- rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15,
- rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19,
- rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23,
- rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
- rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
-
- match(Set dst (LoadBarrierSlowReg mem));
- predicate((UseAVX == 3) && (MaxVectorSize >= 16) && !n->as_LoadBarrierSlowReg()->is_weak());
-
- effect(DEF dst, KILL cr,
- KILL x0, KILL x1, KILL x2, KILL x3,
- KILL x4, KILL x5, KILL x6, KILL x7,
- KILL x8, KILL x9, KILL x10, KILL x11,
- KILL x12, KILL x13, KILL x14, KILL x15,
- KILL x16, KILL x17, KILL x18, KILL x19,
- KILL x20, KILL x21, KILL x22, KILL x23,
- KILL x24, KILL x25, KILL x26, KILL x27,
- KILL x28, KILL x29, KILL x30, KILL x31);
-
- format %{"LoadBarrierSlowRegZmm $dst, $mem" %}
- ins_encode %{
-#if INCLUDE_ZGC
- Register d = $dst$$Register;
- ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
-
- assert(d != r12, "Can't be R12!");
- assert(d != r15, "Can't be R15!");
- assert(d != rsp, "Can't be RSP!");
-
- __ lea(d, $mem$$Address);
- __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
-#else
- ShouldNotReachHere();
-#endif
- %}
- ins_pipe(pipe_slow);
+ ins_pipe(ialu_reg_mem);
%}
-//
-// Execute ZGC load barrier (weak) slow path
-//
-
-// When running without XMM regs
-instruct loadBarrierWeakSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{
+instruct zCompareAndExchangeP(memory mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{
+ match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
+ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
+ effect(KILL cr, TEMP tmp);
- match(Set dst (LoadBarrierSlowReg mem));
- predicate(MaxVectorSize < 16 && n->as_LoadBarrierSlowReg()->is_weak());
+ format %{ "lock\n\t"
+ "cmpxchgq $newval, $mem" %}
- effect(DEF dst, KILL cr);
-
- format %{"LoadBarrierSlowRegNoVec $dst, $mem" %}
ins_encode %{
-#if INCLUDE_ZGC
- Register d = $dst$$Register;
- ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
-
- assert(d != r12, "Can't be R12!");
- assert(d != r15, "Can't be R15!");
- assert(d != rsp, "Can't be RSP!");
-
- __ lea(d, $mem$$Address);
- __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
-#else
- ShouldNotReachHere();
-#endif
+ if (barrier_data() != ZLoadBarrierElided) {
+ __ movptr($tmp$$Register, $oldval$$Register);
+ }
+ __ lock();
+ __ cmpxchgptr($newval$$Register, $mem$$Address);
+ if (barrier_data() != ZLoadBarrierElided) {
+ Label good;
+ __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset()));
+ __ jcc(Assembler::zero, good);
+ z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register);
+ __ movptr($oldval$$Register, $tmp$$Register);
+ __ lock();
+ __ cmpxchgptr($newval$$Register, $mem$$Address);
+ __ bind(good);
+ }
%}
- ins_pipe(pipe_slow);
-%}
-// For XMM and YMM enabled processors
-instruct loadBarrierWeakSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr,
- rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
- rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
- rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
- rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
-
- match(Set dst (LoadBarrierSlowReg mem));
- predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16) && n->as_LoadBarrierSlowReg()->is_weak());
-
- effect(DEF dst, KILL cr,
- KILL x0, KILL x1, KILL x2, KILL x3,
- KILL x4, KILL x5, KILL x6, KILL x7,
- KILL x8, KILL x9, KILL x10, KILL x11,
- KILL x12, KILL x13, KILL x14, KILL x15);
-
- format %{"LoadBarrierWeakSlowRegXmm $dst, $mem" %}
- ins_encode %{
-#if INCLUDE_ZGC
- Register d = $dst$$Register;
- ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
-
- assert(d != r12, "Can't be R12!");
- assert(d != r15, "Can't be R15!");
- assert(d != rsp, "Can't be RSP!");
-
- __ lea(d,$mem$$Address);
- __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
-#else
- ShouldNotReachHere();
-#endif
- %}
- ins_pipe(pipe_slow);
+ ins_pipe(pipe_cmpxchg);
%}
-// For ZMM enabled processors
-instruct loadBarrierWeakSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr,
- rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
- rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
- rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
- rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15,
- rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19,
- rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23,
- rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
- rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
-
- match(Set dst (LoadBarrierSlowReg mem));
- predicate((UseAVX == 3) && (MaxVectorSize >= 16) && n->as_LoadBarrierSlowReg()->is_weak());
-
- effect(DEF dst, KILL cr,
- KILL x0, KILL x1, KILL x2, KILL x3,
- KILL x4, KILL x5, KILL x6, KILL x7,
- KILL x8, KILL x9, KILL x10, KILL x11,
- KILL x12, KILL x13, KILL x14, KILL x15,
- KILL x16, KILL x17, KILL x18, KILL x19,
- KILL x20, KILL x21, KILL x22, KILL x23,
- KILL x24, KILL x25, KILL x26, KILL x27,
- KILL x28, KILL x29, KILL x30, KILL x31);
-
- format %{"LoadBarrierWeakSlowRegZmm $dst, $mem" %}
- ins_encode %{
-#if INCLUDE_ZGC
- Register d = $dst$$Register;
- ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
- assert(d != r12, "Can't be R12!");
- assert(d != r15, "Can't be R15!");
- assert(d != rsp, "Can't be RSP!");
+instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{
+ match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+ match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
+ effect(KILL cr, KILL oldval, TEMP tmp);
+
+ format %{ "lock\n\t"
+ "cmpxchgq $newval, $mem\n\t"
+ "sete $res\n\t"
+ "movzbl $res, $res" %}
- __ lea(d,$mem$$Address);
- __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
-#else
- ShouldNotReachHere();
-#endif
+ ins_encode %{
+ if (barrier_data() != ZLoadBarrierElided) {
+ __ movptr($tmp$$Register, $oldval$$Register);
+ }
+ __ lock();
+ __ cmpxchgptr($newval$$Register, $mem$$Address);
+ if (barrier_data() != ZLoadBarrierElided) {
+ Label good;
+ __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset()));
+ __ jcc(Assembler::zero, good);
+ z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register);
+ __ movptr($oldval$$Register, $tmp$$Register);
+ __ lock();
+ __ cmpxchgptr($newval$$Register, $mem$$Address);
+ __ bind(good);
+ __ cmpptr($tmp$$Register, $oldval$$Register);
+ }
+ __ setb(Assembler::equal, $res$$Register);
+ __ movzbl($res$$Register, $res$$Register);
%}
- ins_pipe(pipe_slow);
-%}
-
-// Specialized versions of compareAndExchangeP that adds a keepalive that is consumed
-// but doesn't affect output.
-
-instruct z_compareAndExchangeP(
- memory mem_ptr,
- rax_RegP oldval, rRegP newval, rRegP keepalive,
- rFlagsReg cr) %{
- predicate(VM_Version::supports_cx8());
- match(Set oldval (ZCompareAndExchangeP (Binary mem_ptr keepalive) (Binary oldval newval)));
- effect(KILL cr);
- format %{ "cmpxchgq $mem_ptr,$newval\t# "
- "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
- opcode(0x0F, 0xB1);
- ins_encode(lock_prefix,
- REX_reg_mem_wide(newval, mem_ptr),
- OpcP, OpcS,
- reg_mem(newval, mem_ptr) // lock cmpxchg
- );
- ins_pipe( pipe_cmpxchg );
+ ins_pipe(pipe_cmpxchg);
%}
-instruct z_compareAndSwapP(rRegI res,
- memory mem_ptr,
- rax_RegP oldval, rRegP newval, rRegP keepalive,
- rFlagsReg cr) %{
- predicate(VM_Version::supports_cx8());
- match(Set res (ZCompareAndSwapP (Binary mem_ptr keepalive) (Binary oldval newval)));
- match(Set res (ZWeakCompareAndSwapP (Binary mem_ptr keepalive) (Binary oldval newval)));
- effect(KILL cr, KILL oldval);
+instruct zXChgP(memory mem, rRegP newval, rFlagsReg cr) %{
+ match(Set newval (GetAndSetP mem newval));
+ predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
+ effect(KILL cr);
- format %{ "cmpxchgq $mem_ptr,$newval\t# "
- "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
- "sete $res\n\t"
- "movzbl $res, $res" %}
- opcode(0x0F, 0xB1);
- ins_encode(lock_prefix,
- REX_reg_mem_wide(newval, mem_ptr),
- OpcP, OpcS,
- reg_mem(newval, mem_ptr),
- REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
- REX_reg_breg(res, res), // movzbl
- Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
- ins_pipe( pipe_cmpxchg );
-%}
+ format %{ "xchgq $newval, $mem" %}
-instruct z_xchgP( memory mem, rRegP newval, rRegP keepalive) %{
- match(Set newval (ZGetAndSetP mem (Binary newval keepalive)));
- format %{ "XCHGQ $newval,[$mem]" %}
ins_encode %{
- __ xchgq($newval$$Register, $mem$$Address);
+ __ xchgptr($newval$$Register, $mem$$Address);
+ if (barrier_data() != ZLoadBarrierElided) {
+ z_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, false /* weak */);
+ }
%}
- ins_pipe( pipe_cmpxchg );
-%}
+
+ ins_pipe(pipe_cmpxchg);
+%}
// ============================================================================
// This name is KNOWN by the ADLC and cannot be changed.
diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp
index ef704f4..5ba1fdc 100644
--- a/src/hotspot/share/adlc/formssel.cpp
+++ b/src/hotspot/share/adlc/formssel.cpp
@@ -774,11 +774,6 @@ bool InstructForm::captures_bottom_type(FormDict &globals) const {
!strcmp(_matrule->_rChild->_opType,"CheckCastPP") ||
!strcmp(_matrule->_rChild->_opType,"GetAndSetP") ||
!strcmp(_matrule->_rChild->_opType,"GetAndSetN") ||
-#if INCLUDE_ZGC
- !strcmp(_matrule->_rChild->_opType,"ZGetAndSetP") ||
- !strcmp(_matrule->_rChild->_opType,"ZCompareAndExchangeP") ||
- !strcmp(_matrule->_rChild->_opType,"LoadBarrierSlowReg") ||
-#endif
#if INCLUDE_SHENANDOAHGC
!strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeP") ||
!strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeN") ||
@@ -3513,9 +3508,6 @@ int MatchNode::needs_ideal_memory_edge(FormDict &globals) const {
"StoreCM",
"GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP",
"GetAndAddB", "GetAndAddS", "GetAndAddL", "GetAndSetL", "GetAndSetN",
-#if INCLUDE_ZGC
- "LoadBarrierSlowReg", "ZGetAndSetP", "ZCompareAndSwapP", "ZCompareAndExchangeP", "ZWeakCompareAndSwapP",
-#endif
"ClearArray"
};
int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*);
diff --git a/src/hotspot/share/compiler/compilerDirectives.hpp b/src/hotspot/share/compiler/compilerDirectives.hpp
index 8eba28f..b20cd73 100644
--- a/src/hotspot/share/compiler/compilerDirectives.hpp
+++ b/src/hotspot/share/compiler/compilerDirectives.hpp
@@ -66,8 +66,7 @@ NOT_PRODUCT(cflags(TraceOptoOutput, bool, TraceOptoOutput, TraceOptoOutput))
cflags(VectorizeDebug, uintx, 0, VectorizeDebug) \
cflags(CloneMapDebug, bool, false, CloneMapDebug) \
cflags(IGVPrintLevel, intx, PrintIdealGraphLevel, IGVPrintLevel) \
- cflags(MaxNodeLimit, intx, MaxNodeLimit, MaxNodeLimit) \
-ZGC_ONLY(cflags(ZTraceLoadBarriers, bool, false, ZTraceLoadBarriers))
+ cflags(MaxNodeLimit, intx, MaxNodeLimit, MaxNodeLimit)
#else
#define compilerdirectives_c2_flags(cflags)
#endif
diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp
index eea7467..487988b 100644
--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp
+++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp
@@ -198,7 +198,7 @@ public:
virtual void clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const;
// Support for GC barriers emitted during parsing
- virtual bool has_load_barriers() const { return false; }
+ virtual bool has_load_barrier_nodes() const { return false; }
virtual bool is_gc_barrier_node(Node* node) const { return false; }
virtual Node* step_over_gc_barrier(Node* c) const { return c; }
@@ -213,12 +213,14 @@ public:
// This could for example comprise macro nodes to be expanded during macro expansion.
virtual void* create_barrier_state(Arena* comp_arena) const { return NULL; }
virtual void optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { }
- virtual void barrier_insertion_phase(Compile* C, PhaseIterGVN &igvn) const { }
// If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be
// expanded later, then now is the time to do so.
virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const { return false; }
virtual void verify_gc_barriers(bool post_parse) const {}
- virtual bool needs_anti_dependence_check(const Node* node) const { return true; }
+
+ virtual void late_barrier_analysis() const { }
+ virtual int estimate_stub_size() const { return 0; }
+ virtual void emit_stubs(CodeBuffer& cb) const { }
};
#endif // SHARE_GC_SHARED_C2_BARRIERSETC2_HPP
diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp
index bf0bd43..a129734 100644
--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp
+++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp
@@ -22,443 +22,156 @@
*/
#include "precompiled.hpp"
-#include "opto/castnode.hpp"
+#include "classfile/javaClasses.hpp"
+#include "gc/z/c2/zBarrierSetC2.hpp"
+#include "gc/z/zBarrierSet.hpp"
+#include "gc/z/zBarrierSetAssembler.hpp"
+#include "gc/z/zBarrierSetRuntime.hpp"
+#include "opto/block.hpp"
#include "opto/compile.hpp"
#include "opto/graphKit.hpp"
-#include "opto/loopnode.hpp"
#include "opto/machnode.hpp"
-#include "opto/macro.hpp"
#include "opto/memnode.hpp"
-#include "opto/movenode.hpp"
#include "opto/node.hpp"
-#include "opto/phase.hpp"
-#include "opto/phaseX.hpp"
+#include "opto/regalloc.hpp"
#include "opto/rootnode.hpp"
-#include "opto/type.hpp"
-#include "utilities/copy.hpp"
#include "utilities/growableArray.hpp"
#include "utilities/macros.hpp"
-#include "gc/z/zBarrierSet.hpp"
-#include "gc/z/c2/zBarrierSetC2.hpp"
-#include "gc/z/zThreadLocalData.hpp"
-#include "gc/z/zBarrierSetRuntime.hpp"
-
-ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena)
- : _load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8, 0, NULL)) {}
-
-int ZBarrierSetC2State::load_barrier_count() const {
- return _load_barrier_nodes->length();
-}
-
-void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) {
- assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list");
- _load_barrier_nodes->append(n);
-}
-
-void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) {
- // this function may be called twice for a node so check
- // that the node is in the array before attempting to remove it
- if (_load_barrier_nodes->contains(n)) {
- _load_barrier_nodes->remove(n);
- }
-}
-
-LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const {
- return _load_barrier_nodes->at(idx);
-}
-
-void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
- return new(comp_arena) ZBarrierSetC2State(comp_arena);
-}
-ZBarrierSetC2State* ZBarrierSetC2::state() const {
- return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
-}
+class ZBarrierSetC2State : public ResourceObj {
+private:
+ GrowableArray<ZLoadBarrierStubC2*>* _stubs;
+ Node_Array _live;
-bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const {
- // 1. This step follows potential oop projections of a load barrier before expansion
- if (node->is_Proj()) {
- node = node->in(0);
- }
+public:
+ ZBarrierSetC2State(Arena* arena) :
+ _stubs(new (arena) GrowableArray<ZLoadBarrierStubC2*>(arena, 8, 0, NULL)),
+ _live(arena) {}
- // 2. This step checks for unexpanded load barriers
- if (node->is_LoadBarrier()) {
- return true;
+ GrowableArray<ZLoadBarrierStubC2*>* stubs() {
+ return _stubs;
}
- // 3. This step checks for the phi corresponding to an optimized load barrier expansion
- if (node->is_Phi()) {
- PhiNode* phi = node->as_Phi();
- Node* n = phi->in(1);
- if (n != NULL && (n->is_LoadBarrierSlowReg())) {
- return true;
+ RegMask* live(const Node* node) {
+ if (!node->is_Mach()) {
+ // Don't need liveness for non-MachNodes
+ return NULL;
}
- }
- return false;
-}
+ const MachNode* const mach = node->as_Mach();
+ if (mach->barrier_data() != ZLoadBarrierStrong &&
+ mach->barrier_data() != ZLoadBarrierWeak) {
+ // Don't need liveness data for nodes without barriers
+ return NULL;
+ }
-void ZBarrierSetC2::register_potential_barrier_node(Node* node) const {
- if (node->is_LoadBarrier()) {
- state()->add_load_barrier_node(node->as_LoadBarrier());
- }
-}
+ RegMask* live = (RegMask*)_live[node->_idx];
+ if (live == NULL) {
+ live = new (Compile::current()->comp_arena()->Amalloc_D(sizeof(RegMask))) RegMask();
+ _live.map(node->_idx, (Node*)live);
+ }
-void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
- if (node->is_LoadBarrier()) {
- state()->remove_load_barrier_node(node->as_LoadBarrier());
+ return live;
}
-}
+};
-void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const {
- // Remove useless LoadBarrier nodes
- ZBarrierSetC2State* s = state();
- for (int i = s->load_barrier_count()-1; i >= 0; i--) {
- LoadBarrierNode* n = s->load_barrier_node(i);
- if (!useful.member(n)) {
- unregister_potential_barrier_node(n);
- }
- }
+static ZBarrierSetC2State* barrier_set_state() {
+ return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
}
-void ZBarrierSetC2::enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const {
- if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) {
- worklist.push(node);
+ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) {
+ ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref, tmp, weak);
+ if (!Compile::current()->in_scratch_emit_size()) {
+ barrier_set_state()->stubs()->append(stub);
}
-}
-
-static bool load_require_barrier(LoadNode* load) { return ((load->barrier_data() & RequireBarrier) != 0); }
-static bool load_has_weak_barrier(LoadNode* load) { return ((load->barrier_data() & WeakBarrier) != 0); }
-static bool load_has_expanded_barrier(LoadNode* load) { return ((load->barrier_data() & ExpandedBarrier) != 0); }
-static void load_set_expanded_barrier(LoadNode* load) { return load->set_barrier_data(ExpandedBarrier); }
-static void load_set_barrier(LoadNode* load, bool weak) {
- if (weak) {
- load->set_barrier_data(WeakBarrier);
- } else {
- load->set_barrier_data(RequireBarrier);
- }
+ return stub;
}
-// == LoadBarrierNode ==
-
-LoadBarrierNode::LoadBarrierNode(Compile* C,
- Node* c,
- Node* mem,
- Node* val,
- Node* adr,
- bool weak) :
- MultiNode(Number_of_Inputs),
- _weak(weak) {
- init_req(Control, c);
- init_req(Memory, mem);
- init_req(Oop, val);
- init_req(Address, adr);
- init_req(Similar, C->top());
-
- init_class_id(Class_LoadBarrier);
- BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
- bs->register_potential_barrier_node(this);
+ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) :
+ _node(node),
+ _ref_addr(ref_addr),
+ _ref(ref),
+ _tmp(tmp),
+ _weak(weak),
+ _entry(),
+ _continuation() {
+ assert_different_registers(ref, ref_addr.base());
+ assert_different_registers(ref, ref_addr.index());
}
-uint LoadBarrierNode::size_of() const {
- return sizeof(*this);
+Address ZLoadBarrierStubC2::ref_addr() const {
+ return _ref_addr;
}
-uint LoadBarrierNode::cmp(const Node& n) const {
- ShouldNotReachHere();
- return 0;
+Register ZLoadBarrierStubC2::ref() const {
+ return _ref;
}
-const Type *LoadBarrierNode::bottom_type() const {
- const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
- Node* in_oop = in(Oop);
- floadbarrier[Control] = Type::CONTROL;
- floadbarrier[Memory] = Type::MEMORY;
- floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type();
- return TypeTuple::make(Number_of_Outputs, floadbarrier);
+Register ZLoadBarrierStubC2::tmp() const {
+ return _tmp;
}
-const TypePtr* LoadBarrierNode::adr_type() const {
- return TypeRawPtr::BOTTOM;
+address ZLoadBarrierStubC2::slow_path() const {
+ const DecoratorSet decorators = _weak ? ON_WEAK_OOP_REF : ON_STRONG_OOP_REF;
+ return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
}
-const Type *LoadBarrierNode::Value(PhaseGVN *phase) const {
- const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*)));
- const Type* val_t = phase->type(in(Oop));
- floadbarrier[Control] = Type::CONTROL;
- floadbarrier[Memory] = Type::MEMORY;
- floadbarrier[Oop] = val_t;
- return TypeTuple::make(Number_of_Outputs, floadbarrier);
+RegMask& ZLoadBarrierStubC2::live() const {
+ return *barrier_set_state()->live(_node);
}
-bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) {
- if (phase != NULL) {
- return phase->is_dominator(d, n);
- }
-
- for (int i = 0; i < 10 && n != NULL; i++) {
- n = IfNode::up_one_dom(n, linear_only);
- if (n == d) {
- return true;
- }
- }
-
- return false;
+Label* ZLoadBarrierStubC2::entry() {
+ // The _entry will never be bound when in_scratch_emit_size() is true.
+ // However, we still need to return a label that is not bound now, but
+ // will eventually be bound. Any lable will do, as it will only act as
+ // a placeholder, so we return the _continuation label.
+ return Compile::current()->in_scratch_emit_size() ? &_continuation : &_entry;
}
-LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) {
- if (is_weak()) {
- // Weak barriers can't be eliminated
- return NULL;
- }
-
- Node* val = in(LoadBarrierNode::Oop);
- if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) {
- LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier();
- assert(lb->in(Address) == in(Address), "");
- // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier.
- if (lb->in(Oop) == in(Oop)) {
- return lb;
- }
- // Follow chain of load barrier through Similar edges
- while (!lb->in(Similar)->is_top()) {
- lb = lb->in(Similar)->in(0)->as_LoadBarrier();
- assert(lb->in(Address) == in(Address), "");
- }
- if (lb != in(Similar)->in(0)) {
- return lb;
- }
- }
- for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
- Node* u = val->fast_out(i);
- if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) {
- Node* this_ctrl = in(LoadBarrierNode::Control);
- Node* other_ctrl = u->in(LoadBarrierNode::Control);
- if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
- return u->as_LoadBarrier();
- }
- }
- }
-
- if (can_be_eliminated()) {
- return NULL;
- }
-
- if (!look_for_similar) {
- return NULL;
- }
-
- Node* addr = in(LoadBarrierNode::Address);
- for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
- Node* u = addr->fast_out(i);
- if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) {
- Node* this_ctrl = in(LoadBarrierNode::Control);
- Node* other_ctrl = u->in(LoadBarrierNode::Control);
- if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) {
- ResourceMark rm;
- Unique_Node_List wq;
- wq.push(in(LoadBarrierNode::Control));
- bool ok = true;
- bool dom_found = false;
- for (uint next = 0; next < wq.size(); ++next) {
- Node *n = wq.at(next);
- if (n->is_top()) {
- return NULL;
- }
- assert(n->is_CFG(), "");
- if (n->is_SafePoint()) {
- ok = false;
- break;
- }
- if (n == u) {
- dom_found = true;
- continue;
- }
- if (n->is_Region()) {
- for (uint i = 1; i < n->req(); i++) {
- Node* m = n->in(i);
- if (m != NULL) {
- wq.push(m);
- }
- }
- } else {
- Node* m = n->in(0);
- if (m != NULL) {
- wq.push(m);
- }
- }
- }
- if (ok) {
- assert(dom_found, "");
- return u->as_LoadBarrier();
- }
- break;
- }
- }
- }
-
- return NULL;
+Label* ZLoadBarrierStubC2::continuation() {
+ return &_continuation;
}
-void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const {
- // Change to that barrier may affect a dominated barrier so re-push those
- assert(!is_weak(), "sanity");
- Node* val = in(LoadBarrierNode::Oop);
-
- for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) {
- Node* u = val->fast_out(i);
- if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) {
- Node* this_ctrl = in(Control);
- Node* other_ctrl = u->in(Control);
- if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
- igvn->_worklist.push(u);
- }
- }
-
- Node* addr = in(LoadBarrierNode::Address);
- for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) {
- Node* u = addr->fast_out(i);
- if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) {
- Node* this_ctrl = in(Control);
- Node* other_ctrl = u->in(Control);
- if (is_dominator(NULL, false, this_ctrl, other_ctrl)) {
- igvn->_worklist.push(u);
- }
- }
- }
- }
+void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
+ return new (comp_arena) ZBarrierSetC2State(comp_arena);
}
-Node *LoadBarrierNode::Identity(PhaseGVN *phase) {
- LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false);
- if (dominating_barrier != NULL) {
- assert(!is_weak(), "Weak barriers cant be eliminated");
- assert(dominating_barrier->in(Oop) == in(Oop), "");
- return dominating_barrier;
- }
-
- return this;
+void ZBarrierSetC2::late_barrier_analysis() const {
+ analyze_dominating_barriers();
+ compute_liveness_at_stubs();
}
-Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) {
- if (remove_dead_region(phase, can_reshape)) {
- return this;
- }
-
- Node *val = in(Oop);
- Node *mem = in(Memory);
- Node *ctrl = in(Control);
- assert(val->Opcode() != Op_DecodeN, "");
-
- if (mem->is_MergeMem()) {
- Node *new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw);
- set_req(Memory, new_mem);
- if (mem->outcnt() == 0 && can_reshape) {
- phase->is_IterGVN()->_worklist.push(mem);
- }
- return this;
- }
-
- LoadBarrierNode *dominating_barrier = NULL;
- if (!is_weak()) {
- dominating_barrier = has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress());
- if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) {
- assert(in(Address) == dominating_barrier->in(Address), "");
- set_req(Similar, dominating_barrier->proj_out(Oop));
- return this;
- }
- }
-
- bool eliminate = can_reshape && (dominating_barrier != NULL || !has_true_uses());
- if (eliminate) {
- if (can_reshape) {
- PhaseIterGVN* igvn = phase->is_IterGVN();
- Node* out_ctrl = proj_out_or_null(Control);
- Node* out_res = proj_out_or_null(Oop);
-
- if (out_ctrl != NULL) {
- igvn->replace_node(out_ctrl, ctrl);
- }
-
- // That transformation may cause the Similar edge on the load barrier to be invalid
- fix_similar_in_uses(igvn);
- if (out_res != NULL) {
- if (dominating_barrier != NULL) {
- assert(!is_weak(), "Sanity");
- igvn->replace_node(out_res, dominating_barrier->proj_out(Oop));
- } else {
- igvn->replace_node(out_res, val);
- }
- }
- }
- return new ConINode(TypeInt::ZERO);
- }
-
- // If the Similar edge is no longer a load barrier, clear it
- Node* similar = in(Similar);
- if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) {
- set_req(Similar, phase->C->top());
- return this;
- }
+void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
+ MacroAssembler masm(&cb);
+ GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
- if (can_reshape && !is_weak()) {
- // If this barrier is linked through the Similar edge by a
- // dominated barrier and both barriers have the same Oop field,
- // the dominated barrier can go away, so push it for reprocessing.
- // We also want to avoid a barrier to depend on another dominating
- // barrier through its Similar edge that itself depend on another
- // barrier through its Similar edge and rather have the first
- // depend on the third.
- PhaseIterGVN* igvn = phase->is_IterGVN();
- Node* out_res = proj_out(Oop);
- for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
- Node* u = out_res->fast_out(i);
- if (u->is_LoadBarrier() && u->in(Similar) == out_res &&
- (u->in(Oop) == val || !u->in(Similar)->is_top())) {
- assert(!u->as_LoadBarrier()->is_weak(), "Sanity");
- igvn->_worklist.push(u);
- }
+ for (int i = 0; i < stubs->length(); i++) {
+ // Make sure there is enough space in the code buffer
+ if (cb.insts()->maybe_expand_to_ensure_remaining(Compile::MAX_inst_size) && cb.blob() == NULL) {
+ ciEnv::current()->record_failure("CodeCache is full");
+ return;
}
- push_dominated_barriers(igvn);
+ ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
}
-
- return NULL;
+ masm.flush();
}
-uint LoadBarrierNode::match_edge(uint idx) const {
- ShouldNotReachHere();
- return 0;
-}
+int ZBarrierSetC2::estimate_stub_size() const {
+ Compile* const C = Compile::current();
+ BufferBlob* const blob = C->scratch_buffer_blob();
+ GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
+ int size = 0;
-void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) {
- Node* out_res = proj_out_or_null(Oop);
- if (out_res == NULL) {
- return;
+ for (int i = 0; i < stubs->length(); i++) {
+ CodeBuffer cb(blob->content_begin(), (address)C->scratch_locs_memory() - blob->content_begin());
+ MacroAssembler masm(&cb);
+ ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
+ size += cb.insts_size();
}
- for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
- Node* u = out_res->fast_out(i);
- if (u->is_LoadBarrier() && u->in(Similar) == out_res) {
- igvn->replace_input_of(u, Similar, igvn->C->top());
- --i;
- --imax;
- }
- }
-}
-
-bool LoadBarrierNode::has_true_uses() const {
- Node* out_res = proj_out_or_null(Oop);
- if (out_res != NULL) {
- for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) {
- Node *u = out_res->fast_out(i);
- if (!u->is_LoadBarrier() || u->in(Similar) != out_res) {
- return true;
- }
- }
- }
- return false;
+ return size;
}
static bool barrier_needed(C2Access access) {
@@ -466,1071 +179,253 @@ static bool barrier_needed(C2Access access) {
}
Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
- Node* p = BarrierSetC2::load_at_resolved(access, val_type);
- if (!barrier_needed(access)) {
- return p;
+ Node* result = BarrierSetC2::load_at_resolved(access, val_type);
+ if (barrier_needed(access) && access.raw_access()->is_Mem()) {
+ if ((access.decorators() & ON_WEAK_OOP_REF) != 0) {
+ access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierWeak);
+ } else {
+ access.raw_access()->as_Load()->set_barrier_data(ZLoadBarrierStrong);
+ }
}
- bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0;
- if (p->isa_Load()) {
- load_set_barrier(p->as_Load(), weak);
- }
- return p;
+ return result;
}
Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val,
Node* new_val, const Type* val_type) const {
Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
- LoadStoreNode* lsn = result->as_LoadStore();
if (barrier_needed(access)) {
- lsn->set_has_barrier();
+ access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);
}
- return lsn;
+ return result;
}
Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
- LoadStoreNode* lsn = result->as_LoadStore();
if (barrier_needed(access)) {
- lsn->set_has_barrier();
+ access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);
}
- return lsn;
+ return result;
}
Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* val_type) const {
Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
- LoadStoreNode* lsn = result->as_LoadStore();
if (barrier_needed(access)) {
- lsn->set_has_barrier();
+ access.raw_access()->as_LoadStore()->set_barrier_data(ZLoadBarrierStrong);
}
- return lsn;
+ return result;
}
-// == Macro Expansion ==
-
-// Optimized, low spill, loadbarrier variant using stub specialized on register used
-void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const {
- PhaseIterGVN &igvn = phase->igvn();
- float unlikely = PROB_UNLIKELY(0.999);
-
- Node* in_ctrl = barrier->in(LoadBarrierNode::Control);
- Node* in_mem = barrier->in(LoadBarrierNode::Memory);
- Node* in_val = barrier->in(LoadBarrierNode::Oop);
- Node* in_adr = barrier->in(LoadBarrierNode::Address);
-
- Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
- Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
-
- assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null");
-
- Node* jthread = igvn.transform(new ThreadLocalNode());
- Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset()));
- Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr,
- TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(),
- MemNode::unordered));
- Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val));
- Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask));
- Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type())));
- Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
- IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
- Node* then = igvn.transform(new IfTrueNode(iff));
- Node* elsen = igvn.transform(new IfFalseNode(iff));
-
- Node* new_loadp = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(),
- (const TypePtr*) in_val->bottom_type(), MemNode::unordered, barrier->is_weak()));
-
- // Create the final region/phi pair to converge cntl/data paths to downstream code
- Node* result_region = igvn.transform(new RegionNode(3));
- result_region->set_req(1, then);
- result_region->set_req(2, elsen);
-
- Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM));
- result_phi->set_req(1, new_loadp);
- result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop));
-
- igvn.replace_node(out_ctrl, result_region);
- igvn.replace_node(out_res, result_phi);
-
- assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!");
-
- igvn.remove_dead_node(barrier);
- igvn.remove_dead_node(out_ctrl);
- igvn.remove_dead_node(out_res);
- assert(is_gc_barrier_node(result_phi), "sanity");
- assert(step_over_gc_barrier(result_phi) == in_val, "sanity");
- phase->C->print_method(PHASE_BEFORE_MACRO_EXPANSION, 4, barrier->_idx);
- return;
-}
-
-bool ZBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const {
- Compile* C = Compile::current();
- PhaseIterGVN &igvn = macro->igvn();
- ZBarrierSetC2State* s = state();
- if (s->load_barrier_count() > 0) {
-#ifdef ASSERT
- verify_gc_barriers(false);
-#endif
- igvn.set_delay_transform(true);
- int skipped = 0;
- while (s->load_barrier_count() > skipped) {
- int load_barrier_count = s->load_barrier_count();
- LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped);
- if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
- // Node is unreachable, so don't try to expand it
- s->remove_load_barrier_node(n);
- continue;
- }
- if (!n->can_be_eliminated()) {
- skipped++;
- continue;
- }
- expand_loadbarrier_node(macro, n);
- assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
- if (C->failing()) return true;
- }
- while (s->load_barrier_count() > 0) {
- int load_barrier_count = s->load_barrier_count();
- LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1);
- assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already");
- assert(!n->can_be_eliminated(), "should have been processed already");
- expand_loadbarrier_node(macro, n);
- assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list");
- if (C->failing()) return true;
- }
- igvn.set_delay_transform(false);
- igvn.optimize();
- if (C->failing()) return true;
- }
- return false;
+bool ZBarrierSetC2::array_copy_requires_gc_barriers(BasicType type) const {
+ return type == T_OBJECT || type == T_ARRAY;
}
-Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const {
- Node* node = c;
-
- // 1. This step follows potential oop projections of a load barrier before expansion
- if (node->is_Proj()) {
- node = node->in(0);
- }
-
- // 2. This step checks for unexpanded load barriers
- if (node->is_LoadBarrier()) {
- return node->in(LoadBarrierNode::Oop);
- }
+// == Dominating barrier elision ==
- // 3. This step checks for the phi corresponding to an optimized load barrier expansion
- if (node->is_Phi()) {
- PhiNode* phi = node->as_Phi();
- Node* n = phi->in(1);
- if (n != NULL && (n->is_LoadBarrierSlowReg())) {
- assert(c == node, "projections from step 1 should only be seen before macro expansion");
- return phi->in(2);
+static bool block_has_safepoint(const Block* block, uint from, uint to) {
+ for (uint i = from; i < to; i++) {
+ if (block->get_node(i)->is_MachSafePoint()) {
+ // Safepoint found
+ return true;
}
}
- return c;
+ // Safepoint not found
+ return false;
}
-// == Verification ==
-
-#ifdef ASSERT
-
-static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) {
- if (visited.test_set(n->_idx)) {
- return true;
- }
+static bool block_has_safepoint(const Block* block) {
+ return block_has_safepoint(block, 0, block->number_of_nodes());
+}
- for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
- Node* u = n->fast_out(i);
- if (u->is_LoadBarrier()) {
- } else if ((u->is_Phi() || u->is_CMove()) && !post_parse) {
- if (!look_for_barrier(u, post_parse, visited)) {
- return false;
- }
- } else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) {
- if (!look_for_barrier(u, post_parse, visited)) {
- return false;
- }
- } else if (u->Opcode() != Op_SCMemProj) {
- tty->print("bad use"); u->dump();
- return false;
+static uint block_index(const Block* block, const Node* node) {
+ for (uint j = 0; j < block->number_of_nodes(); ++j) {
+ if (block->get_node(j) == node) {
+ return j;
}
}
-
- return true;
+ ShouldNotReachHere();
+ return 0;
}
-void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const {
- ZBarrierSetC2State* s = state();
- Compile* C = Compile::current();
+void ZBarrierSetC2::analyze_dominating_barriers() const {
ResourceMark rm;
- VectorSet visited(Thread::current()->resource_area());
- for (int i = 0; i < s->load_barrier_count(); i++) {
- LoadBarrierNode* n = s->load_barrier_node(i);
-
- // The dominating barrier on the same address if it exists and
- // this barrier must not be applied on the value from the same
- // load otherwise the value is not reloaded before it's used the
- // second time.
- assert(n->in(LoadBarrierNode::Similar)->is_top() ||
- (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
- n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) &&
- n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)),
- "broken similar edge");
-
- assert(post_parse || n->as_LoadBarrier()->has_true_uses(),
- "found unneeded load barrier");
-
- // Several load barrier nodes chained through their Similar edge
- // break the code that remove the barriers in final graph reshape.
- assert(n->in(LoadBarrierNode::Similar)->is_top() ||
- (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() &&
- n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()),
- "chain of Similar load barriers");
-
- if (!n->in(LoadBarrierNode::Similar)->is_top()) {
- ResourceMark rm;
- Unique_Node_List wq;
- Node* other = n->in(LoadBarrierNode::Similar)->in(0);
- wq.push(n);
- bool ok = true;
- bool dom_found = false;
- for (uint next = 0; next < wq.size(); ++next) {
- Node *n = wq.at(next);
- assert(n->is_CFG(), "");
- assert(!n->is_SafePoint(), "");
-
- if (n == other) {
- continue;
- }
-
- if (n->is_Region()) {
- for (uint i = 1; i < n->req(); i++) {
- Node* m = n->in(i);
- if (m != NULL) {
- wq.push(m);
- }
- }
- } else {
- Node* m = n->in(0);
- if (m != NULL) {
- wq.push(m);
- }
- }
+ Compile* const C = Compile::current();
+ PhaseCFG* const cfg = C->cfg();
+ Block_List worklist;
+ Node_List mem_ops;
+ Node_List barrier_loads;
+
+ // Step 1 - Find accesses, and track them in lists
+ for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
+ const Block* const block = cfg->get_block(i);
+ for (uint j = 0; j < block->number_of_nodes(); ++j) {
+ const Node* const node = block->get_node(j);
+ if (!node->is_Mach()) {
+ continue;
}
- }
- if (ZVerifyLoadBarriers) {
- if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) {
- visited.Clear();
- bool found = look_for_barrier(n, post_parse, visited);
- if (!found) {
- n->dump(1);
- n->dump(-3);
- stringStream ss;
- C->method()->print_short_name(&ss);
- tty->print_cr("-%s-", ss.as_string());
- assert(found, "");
+ MachNode* const mach = node->as_Mach();
+ switch (mach->ideal_Opcode()) {
+ case Op_LoadP:
+ case Op_CompareAndExchangeP:
+ case Op_CompareAndSwapP:
+ case Op_GetAndSetP:
+ if (mach->barrier_data() == ZLoadBarrierStrong) {
+ barrier_loads.push(mach);
}
- }
- }
- }
-}
-
-#endif
-
-static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl);
-
-// This code is cloning all uses of a load that is between a call and the catch blocks,
-// to each use.
-
-static bool fixup_uses_in_catch(PhaseIdealLoop *phase, Node *start_ctrl, Node *node) {
-
- if (!phase->has_ctrl(node)) {
- // This node is floating - doesn't need to be cloned.
- assert(node != start_ctrl, "check");
- return false;
- }
-
- Node* ctrl = phase->get_ctrl(node);
- if (ctrl != start_ctrl) {
- // We are in a successor block - the node is ok.
- return false; // Unwind
- }
-
- // Process successor nodes
- int outcnt = node->outcnt();
- for (int i = 0; i < outcnt; i++) {
- Node* n = node->raw_out(0);
- assert(!n->is_LoadBarrier(), "Sanity");
- // Calling recursively, visiting leafs first
- fixup_uses_in_catch(phase, start_ctrl, n);
- }
-
- // Now all successors are outside
- // - Clone this node to both successors
- int no_succs = node->outcnt();
- assert(!node->is_Store(), "Stores not expected here");
-
- // In some very rare cases a load that doesn't need a barrier will end up here
- // Treat it as a LoadP and the insertion of phis will be done correctly.
- if (node->is_Load()) {
- assert(node->as_Load()->barrier_data() == 0, "Sanity");
- call_catch_cleanup_one(phase, node->as_Load(), phase->get_ctrl(node));
- } else {
- for (DUIterator_Fast jmax, i = node->fast_outs(jmax); i < jmax; i++) {
- Node* use = node->fast_out(i);
- Node* clone = node->clone();
- assert(clone->outcnt() == 0, "");
-
- assert(use->find_edge(node) != -1, "check");
- phase->igvn().rehash_node_delayed(use);
- use->replace_edge(node, clone);
-
- Node* new_ctrl;
- if (use->is_block_start()) {
- new_ctrl = use;
- } else if (use->is_CFG()) {
- new_ctrl = use->in(0);
- assert (new_ctrl != NULL, "");
- } else {
- new_ctrl = phase->get_ctrl(use);
- }
-
- phase->set_ctrl(clone, new_ctrl);
-
- if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr(" Clone op %i as %i to control %i", node->_idx, clone->_idx, new_ctrl->_idx);
- phase->igvn().register_new_node_with_optimizer(clone);
- --i, --jmax;
- }
- assert(node->outcnt() == 0, "must be empty now");
-
- // Node node is dead.
- phase->igvn().remove_dead_node(node);
- }
- return true; // unwind - return if a use was processed
-}
-
-// Clone a load to a specific catch_proj
-static Node* clone_load_to_catchproj(PhaseIdealLoop* phase, Node* load, Node* catch_proj) {
- Node* cloned_load = load->clone();
- cloned_load->set_req(0, catch_proj); // set explicit control
- phase->set_ctrl(cloned_load, catch_proj); // update
- if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr(" Clone LOAD %i as %i to control %i", load->_idx, cloned_load->_idx, catch_proj->_idx);
- phase->igvn().register_new_node_with_optimizer(cloned_load);
- return cloned_load;
-}
-
-static Node* get_dominating_region(PhaseIdealLoop* phase, Node* node, Node* stop) {
- Node* region = node;
- while (!region->isa_Region()) {
- Node *up = phase->idom(region);
- assert(up != region, "Must not loop");
- assert(up != stop, "Must not find original control");
- region = up;
- }
- return region;
-}
-
-// Clone this load to each catch block
-static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) {
- bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
- phase->igvn().set_delay_transform(true);
-
- // Verify pre conditions
- assert(ctrl->isa_Proj() && ctrl->in(0)->isa_Call(), "Must be a call proj");
- assert(ctrl->raw_out(0)->isa_Catch(), "Must be a catch");
-
- if (ctrl->raw_out(0)->isa_Catch()->outcnt() == 1) {
- if (trace) tty->print_cr("Cleaning up catch: Skipping load %i, call with single catch", load->_idx);
- return;
- }
+ case Op_StoreP:
+ mem_ops.push(mach);
+ break;
- // Process the loads successor nodes - if any is between
- // the call and the catch blocks, they need to be cloned to.
- // This is done recursively
- int outcnt = load->outcnt();
- uint index = 0;
- for (int i = 0; i < outcnt; i++) {
- if (index < load->outcnt()) {
- Node *n = load->raw_out(index);
- assert(!n->is_LoadBarrier(), "Sanity");
- if (!fixup_uses_in_catch(phase, ctrl, n)) {
- // if no successor was cloned, progress to next out.
- index++;
+ default:
+ break;
}
}
}
- // Now all the loads uses has been cloned down
- // Only thing left is to clone the loads, but they must end up
- // first in the catch blocks.
-
- // We clone the loads oo the catch blocks only when needed.
- // An array is used to map the catch blocks to each lazily cloned load.
- // In that way no extra unnecessary loads are cloned.
-
- // Any use dominated by original block must have an phi and a region added
+ // Step 2 - Find dominating accesses for each load
+ for (uint i = 0; i < barrier_loads.size(); i++) {
+ MachNode* const load = barrier_loads.at(i)->as_Mach();
+ const TypePtr* load_adr_type = NULL;
+ intptr_t load_offset = 0;
+ const Node* const load_obj = load->get_base_and_disp(load_offset, load_adr_type);
+ Block* const load_block = cfg->get_block_for_node(load);
+ const uint load_index = block_index(load_block, load);
- Node* catch_node = ctrl->raw_out(0);
- int number_of_catch_projs = catch_node->outcnt();
- Node** proj_to_load_mapping = NEW_RESOURCE_ARRAY(Node*, number_of_catch_projs);
- Copy::zero_to_bytes(proj_to_load_mapping, sizeof(Node*) * number_of_catch_projs);
+ for (uint j = 0; j < mem_ops.size(); j++) {
+ MachNode* mem = mem_ops.at(j)->as_Mach();
+ const TypePtr* mem_adr_type = NULL;
+ intptr_t mem_offset = 0;
+ const Node* mem_obj = mem_obj = mem->get_base_and_disp(mem_offset, mem_adr_type);
+ Block* mem_block = cfg->get_block_for_node(mem);
+ uint mem_index = block_index(mem_block, mem);
- // The phi_map is used to keep track of where phis have already been inserted
- int phi_map_len = phase->C->unique();
- Node** phi_map = NEW_RESOURCE_ARRAY(Node*, phi_map_len);
- Copy::zero_to_bytes(phi_map, sizeof(Node*) * phi_map_len);
-
- for (unsigned int i = 0; i < load->outcnt(); i++) {
- Node* load_use_control = NULL;
- Node* load_use = load->raw_out(i);
-
- if (phase->has_ctrl(load_use)) {
- load_use_control = phase->get_ctrl(load_use);
- } else {
- load_use_control = load_use->in(0);
- }
- assert(load_use_control != NULL, "sanity");
- if (trace) tty->print_cr(" Handling use: %i, with control: %i", load_use->_idx, load_use_control->_idx);
-
- // Some times the loads use is a phi. For them we need to determine from which catch block
- // the use is defined.
- bool load_use_is_phi = false;
- unsigned int load_use_phi_index = 0;
- Node* phi_ctrl = NULL;
- if (load_use->is_Phi()) {
- // Find phi input that matches load
- for (unsigned int u = 1; u < load_use->req(); u++) {
- if (load_use->in(u) == load) {
- load_use_is_phi = true;
- load_use_phi_index = u;
- assert(load_use->in(0)->is_Region(), "Region or broken");
- phi_ctrl = load_use->in(0)->in(u);
- assert(phi_ctrl->is_CFG(), "check");
- assert(phi_ctrl != load, "check");
- break;
- }
- }
- assert(load_use_is_phi, "must find");
- assert(load_use_phi_index > 0, "sanity");
- }
-
- // For each load use, see which catch projs dominates, create load clone lazily and reconnect
- bool found_dominating_catchproj = false;
- for (int c = 0; c < number_of_catch_projs; c++) {
- Node* catchproj = catch_node->raw_out(c);
- assert(catchproj != NULL && catchproj->isa_CatchProj(), "Sanity");
-
- if (!phase->is_dominator(catchproj, load_use_control)) {
- if (load_use_is_phi && phase->is_dominator(catchproj, phi_ctrl)) {
- // The loads use is local to the catchproj.
- // fall out and replace load with catch-local load clone.
- } else {
- continue;
- }
- }
- assert(!found_dominating_catchproj, "Max one should match");
-
- // Clone loads to catch projs
- Node* load_clone = proj_to_load_mapping[c];
- if (load_clone == NULL) {
- load_clone = clone_load_to_catchproj(phase, load, catchproj);
- proj_to_load_mapping[c] = load_clone;
- }
- phase->igvn().rehash_node_delayed(load_use);
-
- if (load_use_is_phi) {
- // phis are special - the load is defined from a specific control flow
- load_use->set_req(load_use_phi_index, load_clone);
- } else {
- // Multipe edges can be replaced at once - on calls for example
- load_use->replace_edge(load, load_clone);
+ if (load_obj == NodeSentinel || mem_obj == NodeSentinel ||
+ load_obj == NULL || mem_obj == NULL ||
+ load_offset < 0 || mem_offset < 0) {
+ continue;
}
- --i; // more than one edge can have been removed, but the next is in later iterations
-
- // We could break the for-loop after finding a dominating match.
- // But keep iterating to catch any bad idom early.
- found_dominating_catchproj = true;
- }
- // We found no single catchproj that dominated the use - The use is at a point after
- // where control flow from multiple catch projs have merged. We will have to create
- // phi nodes before the use and tie the output from the cloned loads together. It
- // can be a single phi or a number of chained phis, depending on control flow
- if (!found_dominating_catchproj) {
-
- // Use phi-control if use is a phi
- if (load_use_is_phi) {
- load_use_control = phi_ctrl;
+ if (mem_obj != load_obj || mem_offset != load_offset) {
+ // Not the same addresses, not a candidate
+ continue;
}
- assert(phase->is_dominator(ctrl, load_use_control), "Common use but no dominator");
- // Clone a load on all paths
- for (int c = 0; c < number_of_catch_projs; c++) {
- Node* catchproj = catch_node->raw_out(c);
- Node* load_clone = proj_to_load_mapping[c];
- if (load_clone == NULL) {
- load_clone = clone_load_to_catchproj(phase, load, catchproj);
- proj_to_load_mapping[c] = load_clone;
+ if (load_block == mem_block) {
+ // Earlier accesses in the same block
+ if (mem_index < load_index && !block_has_safepoint(mem_block, mem_index + 1, load_index)) {
+ load->set_barrier_data(ZLoadBarrierElided);
}
- }
-
- // Move up dominator tree from use until dom front is reached
- Node* next_region = get_dominating_region(phase, load_use_control, ctrl);
- while (phase->idom(next_region) != catch_node) {
- next_region = phase->idom(next_region);
- if (trace) tty->print_cr("Moving up idom to region ctrl %i", next_region->_idx);
- }
- assert(phase->is_dominator(catch_node, next_region), "Sanity");
-
- // Create or reuse phi node that collect all cloned loads and feed it to the use.
- Node* test_phi = phi_map[next_region->_idx];
- if ((test_phi != NULL) && test_phi->is_Phi()) {
- // Reuse an already created phi
- if (trace) tty->print_cr(" Using cached Phi %i on load_use %i", test_phi->_idx, load_use->_idx);
- phase->igvn().rehash_node_delayed(load_use);
- load_use->replace_edge(load, test_phi);
- // Now this use is done
- } else {
- // Otherwise we need to create one or more phis
- PhiNode* next_phi = new PhiNode(next_region, load->type());
- phi_map[next_region->_idx] = next_phi; // cache new phi
- phase->igvn().rehash_node_delayed(load_use);
- load_use->replace_edge(load, next_phi);
-
- int dominators_of_region = 0;
- do {
- // New phi, connect to region and add all loads as in.
- Node* region = next_region;
- assert(region->isa_Region() && region->req() > 2, "Catch dead region nodes");
- PhiNode* new_phi = next_phi;
-
- if (trace) tty->print_cr("Created Phi %i on load %i with control %i", new_phi->_idx, load->_idx, region->_idx);
-
- // Need to add all cloned loads to the phi, taking care that the right path is matched
- dominators_of_region = 0; // reset for new region
- for (unsigned int reg_i = 1; reg_i < region->req(); reg_i++) {
- Node* region_pred = region->in(reg_i);
- assert(region_pred->is_CFG(), "check");
- bool pred_has_dominator = false;
- for (int c = 0; c < number_of_catch_projs; c++) {
- Node* catchproj = catch_node->raw_out(c);
- if (phase->is_dominator(catchproj, region_pred)) {
- new_phi->set_req(reg_i, proj_to_load_mapping[c]);
- if (trace) tty->print_cr(" - Phi in(%i) set to load %i", reg_i, proj_to_load_mapping[c]->_idx);
- pred_has_dominator = true;
- dominators_of_region++;
- break;
- }
- }
-
- // Sometimes we need to chain several phis.
- if (!pred_has_dominator) {
- assert(dominators_of_region <= 1, "More than one region can't require extra phi");
- if (trace) tty->print_cr(" - Region %i pred %i not dominated by catch proj", region->_idx, region_pred->_idx);
- // Continue search on on this region_pred
- // - walk up to next region
- // - create a new phi and connect to first new_phi
- next_region = get_dominating_region(phase, region_pred, ctrl);
-
- // Lookup if there already is a phi, create a new otherwise
- Node* test_phi = phi_map[next_region->_idx];
- if ((test_phi != NULL) && test_phi->is_Phi()) {
- next_phi = test_phi->isa_Phi();
- dominators_of_region++; // record that a match was found and that we are done
- if (trace) tty->print_cr(" Using cached phi Phi %i on control %i", next_phi->_idx, next_region->_idx);
- } else {
- next_phi = new PhiNode(next_region, load->type());
- phi_map[next_region->_idx] = next_phi;
- }
- new_phi->set_req(reg_i, next_phi);
- }
+ } else if (mem_block->dominates(load_block)) {
+ // Dominating block? Look around for safepoints
+ ResourceMark rm;
+ Block_List stack;
+ VectorSet visited(Thread::current()->resource_area());
+ stack.push(load_block);
+ bool safepoint_found = block_has_safepoint(load_block);
+ while (!safepoint_found && stack.size() > 0) {
+ Block* block = stack.pop();
+ if (visited.test_set(block->_pre_order)) {
+ continue;
+ }
+ if (block_has_safepoint(block)) {
+ safepoint_found = true;
+ break;
+ }
+ if (block == mem_block) {
+ continue;
}
- new_phi->set_req(0, region);
- phase->igvn().register_new_node_with_optimizer(new_phi);
- phase->set_ctrl(new_phi, region);
-
- assert(dominators_of_region != 0, "Must have found one this iteration");
- } while (dominators_of_region == 1);
- }
- --i;
- }
- } // end of loop over uses
-
- assert(load->outcnt() == 0, "All uses should be handled");
- phase->igvn().remove_dead_node(load);
- phase->C->print_method(PHASE_CALL_CATCH_CLEANUP, 4, load->_idx);
-
- // Now we should be home
- phase->igvn().set_delay_transform(false);
-}
-
-// Sort out the loads that are between a call ant its catch blocks
-static void process_catch_cleanup_candidate(PhaseIdealLoop* phase, LoadNode* load) {
- bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
-
- Node* ctrl = phase->get_ctrl(load);
- if (!ctrl->is_Proj() || (ctrl->in(0) == NULL) || !ctrl->in(0)->isa_Call()) {
- return;
- }
-
- Node* catch_node = ctrl->isa_Proj()->raw_out(0);
- if (catch_node->is_Catch()) {
- if (catch_node->outcnt() > 1) {
- call_catch_cleanup_one(phase, load, ctrl);
- } else {
- if (trace) tty->print_cr("Call catch cleanup with only one catch: load %i ", load->_idx);
- }
- }
-}
-
-void ZBarrierSetC2::barrier_insertion_phase(Compile* C, PhaseIterGVN& igvn) const {
- ResourceMark rm;
- PhaseIdealLoop v(igvn, false, false, false, true);
- if (C->failing()) return;
-}
-
-void ZBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {
- // First make sure all loads between call and catch are moved to the catch block
- clean_catch_blocks(phase);
-
- // Then expand barriers on all loads
- insert_load_barriers(phase);
-
- // Handle all Unsafe that need barriers.
- insert_barriers_on_unsafe(phase);
-
- phase->C->clear_major_progress();
-}
-
-static bool can_simplify_cas(LoadStoreNode* node) {
- if (node->isa_LoadStoreConditional()) {
- Node *expected_in = node->as_LoadStoreConditional()->in(LoadStoreConditionalNode::ExpectedIn);
- return (expected_in->get_ptr_type() == TypePtr::NULL_PTR);
- } else {
- return false;
- }
-}
-
-static void insert_barrier_before_unsafe(PhaseIdealLoop* phase, LoadStoreNode* old_node) {
-
- Compile *C = phase->C;
- PhaseIterGVN &igvn = phase->igvn();
- LoadStoreNode* zclone = NULL;
- bool is_weak = false;
-
- Node *in_ctrl = old_node->in(MemNode::Control);
- Node *in_mem = old_node->in(MemNode::Memory);
- Node *in_adr = old_node->in(MemNode::Address);
- Node *in_val = old_node->in(MemNode::ValueIn);
- const TypePtr *adr_type = old_node->adr_type();
- const TypePtr* load_type = TypeOopPtr::BOTTOM; // The type for the load we are adding
-
- switch (old_node->Opcode()) {
- case Op_CompareAndExchangeP: {
- zclone = new ZCompareAndExchangePNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn),
- adr_type, old_node->get_ptr_type(), ((CompareAndExchangeNode*)old_node)->order());
- load_type = old_node->bottom_type()->is_ptr();
- break;
- }
- case Op_WeakCompareAndSwapP: {
- if (can_simplify_cas(old_node)) {
- break;
- }
- is_weak = true;
- zclone = new ZWeakCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn),
- ((CompareAndSwapNode*)old_node)->order());
- adr_type = TypePtr::BOTTOM;
- break;
- }
- case Op_CompareAndSwapP: {
- if (can_simplify_cas(old_node)) {
- break;
- }
- zclone = new ZCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn),
- ((CompareAndSwapNode*)old_node)->order());
- adr_type = TypePtr::BOTTOM;
- break;
- }
- case Op_GetAndSetP: {
- zclone = new ZGetAndSetPNode(in_ctrl, in_mem, in_adr, in_val, old_node->adr_type(), old_node->get_ptr_type());
- load_type = old_node->bottom_type()->is_ptr();
- break;
- }
- }
- if (zclone != NULL) {
- igvn.register_new_node_with_optimizer(zclone, old_node);
-
- // Make load
- LoadPNode *load = new LoadPNode(NULL, in_mem, in_adr, adr_type, load_type, MemNode::unordered,
- LoadNode::DependsOnlyOnTest);
- load_set_expanded_barrier(load);
- igvn.register_new_node_with_optimizer(load);
- igvn.replace_node(old_node, zclone);
-
- Node *barrier = new LoadBarrierNode(C, NULL, in_mem, load, in_adr, is_weak);
- Node *barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop);
- Node *barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control);
-
- igvn.register_new_node_with_optimizer(barrier);
- igvn.register_new_node_with_optimizer(barrier_val);
- igvn.register_new_node_with_optimizer(barrier_ctrl);
-
- // loop over all of in_ctrl usages and move to barrier_ctrl
- for (DUIterator_Last imin, i = in_ctrl->last_outs(imin); i >= imin; --i) {
- Node *use = in_ctrl->last_out(i);
- uint l;
- for (l = 0; use->in(l) != in_ctrl; l++) {}
- igvn.replace_input_of(use, l, barrier_ctrl);
- }
-
- load->set_req(MemNode::Control, in_ctrl);
- barrier->set_req(LoadBarrierNode::Control, in_ctrl);
- zclone->add_req(barrier_val); // add req as keep alive.
-
- C->print_method(PHASE_ADD_UNSAFE_BARRIER, 4, zclone->_idx);
- }
-}
-
-void ZBarrierSetC2::insert_barriers_on_unsafe(PhaseIdealLoop* phase) const {
- Compile *C = phase->C;
- PhaseIterGVN &igvn = phase->igvn();
- uint new_ids = C->unique();
- VectorSet visited(Thread::current()->resource_area());
- GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL);
- nodeStack.push(C->root());
- visited.test_set(C->root()->_idx);
-
- // Traverse all nodes, visit all unsafe ops that require a barrier
- while (nodeStack.length() > 0) {
- Node *n = nodeStack.pop();
-
- bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup
- if (is_old_node) {
- if (n->is_LoadStore()) {
- LoadStoreNode* lsn = n->as_LoadStore();
- if (lsn->has_barrier()) {
- BasicType bt = lsn->in(MemNode::Address)->bottom_type()->basic_type();
- assert ((bt == T_OBJECT || bt == T_ARRAY), "Sanity test");
- insert_barrier_before_unsafe(phase, lsn);
- }
- }
- }
- for (uint i = 0; i < n->len(); i++) {
- if (n->in(i)) {
- if (!visited.test_set(n->in(i)->_idx)) {
- nodeStack.push(n->in(i));
+ // Push predecessor blocks
+ for (uint p = 1; p < block->num_preds(); ++p) {
+ Block* pred = cfg->get_block_for_node(block->pred(p));
+ stack.push(pred);
+ }
}
- }
- }
- }
-
- igvn.optimize();
- C->print_method(PHASE_ADD_UNSAFE_BARRIER, 2);
-}
-
-// The purpose of ZBarrierSetC2::clean_catch_blocks is to prepare the IR for
-// splicing in load barrier nodes.
-//
-// The problem is that we might have instructions between a call and its catch nodes.
-// (This is usually handled in PhaseCFG:call_catch_cleanup, which clones mach nodes in
-// already scheduled blocks.) We can't have loads that require barriers there,
-// because we need to splice in new control flow, and that would violate the IR.
-//
-// clean_catch_blocks find all Loads that require a barrier and clone them and any
-// dependent instructions to each use. The loads must be in the beginning of the catch block
-// before any store.
-//
-// Sometimes the loads use will be at a place dominated by all catch blocks, then we need
-// a load in each catch block, and a Phi at the dominated use.
-
-void ZBarrierSetC2::clean_catch_blocks(PhaseIdealLoop* phase) const {
-
- Compile *C = phase->C;
- uint new_ids = C->unique();
- PhaseIterGVN &igvn = phase->igvn();
- VectorSet visited(Thread::current()->resource_area());
- GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL);
- nodeStack.push(C->root());
- visited.test_set(C->root()->_idx);
-
- // Traverse all nodes, visit all loads that require a barrier
- while(nodeStack.length() > 0) {
- Node *n = nodeStack.pop();
-
- for (uint i = 0; i < n->len(); i++) {
- if (n->in(i)) {
- if (!visited.test_set(n->in(i)->_idx)) {
- nodeStack.push(n->in(i));
+ if (!safepoint_found) {
+ load->set_barrier_data(ZLoadBarrierElided);
}
}
}
-
- bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup
- if (n->is_Load() && is_old_node) {
- LoadNode* load = n->isa_Load();
- // only care about loads that will have a barrier
- if (load_require_barrier(load)) {
- process_catch_cleanup_candidate(phase, load);
- }
- }
}
-
- C->print_method(PHASE_CALL_CATCH_CLEANUP, 2);
}
-class DomDepthCompareClosure : public CompareClosure<LoadNode*> {
- PhaseIdealLoop* _phase;
-
-public:
- DomDepthCompareClosure(PhaseIdealLoop* phase) : _phase(phase) { }
-
- int do_compare(LoadNode* const &n1, LoadNode* const &n2) {
- int d1 = _phase->dom_depth(_phase->get_ctrl(n1));
- int d2 = _phase->dom_depth(_phase->get_ctrl(n2));
- if (d1 == d2) {
- // Compare index if the depth is the same, ensures all entries are unique.
- return n1->_idx - n2->_idx;
- } else {
- return d2 - d1;
- }
- }
-};
-
-// Traverse graph and add all loadPs to list, sorted by dom depth
-void gather_loadnodes_sorted(PhaseIdealLoop* phase, GrowableArray<LoadNode*>* loadList) {
-
- VectorSet visited(Thread::current()->resource_area());
- GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL);
- DomDepthCompareClosure ddcc(phase);
+// == Reduced spilling optimization ==
- nodeStack.push(phase->C->root());
- while(nodeStack.length() > 0) {
- Node *n = nodeStack.pop();
- if (visited.test(n->_idx)) {
- continue;
- }
-
- if (n->isa_Load()) {
- LoadNode *load = n->as_Load();
- if (load_require_barrier(load)) {
- assert(phase->get_ctrl(load) != NULL, "sanity");
- assert(phase->dom_depth(phase->get_ctrl(load)) != 0, "sanity");
- loadList->insert_sorted(&ddcc, load);
- }
- }
+void ZBarrierSetC2::compute_liveness_at_stubs() const {
+ ResourceMark rm;
+ Compile* const C = Compile::current();
+ Arena* const A = Thread::current()->resource_area();
+ PhaseCFG* const cfg = C->cfg();
+ PhaseRegAlloc* const regalloc = C->regalloc();
+ RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask));
+ ZBarrierSetAssembler* const bs = ZBarrierSet::assembler();
+ Block_List worklist;
+
+ for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
+ new ((void*)(live + i)) RegMask();
+ worklist.push(cfg->get_block(i));
+ }
+
+ while (worklist.size() > 0) {
+ const Block* const block = worklist.pop();
+ RegMask& old_live = live[block->_pre_order];
+ RegMask new_live;
+
+ // Initialize to union of successors
+ for (uint i = 0; i < block->_num_succs; i++) {
+ const uint succ_id = block->_succs[i]->_pre_order;
+ new_live.OR(live[succ_id]);
+ }
+
+ // Walk block backwards, computing liveness
+ for (int i = block->number_of_nodes() - 1; i >= 0; --i) {
+ const Node* const node = block->get_node(i);
+
+ // Remove def bits
+ const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));
+ const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));
+ if (first != OptoReg::Bad) {
+ new_live.Remove(first);
+ }
+ if (second != OptoReg::Bad) {
+ new_live.Remove(second);
+ }
+
+ // Add use bits
+ for (uint j = 1; j < node->req(); ++j) {
+ const Node* const use = node->in(j);
+ const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use));
+ const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use));
+ if (first != OptoReg::Bad) {
+ new_live.Insert(first);
+ }
- visited.set(n->_idx);
- for (uint i = 0; i < n->req(); i++) {
- if (n->in(i)) {
- if (!visited.test(n->in(i)->_idx)) {
- nodeStack.push(n->in(i));
+ if (second != OptoReg::Bad) {
+ new_live.Insert(second);
}
}
- }
- }
-}
-
-// Add LoadBarriers to all LoadPs
-void ZBarrierSetC2::insert_load_barriers(PhaseIdealLoop* phase) const {
- bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
- GrowableArray<LoadNode *> loadList(Thread::current()->resource_area(), 0, 0, NULL);
- gather_loadnodes_sorted(phase, &loadList);
- PhaseIterGVN &igvn = phase->igvn();
- int count = 0;
-
- for (GrowableArrayIterator<LoadNode *> loadIter = loadList.begin(); loadIter != loadList.end(); ++loadIter) {
- LoadNode *load = *loadIter;
- if (load_has_expanded_barrier(load)) {
- continue;
- }
-
- do {
- // Insert a barrier on a loadP
- // if another load is found that needs to be expanded first, retry on that one
- LoadNode* result = insert_one_loadbarrier(phase, load, phase->get_ctrl(load));
- while (result != NULL) {
- result = insert_one_loadbarrier(phase, result, phase->get_ctrl(result));
+ // If this node tracks liveness, update it
+ RegMask* const regs = barrier_set_state()->live(node);
+ if (regs != NULL) {
+ regs->OR(new_live);
}
- } while (!load_has_expanded_barrier(load));
- }
- phase->C->print_method(PHASE_INSERT_BARRIER, 2);
-}
-
-void push_antidependent_stores(PhaseIdealLoop* phase, Node_Stack& nodestack, LoadNode* start_load) {
- // push all stores on the same mem, that can_alias
- // Any load found must be handled first
- PhaseIterGVN &igvn = phase->igvn();
- int load_alias_idx = igvn.C->get_alias_index(start_load->adr_type());
-
- Node *mem = start_load->in(1);
- for (DUIterator_Fast imax, u = mem->fast_outs(imax); u < imax; u++) {
- Node *mem_use = mem->fast_out(u);
-
- if (mem_use == start_load) continue;
- if (!mem_use->is_Store()) continue;
- if (!phase->has_ctrl(mem_use)) continue;
- if (phase->get_ctrl(mem_use) != phase->get_ctrl(start_load)) continue;
-
- // add any aliasing store in this block
- StoreNode *store = mem_use->isa_Store();
- const TypePtr *adr_type = store->adr_type();
- if (igvn.C->can_alias(adr_type, load_alias_idx)) {
- nodestack.push(store, 0);
- }
- }
-}
-
-LoadNode* ZBarrierSetC2::insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* start_load, Node* ctrl) const {
- bool trace = phase->C->directive()->ZTraceLoadBarriersOption;
- PhaseIterGVN &igvn = phase->igvn();
-
- // Check for other loadPs at the same loop depth that is reachable by a DFS
- // - if found - return it. It needs to be inserted first
- // - otherwise proceed and insert barrier
-
- VectorSet visited(Thread::current()->resource_area());
- Node_Stack nodestack(100);
-
- nodestack.push(start_load, 0);
- push_antidependent_stores(phase, nodestack, start_load);
-
- while(!nodestack.is_empty()) {
- Node* n = nodestack.node(); // peek
- nodestack.pop();
- if (visited.test(n->_idx)) {
- continue;
- }
-
- if (n->is_Load() && n != start_load && load_require_barrier(n->as_Load()) && !load_has_expanded_barrier(n->as_Load())) {
- // Found another load that needs a barrier in the same block. Must expand later loads first.
- if (trace) tty->print_cr(" * Found LoadP %i on DFS", n->_idx);
- return n->as_Load(); // return node that should be expanded first
}
- if (!phase->has_ctrl(n)) continue;
- if (phase->get_ctrl(n) != phase->get_ctrl(start_load)) continue;
- if (n->is_Phi()) continue;
-
- visited.set(n->_idx);
- // push all children
- for (DUIterator_Fast imax, ii = n->fast_outs(imax); ii < imax; ii++) {
- Node* c = n->fast_out(ii);
- if (c != NULL) {
- nodestack.push(c, 0);
- }
- }
- }
- insert_one_loadbarrier_inner(phase, start_load, ctrl, visited);
- return NULL;
-}
-
-void ZBarrierSetC2::insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited2) const {
- PhaseIterGVN &igvn = phase->igvn();
- Compile* C = igvn.C;
- bool trace = C->directive()->ZTraceLoadBarriersOption;
-
- // create barrier
- Node* barrier = new LoadBarrierNode(C, NULL, load->in(LoadNode::Memory), NULL, load->in(LoadNode::Address), load_has_weak_barrier(load));
- Node* barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop);
- Node* barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control);
-
- if (trace) tty->print_cr("Insert load %i with barrier: %i and ctrl : %i", load->_idx, barrier->_idx, ctrl->_idx);
-
- // Splice control
- // - insert barrier control diamond between loads ctrl and ctrl successor on path to block end.
- // - If control successor is a catch, step over to next.
- Node* ctrl_succ = NULL;
- for (DUIterator_Fast imax, j = ctrl->fast_outs(imax); j < imax; j++) {
- Node* tmp = ctrl->fast_out(j);
-
- // - CFG nodes is the ones we are going to splice (1 only!)
- // - Phi nodes will continue to hang from the region node!
- // - self loops should be skipped
- if (tmp->is_Phi() || tmp == ctrl) {
- continue;
- }
-
- if (tmp->is_CFG()) {
- assert(ctrl_succ == NULL, "There can be only one");
- ctrl_succ = tmp;
- continue;
- }
- }
-
- // Now splice control
- assert(ctrl_succ != load, "sanity");
- assert(ctrl_succ != NULL, "Broken IR");
- bool found = false;
- for(uint k = 0; k < ctrl_succ->req(); k++) {
- if (ctrl_succ->in(k) == ctrl) {
- assert(!found, "sanity");
- if (trace) tty->print_cr(" Move CFG ctrl_succ %i to barrier_ctrl", ctrl_succ->_idx);
- igvn.replace_input_of(ctrl_succ, k, barrier_ctrl);
- found = true;
- k--;
- }
- }
-
- // For all successors of ctrl - move all visited to become successors of barrier_ctrl instead
- for (DUIterator_Fast imax, r = ctrl->fast_outs(imax); r < imax; r++) {
- Node* tmp = ctrl->fast_out(r);
- if (visited2.test(tmp->_idx) && (tmp != load)) {
- if (trace) tty->print_cr(" Move ctrl_succ %i to barrier_ctrl", tmp->_idx);
- igvn.replace_input_of(tmp, 0, barrier_ctrl);
- --r; --imax;
- }
- }
-
- // Move the loads user to the barrier
- for (DUIterator_Fast imax, i = load->fast_outs(imax); i < imax; i++) {
- Node* u = load->fast_out(i);
- if (u->isa_LoadBarrier()) {
- continue;
- }
-
- // find correct input - replace with iterator?
- for(uint j = 0; j < u->req(); j++) {
- if (u->in(j) == load) {
- igvn.replace_input_of(u, j, barrier_val);
- --i; --imax; // Adjust the iterator of the *outer* loop
- break; // some nodes (calls) might have several uses from the same node
+ // Now at block top, see if we have any changes
+ new_live.SUBTRACT(old_live);
+ if (new_live.is_NotEmpty()) {
+ // Liveness has refined, update and propagate to prior blocks
+ old_live.OR(new_live);
+ for (uint i = 1; i < block->num_preds(); ++i) {
+ Block* const pred = cfg->get_block_for_node(block->pred(i));
+ worklist.push(pred);
}
}
}
-
- // Connect barrier to load and control
- barrier->set_req(LoadBarrierNode::Oop, load);
- barrier->set_req(LoadBarrierNode::Control, ctrl);
-
- igvn.rehash_node_delayed(load);
- igvn.register_new_node_with_optimizer(barrier);
- igvn.register_new_node_with_optimizer(barrier_val);
- igvn.register_new_node_with_optimizer(barrier_ctrl);
- load_set_expanded_barrier(load);
-
- C->print_method(PHASE_INSERT_BARRIER, 3, load->_idx);
-}
-
-// The bad_mask in the ThreadLocalData shouldn't have an anti-dep-check.
-// The bad_mask address if of type TypeRawPtr, but that will alias
-// InitializeNodes until the type system is expanded.
-bool ZBarrierSetC2::needs_anti_dependence_check(const Node* node) const {
- MachNode* mnode = node->as_Mach();
- if (mnode != NULL) {
- intptr_t offset = 0;
- const TypePtr *adr_type2 = NULL;
- const Node* base = mnode->get_base_and_disp(offset, adr_type2);
- if ((base != NULL) &&
- (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_ThreadLocal) &&
- (offset == in_bytes(ZThreadLocalData::address_bad_mask_offset()))) {
- return false;
- }
- }
- return true;
}
diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp
index bef6a70..accae1f 100644
--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp
+++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp
@@ -29,134 +29,38 @@
#include "opto/node.hpp"
#include "utilities/growableArray.hpp"
-class ZCompareAndSwapPNode : public CompareAndSwapPNode {
-public:
- ZCompareAndSwapPNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { }
- virtual int Opcode() const;
-};
-
-class ZWeakCompareAndSwapPNode : public WeakCompareAndSwapPNode {
-public:
- ZWeakCompareAndSwapPNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : WeakCompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { }
- virtual int Opcode() const;
-};
-
-class ZCompareAndExchangePNode : public CompareAndExchangePNode {
-public:
- ZCompareAndExchangePNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) : CompareAndExchangePNode(c, mem, adr, val, ex, at, t, mem_ord) { }
- virtual int Opcode() const;
-};
-
-class ZGetAndSetPNode : public GetAndSetPNode {
-public:
- ZGetAndSetPNode(Node* c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t) : GetAndSetPNode(c, mem, adr, val, at, t) { }
- virtual int Opcode() const;
-};
-
-class LoadBarrierNode : public MultiNode {
-private:
- bool _weak;
-
- static bool is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n);
- void push_dominated_barriers(PhaseIterGVN* igvn) const;
-
-public:
- enum {
- Control,
- Memory,
- Oop,
- Address,
- Number_of_Outputs = Address,
- Similar,
- Number_of_Inputs
- };
-
- LoadBarrierNode(Compile* C,
- Node* c,
- Node* mem,
- Node* val,
- Node* adr,
- bool weak);
-
- virtual int Opcode() const;
- virtual uint size_of() const;
- virtual uint cmp(const Node& n) const;
- virtual const Type *bottom_type() const;
- virtual const TypePtr* adr_type() const;
- virtual const Type *Value(PhaseGVN *phase) const;
- virtual Node *Identity(PhaseGVN *phase);
- virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
- virtual uint match_edge(uint idx) const;
-
- LoadBarrierNode* has_dominating_barrier(PhaseIdealLoop* phase,
- bool linear_only,
- bool look_for_similar);
-
- void fix_similar_in_uses(PhaseIterGVN* igvn);
+const uint8_t ZLoadBarrierStrong = 1;
+const uint8_t ZLoadBarrierWeak = 2;
+const uint8_t ZLoadBarrierElided = 3;
- bool has_true_uses() const;
-
- bool can_be_eliminated() const {
- return !in(Similar)->is_top();
- }
-
- bool is_weak() const {
- return _weak;
- }
-};
-
-class LoadBarrierSlowRegNode : public LoadPNode {
-private:
- bool _is_weak;
-public:
- LoadBarrierSlowRegNode(Node *c,
- Node *mem,
- Node *adr,
- const TypePtr *at,
- const TypePtr* t,
- MemOrd mo,
- bool weak = false,
- ControlDependency control_dependency = DependsOnlyOnTest)
- : LoadPNode(c, mem, adr, at, t, mo, control_dependency), _is_weak(weak) {
- init_class_id(Class_LoadBarrierSlowReg);
- }
-
- virtual const char * name() {
- return "LoadBarrierSlowRegNode";
- }
-
- virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) {
- return NULL;
- }
-
- virtual int Opcode() const;
- bool is_weak() { return _is_weak; }
-};
-
-class ZBarrierSetC2State : public ResourceObj {
+class ZLoadBarrierStubC2 : public ResourceObj {
private:
- // List of load barrier nodes which need to be expanded before matching
- GrowableArray<LoadBarrierNode*>* _load_barrier_nodes;
-
+ const MachNode* _node;
+ const Address _ref_addr;
+ const Register _ref;
+ const Register _tmp;
+ const bool _weak;
+ Label _entry;
+ Label _continuation;
+
+ ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak);
+
public:
- ZBarrierSetC2State(Arena* comp_arena);
- int load_barrier_count() const;
- void add_load_barrier_node(LoadBarrierNode* n);
- void remove_load_barrier_node(LoadBarrierNode* n);
- LoadBarrierNode* load_barrier_node(int idx) const;
-};
-
-enum BarrierInfo {
- NoBarrier = 0,
- RequireBarrier = 1,
- WeakBarrier = 3, // Inclusive with RequireBarrier
- ExpandedBarrier = 4
+ static ZLoadBarrierStubC2* create(const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak);
+
+ Address ref_addr() const;
+ Register ref() const;
+ Register tmp() const;
+ address slow_path() const;
+ RegMask& live() const;
+ Label* entry();
+ Label* continuation();
};
class ZBarrierSetC2 : public BarrierSetC2 {
private:
- ZBarrierSetC2State* state() const;
- void expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const;
+ void compute_liveness_at_stubs() const;
+ void analyze_dominating_barriers() const;
protected:
virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const;
@@ -174,38 +78,11 @@ protected:
public:
virtual void* create_barrier_state(Arena* comp_arena) const;
- virtual bool has_load_barriers() const { return true; }
- virtual bool is_gc_barrier_node(Node* node) const;
-
- virtual void register_potential_barrier_node(Node* node) const;
- virtual void unregister_potential_barrier_node(Node* node) const;
- virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { }
- virtual void enqueue_useful_gc_barrier(Unique_Node_List &worklist, Node* node) const;
- virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful) const;
-
- virtual bool array_copy_requires_gc_barriers(BasicType type) const { return true; }
- virtual Node* step_over_gc_barrier(Node* c) const;
- // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be
- // expanded later, then now is the time to do so.
- virtual bool expand_macro_nodes(PhaseMacroExpand* macro) const;
- virtual bool needs_anti_dependence_check(const Node* node) const;
-
-#ifdef ASSERT
- virtual void verify_gc_barriers(bool post_parse) const;
-#endif
-
- // Load barrier insertion and expansion external
- virtual void barrier_insertion_phase(Compile* C, PhaseIterGVN &igvn) const;
- virtual void optimize_loops(PhaseIdealLoop* phase, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const;
-
-private:
- // Load barrier insertion and expansion internal
- void insert_barriers_on_unsafe(PhaseIdealLoop* phase) const;
- void clean_catch_blocks(PhaseIdealLoop* phase) const;
- void insert_load_barriers(PhaseIdealLoop* phase) const;
- LoadNode* insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) const;
- void insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited) const;
+ virtual bool array_copy_requires_gc_barriers(BasicType type) const;
+ virtual void late_barrier_analysis() const;
+ virtual int estimate_stub_size() const;
+ virtual void emit_stubs(CodeBuffer& cb) const;
};
#endif // SHARE_GC_Z_C2_ZBARRIERSETC2_HPP
diff --git a/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp b/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp
index 52133c0..9d07f9e 100644
--- a/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp
+++ b/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,10 +24,7 @@
#ifndef SHARE_GC_Z_ZBARRIERSETASSEMBLER_HPP
#define SHARE_GC_Z_ZBARRIERSETASSEMBLER_HPP
-#include "asm/macroAssembler.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
-#include "oops/accessDecorators.hpp"
-#include "utilities/globalDefinitions.hpp"
#include "utilities/macros.hpp"
class ZBarrierSetAssemblerBase : public BarrierSetAssembler {
diff --git a/src/hotspot/share/opto/classes.cpp b/src/hotspot/share/opto/classes.cpp
index 75f070f..d1282ac 100644
--- a/src/hotspot/share/opto/classes.cpp
+++ b/src/hotspot/share/opto/classes.cpp
@@ -48,9 +48,6 @@
#include "opto/subnode.hpp"
#include "opto/vectornode.hpp"
#include "utilities/macros.hpp"
-#if INCLUDE_ZGC
-#include "gc/z/c2/zBarrierSetC2.hpp"
-#endif
#if INCLUDE_SHENANDOAHGC
#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
#endif
diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp
index 69f8a3c..5ee23f7 100644
--- a/src/hotspot/share/opto/classes.hpp
+++ b/src/hotspot/share/opto/classes.hpp
@@ -189,17 +189,6 @@ macro(LoadP)
macro(LoadN)
macro(LoadRange)
macro(LoadS)
-#if INCLUDE_ZGC
-#define zgcmacro(x) macro(x)
-#else
-#define zgcmacro(x) optionalmacro(x)
-#endif
-zgcmacro(LoadBarrier)
-zgcmacro(LoadBarrierSlowReg)
-zgcmacro(ZCompareAndSwapP)
-zgcmacro(ZWeakCompareAndSwapP)
-zgcmacro(ZCompareAndExchangeP)
-zgcmacro(ZGetAndSetP)
macro(Lock)
macro(Loop)
macro(LoopLimit)
diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp
index 45e2d11..e5559a2 100644
--- a/src/hotspot/share/opto/compile.cpp
+++ b/src/hotspot/share/opto/compile.cpp
@@ -79,9 +79,6 @@
#if INCLUDE_G1GC
#include "gc/g1/g1ThreadLocalData.hpp"
#endif // INCLUDE_G1GC
-#if INCLUDE_ZGC
-#include "gc/z/c2/zBarrierSetC2.hpp"
-#endif
#if INCLUDE_SHENANDOAHGC
#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
#endif
@@ -639,16 +636,12 @@ debug_only( int Compile::_debug_idx = 100000; )
Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr_bci,
bool subsume_loads, bool do_escape_analysis, bool eliminate_boxing, DirectiveSet* directive)
: Phase(Compiler),
- _env(ci_env),
- _directive(directive),
- _log(ci_env->log()),
_compile_id(ci_env->compile_id()),
_save_argument_registers(false),
_stub_name(NULL),
_stub_function(NULL),
_stub_entry_point(NULL),
_method(target),
- _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
_entry_bci(osr_bci),
_initial_gvn(NULL),
_for_igvn(NULL),
@@ -668,8 +661,6 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr
_inner_loops(0),
_scratch_const_size(-1),
_in_scratch_emit_size(false),
- _dead_node_list(comp_arena()),
- _dead_node_count(0),
#ifndef PRODUCT
_trace_opto_output(directive->TraceOptoOutputOption),
_in_dump_cnt(0),
@@ -677,9 +668,15 @@ Compile::Compile( ciEnv* ci_env, C2Compiler* compiler, ciMethod* target, int osr
#endif
_congraph(NULL),
_comp_arena(mtCompiler),
+ _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
+ _env(ci_env),
+ _directive(directive),
+ _log(ci_env->log()),
_node_arena(mtCompiler),
_old_arena(mtCompiler),
_Compile_types(mtCompiler),
+ _dead_node_list(comp_arena()),
+ _dead_node_count(0),
_replay_inline_data(NULL),
_late_inlines(comp_arena(), 2, 0, NULL),
_string_late_inlines(comp_arena(), 2, 0, NULL),
@@ -962,9 +959,6 @@ Compile::Compile( ciEnv* ci_env,
bool return_pc,
DirectiveSet* directive)
: Phase(Compiler),
- _env(ci_env),
- _directive(directive),
- _log(ci_env->log()),
_compile_id(0),
_save_argument_registers(save_arg_registers),
_method(NULL),
@@ -994,6 +988,10 @@ Compile::Compile( ciEnv* ci_env,
_printer(NULL),
#endif
_comp_arena(mtCompiler),
+ _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
+ _env(ci_env),
+ _directive(directive),
+ _log(ci_env->log()),
_node_arena(mtCompiler),
_old_arena(mtCompiler),
_Compile_types(mtCompiler),
@@ -2408,13 +2406,6 @@ void Compile::Optimize() {
bs->verify_gc_barriers(false);
#endif
- bs->barrier_insertion_phase(C, igvn);
- if (failing()) return;
-
-#ifdef ASSERT
- bs->verify_gc_barriers(false);
-#endif
-
{
TracePhase tp("macroExpand", &timers[_t_macroExpand]);
PhaseMacroExpand mex(igvn);
@@ -2996,29 +2987,6 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) {
break;
}
-#if INCLUDE_ZGC
- case Op_LoadBarrier: {
- assert(0, "There should be no load barriers left");
- }
- case Op_ZGetAndSetP:
- case Op_ZCompareAndExchangeP:
- case Op_ZCompareAndSwapP:
- case Op_ZWeakCompareAndSwapP:
- case Op_LoadBarrierSlowReg: {
-#ifdef ASSERT
- if (VerifyOptoOopOffsets) {
- MemNode *mem = n->as_Mem();
- // Check to see if address types have grounded out somehow.
- const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr();
- ciInstanceKlass *k = tp->klass()->as_instance_klass();
- bool oop_offset_is_sane = k->contains_field_offset(tp->offset());
- assert(!tp || oop_offset_is_sane, "");
- }
-#endif
- break;
- }
-#endif
-
case Op_AddP: { // Assert sane base pointers
Node *addp = n->in(AddPNode::Address);
assert( !addp->is_AddP() ||
diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp
index 320030e..1246b1b 100644
--- a/src/hotspot/share/opto/compile.hpp
+++ b/src/hotspot/share/opto/compile.hpp
@@ -55,7 +55,6 @@ class ConnectionGraph;
class IdealGraphPrinter;
class InlineTree;
class Int_Array;
-class LoadBarrierNode;
class Matcher;
class MachConstantNode;
class MachConstantBaseNode;
@@ -1185,11 +1184,7 @@ class Compile : public Phase {
bool in_scratch_emit_size() const { return _in_scratch_emit_size; }
enum ScratchBufferBlob {
-#if defined(PPC64)
MAX_inst_size = 2048,
-#else
- MAX_inst_size = 1024,
-#endif
MAX_locs_size = 128, // number of relocInfo elements
MAX_const_size = 128,
MAX_stubs_size = 128
@@ -1264,14 +1259,30 @@ class Compile : public Phase {
// Process an OopMap Element while emitting nodes
void Process_OopMap_Node(MachNode *mach, int code_offset);
+ class BufferSizingData {
+ public:
+ int _stub;
+ int _code;
+ int _const;
+ int _reloc;
+
+ BufferSizingData() :
+ _stub(0),
+ _code(0),
+ _const(0),
+ _reloc(0)
+ { };
+ };
+
// Initialize code buffer
- CodeBuffer* init_buffer(uint* blk_starts);
+ void estimate_buffer_size(int& const_req);
+ CodeBuffer* init_buffer(BufferSizingData& buf_sizes);
// Write out basic block data to code buffer
void fill_buffer(CodeBuffer* cb, uint* blk_starts);
// Determine which variable sized branches can be shortened
- void shorten_branches(uint* blk_starts, int& code_size, int& reloc_size, int& stub_size);
+ void shorten_branches(uint* blk_starts, BufferSizingData& buf_sizes);
// Compute the size of first NumberOfLoopInstrToAlign instructions
// at the head of a loop.
diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp
index cd0ef26..0a22c89 100644
--- a/src/hotspot/share/opto/escape.cpp
+++ b/src/hotspot/share/opto/escape.cpp
@@ -490,13 +490,6 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de
add_local_var_and_edge(n, PointsToNode::NoEscape,
n->in(0), delayed_worklist);
}
-#if INCLUDE_ZGC
- else if (UseZGC) {
- if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) {
- add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), delayed_worklist);
- }
- }
-#endif
break;
}
case Op_Rethrow: // Exception object escapes
@@ -731,14 +724,6 @@ void ConnectionGraph::add_final_edges(Node *n) {
add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), NULL);
break;
}
-#if INCLUDE_ZGC
- else if (UseZGC) {
- if (n->as_Proj()->_con == LoadBarrierNode::Oop && n->in(0)->is_LoadBarrier()) {
- add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), NULL);
- break;
- }
- }
-#endif
ELSE_FAIL("Op_Proj");
}
case Op_Rethrow: // Exception object escapes
diff --git a/src/hotspot/share/opto/lcm.cpp b/src/hotspot/share/opto/lcm.cpp
index 05ec9fa..16b80bf 100644
--- a/src/hotspot/share/opto/lcm.cpp
+++ b/src/hotspot/share/opto/lcm.cpp
@@ -169,7 +169,6 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo
case Op_LoadI:
case Op_LoadL:
case Op_LoadP:
- case Op_LoadBarrierSlowReg:
case Op_LoadN:
case Op_LoadS:
case Op_LoadKlass:
diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp
index 6aff3d5..38f0449 100644
--- a/src/hotspot/share/opto/loopnode.cpp
+++ b/src/hotspot/share/opto/loopnode.cpp
@@ -1005,18 +1005,6 @@ void LoopNode::verify_strip_mined(int expect_skeleton) const {
}
}
- if (UseZGC && !inner_out->in(0)->is_CountedLoopEnd()) {
- // In some very special cases there can be a load that has no other uses than the
- // counted loop safepoint. Then its loadbarrier will be placed between the inner
- // loop exit and the safepoint. This is very rare
-
- Node* ifnode = inner_out->in(1)->in(0);
- // Region->IfTrue->If == Region->Iffalse->If
- if (ifnode == inner_out->in(2)->in(0)) {
- inner_out = ifnode->in(0);
- }
- }
-
CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd();
assert(cle == inner->loopexit_or_null(), "mismatch");
bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0;
@@ -4273,7 +4261,6 @@ void PhaseIdealLoop::build_loop_late_post( Node *n ) {
case Op_LoadL:
case Op_LoadS:
case Op_LoadP:
- case Op_LoadBarrierSlowReg:
case Op_LoadN:
case Op_LoadRange:
case Op_LoadD_unaligned:
diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp
index 5793f51..6b85529 100644
--- a/src/hotspot/share/opto/loopopts.cpp
+++ b/src/hotspot/share/opto/loopopts.cpp
@@ -41,9 +41,6 @@
#include "opto/rootnode.hpp"
#include "opto/subnode.hpp"
#include "utilities/macros.hpp"
-#if INCLUDE_ZGC
-#include "gc/z/c2/zBarrierSetC2.hpp"
-#endif
//=============================================================================
//------------------------------split_thru_phi---------------------------------
diff --git a/src/hotspot/share/opto/machnode.hpp b/src/hotspot/share/opto/machnode.hpp
index 66adf9b..90d5b0a 100644
--- a/src/hotspot/share/opto/machnode.hpp
+++ b/src/hotspot/share/opto/machnode.hpp
@@ -197,7 +197,7 @@ public:
// ADLC inherit from this class.
class MachNode : public Node {
public:
- MachNode() : Node((uint)0), _num_opnds(0), _opnds(NULL) {
+ MachNode() : Node((uint)0), _barrier(0), _num_opnds(0), _opnds(NULL) {
init_class_id(Class_Mach);
}
// Required boilerplate
@@ -211,6 +211,10 @@ public:
// no constant base node input.
virtual uint mach_constant_base_node_input() const { return (uint)-1; }
+ uint8_t barrier_data() const { return _barrier; }
+ void set_barrier_data(uint data) { _barrier = data; }
+
+
// Copy inputs and operands to new node of instruction.
// Called from cisc_version() and short_branch_version().
// !!!! The method's body is defined in ad_<arch>.cpp file.
@@ -255,6 +259,9 @@ public:
// output have choices - but they must use the same choice.
virtual uint two_adr( ) const { return 0; }
+ // The GC might require some barrier metadata for machine code emission.
+ uint8_t _barrier;
+
// Array of complex operand pointers. Each corresponds to zero or
// more leafs. Must be set by MachNode constructor to point to an
// internal array of MachOpers. The MachOper array is sized by
diff --git a/src/hotspot/share/opto/matcher.cpp b/src/hotspot/share/opto/matcher.cpp
index 45d2627..05fdab2 100644
--- a/src/hotspot/share/opto/matcher.cpp
+++ b/src/hotspot/share/opto/matcher.cpp
@@ -1752,6 +1752,13 @@ MachNode *Matcher::ReduceInst( State *s, int rule, Node *&mem ) {
_shared_nodes.map(leaf->_idx, ex);
}
+ // Have mach nodes inherit GC barrier data
+ if (leaf->is_LoadStore()) {
+ mach->set_barrier_data(leaf->as_LoadStore()->barrier_data());
+ } else if (leaf->is_Mem()) {
+ mach->set_barrier_data(leaf->as_Mem()->barrier_data());
+ }
+
return ex;
}
@@ -2171,17 +2178,6 @@ void Matcher::find_shared( Node *n ) {
case Op_SafePoint:
mem_op = true;
break;
-#if INCLUDE_ZGC
- case Op_CallLeaf:
- if (UseZGC) {
- if (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() ||
- n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr()) {
- mem_op = true;
- mem_addr_idx = TypeFunc::Parms+1;
- }
- break;
- }
-#endif
default:
if( n->is_Store() ) {
// Do match stores, despite no ideal reg
@@ -2294,33 +2290,6 @@ void Matcher::find_shared( Node *n ) {
n->del_req(LoadStoreConditionalNode::ExpectedIn);
break;
}
-#if INCLUDE_ZGC
- case Op_ZCompareAndExchangeP:
- case Op_ZCompareAndSwapP:
- case Op_ZWeakCompareAndSwapP: {
- Node *mem = n->in(MemNode::Address);
- Node *keepalive = n->in(5);
- Node *pair1 = new BinaryNode(mem, keepalive);
-
- Node *newval = n->in(MemNode::ValueIn);
- Node *oldval = n->in(LoadStoreConditionalNode::ExpectedIn);
- Node *pair2 = new BinaryNode(oldval, newval);
-
- n->set_req(MemNode::Address, pair1);
- n->set_req(MemNode::ValueIn, pair2);
- n->del_req(5);
- n->del_req(LoadStoreConditionalNode::ExpectedIn);
- break;
- }
- case Op_ZGetAndSetP: {
- Node *keepalive = n->in(4);
- Node *newval = n->in(MemNode::ValueIn);
- Node *pair = new BinaryNode(newval, keepalive);
- n->set_req(MemNode::ValueIn, pair);
- n->del_req(4);
- break;
- }
-#endif // INCLUDE_ZGC
case Op_CMoveD: // Convert trinary to binary-tree
case Op_CMoveF:
case Op_CMoveI:
diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp
index 76e950c..028d1a9 100644
--- a/src/hotspot/share/opto/memnode.cpp
+++ b/src/hotspot/share/opto/memnode.cpp
@@ -49,9 +49,6 @@
#include "utilities/copy.hpp"
#include "utilities/macros.hpp"
#include "utilities/vmError.hpp"
-#if INCLUDE_ZGC
-#include "gc/z/c2/zBarrierSetC2.hpp"
-#endif
#if INCLUDE_SHENANDOAHGC
#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
#endif
@@ -2826,7 +2823,7 @@ LoadStoreNode::LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const Ty
: Node(required),
_type(rt),
_adr_type(at),
- _has_barrier(false)
+ _barrier(0)
{
init_req(MemNode::Control, c );
init_req(MemNode::Memory , mem);
diff --git a/src/hotspot/share/opto/memnode.hpp b/src/hotspot/share/opto/memnode.hpp
index e2bd6a6..3bf3dc4 100644
--- a/src/hotspot/share/opto/memnode.hpp
+++ b/src/hotspot/share/opto/memnode.hpp
@@ -43,6 +43,8 @@ private:
bool _unaligned_access; // Unaligned access from unsafe
bool _mismatched_access; // Mismatched access from unsafe: byte read in integer array for instance
bool _unsafe_access; // Access of unsafe origin.
+ uint8_t _barrier; // Bit field with barrier information
+
protected:
#ifdef ASSERT
const TypePtr* _adr_type; // What kind of memory is being addressed?
@@ -62,18 +64,30 @@ public:
unset // The memory ordering is not set (used for testing)
} MemOrd;
protected:
- MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at )
- : Node(c0,c1,c2 ), _unaligned_access(false), _mismatched_access(false), _unsafe_access(false) {
+ MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at ) :
+ Node(c0,c1,c2),
+ _unaligned_access(false),
+ _mismatched_access(false),
+ _unsafe_access(false),
+ _barrier(0) {
init_class_id(Class_Mem);
debug_only(_adr_type=at; adr_type();)
}
- MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3 )
- : Node(c0,c1,c2,c3), _unaligned_access(false), _mismatched_access(false), _unsafe_access(false) {
+ MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3 ) :
+ Node(c0,c1,c2,c3),
+ _unaligned_access(false),
+ _mismatched_access(false),
+ _unsafe_access(false),
+ _barrier(0) {
init_class_id(Class_Mem);
debug_only(_adr_type=at; adr_type();)
}
- MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3, Node *c4)
- : Node(c0,c1,c2,c3,c4), _unaligned_access(false), _mismatched_access(false), _unsafe_access(false) {
+ MemNode( Node *c0, Node *c1, Node *c2, const TypePtr* at, Node *c3, Node *c4) :
+ Node(c0,c1,c2,c3,c4),
+ _unaligned_access(false),
+ _mismatched_access(false),
+ _unsafe_access(false),
+ _barrier(0) {
init_class_id(Class_Mem);
debug_only(_adr_type=at; adr_type();)
}
@@ -125,6 +139,9 @@ public:
#endif
}
+ uint8_t barrier_data() { return _barrier; }
+ void set_barrier_data(uint8_t barrier_data) { _barrier = barrier_data; }
+
// Search through memory states which precede this node (load or store).
// Look for an exact match for the address, with no intervening
// aliased stores.
@@ -182,8 +199,6 @@ private:
// this field.
const MemOrd _mo;
- uint _barrier; // Bit field with barrier information
-
protected:
virtual uint cmp(const Node &n) const;
virtual uint size_of() const; // Size is bigger
@@ -195,7 +210,7 @@ protected:
public:
LoadNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *rt, MemOrd mo, ControlDependency control_dependency)
- : MemNode(c,mem,adr,at), _type(rt), _mo(mo), _control_dependency(control_dependency), _barrier(0) {
+ : MemNode(c,mem,adr,at), _control_dependency(control_dependency), _mo(mo), _type(rt) {
init_class_id(Class_Load);
}
inline bool is_unordered() const { return !is_acquire(); }
@@ -264,10 +279,6 @@ public:
Node* convert_to_unsigned_load(PhaseGVN& gvn);
Node* convert_to_signed_load(PhaseGVN& gvn);
- void copy_barrier_info(const Node* src) { _barrier = src->as_Load()->_barrier; }
- uint barrier_data() { return _barrier; }
- void set_barrier_data(uint barrier_data) { _barrier |= barrier_data; }
-
#ifndef PRODUCT
virtual void dump_spec(outputStream *st) const;
#endif
@@ -816,7 +827,7 @@ class LoadStoreNode : public Node {
private:
const Type* const _type; // What kind of value is loaded?
const TypePtr* _adr_type; // What kind of memory is being addressed?
- bool _has_barrier;
+ uint8_t _barrier; // Bit field with barrier information
virtual uint size_of() const; // Size is bigger
public:
LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required );
@@ -829,8 +840,9 @@ public:
bool result_not_used() const;
MemBarNode* trailing_membar() const;
- void set_has_barrier() { _has_barrier = true; };
- bool has_barrier() const { return _has_barrier; };
+
+ uint8_t barrier_data() { return _barrier; }
+ void set_barrier_data(uint8_t barrier_data) { _barrier = barrier_data; }
};
class LoadStoreConditionalNode : public LoadStoreNode {
@@ -882,6 +894,7 @@ public:
MemNode::MemOrd order() const {
return _mem_ord;
}
+ virtual uint size_of() const { return sizeof(*this); }
};
class CompareAndExchangeNode : public LoadStoreNode {
@@ -899,6 +912,7 @@ public:
MemNode::MemOrd order() const {
return _mem_ord;
}
+ virtual uint size_of() const { return sizeof(*this); }
};
//------------------------------CompareAndSwapBNode---------------------------
diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp
index bb020c4..5a5e44e 100644
--- a/src/hotspot/share/opto/node.cpp
+++ b/src/hotspot/share/opto/node.cpp
@@ -546,9 +546,6 @@ Node *Node::clone() const {
if (n->is_SafePoint()) {
n->as_SafePoint()->clone_replaced_nodes();
}
- if (n->is_Load()) {
- n->as_Load()->copy_barrier_info(this);
- }
return n; // Return the clone
}
@@ -1471,10 +1468,6 @@ bool Node::needs_anti_dependence_check() const {
if( req() < 2 || (_flags & Flag_needs_anti_dependence_check) == 0 ) {
return false;
}
- BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
- if (!bs->needs_anti_dependence_check(this)) {
- return false;
- }
return in(1)->bottom_type()->has_memory();
}
diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp
index a4c5fd8..2a42e99 100644
--- a/src/hotspot/share/opto/node.hpp
+++ b/src/hotspot/share/opto/node.hpp
@@ -82,8 +82,6 @@ class JVMState;
class JumpNode;
class JumpProjNode;
class LoadNode;
-class LoadBarrierNode;
-class LoadBarrierSlowRegNode;
class LoadStoreNode;
class LoadStoreConditionalNode;
class LockNode;
@@ -644,7 +642,6 @@ public:
DEFINE_CLASS_ID(MemBar, Multi, 3)
DEFINE_CLASS_ID(Initialize, MemBar, 0)
DEFINE_CLASS_ID(MemBarStoreStore, MemBar, 1)
- DEFINE_CLASS_ID(LoadBarrier, Multi, 4)
DEFINE_CLASS_ID(Mach, Node, 1)
DEFINE_CLASS_ID(MachReturn, Mach, 0)
@@ -696,7 +693,6 @@ public:
DEFINE_CLASS_ID(Mem, Node, 4)
DEFINE_CLASS_ID(Load, Mem, 0)
DEFINE_CLASS_ID(LoadVector, Load, 0)
- DEFINE_CLASS_ID(LoadBarrierSlowReg, Load, 1)
DEFINE_CLASS_ID(Store, Mem, 1)
DEFINE_CLASS_ID(StoreVector, Store, 0)
DEFINE_CLASS_ID(LoadStore, Mem, 2)
@@ -839,8 +835,6 @@ public:
DEFINE_CLASS_QUERY(Load)
DEFINE_CLASS_QUERY(LoadStore)
DEFINE_CLASS_QUERY(LoadStoreConditional)
- DEFINE_CLASS_QUERY(LoadBarrier)
- DEFINE_CLASS_QUERY(LoadBarrierSlowReg)
DEFINE_CLASS_QUERY(Lock)
DEFINE_CLASS_QUERY(Loop)
DEFINE_CLASS_QUERY(Mach)
diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp
index 3c02a6f..e3933f4 100644
--- a/src/hotspot/share/opto/output.cpp
+++ b/src/hotspot/share/opto/output.cpp
@@ -32,6 +32,8 @@
#include "compiler/compilerDirectives.hpp"
#include "compiler/disassembler.hpp"
#include "compiler/oopMap.hpp"
+#include "gc/shared/barrierSet.hpp"
+#include "gc/shared/c2/barrierSetC2.hpp"
#include "memory/allocation.inline.hpp"
#include "opto/ad.hpp"
#include "opto/callnode.hpp"
@@ -115,35 +117,35 @@ void Compile::Output() {
}
}
+ // Keeper of sizing aspects
+ BufferSizingData buf_sizes = BufferSizingData();
+
+ // Initialize code buffer
+ estimate_buffer_size(buf_sizes._const);
+ if (failing()) return;
+
+ // Pre-compute the length of blocks and replace
+ // long branches with short if machine supports it.
+ // Must be done before ScheduleAndBundle due to SPARC delay slots
+
uint* blk_starts = NEW_RESOURCE_ARRAY(uint, _cfg->number_of_blocks() + 1);
blk_starts[0] = 0;
- // Initialize code buffer and process short branches.
- CodeBuffer* cb = init_buffer(blk_starts);
+ shorten_branches(blk_starts, buf_sizes);
- if (cb == NULL || failing()) {
+ ScheduleAndBundle();
+ if (failing()) {
return;
}
- ScheduleAndBundle();
-
-#ifndef PRODUCT
- if (trace_opto_output()) {
- tty->print("\n---- After ScheduleAndBundle ----\n");
- for (uint i = 0; i < _cfg->number_of_blocks(); i++) {
- tty->print("\nBB#%03d:\n", i);
- Block* block = _cfg->get_block(i);
- for (uint j = 0; j < block->number_of_nodes(); j++) {
- Node* n = block->get_node(j);
- OptoReg::Name reg = _regalloc->get_reg_first(n);
- tty->print(" %-6s ", reg >= 0 && reg < REG_COUNT ? Matcher::regName[reg] : "");
- n->dump();
- }
- }
- }
-#endif
+ // Late barrier analysis must be done after schedule and bundle
+ // Otherwise liveness based spilling will fail
+ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
+ bs->late_barrier_analysis();
- if (failing()) {
+ // Complete sizing of codebuffer
+ CodeBuffer* cb = init_buffer(buf_sizes);
+ if (cb == NULL || failing()) {
return;
}
@@ -224,7 +226,7 @@ void Compile::compute_loop_first_inst_sizes() {
// The architecture description provides short branch variants for some long
// branch instructions. Replace eligible long branches with short branches.
-void Compile::shorten_branches(uint* blk_starts, int& code_size, int& reloc_size, int& stub_size) {
+void Compile::shorten_branches(uint* blk_starts, BufferSizingData& buf_sizes) {
// Compute size of each block, method size, and relocation information size
uint nblocks = _cfg->number_of_blocks();
@@ -242,11 +244,11 @@ void Compile::shorten_branches(uint* blk_starts, int& code_size, int& reloc_size
bool has_short_branch_candidate = false;
// Initialize the sizes to 0
- code_size = 0; // Size in bytes of generated code
- stub_size = 0; // Size in bytes of all stub entries
+ int code_size = 0; // Size in bytes of generated code
+ int stub_size = 0; // Size in bytes of all stub entries
// Size in bytes of all relocation entries, including those in local stubs.
// Start with 2-bytes of reloc info for the unvalidated entry point
- reloc_size = 1; // Number of relocation entries
+ int reloc_size = 1; // Number of relocation entries
// Make three passes. The first computes pessimistic blk_starts,
// relative jmp_offset and reloc_size information. The second performs
@@ -480,6 +482,10 @@ void Compile::shorten_branches(uint* blk_starts, int& code_size, int& reloc_size
// a relocation index.
// The CodeBuffer will expand the locs array if this estimate is too low.
reloc_size *= 10 / sizeof(relocInfo);
+
+ buf_sizes._reloc = reloc_size;
+ buf_sizes._code = code_size;
+ buf_sizes._stub = stub_size;
}
//------------------------------FillLocArray-----------------------------------
@@ -491,8 +497,8 @@ static LocationValue *new_loc_value( PhaseRegAlloc *ra, OptoReg::Name regnum, Lo
// This should never have accepted Bad before
assert(OptoReg::is_valid(regnum), "location must be valid");
return (OptoReg::is_reg(regnum))
- ? new LocationValue(Location::new_reg_loc(l_type, OptoReg::as_VMReg(regnum)) )
- : new LocationValue(Location::new_stk_loc(l_type, ra->reg2offset(regnum)));
+ ? new LocationValue(Location::new_reg_loc(l_type, OptoReg::as_VMReg(regnum)) )
+ : new LocationValue(Location::new_stk_loc(l_type, ra->reg2offset(regnum)));
}
@@ -611,12 +617,12 @@ void Compile::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local,
}
#endif //_LP64
else if( (t->base() == Type::FloatBot || t->base() == Type::FloatCon) &&
- OptoReg::is_reg(regnum) ) {
+ OptoReg::is_reg(regnum) ) {
array->append(new_loc_value( _regalloc, regnum, Matcher::float_in_double()
- ? Location::float_in_dbl : Location::normal ));
+ ? Location::float_in_dbl : Location::normal ));
} else if( t->base() == Type::Int && OptoReg::is_reg(regnum) ) {
array->append(new_loc_value( _regalloc, regnum, Matcher::int_in_long
- ? Location::int_in_long : Location::normal ));
+ ? Location::int_in_long : Location::normal ));
} else if( t->base() == Type::NarrowOop ) {
array->append(new_loc_value( _regalloc, regnum, Location::narrowoop ));
} else {
@@ -627,48 +633,48 @@ void Compile::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local,
// No register. It must be constant data.
switch (t->base()) {
- case Type::Half: // Second half of a double
- ShouldNotReachHere(); // Caller should skip 2nd halves
- break;
- case Type::AnyPtr:
- array->append(new ConstantOopWriteValue(NULL));
- break;
- case Type::AryPtr:
- case Type::InstPtr: // fall through
- array->append(new ConstantOopWriteValue(t->isa_oopptr()->const_oop()->constant_encoding()));
- break;
- case Type::NarrowOop:
- if (t == TypeNarrowOop::NULL_PTR) {
+ case Type::Half: // Second half of a double
+ ShouldNotReachHere(); // Caller should skip 2nd halves
+ break;
+ case Type::AnyPtr:
array->append(new ConstantOopWriteValue(NULL));
- } else {
- array->append(new ConstantOopWriteValue(t->make_ptr()->isa_oopptr()->const_oop()->constant_encoding()));
- }
- break;
- case Type::Int:
- array->append(new ConstantIntValue(t->is_int()->get_con()));
- break;
- case Type::RawPtr:
- // A return address (T_ADDRESS).
- assert((intptr_t)t->is_ptr()->get_con() < (intptr_t)0x10000, "must be a valid BCI");
+ break;
+ case Type::AryPtr:
+ case Type::InstPtr: // fall through
+ array->append(new ConstantOopWriteValue(t->isa_oopptr()->const_oop()->constant_encoding()));
+ break;
+ case Type::NarrowOop:
+ if (t == TypeNarrowOop::NULL_PTR) {
+ array->append(new ConstantOopWriteValue(NULL));
+ } else {
+ array->append(new ConstantOopWriteValue(t->make_ptr()->isa_oopptr()->const_oop()->constant_encoding()));
+ }
+ break;
+ case Type::Int:
+ array->append(new ConstantIntValue(t->is_int()->get_con()));
+ break;
+ case Type::RawPtr:
+ // A return address (T_ADDRESS).
+ assert((intptr_t)t->is_ptr()->get_con() < (intptr_t)0x10000, "must be a valid BCI");
#ifdef _LP64
- // Must be restored to the full-width 64-bit stack slot.
- array->append(new ConstantLongValue(t->is_ptr()->get_con()));
+ // Must be restored to the full-width 64-bit stack slot.
+ array->append(new ConstantLongValue(t->is_ptr()->get_con()));
#else
- array->append(new ConstantIntValue(t->is_ptr()->get_con()));
+ array->append(new ConstantIntValue(t->is_ptr()->get_con()));
#endif
- break;
- case Type::FloatCon: {
- float f = t->is_float_constant()->getf();
- array->append(new ConstantIntValue(jint_cast(f)));
- break;
- }
- case Type::DoubleCon: {
- jdouble d = t->is_double_constant()->getd();
+ break;
+ case Type::FloatCon: {
+ float f = t->is_float_constant()->getf();
+ array->append(new ConstantIntValue(jint_cast(f)));
+ break;
+ }
+ case Type::DoubleCon: {
+ jdouble d = t->is_double_constant()->getd();
#ifdef _LP64
- array->append(new ConstantIntValue((jint)0));
- array->append(new ConstantDoubleValue(d));
+ array->append(new ConstantIntValue((jint)0));
+ array->append(new ConstantDoubleValue(d));
#else
- // Repack the double as two jints.
+ // Repack the double as two jints.
// The convention the interpreter uses is that the second local
// holds the first raw word of the native double representation.
// This is actually reasonable, since locals and stack arrays
@@ -680,15 +686,15 @@ void Compile::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local,
array->append(new ConstantIntValue(acc.words[1]));
array->append(new ConstantIntValue(acc.words[0]));
#endif
- break;
- }
- case Type::Long: {
- jlong d = t->is_long()->get_con();
+ break;
+ }
+ case Type::Long: {
+ jlong d = t->is_long()->get_con();
#ifdef _LP64
- array->append(new ConstantIntValue((jint)0));
- array->append(new ConstantLongValue(d));
+ array->append(new ConstantIntValue((jint)0));
+ array->append(new ConstantLongValue(d));
#else
- // Repack the long as two jints.
+ // Repack the long as two jints.
// The convention the interpreter uses is that the second local
// holds the first raw word of the native double representation.
// This is actually reasonable, since locals and stack arrays
@@ -700,14 +706,14 @@ void Compile::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local,
array->append(new ConstantIntValue(acc.words[1]));
array->append(new ConstantIntValue(acc.words[0]));
#endif
- break;
- }
- case Type::Top: // Add an illegal value here
- array->append(new LocationValue(Location()));
- break;
- default:
- ShouldNotReachHere();
- break;
+ break;
+ }
+ case Type::Top: // Add an illegal value here
+ array->append(new LocationValue(Location()));
+ break;
+ default:
+ ShouldNotReachHere();
+ break;
}
}
@@ -872,58 +878,58 @@ void Compile::Process_OopMap_Node(MachNode *mach, int current_offset) {
// A simplified version of Process_OopMap_Node, to handle non-safepoints.
class NonSafepointEmitter {
- Compile* C;
- JVMState* _pending_jvms;
- int _pending_offset;
+ Compile* C;
+ JVMState* _pending_jvms;
+ int _pending_offset;
- void emit_non_safepoint();
+ void emit_non_safepoint();
public:
- NonSafepointEmitter(Compile* compile) {
- this->C = compile;
- _pending_jvms = NULL;
- _pending_offset = 0;
- }
+ NonSafepointEmitter(Compile* compile) {
+ this->C = compile;
+ _pending_jvms = NULL;
+ _pending_offset = 0;
+ }
- void observe_instruction(Node* n, int pc_offset) {
- if (!C->debug_info()->recording_non_safepoints()) return;
+ void observe_instruction(Node* n, int pc_offset) {
+ if (!C->debug_info()->recording_non_safepoints()) return;
- Node_Notes* nn = C->node_notes_at(n->_idx);
- if (nn == NULL || nn->jvms() == NULL) return;
- if (_pending_jvms != NULL &&
- _pending_jvms->same_calls_as(nn->jvms())) {
- // Repeated JVMS? Stretch it up here.
- _pending_offset = pc_offset;
- } else {
+ Node_Notes* nn = C->node_notes_at(n->_idx);
+ if (nn == NULL || nn->jvms() == NULL) return;
if (_pending_jvms != NULL &&
- _pending_offset < pc_offset) {
- emit_non_safepoint();
- }
- _pending_jvms = NULL;
- if (pc_offset > C->debug_info()->last_pc_offset()) {
- // This is the only way _pending_jvms can become non-NULL:
- _pending_jvms = nn->jvms();
+ _pending_jvms->same_calls_as(nn->jvms())) {
+ // Repeated JVMS? Stretch it up here.
_pending_offset = pc_offset;
+ } else {
+ if (_pending_jvms != NULL &&
+ _pending_offset < pc_offset) {
+ emit_non_safepoint();
+ }
+ _pending_jvms = NULL;
+ if (pc_offset > C->debug_info()->last_pc_offset()) {
+ // This is the only way _pending_jvms can become non-NULL:
+ _pending_jvms = nn->jvms();
+ _pending_offset = pc_offset;
+ }
}
}
- }
// Stay out of the way of real safepoints:
- void observe_safepoint(JVMState* jvms, int pc_offset) {
- if (_pending_jvms != NULL &&
- !_pending_jvms->same_calls_as(jvms) &&
- _pending_offset < pc_offset) {
- emit_non_safepoint();
+ void observe_safepoint(JVMState* jvms, int pc_offset) {
+ if (_pending_jvms != NULL &&
+ !_pending_jvms->same_calls_as(jvms) &&
+ _pending_offset < pc_offset) {
+ emit_non_safepoint();
+ }
+ _pending_jvms = NULL;
}
- _pending_jvms = NULL;
- }
- void flush_at_end() {
- if (_pending_jvms != NULL) {
- emit_non_safepoint();
+ void flush_at_end() {
+ if (_pending_jvms != NULL) {
+ emit_non_safepoint();
+ }
+ _pending_jvms = NULL;
}
- _pending_jvms = NULL;
- }
};
void NonSafepointEmitter::emit_non_safepoint() {
@@ -953,15 +959,11 @@ void NonSafepointEmitter::emit_non_safepoint() {
}
//------------------------------init_buffer------------------------------------
-CodeBuffer* Compile::init_buffer(uint* blk_starts) {
+void Compile::estimate_buffer_size(int& const_req) {
// Set the initially allocated size
- int code_req = initial_code_capacity;
- int locs_req = initial_locs_capacity;
- int stub_req = initial_stub_capacity;
- int const_req = initial_const_capacity;
+ const_req = initial_const_capacity;
- int pad_req = NativeCall::instruction_size;
// The extra spacing after the code is necessary on some platforms.
// Sometimes we need to patch in a jump after the last instruction,
// if the nmethod has been deoptimized. (See 4932387, 4894843.)
@@ -973,7 +975,7 @@ CodeBuffer* Compile::init_buffer(uint* blk_starts) {
// Compute prolog code size
_method_size = 0;
- _frame_slots = OptoReg::reg2stack(_matcher->_old_SP)+_regalloc->_framesize;
+ _frame_slots = OptoReg::reg2stack(_matcher->_old_SP) + _regalloc->_framesize;
#if defined(IA64) && !defined(AIX)
if (save_argument_registers()) {
// 4815101: this is a stub with implicit and unknown precision fp args.
@@ -1022,12 +1024,18 @@ CodeBuffer* Compile::init_buffer(uint* blk_starts) {
// Initialize the space for the BufferBlob used to find and verify
// instruction size in MachNode::emit_size()
init_scratch_buffer_blob(const_req);
- if (failing()) return NULL; // Out of memory
+}
- // Pre-compute the length of blocks and replace
- // long branches with short if machine supports it.
- shorten_branches(blk_starts, code_req, locs_req, stub_req);
+CodeBuffer* Compile::init_buffer(BufferSizingData& buf_sizes) {
+ int stub_req = buf_sizes._stub;
+ int code_req = buf_sizes._code;
+ int const_req = buf_sizes._const;
+
+ int pad_req = NativeCall::instruction_size;
+
+ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
+ stub_req += bs->estimate_stub_size();
// nmethod and CodeBuffer count stubs & constants as part of method's code.
// class HandlerImpl is platform-specific and defined in the *.ad files.
int exception_handler_req = HandlerImpl::size_exception_handler() + MAX_stubs_size; // add marginal slop for handler
@@ -1039,18 +1047,18 @@ CodeBuffer* Compile::init_buffer(uint* blk_starts) {
code_req = const_req = stub_req = exception_handler_req = deopt_handler_req = 0x10; // force expansion
int total_req =
- const_req +
- code_req +
- pad_req +
- stub_req +
- exception_handler_req +
- deopt_handler_req; // deopt handler
+ const_req +
+ code_req +
+ pad_req +
+ stub_req +
+ exception_handler_req +
+ deopt_handler_req; // deopt handler
if (has_method_handle_invokes())
total_req += deopt_handler_req; // deopt MH handler
CodeBuffer* cb = code_buffer();
- cb->initialize(total_req, locs_req);
+ cb->initialize(total_req, buf_sizes._reloc);
// Have we run out of code space?
if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) {
@@ -1264,12 +1272,12 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) {
Process_OopMap_Node(mach, current_offset);
} // End if safepoint
- // If this is a null check, then add the start of the previous instruction to the list
+ // If this is a null check, then add the start of the previous instruction to the list
else if( mach->is_MachNullCheck() ) {
inct_starts[inct_cnt++] = previous_offset;
}
- // If this is a branch, then fill in the label with the target BB's label
+ // If this is a branch, then fill in the label with the target BB's label
else if (mach->is_MachBranch()) {
// This requires the TRUE branch target be in succs[0]
uint block_num = block->non_connector_successor(0)->_pre_order;
@@ -1280,8 +1288,8 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) {
bool delay_slot_is_used = valid_bundle_info(n) &&
node_bundling(n)->use_unconditional_delay();
if (!delay_slot_is_used && mach->may_be_short_branch()) {
- assert(delay_slot == NULL, "not expecting delay slot node");
- int br_size = n->size(_regalloc);
+ assert(delay_slot == NULL, "not expecting delay slot node");
+ int br_size = n->size(_regalloc);
int offset = blk_starts[block_num] - current_offset;
if (block_num >= i) {
// Current and following block's offset are not
@@ -1339,7 +1347,7 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) {
}
}
#ifdef ASSERT
- // Check that oop-store precedes the card-mark
+ // Check that oop-store precedes the card-mark
else if (mach->ideal_Opcode() == Op_StoreCM) {
uint storeCM_idx = j;
int count = 0;
@@ -1517,6 +1525,10 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) {
}
#endif
+ BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
+ bs->emit_stubs(*cb);
+ if (failing()) return;
+
#ifndef PRODUCT
// Information on the size of the method, without the extraneous code
Scheduling::increment_method_size(cb->insts_size());
@@ -1681,20 +1693,20 @@ uint Scheduling::_total_instructions_per_bundle[Pipeline::_max_instrs_per_cycle+
// Initializer for class Scheduling
Scheduling::Scheduling(Arena *arena, Compile &compile)
- : _arena(arena),
- _cfg(compile.cfg()),
- _regalloc(compile.regalloc()),
- _reg_node(arena),
- _bundle_instr_count(0),
- _bundle_cycle_number(0),
- _scheduled(arena),
- _available(arena),
- _next_node(NULL),
- _bundle_use(0, 0, resource_count, &_bundle_use_elements[0]),
- _pinch_free_list(arena)
+ : _arena(arena),
+ _cfg(compile.cfg()),
+ _regalloc(compile.regalloc()),
+ _reg_node(arena),
+ _bundle_instr_count(0),
+ _bundle_cycle_number(0),
+ _scheduled(arena),
+ _available(arena),
+ _next_node(NULL),
+ _bundle_use(0, 0, resource_count, &_bundle_use_elements[0]),
+ _pinch_free_list(arena)
#ifndef PRODUCT
- , _branches(0)
- , _unconditional_delays(0)
+ , _branches(0)
+ , _unconditional_delays(0)
#endif
{
// Create a MachNopNode
@@ -1773,8 +1785,8 @@ void Scheduling::step_and_clear() {
_bundle_use.reset();
memcpy(_bundle_use_elements,
- Pipeline_Use::elaborated_elements,
- sizeof(Pipeline_Use::elaborated_elements));
+ Pipeline_Use::elaborated_elements,
+ sizeof(Pipeline_Use::elaborated_elements));
}
// Perform instruction scheduling and bundling over the sequence of
@@ -1801,6 +1813,22 @@ void Compile::ScheduleAndBundle() {
// Walk backwards over each basic block, computing the needed alignment
// Walk over all the basic blocks
scheduling.DoScheduling();
+
+#ifndef PRODUCT
+ if (trace_opto_output()) {
+ tty->print("\n---- After ScheduleAndBundle ----\n");
+ for (uint i = 0; i < _cfg->number_of_blocks(); i++) {
+ tty->print("\nBB#%03d:\n", i);
+ Block* block = _cfg->get_block(i);
+ for (uint j = 0; j < block->number_of_nodes(); j++) {
+ Node* n = block->get_node(j);
+ OptoReg::Name reg = _regalloc->get_reg_first(n);
+ tty->print(" %-6s ", reg >= 0 && reg < REG_COUNT ? Matcher::regName[reg] : "");
+ n->dump();
+ }
+ }
+ }
+#endif
}
// Compute the latency of all the instructions. This is fairly simple,
@@ -1869,7 +1897,7 @@ bool Scheduling::NodeFitsInBundle(Node *n) {
#ifndef PRODUCT
if (_cfg->C->trace_opto_output())
tty->print("# NodeFitsInBundle [%4d]: FALSE; latency %4d > %d\n",
- n->_idx, _current_latency[n_idx], _bundle_cycle_number);
+ n->_idx, _current_latency[n_idx], _bundle_cycle_number);
#endif
return (false);
}
@@ -2094,12 +2122,12 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) {
// Don't allow safepoints in the branch shadow, that will
// cause a number of difficulties
if ( avail_pipeline->instructionCount() == 1 &&
- !avail_pipeline->hasMultipleBundles() &&
- !avail_pipeline->hasBranchDelay() &&
- Pipeline::instr_has_unit_size() &&
- d->size(_regalloc) == Pipeline::instr_unit_size() &&
- NodeFitsInBundle(d) &&
- !node_bundling(d)->used_in_delay()) {
+ !avail_pipeline->hasMultipleBundles() &&
+ !avail_pipeline->hasBranchDelay() &&
+ Pipeline::instr_has_unit_size() &&
+ d->size(_regalloc) == Pipeline::instr_unit_size() &&
+ NodeFitsInBundle(d) &&
+ !node_bundling(d)->used_in_delay()) {
if (d->is_Mach() && !d->is_MachSafePoint()) {
// A node that fits in the delay slot was found, so we need to
@@ -2144,13 +2172,13 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) {
// step of the bundles
if (!NodeFitsInBundle(n)) {
#ifndef PRODUCT
- if (_cfg->C->trace_opto_output())
- tty->print("# *** STEP(branch won't fit) ***\n");
+ if (_cfg->C->trace_opto_output())
+ tty->print("# *** STEP(branch won't fit) ***\n");
#endif
// Update the state information
- _bundle_instr_count = 0;
- _bundle_cycle_number += 1;
- _bundle_use.step(1);
+ _bundle_instr_count = 0;
+ _bundle_cycle_number += 1;
+ _bundle_use.step(1);
}
}
@@ -2196,8 +2224,8 @@ void Scheduling::AddNodeToBundle(Node *n, const Block *bb) {
#ifndef PRODUCT
if (_cfg->C->trace_opto_output())
tty->print("# *** STEP(%d >= %d instructions) ***\n",
- instruction_count + _bundle_instr_count,
- Pipeline::_max_instrs_per_cycle);
+ instruction_count + _bundle_instr_count,
+ Pipeline::_max_instrs_per_cycle);
#endif
step(1);
}
@@ -2403,7 +2431,7 @@ void Scheduling::DoScheduling() {
}
assert(!last->is_Mach() || last->as_Mach()->ideal_Opcode() != Op_Con, "");
if( last->is_Catch() ||
- (last->is_Mach() && last->as_Mach()->ideal_Opcode() == Op_Halt) ) {
+ (last->is_Mach() && last->as_Mach()->ideal_Opcode() == Op_Halt) ) {
// There might be a prior call. Skip it.
while (_bb_start < _bb_end && bb->get_node(--_bb_end)->is_MachProj());
} else if( last->is_MachNullCheck() ) {
@@ -2473,7 +2501,7 @@ void Scheduling::DoScheduling() {
}
#endif
#ifdef ASSERT
- verify_good_schedule(bb,"after block local scheduling");
+ verify_good_schedule(bb,"after block local scheduling");
#endif
}
@@ -2821,31 +2849,31 @@ void Scheduling::ComputeRegisterAntidependencies(Block *b) {
//
void Scheduling::garbage_collect_pinch_nodes() {
#ifndef PRODUCT
- if (_cfg->C->trace_opto_output()) tty->print("Reclaimed pinch nodes:");
-#endif
- int trace_cnt = 0;
- for (uint k = 0; k < _reg_node.Size(); k++) {
- Node* pinch = _reg_node[k];
- if ((pinch != NULL) && pinch->Opcode() == Op_Node &&
- // no predecence input edges
- (pinch->req() == pinch->len() || pinch->in(pinch->req()) == NULL) ) {
- cleanup_pinch(pinch);
- _pinch_free_list.push(pinch);
- _reg_node.map(k, NULL);
+ if (_cfg->C->trace_opto_output()) tty->print("Reclaimed pinch nodes:");
+#endif
+ int trace_cnt = 0;
+ for (uint k = 0; k < _reg_node.Size(); k++) {
+ Node* pinch = _reg_node[k];
+ if ((pinch != NULL) && pinch->Opcode() == Op_Node &&
+ // no predecence input edges
+ (pinch->req() == pinch->len() || pinch->in(pinch->req()) == NULL) ) {
+ cleanup_pinch(pinch);
+ _pinch_free_list.push(pinch);
+ _reg_node.map(k, NULL);
#ifndef PRODUCT
- if (_cfg->C->trace_opto_output()) {
- trace_cnt++;
- if (trace_cnt > 40) {
- tty->print("\n");
- trace_cnt = 0;
- }
- tty->print(" %d", pinch->_idx);
+ if (_cfg->C->trace_opto_output()) {
+ trace_cnt++;
+ if (trace_cnt > 40) {
+ tty->print("\n");
+ trace_cnt = 0;
}
-#endif
+ tty->print(" %d", pinch->_idx);
}
+#endif
}
+ }
#ifndef PRODUCT
- if (_cfg->C->trace_opto_output()) tty->print("\n");
+ if (_cfg->C->trace_opto_output()) tty->print("\n");
#endif
}
@@ -2882,19 +2910,19 @@ void Scheduling::dump_available() const {
void Scheduling::print_statistics() {
// Print the size added by nops for bundling
tty->print("Nops added %d bytes to total of %d bytes",
- _total_nop_size, _total_method_size);
+ _total_nop_size, _total_method_size);
if (_total_method_size > 0)
tty->print(", for %.2f%%",
- ((double)_total_nop_size) / ((double) _total_method_size) * 100.0);
+ ((double)_total_nop_size) / ((double) _total_method_size) * 100.0);
tty->print("\n");
// Print the number of branch shadows filled
if (Pipeline::_branch_has_delay_slot) {
tty->print("Of %d branches, %d had unconditional delay slots filled",
- _total_branches, _total_unconditional_delays);
+ _total_branches, _total_unconditional_delays);
if (_total_branches > 0)
tty->print(", for %.2f%%",
- ((double)_total_unconditional_delays) / ((double)_total_branches) * 100.0);
+ ((double)_total_unconditional_delays) / ((double)_total_branches) * 100.0);
tty->print("\n");
}
@@ -2908,6 +2936,6 @@ void Scheduling::print_statistics() {
if (total_bundles > 0)
tty->print("Average ILP (excluding nops) is %.2f\n",
- ((double)total_instructions) / ((double)total_bundles));
+ ((double)total_instructions) / ((double)total_bundles));
}
#endif
diff --git a/src/hotspot/share/opto/output.hpp b/src/hotspot/share/opto/output.hpp
index ab3c1a3..ec3cc29 100644
--- a/src/hotspot/share/opto/output.hpp
+++ b/src/hotspot/share/opto/output.hpp
@@ -49,10 +49,7 @@ class Pipeline_Use;
// Define the initial sizes for allocation of the resizable code buffer
enum {
- initial_code_capacity = 16 * 1024,
- initial_stub_capacity = 4 * 1024,
- initial_const_capacity = 4 * 1024,
- initial_locs_capacity = 3 * 1024
+ initial_const_capacity = 4 * 1024
};
//------------------------------Scheduling----------------------------------
diff --git a/src/hotspot/share/opto/phaseX.cpp b/src/hotspot/share/opto/phaseX.cpp
index 07b8490..9d5d4de 100644
--- a/src/hotspot/share/opto/phaseX.cpp
+++ b/src/hotspot/share/opto/phaseX.cpp
@@ -1648,14 +1648,14 @@ void PhaseIterGVN::add_users_to_worklist( Node *n ) {
// of the mirror load depends on the type of 'n'. See LoadNode::Value().
// LoadBarrier?(LoadP(LoadP(AddP(foo:Klass, #java_mirror))))
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
- bool has_load_barriers = bs->has_load_barriers();
+ bool has_load_barrier_nodes = bs->has_load_barrier_nodes();
if (use_op == Op_LoadP && use->bottom_type()->isa_rawptr()) {
for (DUIterator_Fast i2max, i2 = use->fast_outs(i2max); i2 < i2max; i2++) {
Node* u = use->fast_out(i2);
const Type* ut = u->bottom_type();
if (u->Opcode() == Op_LoadP && ut->isa_instptr()) {
- if (has_load_barriers) {
+ if (has_load_barrier_nodes) {
// Search for load barriers behind the load
for (DUIterator_Fast i3max, i3 = u->fast_outs(i3max); i3 < i3max; i3++) {
Node* b = u->fast_out(i3);
@@ -1818,14 +1818,14 @@ void PhaseCCP::analyze() {
// Loading the java mirror from a Klass requires two loads and the type
// of the mirror load depends on the type of 'n'. See LoadNode::Value().
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
- bool has_load_barriers = bs->has_load_barriers();
+ bool has_load_barrier_nodes = bs->has_load_barrier_nodes();
if (m_op == Op_LoadP && m->bottom_type()->isa_rawptr()) {
for (DUIterator_Fast i2max, i2 = m->fast_outs(i2max); i2 < i2max; i2++) {
Node* u = m->fast_out(i2);
const Type* ut = u->bottom_type();
if (u->Opcode() == Op_LoadP && ut->isa_instptr() && ut != type(u)) {
- if (has_load_barriers) {
+ if (has_load_barrier_nodes) {
// Search for load barriers behind the load
for (DUIterator_Fast i3max, i3 = u->fast_outs(i3max); i3 < i3max; i3++) {
Node* b = u->fast_out(i3);
diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp
index 0258db3..fae147f 100644
--- a/src/hotspot/share/opto/vectornode.cpp
+++ b/src/hotspot/share/opto/vectornode.cpp
@@ -264,7 +264,6 @@ void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
case Op_LoadI: case Op_LoadL:
case Op_LoadF: case Op_LoadD:
case Op_LoadP: case Op_LoadN:
- case Op_LoadBarrierSlowReg:
*start = 0;
*end = 0; // no vector operands
break;
此处可能存在不合适展示的内容，页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容，可点击提交进行申诉，我们将尽快为您处理。