6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.inline.hpp"
26 #include "gc/g1/g1BarrierSet.hpp"
27 #include "gc/g1/g1BarrierSetAssembler.hpp"
28 #include "gc/g1/g1BarrierSetRuntime.hpp"
29 #include "gc/g1/g1CardTable.hpp"
30 #include "gc/g1/g1HeapRegion.hpp"
31 #include "gc/g1/g1ThreadLocalData.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "utilities/debug.hpp"
35 #include "utilities/macros.hpp"
36 #ifdef COMPILER1
37 #include "c1/c1_LIRAssembler.hpp"
38 #include "c1/c1_MacroAssembler.hpp"
39 #include "gc/g1/c1/g1BarrierSetC1.hpp"
40 #endif // COMPILER1
41 #ifdef COMPILER2
42 #include "gc/g1/c2/g1BarrierSetC2.hpp"
43 #endif // COMPILER2
44
45 #define __ masm->
218 // expand_call should be passed true.
219
220 if (expand_call) {
221 assert(pre_val != c_rarg1, "smashed arg");
222 if (c_rarg1 != thread) {
223 __ mov(c_rarg1, thread);
224 }
225 if (c_rarg0 != pre_val) {
226 __ mov(c_rarg0, pre_val);
227 }
228 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
229 } else {
230 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
231 }
232
233 __ pop_call_clobbered_registers();
234
235 __ bind(done);
236 }
237
238 static void generate_post_barrier_fast_path(MacroAssembler* masm,
239 const Register store_addr,
240 const Register new_val,
241 const Register tmp,
242 const Register tmp2,
243 Label& done,
244 bool new_val_may_be_null) {
245 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
246 // Does store cross heap regions?
247 __ movptr(tmp, store_addr); // tmp := store address
248 __ xorptr(tmp, new_val); // tmp := store address ^ new value
249 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
250 __ jcc(Assembler::equal, done);
251 // Crosses regions, storing null?
252 if (new_val_may_be_null) {
253 __ cmpptr(new_val, NULL_WORD); // new value == null?
254 __ jcc(Assembler::equal, done);
255 }
256 // Storing region crossing non-null, is card young?
257 __ movptr(tmp, store_addr); // tmp := store address
258 __ shrptr(tmp, CardTable::card_shift()); // tmp := card address relative to card table base
259 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
260 // a valid address and therefore is not properly handled by the relocation code.
261 __ movptr(tmp2, (intptr_t)ct->card_table()->byte_map_base()); // tmp2 := card table base address
262 __ addptr(tmp, tmp2); // tmp := card address
263 __ cmpb(Address(tmp, 0), G1CardTable::g1_young_card_val()); // *(card address) == young_card_val?
264 }
265
266 static void generate_post_barrier_slow_path(MacroAssembler* masm,
267 const Register thread,
268 const Register tmp,
269 const Register tmp2,
270 Label& done,
271 Label& runtime) {
272 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); // StoreLoad membar
273 __ cmpb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) == dirty_card_val?
274 __ jcc(Assembler::equal, done);
275 // Storing a region crossing, non-null oop, card is clean.
276 // Dirty card and log.
277 __ movb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val
278 generate_queue_insertion(masm,
279 G1ThreadLocalData::dirty_card_queue_index_offset(),
280 G1ThreadLocalData::dirty_card_queue_buffer_offset(),
281 runtime,
566 Label runtime;
567
568 // At this point we know new_value is non-null and the new_value crosses regions.
569 // Must check to see if card is already dirty
570
571 const Register thread = r15_thread;
572
573 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
574 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
575
576 __ push(rax);
577 __ push(rcx);
578
579 const Register cardtable = rax;
580 const Register card_addr = rcx;
581
582 __ load_parameter(0, card_addr);
583 __ shrptr(card_addr, CardTable::card_shift());
584 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
585 // a valid address and therefore is not properly handled by the relocation code.
586 __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
587 __ addptr(card_addr, cardtable);
588
589 __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
590 __ jcc(Assembler::equal, done);
591
592 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
593 __ cmpb(Address(card_addr, 0), CardTable::dirty_card_val());
594 __ jcc(Assembler::equal, done);
595
596 // storing region crossing non-null, card is clean.
597 // dirty card and log.
598
599 __ movb(Address(card_addr, 0), CardTable::dirty_card_val());
600
601 const Register tmp = rdx;
602 __ push(rdx);
603
604 __ movptr(tmp, queue_index);
605 __ testptr(tmp, tmp);
606 __ jcc(Assembler::zero, runtime);
|
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.inline.hpp"
26 #include "code/aotCodeCache.hpp"
27 #include "gc/g1/g1BarrierSet.hpp"
28 #include "gc/g1/g1BarrierSetAssembler.hpp"
29 #include "gc/g1/g1BarrierSetRuntime.hpp"
30 #include "gc/g1/g1CardTable.hpp"
31 #include "gc/g1/g1HeapRegion.hpp"
32 #include "gc/g1/g1ThreadLocalData.hpp"
33 #include "interpreter/interp_masm.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "utilities/debug.hpp"
36 #include "utilities/macros.hpp"
37 #ifdef COMPILER1
38 #include "c1/c1_LIRAssembler.hpp"
39 #include "c1/c1_MacroAssembler.hpp"
40 #include "gc/g1/c1/g1BarrierSetC1.hpp"
41 #endif // COMPILER1
42 #ifdef COMPILER2
43 #include "gc/g1/c2/g1BarrierSetC2.hpp"
44 #endif // COMPILER2
45
46 #define __ masm->
219 // expand_call should be passed true.
220
221 if (expand_call) {
222 assert(pre_val != c_rarg1, "smashed arg");
223 if (c_rarg1 != thread) {
224 __ mov(c_rarg1, thread);
225 }
226 if (c_rarg0 != pre_val) {
227 __ mov(c_rarg0, pre_val);
228 }
229 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), 2);
230 } else {
231 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread);
232 }
233
234 __ pop_call_clobbered_registers();
235
236 __ bind(done);
237 }
238
239 // return a register that differs from reg1, reg2, reg3 and is not rcx
240
241 static Register pick_different_reg(Register reg1, Register reg2 = noreg, Register reg3= noreg, Register reg4 = noreg) {
242 RegSet available = (RegSet::of(rscratch1, rscratch2, rax, rbx) + rdx -
243 RegSet::of(reg1, reg2, reg3, reg4));
244 return *(available.begin());
245 }
246
247 static void generate_post_barrier_fast_path(MacroAssembler* masm,
248 const Register store_addr,
249 const Register new_val,
250 const Register tmp,
251 const Register tmp2,
252 Label& done,
253 bool new_val_may_be_null) {
254 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
255 // Does store cross heap regions?
256 #if INCLUDE_CDS
257 // AOT code needs to load the barrier grain shift from the aot
258 // runtime constants area in the code cache otherwise we can compile
259 // it as an immediate operand
260
261 if (AOTCodeCache::is_on_for_dump()) {
262 address grain_shift_addr = AOTRuntimeConstants::grain_shift_address();
263 Register save = pick_different_reg(rcx, tmp, new_val, store_addr);
264 __ push(save);
265 __ movptr(save, store_addr);
266 __ xorptr(save, new_val);
267 __ push(rcx);
268 __ lea(rcx, ExternalAddress(grain_shift_addr));
269 __ movptr(rcx, Address(rcx, 0));
270 __ shrptr(save);
271 __ pop(rcx);
272 __ mov(tmp, save);
273 __ pop(save);
274 __ jcc(Assembler::equal, done);
275 } else
276 #endif // INCLUDE_CDS
277 {
278 __ movptr(tmp, store_addr); // tmp := store address
279 __ xorptr(tmp, new_val); // tmp := store address ^ new value
280 __ shrptr(tmp, G1HeapRegion::LogOfHRGrainBytes); // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
281 __ jcc(Assembler::equal, done);
282 }
283
284 // Crosses regions, storing null?
285 if (new_val_may_be_null) {
286 __ cmpptr(new_val, NULL_WORD); // new value == null?
287 __ jcc(Assembler::equal, done);
288 }
289 // Storing region crossing non-null, is card young?
290 __ movptr(tmp, store_addr); // tmp := store address
291 #if INCLUDE_CDS
292 // AOT code needs to load the barrier card shift from the aot
293 // runtime constants area in the code cache otherwise we can compile
294 // it as an immediate operand
295 if (AOTCodeCache::is_on_for_dump()) {
296 address card_shift_addr = AOTRuntimeConstants::card_shift_address();
297 Register save = pick_different_reg(rcx, tmp);
298 __ push(save);
299 __ mov(save, tmp);
300 __ push(rcx);
301 __ lea(rcx, ExternalAddress(card_shift_addr));
302 __ movptr(rcx, Address(rcx, 0));
303 __ shrptr(save);
304 __ pop(rcx);
305 __ mov(tmp, save);
306 __ pop(save);
307 } else
308 #endif // INCLUDE_CDS
309 {
310 __ shrptr(tmp, CardTable::card_shift()); // tmp := card address relative to card table base
311 }
312 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
313 // a valid address and therefore is not properly handled by the relocation code.
314 if (AOTCodeCache::is_on_for_dump()) {
315 // AOT code needs relocation info for this address
316 __ lea(tmp2, ExternalAddress((address)ct->card_table()->byte_map_base())); // tmp2 := card table base address
317 } else {
318 __ movptr(tmp2, (intptr_t)ct->card_table()->byte_map_base()); // tmp2 := card table base address
319 }
320 __ addptr(tmp, tmp2); // tmp := card address
321 __ cmpb(Address(tmp, 0), G1CardTable::g1_young_card_val()); // *(card address) == young_card_val?
322 }
323
324 static void generate_post_barrier_slow_path(MacroAssembler* masm,
325 const Register thread,
326 const Register tmp,
327 const Register tmp2,
328 Label& done,
329 Label& runtime) {
330 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); // StoreLoad membar
331 __ cmpb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) == dirty_card_val?
332 __ jcc(Assembler::equal, done);
333 // Storing a region crossing, non-null oop, card is clean.
334 // Dirty card and log.
335 __ movb(Address(tmp, 0), G1CardTable::dirty_card_val()); // *(card address) := dirty_card_val
336 generate_queue_insertion(masm,
337 G1ThreadLocalData::dirty_card_queue_index_offset(),
338 G1ThreadLocalData::dirty_card_queue_buffer_offset(),
339 runtime,
624 Label runtime;
625
626 // At this point we know new_value is non-null and the new_value crosses regions.
627 // Must check to see if card is already dirty
628
629 const Register thread = r15_thread;
630
631 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
632 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
633
634 __ push(rax);
635 __ push(rcx);
636
637 const Register cardtable = rax;
638 const Register card_addr = rcx;
639
640 __ load_parameter(0, card_addr);
641 __ shrptr(card_addr, CardTable::card_shift());
642 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
643 // a valid address and therefore is not properly handled by the relocation code.
644 if (AOTCodeCache::is_on()) {
645 // AOT code needs relocation info for this address
646 __ lea(cardtable, ExternalAddress((address)ct->card_table()->byte_map_base()));
647 } else {
648 __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
649 }
650 __ addptr(card_addr, cardtable);
651
652 __ cmpb(Address(card_addr, 0), G1CardTable::g1_young_card_val());
653 __ jcc(Assembler::equal, done);
654
655 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
656 __ cmpb(Address(card_addr, 0), CardTable::dirty_card_val());
657 __ jcc(Assembler::equal, done);
658
659 // storing region crossing non-null, card is clean.
660 // dirty card and log.
661
662 __ movb(Address(card_addr, 0), CardTable::dirty_card_val());
663
664 const Register tmp = rdx;
665 __ push(rdx);
666
667 __ movptr(tmp, queue_index);
668 __ testptr(tmp, tmp);
669 __ jcc(Assembler::zero, runtime);
|