csu3_am335x/EVSE/GPL/php-8.1.12/ext/opcache/jit/zend_jit_arm64.dasc

/*
 *  +----------------------------------------------------------------------+
 *  | Zend JIT                                                             |
 *  +----------------------------------------------------------------------+
 *  | Copyright (c) The PHP Group                                          |
 *  +----------------------------------------------------------------------+
 *  | This source file is subject to version 3.01 of the PHP license,      |
 *  | that is bundled with this package in the file LICENSE, and is        |
 *  | available through the world-wide-web at the following url:           |
 *  | https://www.php.net/license/3_01.txt                                 |
 *  | If you did not receive a copy of the PHP license and are unable to   |
 *  | obtain it through the world-wide-web, please send a note to          |
 *  | license@php.net so we can mail you a copy immediately.               |
 *  +----------------------------------------------------------------------+
 *  | Authors: Dmitry Stogov <dmitry@php.net>                              |
 *  |          Xinchen Hui <laruence@php.net>                              |
 *  |          Hao Sun <hao.sun@arm.com>                                   |
 *  +----------------------------------------------------------------------+
 */

|.arch arm64

|.define FP,      x27
|.define IP,      x28
|.define IPl,     w28
|.define RX,      x28         // the same as VM IP reused as a general purpose reg
|.define LR,      x30
|.define CARG1,   x0
|.define CARG2,   x1
|.define CARG3,   x2
|.define CARG4,   x3
|.define CARG5,   x4
|.define CARG6,   x5
|.define CARG1w,  w0
|.define CARG2w,  w1
|.define CARG3w,  w2
|.define CARG4w,  w3
|.define CARG5w,  w4
|.define CARG6w,  w5
|.define RETVALx, x0
|.define RETVALw, w0
|.define FCARG1x, x0
|.define FCARG1w, w0
|.define FCARG2x, x1
|.define FCARG2w, w1
|.define SPAD,    0x20        // padding for CPU stack alignment
|.define NR_SPAD, 0x30        // padding for CPU stack alignment
|.define T3,      [sp, #0x28] // Used to store old value of IP (CALL VM only)
|.define T2,      [sp, #0x20] // Used to store old value of FP (CALL VM only)
|.define T1,      [sp, #0x10]

// We use REG0/1/2 and FPR0/1 to replace r0/1/2 and xmm0/1 in the x86 implementation.
// Scratch registers
|.define REG0,    x8
|.define REG0w,   w8
|.define REG1,    x9
|.define REG1w,   w9
|.define REG2,    x10
|.define REG2w,   w10
|.define FPR0,    d0
|.define FPR1,    d1

|.define ZREG_REG0,   ZREG_X8
|.define ZREG_REG1,   ZREG_X9
|.define ZREG_REG2,   ZREG_X10
|.define ZREG_FPR0,   ZREG_V0
|.define ZREG_FPR1,   ZREG_V1

// Temporaries, not preserved across calls
|.define TMP1,    x15
|.define TMP1w,   w15
|.define TMP2,    x16
|.define TMP2w,   w16
|.define TMP3,    x17 // TODO: remember about hard-coded: mrs TMP3, tpidr_el0
|.define TMP3w,   w17
|.define FPTMP,   d16

|.define ZREG_TMP1,   ZREG_X15
|.define ZREG_TMP2,   ZREG_X16
|.define ZREG_TMP3,   ZREG_X17
|.define ZREG_FPTMP,  ZREG_V16

|.define HYBRID_SPAD, 32     // padding for stack alignment

#define TMP_ZVAL_OFFSET 16
#define DASM_ALIGNMENT  16

const char* zend_reg_name[] = {
	"x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
	"x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
	"x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
	"x24", "x25", "x26", "x27", "x28", "x29", "x30", "sp",
	"d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
	"d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15",
	"d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23",
	"d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31"
};

#define ZREG_FCARG1 ZREG_X0
#define ZREG_FCARG2 ZREG_X1

|.type EX, zend_execute_data, FP
|.type OP, zend_op
|.type ZVAL, zval
|.actionlist dasm_actions
|.globals zend_lb
|.section code, cold_code, jmp_table

static void* dasm_labels[zend_lb_MAX];

#if ZTS
static size_t tsrm_ls_cache_tcb_offset = 0;
# ifdef __APPLE__
struct TLVDescriptor {
	void*       (*thunk)(struct TLVDescriptor*);
	uint64_t    key;
	uint64_t    offset;
};
typedef struct TLVDescriptor TLVDescriptor;
# endif
#endif

#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1)))

/* Encoding of immediate. */
#define MAX_IMM12       0xfff          // maximum value for imm12
#define MAX_IMM16       0xffff         // maximum value for imm16
#define MOVZ_IMM        MAX_IMM16      // movz insn
#define LDR_STR_PIMM64  (MAX_IMM12*8)  // ldr/str insn for 64-bit register: pimm is imm12 * 8
#define LDR_STR_PIMM32  (MAX_IMM12*4)  // ldr/str insn for 32-bit register: pimm is imm12 * 4
#define LDRB_STRB_PIMM  MAX_IMM12      // ldrb/strb insn

#define B_IMM           (1<<27)        // signed imm26 * 4
#define ADR_IMM         (1<<20)        // signed imm21
#define ADRP_IMM        (1LL<<32)      // signed imm21 * 4096

static bool arm64_may_use_b(const void *addr)
{
	if (addr >= dasm_buf && addr < dasm_end) {
		return (((char*)dasm_end - (char*)dasm_buf) < B_IMM);
	} else if (addr >= dasm_end) {
		return (((char*)addr - (char*)dasm_buf) < B_IMM);
	} else if (addr < dasm_buf) {
		return (((char*)dasm_end - (char*)addr) < B_IMM);
	}
	return 0;
}

static bool arm64_may_use_adr(const void *addr)
{
	if (addr >= dasm_buf && addr < dasm_end) {
		return (((char*)dasm_end - (char*)dasm_buf) < ADR_IMM);
	} else if (addr >= dasm_end) {
		return (((char*)addr - (char*)dasm_buf) < ADR_IMM);
	} else if (addr < dasm_buf) {
		return (((char*)dasm_end - (char*)addr) < ADR_IMM);
	}
	return 0;
}

static bool arm64_may_use_adrp(const void *addr)
{
	if (addr >= dasm_buf && addr < dasm_end) {
		return (((char*)dasm_end - (char*)dasm_buf) < ADRP_IMM);
	} else if (addr >= dasm_end) {
		return (((char*)addr - (char*)dasm_buf) < ADRP_IMM);
	} else if (addr < dasm_buf) {
		return (((char*)dasm_end - (char*)addr) < ADRP_IMM);
	}
	return 0;
}

/* Determine whether "val" falls into two allowed ranges:
 *   Range 1: [0, 0xfff]
 *   Range 2: LSL #12 to Range 1
 * Used to guard the immediate encoding for add/adds/sub/subs/cmp/cmn instructions. */
static bool arm64_may_encode_imm12(const int64_t val)
{
	return (val >= 0 && (val <= MAX_IMM12 || !(val & 0xffffffffff000fff)));
}

/* Determine whether an immediate value can be encoded as the immediate operand of logical instructions. */
static bool logical_immediate_p(uint64_t value, uint32_t reg_size)
{
	/* fast path: power of two */
	if (value > 0 && !(value & (value - 1))) {
		return true;
	}

	if (reg_size == 32) {
		if (dasm_imm13((uint32_t)value, (uint32_t)value) != -1) {
			return true;
		}
	} else if (reg_size == 64) {
		if (dasm_imm13((uint32_t)value, (uint32_t)(value >> 32)) != -1) {
			return true;
		}
	} else {
		ZEND_UNREACHABLE();
	}

	return false;
}

/* Not Implemented Yet */
|.macro NIY
||	//ZEND_ASSERT(0);
|	brk #0
|.endmacro

|.macro NIY_STUB
||	//ZEND_ASSERT(0);
|	brk #0
|.endmacro

|.macro ADD_HYBRID_SPAD
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
|	add sp, sp, # HYBRID_SPAD
||#endif
|.endmacro

|.macro SUB_HYBRID_SPAD
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
|	sub sp, sp, # HYBRID_SPAD
||#endif
|.endmacro

/* Move address into register. TODO: Support 52-bit address */
|.macro LOAD_ADDR, reg, addr
|	// 48-bit virtual address
||	if (((uintptr_t)(addr)) == 0) {
|		mov reg, xzr
||	} else if (((uintptr_t)(addr)) <= MOVZ_IMM) {
|		movz reg, #((uint64_t)(addr))
||	} else if (arm64_may_use_adr((void*)(addr))) {
|		adr reg, &addr
||	} else if (arm64_may_use_adrp((void*)(addr))) {
|		adrp reg, &(((uintptr_t)(addr)))
||		if (((uintptr_t)(addr)) & 0xfff) {
|			add reg, reg, #(((uintptr_t)(addr)) & 0xfff)
||		}
||	} else if ((uintptr_t)(addr) & 0xffff) {
|		movz reg, #((uintptr_t)(addr) & 0xffff)
||		if (((uintptr_t)(addr) >> 16) & 0xffff) {
|			movk reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16
||		}
||		if (((uintptr_t)(addr) >> 32) & 0xffff) {
|			movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32
||		}
||	} else if (((uintptr_t)(addr) >> 16) & 0xffff) {
|		movz reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16
||		if (((uintptr_t)(addr) >> 32) & 0xffff) {
|			movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32
||		}
||	} else {
|		movz reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32
||	}
|.endmacro

/* Move 32-bit immediate value into register. */
|.macro LOAD_32BIT_VAL, reg, val
||	if (((uint32_t)(val)) <= MOVZ_IMM) {
|		movz reg, #((uint32_t)(val))
||	} else if (((uint32_t)(val) & 0xffff)) {
|		movz reg, #((uint32_t)(val) & 0xffff)
||		if ((((uint32_t)(val) >> 16) & 0xffff)) {
|			movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16
||		}
||	} else {
|		movz reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16
||	}
|.endmacro

/* Move 64-bit immediate value into register. */
|.macro LOAD_64BIT_VAL, reg, val
||	if (((uint64_t)(val)) == 0) {
|		mov reg, xzr
||	} else if (((uint64_t)(val)) <= MOVZ_IMM) {
|		movz reg, #((uint64_t)(val))
||	} else if (~((uint64_t)(val)) <= MOVZ_IMM) {
|		movn reg, #(~((uint64_t)(val)))
||	} else if ((uint64_t)(val) & 0xffff) {
|		movz reg, #((uint64_t)(val) & 0xffff)
||		if (((uint64_t)(val) >> 16) & 0xffff) {
|			movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16
||		}
||		if (((uint64_t)(val) >> 32) & 0xffff) {
|			movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32
||		}
||		if ((((uint64_t)(val) >> 48) & 0xffff)) {
|			movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48
||		}
||	} else if (((uint64_t)(val) >> 16) & 0xffff) {
|		movz reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16
||		if (((uint64_t)(val) >> 32) & 0xffff) {
|			movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32
||		}
||		if ((((uint64_t)(val) >> 48) & 0xffff)) {
|			movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48
||		}
||	} else if (((uint64_t)(val) >> 32) & 0xffff) {
|		movz reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32
||		if ((((uint64_t)(val) >> 48) & 0xffff)) {
|			movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48
||		}
||	} else {
|		movz reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48
||	}
|.endmacro

/* Extract the low 8 bits from 'src_reg' into 'dst_reg'.
 * Note: 0xff can be encoded as imm for 'and' instruction. */
|.macro GET_LOW_8BITS, dst_reg, src_reg
|	and dst_reg, src_reg, #0xff
|.endmacro

/* Bitwise operation with immediate. 'bw_ins' can be and/orr/eor/ands.
 * 32-bit and 64-bit registers are distinguished. */
|.macro BW_OP_32_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg
||	if (val == 0) {
|		bw_ins dst_reg, src_reg1, wzr
||	} else if (logical_immediate_p((uint32_t)val, 32)) {
|		bw_ins dst_reg, src_reg1, #val
||	} else {
|		LOAD_32BIT_VAL tmp_reg, val
|		bw_ins dst_reg, src_reg1, tmp_reg
||	}
|.endmacro

|.macro BW_OP_64_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg
||	if (val == 0) {
|		bw_ins dst_reg, src_reg1, xzr
||	} else if (logical_immediate_p(val, 64)) {
|		bw_ins dst_reg, src_reg1, #val
||	} else {
|		LOAD_64BIT_VAL tmp_reg, val
|		bw_ins dst_reg, src_reg1, tmp_reg
||	}
|.endmacro

/* Test bits 'tst' with immediate. 32-bit and 64-bit registers are distinguished. */
|.macro TST_32_WITH_CONST, reg, val, tmp_reg
||	if (val == 0) {
|		tst reg, wzr
||	} else if (logical_immediate_p((uint32_t)val, 32)) {
|		tst reg, #val
||	} else {
|		LOAD_32BIT_VAL tmp_reg, val
|		tst reg, tmp_reg
||	}
|.endmacro

|.macro TST_64_WITH_CONST, reg, val, tmp_reg
||	if (val == 0) {
|		tst reg, xzr
||	} else if (logical_immediate_p(val, 64)) {
|		tst reg, #val
||	} else {
|		LOAD_64BIT_VAL tmp_reg, val
|		tst reg, tmp_reg
||	}
|.endmacro

/* Test bits between 64-bit register with constant 1. */
|.macro TST_64_WITH_ONE, reg
|	tst reg, #1
|.endmacro

/* Compare a register value with immediate. 32-bit and 64-bit registers are distinguished.
 * Note: Comparing 64-bit register with 32-bit immediate is handled in CMP_64_WITH_CONST_32. */
|.macro CMP_32_WITH_CONST, reg, val, tmp_reg
||	if (val == 0) {
|		cmp reg, wzr
||	} else if (arm64_may_encode_imm12((int64_t)(val))) {
|		cmp reg, #val
||	} else if (arm64_may_encode_imm12((int64_t)(-val))) {
|		cmn reg, #-val
||	} else {
|		LOAD_32BIT_VAL tmp_reg, val
|		cmp reg, tmp_reg
||	}
|.endmacro

|.macro CMP_64_WITH_CONST_32, reg, val, tmp_reg
||	if (val == 0) {
|		cmp reg, xzr
||	} else if (arm64_may_encode_imm12((int64_t)(val))) {
|		cmp reg, #val
||	} else if (arm64_may_encode_imm12((int64_t)(-val))) {
|		cmn reg, #-val
||	} else {
|		LOAD_32BIT_VAL tmp_reg, val
|		cmp reg, tmp_reg
||	}
|.endmacro

|.macro CMP_64_WITH_CONST, reg, val, tmp_reg
||	if (val == 0) {
|		cmp reg, xzr
||	} else if (arm64_may_encode_imm12((int64_t)(val))) {
|		cmp reg, #val
||	} else if (arm64_may_encode_imm12((int64_t)(-val))) {
|		cmn reg, #-val
||	} else {
|		LOAD_64BIT_VAL tmp_reg, val
|		cmp reg, tmp_reg
||	}
|.endmacro

/* Add/sub a register value with immediate. 'add_sub_ins' can be add/sub/adds/subs.
 * 32-bit and 64-bit registers are distinguished.
 * Note: Case of 64-bit register with 32-bit immediate is handled in ADD_SUB_64_WITH_CONST_32. */
|.macro ADD_SUB_32_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg
||	if (val == 0) {
|		add_sub_ins dst_reg, src_reg1, wzr
||	} else if (arm64_may_encode_imm12((int64_t)(val))) {
|		add_sub_ins dst_reg, src_reg1, #val
||	} else {
|		LOAD_32BIT_VAL tmp_reg, val
|		add_sub_ins dst_reg, src_reg1, tmp_reg
||	}
|.endmacro

|.macro ADD_SUB_64_WITH_CONST_32, add_sub_ins, dst_reg, src_reg1, val, tmp_reg
||	if (val == 0) {
|		add_sub_ins dst_reg, src_reg1, xzr
||	} else if (arm64_may_encode_imm12((int64_t)(val))) {
|		add_sub_ins dst_reg, src_reg1, #val
||	} else {
|		LOAD_32BIT_VAL tmp_reg, val
|		add_sub_ins dst_reg, src_reg1, tmp_reg
||	}
|.endmacro

|.macro ADD_SUB_64_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg
||	if (val == 0) {
|		add_sub_ins dst_reg, src_reg1, xzr
||	} else if (arm64_may_encode_imm12((int64_t)(val))) {
|		add_sub_ins dst_reg, src_reg1, #val
||	} else {
|		LOAD_64BIT_VAL tmp_reg, val
|		add_sub_ins dst_reg, src_reg1, tmp_reg
||	}
|.endmacro

/* Memory access(load/store) with 32-bit 'offset'.
 * Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register.
 * 8-bit, 32-bit and 64-bit registers to be transferred are distinguished.
 * Note: 'reg' can be used as 'tmp_reg' if 1) 'reg' is one GPR, AND 2) 'reg' != 'base_reg', AND 3) ins is 'ldr'. */
|.macro MEM_ACCESS_64_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg
||	if (((uintptr_t)(offset)) > LDR_STR_PIMM64) {
|		LOAD_32BIT_VAL tmp_reg, offset
|		ldr_str_ins reg, [base_reg, tmp_reg]
||	} else {
|		ldr_str_ins reg, [base_reg, #(offset)]
||	}
|.endmacro

|.macro MEM_ACCESS_32_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg
||	if (((uintptr_t)(offset)) > LDR_STR_PIMM32) {
|		LOAD_32BIT_VAL tmp_reg, offset
|		ldr_str_ins reg, [base_reg, tmp_reg]
||	} else {
|		ldr_str_ins reg, [base_reg, #(offset)]
||	}
|.endmacro

|.macro MEM_ACCESS_8_WITH_UOFFSET, ldrb_strb_ins, reg, base_reg, offset, tmp_reg
||	if (((uintptr_t)(offset)) > LDRB_STRB_PIMM) {
|		LOAD_32BIT_VAL tmp_reg, offset
|		ldrb_strb_ins reg, [base_reg, tmp_reg]
||	} else {
|		ldrb_strb_ins reg, [base_reg, #(offset)]
||	}
|.endmacro

/* Memory access(load/store) with 64-bit 'offset'.
 * Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register. */
|.macro MEM_ACCESS_64_WITH_UOFFSET_64, ldr_str_ins, reg, base_reg, offset, tmp_reg
||	if (((uintptr_t)(offset)) > LDR_STR_PIMM64) {
|		LOAD_64BIT_VAL tmp_reg, offset
|		ldr_str_ins reg, [base_reg, tmp_reg]
||	} else {
|		ldr_str_ins reg, [base_reg, #(offset)]
||	}
|.endmacro

/* ZTS: get thread local variable "_tsrm_ls_cache" */
|.macro LOAD_TSRM_CACHE, reg
||#ifdef __APPLE__
|	.long 0xd53bd071 // TODO: hard-coded: mrs TMP3, tpidrro_el0
|	and TMP3, TMP3, #0xfffffffffffffff8
|	MEM_ACCESS_64_WITH_UOFFSET_64 ldr, TMP3, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->key << 3), TMP1
|	MEM_ACCESS_64_WITH_UOFFSET_64 ldr, reg, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->offset), TMP1
||#else
|	.long 0xd53bd051 // TODO: hard-coded: mrs TMP3, tpidr_el0
||	ZEND_ASSERT(tsrm_ls_cache_tcb_offset <= LDR_STR_PIMM64);
|	ldr reg, [TMP3, #tsrm_ls_cache_tcb_offset]
||#endif
|.endmacro

|.macro LOAD_ADDR_ZTS, reg, struct, field
|	.if ZTS
|		LOAD_TSRM_CACHE TMP3
|		ADD_SUB_64_WITH_CONST_32 add, reg, TMP3, (struct.._offset + offsetof(zend_..struct, field)), reg
|	.else
|		LOAD_ADDR reg, &struct.field
|	.endif
|.endmacro

/* Store address 'addr' into memory 'mem'. */
|.macro ADDR_STORE, mem, addr, tmp_reg
|	LOAD_ADDR tmp_reg, addr
|	str tmp_reg, mem
|.endmacro

/* Store a register value 'reg' into memory 'addr'.
 * For ZTS mode, base register with unsigned offset variant is used,
 * and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */
|.macro MEM_STORE, str_ins, reg, addr, tmp_reg
||	if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) {
|		adr tmp_reg, &addr
|		str_ins reg, [tmp_reg]
||	} else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) {
|		adrp tmp_reg, &(((uintptr_t)(addr)))
|		str_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)]
||	} else {
|		LOAD_ADDR tmp_reg, addr
|		str_ins reg, [tmp_reg]
||	}
|.endmacro

|.macro MEM_STORE_64_ZTS, str_ins, reg, struct, field, tmp_reg
|	.if ZTS
|		LOAD_TSRM_CACHE TMP3
|		MEM_ACCESS_64_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
|	.else
|		MEM_STORE str_ins, reg, &struct.field, tmp_reg
|	.endif
|.endmacro

|.macro MEM_STORE_32_ZTS, str_ins, reg, struct, field, tmp_reg
|	.if ZTS
|		LOAD_TSRM_CACHE TMP3
|		MEM_ACCESS_32_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
|	.else
|		MEM_STORE str_ins, reg, &struct.field, tmp_reg
|	.endif
|.endmacro

|.macro MEM_STORE_8_ZTS, strb_ins, reg, struct, field, tmp_reg
|	.if ZTS
|		LOAD_TSRM_CACHE TMP3
|		MEM_ACCESS_8_WITH_UOFFSET strb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
|	.else
|		MEM_STORE strb_ins, reg, &struct.field, tmp_reg
|	.endif
|.endmacro

/* Load value from memory 'addr' and write it into register 'reg'.
 * For ZTS mode, base register with unsigned offset variant is used,
 * and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */
|.macro MEM_LOAD, ldr_ins, reg, addr, tmp_reg
||	if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) {
|		adr tmp_reg, &addr
|		ldr_ins reg, [tmp_reg]
||	} else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) {
|		adrp tmp_reg, &(((uintptr_t)(addr)))
|		ldr_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)]
||	} else {
|		LOAD_ADDR tmp_reg, addr
|		ldr_ins reg, [tmp_reg]
||	}
|.endmacro

|.macro MEM_LOAD_64_ZTS, ldr_ins, reg, struct, field, tmp_reg
|	.if ZTS
|		LOAD_TSRM_CACHE TMP3
|		MEM_ACCESS_64_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
|	.else
|		MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg
|	.endif
|.endmacro

|.macro MEM_LOAD_32_ZTS, ldr_ins, reg, struct, field, tmp_reg
|	.if ZTS
|		LOAD_TSRM_CACHE TMP3
|		MEM_ACCESS_32_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
|	.else
|		MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg
|	.endif
|.endmacro

|.macro MEM_LOAD_8_ZTS, ldrb_ins, reg, struct, field, tmp_reg
|	.if ZTS
|		LOAD_TSRM_CACHE TMP3
|		MEM_ACCESS_8_WITH_UOFFSET ldrb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
|	.else
|		MEM_LOAD ldrb_ins, reg, &struct.field, tmp_reg
|	.endif
|.endmacro

/* Conduct arithmetic operation between the value in memory 'addr' and register value in 'reg',
 * and the computation result is stored back in 'reg'. 'op_ins' can be add/sub. */
|.macro MEM_LOAD_OP, op_ins, ldr_ins, reg, addr, tmp_reg1, tmp_reg2
|	MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2
|	op_ins reg, reg, tmp_reg1
|.endmacro

|.macro MEM_LOAD_OP_ZTS, op_ins, ldr_ins, reg, struct, field, tmp_reg1, tmp_reg2
|	.if ZTS
|		LOAD_TSRM_CACHE TMP3
|		MEM_ACCESS_64_WITH_UOFFSET ldr_ins, tmp_reg2, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg1
|		op_ins reg, reg, tmp_reg2
|	.else
|		MEM_LOAD_OP op_ins, ldr_ins, reg, &struct.field, tmp_reg1, tmp_reg2
|	.endif
|.endmacro

/* Conduct arithmetic operation between the value in memory 'addr' and operand 'op', and the computation
 * result is stored back to memory 'addr'. Operand 'op' can be either a register value or an immediate value.
 * Currently, only add instruction is used as 'op_ins'.
 * Note: It should be guaranteed that the immediate value can be encoded for 'op_ins'. */
|.macro MEM_UPDATE, op_ins, ldr_ins, str_ins, op, addr, tmp_reg1, tmp_reg2
|	LOAD_ADDR tmp_reg2, addr
|	ldr_ins, tmp_reg1, [tmp_reg2]
|	op_ins tmp_reg1, tmp_reg1, op
|	str_ins tmp_reg1, [tmp_reg2]
|.endmacro

|.macro MEM_UPDATE_ZTS, op_ins, ldr_ins, str_ins, op, struct, field, tmp_reg1, tmp_reg2
|	.if ZTS
|		LOAD_TSRM_CACHE TMP3
||		if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > LDR_STR_PIMM64) {
|			LOAD_32BIT_VAL tmp_reg1, (struct.._offset+offsetof(zend_..struct, field))
|			ldr_ins tmp_reg2, [TMP3, tmp_reg1]
|			op_ins tmp_reg2, tmp_reg2, op
|			str_ins tmp_reg2, [TMP3, tmp_reg1]
||		} else {
|			ldr_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))]
|			op_ins tmp_reg2, tmp_reg2, op
|			str_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))]
||		}
|	.else
|		MEM_UPDATE op_ins, ldr_ins, str_ins, op, &struct.field, tmp_reg1, tmp_reg2
|	.endif
|.endmacro

|.macro EXT_CALL, func, tmp_reg
||	if (arm64_may_use_b(func)) {
|		bl &func
||	} else {
|		LOAD_ADDR tmp_reg, func
|		blr tmp_reg
||	}
|.endmacro

|.macro EXT_JMP, func, tmp_reg
||	if (arm64_may_use_b(func)) {
|		b &func
||	} else {
|		LOAD_ADDR tmp_reg, func
|		br tmp_reg
||	}
|.endmacro

|.macro SAVE_IP
||	if (GCC_GLOBAL_REGS) {
|		str IP, EX->opline
||	}
|.endmacro

|.macro LOAD_IP
||	if (GCC_GLOBAL_REGS) {
|		ldr IP, EX->opline
||	}
|.endmacro

|.macro LOAD_IP_ADDR, addr
||	if (GCC_GLOBAL_REGS) {
|		LOAD_ADDR IP, addr
||	} else {
|		ADDR_STORE EX->opline, addr, RX
||	}
|.endmacro

|.macro LOAD_IP_ADDR_ZTS, struct, field, tmp_reg
|	.if ZTS
||		if (GCC_GLOBAL_REGS) {
|			LOAD_TSRM_CACHE IP
|	   		MEM_ACCESS_64_WITH_UOFFSET ldr, IP, IP, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
||		} else {
|			LOAD_TSRM_CACHE RX
|			ADD_SUB_64_WITH_CONST_32 add, RX, RX, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
|			str RX, EX->opline
||		}
|	.else
|		LOAD_IP_ADDR &struct.field
|	.endif
|.endmacro

|.macro GET_IP, reg
||	if (GCC_GLOBAL_REGS) {
|		mov reg, IP
||	} else {
|		ldr reg, EX->opline
||	}
|.endmacro

/* Update IP with register 'reg'. Note: shift variant is handled by ADD_IP_SHIFT. */
|.macro ADD_IP, reg, tmp_reg
||	if (GCC_GLOBAL_REGS) {
|		add IP, IP, reg
||	} else {
|		ldr tmp_reg, EX->opline
|		add tmp_reg, tmp_reg, reg
|		str tmp_reg, EX->opline
||	}
|.endmacro

|.macro ADD_IP_SHIFT, reg, shift, tmp_reg
||	if (GCC_GLOBAL_REGS) {
|		add IP, IP, reg, shift
||	} else {
|		ldr tmp_reg, EX->opline
|		add tmp_reg, tmp_reg, reg, shift
|		str tmp_reg, EX->opline
||	}
|.endmacro

/* Update IP with 32-bit immediate 'val'. */
|.macro ADD_IP_WITH_CONST, val, tmp_reg
||	ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val)));
||	if (GCC_GLOBAL_REGS) {
|		add IP, IP, #val
||	} else {
|		ldr tmp_reg, EX->opline
|		add tmp_reg, tmp_reg, #val
|		str tmp_reg, EX->opline
||	}
|.endmacro

|.macro JMP_IP, tmp_reg
||	if (GCC_GLOBAL_REGS) {
|		ldr tmp_reg, [IP]
|		br tmp_reg
||	} else {
|		ldr tmp_reg, EX:CARG1->opline
|		br tmp_reg
||	}
|.endmacro

|.macro CMP_IP, addr, tmp_reg1, tmp_reg2
|	LOAD_ADDR tmp_reg1, addr
||	if (GCC_GLOBAL_REGS) {
|		cmp IP, tmp_reg1
||	} else {
|		ldr tmp_reg2, EX->opline
|		cmp tmp_reg2, tmp_reg1
||	}
|.endmacro

|.macro LOAD_ZVAL_ADDR, reg, addr
||	if (Z_MODE(addr) == IS_CONST_ZVAL) {
|		LOAD_ADDR reg, Z_ZV(addr)
||	} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
||		if (Z_OFFSET(addr)) {
|			ADD_SUB_64_WITH_CONST_32 add, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), reg
||		} else {
||			if (Z_REG(addr) == ZREG_RSP) {
|				mov reg, sp
||			} else {
|				mov reg, Rx(Z_REG(addr))
||			}
||		}
||	} else {
||		ZEND_UNREACHABLE();
||	}
|.endmacro

|.macro GET_Z_TYPE_INFO, reg, zv
|	ldr reg, [zv, #offsetof(zval,u1.type_info)]
|.endmacro

|.macro SET_Z_TYPE_INFO, zv, type, tmp_reg
|	LOAD_32BIT_VAL tmp_reg, type
|	str tmp_reg, [zv, #offsetof(zval,u1.type_info)]
|.endmacro

|.macro GET_ZVAL_TYPE, reg, addr, tmp_reg
||	ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|	MEM_ACCESS_8_WITH_UOFFSET ldrb, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.v.type), tmp_reg
|.endmacro

|.macro GET_ZVAL_TYPE_INFO, reg, addr, tmp_reg
||	ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|	MEM_ACCESS_32_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg
|.endmacro

|.macro SET_ZVAL_TYPE_INFO, addr, type, tmp_reg1, tmp_reg2
||	ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|	LOAD_32BIT_VAL tmp_reg1, type
|	MEM_ACCESS_32_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2
|.endmacro

|.macro SET_ZVAL_TYPE_INFO_FROM_REG, addr, reg, tmp_reg
||	ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|	MEM_ACCESS_32_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg
|.endmacro

|.macro GET_Z_PTR, reg, zv
|	ldr reg, [zv]
|.endmacro

|.macro GET_ZVAL_PTR, reg, addr, tmp_reg
||	ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|	MEM_ACCESS_64_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
|.endmacro

|.macro SET_ZVAL_PTR, addr, reg, tmp_reg
||	ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|	MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
|.endmacro

|.macro UNDEF_OPLINE_RESULT, tmp_reg
|	ldr REG0, EX->opline
|	ldr REG0w, OP:REG0->result.var
|	add REG0, FP, REG0
|	SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg
|.endmacro

|.macro UNDEF_OPLINE_RESULT_IF_USED, tmp_reg1, tmp_reg2
|	ldrb tmp_reg1, OP:RX->result_type
|	TST_32_WITH_CONST tmp_reg1, (IS_TMP_VAR|IS_VAR), tmp_reg2
|	beq >1
|	ldr REG0w, OP:RX->result.var
|	add REG0, FP, REG0
|	SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg1
|1:
|.endmacro

/* Floating-point comparison between register 'reg' and value from memory 'addr'.
 * Note: the equivalent macros in JIT/x86 are SSE_AVX_OP and SSE_OP. */
|.macro DOUBLE_CMP, reg, addr, tmp_reg, fp_tmp_reg
||	if (Z_MODE(addr) == IS_CONST_ZVAL) {
|		MEM_LOAD ldr, Rd(fp_tmp_reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg)
|		fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0)
||	} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|		MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg)
|		fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0)
||	} else if (Z_MODE(addr) == IS_REG) {
|		fcmp Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0)
||	} else {
||		ZEND_UNREACHABLE();
||	}
|.endmacro

/* Convert LONG value 'val' into DOUBLE type, and move it into FP register 'reg'.
 * Note: the equivalent macro in JIT/x86 is SSE_GET_LONG. */
|.macro DOUBLE_GET_LONG, reg, val, tmp_reg
||	if (val == 0) {
|		fmov Rd(reg-ZREG_V0), xzr  // TODO: "movi d0, #0" is not recognized by DynASM/arm64
||	} else {
|		LOAD_64BIT_VAL Rx(tmp_reg), val
|		scvtf Rd(reg-ZREG_V0), Rx(tmp_reg)
||	}
|.endmacro

/* Convert LONG value from memory 'addr' into DOUBLE type, and move it into FP register 'reg'.
 * Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_LVAL. */
|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg1, tmp_reg2
||	if (Z_MODE(addr) == IS_CONST_ZVAL) {
|		DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg1
||	} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|		MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg2)
|		scvtf Rd(reg-ZREG_V0), Rx(tmp_reg1)
||	} else if (Z_MODE(addr) == IS_REG) {
|		scvtf Rd(reg-ZREG_V0), Rx(Z_REG(addr))
||	} else {
||		ZEND_UNREACHABLE();
||	}
|.endmacro

/* Floating-point arithmetic operation between two FP registers.
 * Note: the equivalent macro in JIT/x86 is AVX_MATH_REG. */
|.macro DOUBLE_MATH_REG, opcode, dst_reg, op1_reg, op2_reg
||	switch (opcode) {
||		case ZEND_ADD:
|			fadd Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0)
||			break;
||		case ZEND_SUB:
|			fsub Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0)
||			break;
||		case ZEND_MUL:
|			fmul Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0)
||			break;
||		case ZEND_DIV:
|			fdiv Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0)
||			break;
||	}
|.endmacro

/* Conduct binary operation between register 'reg' and value from memory 'addr',
 * and the computation result is stored in 'reg'.
 * For LONG_ADD_SUB, 'add_sub_ins' can be adds/subs. For LONG_BW_OP, 'bw_ins' can be and/orr/eor.
 * For LONG_CMP, 'cmp' instruction is used by default and only flag registers are affected.
 * Note: the equivalent macro in JIT/x86 is LONG_OP. */
|.macro LONG_ADD_SUB, add_sub_ins, reg, addr, tmp_reg
||	if (Z_MODE(addr) == IS_CONST_ZVAL) {
|		ADD_SUB_64_WITH_CONST add_sub_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg
||	} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|		MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
|		add_sub_ins Rx(reg), Rx(reg), tmp_reg
||	} else if (Z_MODE(addr) == IS_REG) {
|		add_sub_ins Rx(reg), Rx(reg), Rx(Z_REG(addr))
||	} else {
||		ZEND_UNREACHABLE();
||	}
|.endmacro

|.macro LONG_BW_OP, bw_ins, reg, addr, tmp_reg
||	if (Z_MODE(addr) == IS_CONST_ZVAL) {
|		BW_OP_64_WITH_CONST bw_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg
||	} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|		MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
|		bw_ins Rx(reg), Rx(reg), tmp_reg
||	} else if (Z_MODE(addr) == IS_REG) {
|		bw_ins Rx(reg), Rx(reg), Rx(Z_REG(addr))
||	} else {
||		ZEND_UNREACHABLE();
||	}
|.endmacro

|.macro LONG_CMP, reg, addr, tmp_reg
||	if (Z_MODE(addr) == IS_CONST_ZVAL) {
|		CMP_64_WITH_CONST Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg
||	} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|		MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
|		cmp Rx(reg), tmp_reg
||	} else if (Z_MODE(addr) == IS_REG) {
|		cmp Rx(reg), Rx(Z_REG(addr))
||	} else {
||		ZEND_UNREACHABLE();
||	}
|.endmacro

/* Conduct add/sub between value from memory 'addr' and an immediate value 'val', and
 * the computation result is stored back into 'addr'.
 * Note: it should be guaranteed that 'val' can be encoded into add/sub instruction. */
|.macro LONG_ADD_SUB_WITH_IMM, add_sub_ins, addr, val, tmp_reg1, tmp_reg2
||	ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val)));
||	if (Z_MODE(addr) == IS_MEM_ZVAL) {
||		if (((uint32_t)(Z_OFFSET(addr))) > LDR_STR_PIMM64) {
|			LOAD_32BIT_VAL tmp_reg2, Z_OFFSET(addr)
|			ldr tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2]
|			add_sub_ins tmp_reg1, tmp_reg1, #val
|			str tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2]
||		} else {
|			ldr tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)]
|			add_sub_ins tmp_reg1, tmp_reg1, #val
|			str tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)]
||		}
||	} else if (Z_MODE(addr) == IS_REG) {
|		add_sub_ins Rx(Z_REG(addr)), Rx(Z_REG(addr)), #val
||	} else {
||		ZEND_UNREACHABLE();
||	}
|.endmacro

/* Compare value from memory 'addr' with immediate value 'val'.
 * Note: the equivalent macro in JIT/x86 is LONG_OP_WITH_CONST. */
|.macro LONG_CMP_WITH_CONST, addr, val, tmp_reg1, tmp_reg2
||	if (Z_MODE(addr) == IS_MEM_ZVAL) {
|		MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2
|		CMP_64_WITH_CONST tmp_reg1, val, tmp_reg2
||	} else if (Z_MODE(addr) == IS_REG) {
|		CMP_64_WITH_CONST Rx(Z_REG(addr)), val, tmp_reg1
||	} else {
||		ZEND_UNREACHABLE();
||	}
|.endmacro

|.macro GET_ZVAL_LVAL, reg, addr, tmp_reg
||	if (Z_MODE(addr) == IS_CONST_ZVAL) {
||		if (Z_LVAL_P(Z_ZV(addr)) == 0) {
|			mov Rx(reg), xzr
||		} else {
|			LOAD_64BIT_VAL Rx(reg), Z_LVAL_P(Z_ZV(addr))
||		}
||	} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|		MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
||	} else if (Z_MODE(addr) == IS_REG) {
||		if (reg != Z_REG(addr)) {
|			mov Rx(reg), Rx(Z_REG(addr))
||		}
||	} else {
||		ZEND_UNREACHABLE();
||	}
|.endmacro

|.macro LONG_MATH, opcode, reg, addr, tmp_reg1
||	switch (opcode) {
||		case ZEND_ADD:
|			LONG_ADD_SUB adds, reg, addr, tmp_reg1
||			break;
||		case ZEND_SUB:
|			LONG_ADD_SUB subs, reg, addr, tmp_reg1
||			break;
||		case ZEND_BW_OR:
|			LONG_BW_OP orr, reg, addr, tmp_reg1
||			break;
||		case ZEND_BW_AND:
|			LONG_BW_OP and, reg, addr, tmp_reg1
||			break;
||		case ZEND_BW_XOR:
|			LONG_BW_OP eor, reg, addr, tmp_reg1
||			break;
||		default:
||			ZEND_UNREACHABLE();
||	}
|.endmacro

|.macro LONG_MATH_REG, opcode, dst_reg, src_reg1, src_reg2
||	switch (opcode) {
||		case ZEND_ADD:
|			adds dst_reg, src_reg1, src_reg2
||			break;
||		case ZEND_SUB:
|			subs dst_reg, src_reg1, src_reg2
||			break;
||		case ZEND_BW_OR:
|			orr dst_reg, src_reg1, src_reg2
||			break;
||		case ZEND_BW_AND:
|			and dst_reg, src_reg1, src_reg2
||			break;
||		case ZEND_BW_XOR:
|			eor dst_reg, src_reg1, src_reg2
||			break;
||		default:
||			ZEND_UNREACHABLE();
||	}
|.endmacro

/* Store LONG value into memory 'addr'.
 * This LONG value can be an immediate value i.e. 'val' in macro SET_ZVAL_LVAL, or
 * a register value i.e. 'reg' in macro SET_ZVAL_LVAL_FROM_REG. */
|.macro SET_ZVAL_LVAL_FROM_REG, addr, reg, tmp_reg
||	if (Z_MODE(addr) == IS_REG) {
|		mov Rx(Z_REG(addr)), reg
||	} else {
||		ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|		MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
||	}
|.endmacro

|.macro SET_ZVAL_LVAL, addr, val, tmp_reg1, tmp_reg2
||	if (val == 0) {
|		SET_ZVAL_LVAL_FROM_REG addr, xzr, tmp_reg2
||	} else {
|		LOAD_64BIT_VAL tmp_reg1, val
|		SET_ZVAL_LVAL_FROM_REG addr, tmp_reg1, tmp_reg2
||	}
|.endmacro

/* Store DOUBLE value from FP register 'reg' into memory 'addr'.
 * Note: the equivalent macro in JIT/x86 is SSE_SET_ZVAL_DVAL. */
|.macro SET_ZVAL_DVAL, addr, reg, tmp_reg
||	if (Z_MODE(addr) == IS_REG) {
||		if (reg != Z_REG(addr)) {
|			fmov Rd(Z_REG(addr)-ZREG_V0), Rd(reg-ZREG_V0)
||		}
||	} else {
||		ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|		MEM_ACCESS_64_WITH_UOFFSET str, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg)
||	}
|.endmacro

/* Load DOUBLE value from memory 'addr' into FP register 'reg'.
 * Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_DVAL. */
|.macro GET_ZVAL_DVAL, reg, addr, tmp_reg
||	if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) {
||		if (Z_MODE(addr) == IS_CONST_ZVAL) {
|			MEM_LOAD ldr, Rd(reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg)
||		} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|			MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg)
||		} else if (Z_MODE(addr) == IS_REG) {
|			fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0)
||		} else {
||			ZEND_UNREACHABLE();
||		}
||	}
|.endmacro

|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg
||	if (Z_TYPE_P(zv) > IS_TRUE) {
||		if (Z_TYPE_P(zv) == IS_DOUBLE) {
||			zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg;
|			MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1)
|			SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2
||		} else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
||			zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg;
|			DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), tmp_reg1
|			SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2
||		} else {
|			// In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr.
|			// Note that imm32 is signed extended to 64 bits during mov.
|			// In aarch64, we choose to handle both cases in the same way. Even though 4 mov's are used for 64-bit value and 2 mov's are
|			// needed for 32-bit value, an extra ext insn is needed for 32-bit vlaue.
|			SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2)
||		}
||	}
||	if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
||		if (dst_def_info == MAY_BE_DOUBLE) {
||			if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
|				SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2)
||			}
||		} else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
|			SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2)
||		}
||	}
|.endmacro

|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg
||	if (Z_TYPE_P(zv) > IS_TRUE) {
||		if (Z_TYPE_P(zv) == IS_DOUBLE) {
||			zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ?
||				Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : fp_tmp_reg);
|			MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1)
|			SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2
|			SET_ZVAL_DVAL res_addr, dst_reg, tmp_reg2
||		} else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
||			if (Z_MODE(dst_addr) == IS_REG) {
|				DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), tmp_reg1
|				SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg2
||			} else if (Z_MODE(res_addr) == IS_REG) {
|				DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), tmp_reg1
|				SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg2
||			} else {
|				DOUBLE_GET_LONG fp_tmp_reg, Z_LVAL_P(zv), tmp_reg1
|				SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg2
|				SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg2
||			}
||		} else {
||			if (Z_MODE(dst_addr) == IS_REG) {
|				SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2)
|				SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg1)
||			} else if (Z_MODE(res_addr) == IS_REG) {
|				SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2)
|				SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg1)
||			} else {
|				SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2)
|				SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2)
||			}
||		}
||	}
||	if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
||		if (dst_def_info == MAY_BE_DOUBLE) {
||			if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
|				SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2)
||			}
||		} else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
|			SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2)
||		}
||	}
||	if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
||		if (dst_def_info == MAY_BE_DOUBLE) {
|			SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2)
||		} else {
|			SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2)
||		}
||	}
|.endmacro

// the same as above, but "src" may overlap with "reg1"
// Useful info would be stored into reg1 and reg2, and they might be used afterward.
|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg1, tmp_reg2, fp_tmp_reg
|	ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg1, fp_tmp_reg
||	if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) &&
||		!(src_info & MAY_BE_GUARD) &&
||		has_concrete_type(src_info & MAY_BE_ANY)) {
||		if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
||			if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
||				uint32_t type = concrete_type(src_info);
|				SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg1), Rx(tmp_reg2)
||			}
||		}
||	} else {
|		GET_ZVAL_TYPE_INFO Rw(reg1), src_addr, Rx(tmp_reg1)
|		SET_ZVAL_TYPE_INFO_FROM_REG dst_addr, Rw(reg1), Rx(tmp_reg1)
||	}
|.endmacro

|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg, fp_tmp_reg
||	if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
||		if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
||			if (Z_MODE(src_addr) == IS_REG) {
||				if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) {
|					SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg)
||				}
||			} else if (Z_MODE(dst_addr) == IS_REG) {
|				GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg)
||			} else {
|				GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg)
|				SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg)
||			}
||		} else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
||			if (Z_MODE(src_addr) == IS_REG) {
|				SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg
||			} else if (Z_MODE(dst_addr) == IS_REG) {
|				GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg
||			} else {
|				GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg
|				SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg
||			}
||		// Combine the following two branches.
||		// } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) {
||		} else {
|			GET_ZVAL_PTR Rx(reg2), src_addr, Rx(tmp_reg)
|			SET_ZVAL_PTR dst_addr, Rx(reg2), Rx(tmp_reg)
||		}
||	}
|.endmacro

|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, reg1, reg2, tmp_reg, tmp_reg2, fp_tmp_reg
||	if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
||		if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) {
||			if (Z_MODE(src_addr) == IS_REG) {
||				if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) {
|					SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg)
||				}
||				if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) {
|					SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg)
||				}
||			} else if (Z_MODE(dst_addr) == IS_REG) {
|				GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg)
||				if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) {
|					SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg)
||				}
||			} else if (Z_MODE(res_addr) == IS_REG) {
|				GET_ZVAL_LVAL Z_REG(res_addr), src_addr, Rx(tmp_reg)
|				SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg)
||			} else {
|				GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg)
|				SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg)
|				SET_ZVAL_LVAL_FROM_REG res_addr, Rx(reg2), Rx(tmp_reg)
||			}
||		} else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
||			if (Z_MODE(src_addr) == IS_REG) {
|				SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg
|				SET_ZVAL_DVAL res_addr, Z_REG(src_addr), tmp_reg
||			} else if (Z_MODE(dst_addr) == IS_REG) {
|				GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg
|				SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg
||			} else if (Z_MODE(res_addr) == IS_REG) {
|				GET_ZVAL_DVAL Z_REG(res_addr), src_addr, tmp_reg
|				SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg
||			} else {
|				GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg
|				SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg
|				SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg
||			}
||		} else {
|			GET_ZVAL_PTR Rx(reg2), src_addr, Rx(tmp_reg)
|			SET_ZVAL_PTR dst_addr, Rx(reg2), Rx(tmp_reg)
|			SET_ZVAL_PTR res_addr, Rx(reg2), Rx(tmp_reg)
||		}
||	}
||	if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) &&
||	    has_concrete_type(src_info & MAY_BE_ANY)) {
||		uint32_t type = concrete_type(src_info);
||		if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
||			if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
|				SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg), Rx(tmp_reg2)
||			}
||		}
||		if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
|			SET_ZVAL_TYPE_INFO res_addr, type, Rw(tmp_reg), Rx(tmp_reg2)
||		}
||	} else {
|		GET_ZVAL_TYPE_INFO Rw(reg1), src_addr, Rx(tmp_reg)
|		SET_ZVAL_TYPE_INFO_FROM_REG dst_addr, Rw(reg1), Rx(tmp_reg)
|		SET_ZVAL_TYPE_INFO_FROM_REG res_addr, Rw(reg1), Rx(tmp_reg)
||	}
|.endmacro

|.macro IF_UNDEF, type_reg, label
|	cbz type_reg, label
|.endmacro

|.macro IF_TYPE, type, val, label
||	if (val == 0) {
|		cbz type, label
||	} else {
|		cmp type, #val
|		beq label
||	}
|.endmacro

|.macro IF_NOT_TYPE, type, val, label
||	if (val == 0) {
|		cbnz type, label
||	} else {
|		cmp type, #val
|		bne label
||	}
|.endmacro

|.macro IF_Z_TYPE, zv, val, label, tmp_reg
|	ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)]
|	IF_TYPE tmp_reg, val, label
|.endmacro

|.macro IF_NOT_Z_TYPE, zv, val, label, tmp_reg
|	ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)]
|	IF_NOT_TYPE tmp_reg, val, label
|.endmacro

|.macro CMP_ZVAL_TYPE, addr, val, tmp_reg
||	ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|	MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg)
|	cmp Rw(tmp_reg), #val
|.endmacro

|.macro IF_ZVAL_TYPE, addr, val, label, tmp_reg
||	ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|	MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg)
|	IF_TYPE Rw(tmp_reg), val, label
|.endmacro

|.macro IF_NOT_ZVAL_TYPE, addr, val, label, tmp_reg
||	ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|	MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg)
|	IF_NOT_TYPE Rw(tmp_reg), val, label
|.endmacro

|.macro IF_FLAGS, type_flags, mask, label, tmp_reg
|	TST_32_WITH_CONST type_flags, mask, tmp_reg
|	bne label
|.endmacro

|.macro IF_NOT_FLAGS, type_flags, mask, label, tmp_reg
|	TST_32_WITH_CONST type_flags, mask, tmp_reg
|	beq label
|.endmacro

|.macro IF_REFCOUNTED, type_flags, label, tmp_reg
|	TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg
|	bne label
|.endmacro

|.macro IF_NOT_REFCOUNTED, type_flags, label, tmp_reg
|	TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg
|	beq label
|.endmacro

|.macro IF_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2
||	ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|	MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2)
|	IF_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2)
|.endmacro

|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2
||	ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
|	MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2)
|	IF_NOT_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2)
|.endmacro

|.macro IF_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2
|	IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2
|.endmacro

|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2
|	IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2
|.endmacro

|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label, tmp_reg1, tmp_reg2
|	IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label, tmp_reg1, tmp_reg2
|.endmacro

|.macro GC_ADDREF, zv, tmp_reg
|	ldr tmp_reg, [zv]
|	add tmp_reg, tmp_reg, #1
|	str tmp_reg, [zv]
|.endmacro

|.macro GC_ADDREF_2, zv, tmp_reg
|	ldr tmp_reg, [zv]
|	add tmp_reg, tmp_reg, #2
|	str tmp_reg, [zv]
|.endmacro

|.macro GC_DELREF, zv, tmp_reg
|	ldr tmp_reg, [zv]
|	subs tmp_reg, tmp_reg, #1
|	str tmp_reg, [zv]
|.endmacro

|.macro IF_GC_MAY_NOT_LEAK, ptr, label, tmp_reg1, tmp_reg2
|	ldr tmp_reg1, [ptr, #4]
|	TST_32_WITH_CONST tmp_reg1, (GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)), tmp_reg2
|	bne label
|.endmacro

|.macro ADDREF_CONST, zv, tmp_reg1, tmp_reg2
|	LOAD_ADDR tmp_reg1, Z_PTR_P(zv)
|	ldr tmp_reg2, [tmp_reg1]
|	add tmp_reg2, tmp_reg2, #1
|	str tmp_reg2, [tmp_reg1]
|.endmacro

|.macro ADDREF_CONST_2, zv, tmp_reg1, tmp_reg2
|	LOAD_ADDR tmp_reg1, Z_PTR_P(zv)
|	ldr tmp_reg2, [tmp_reg1]
|	add tmp_reg2, tmp_reg2, #2
|	str tmp_reg2, [tmp_reg1]
|.endmacro

|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg, tmp_reg
||	if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
||		if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
|			IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg
||		}
|		GC_ADDREF value_ptr_reg, tmp_reg
|1:
||	}
|.endmacro

|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg, tmp_reg
||	if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
||		if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
|			IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg
||		}
|		ldr tmp_reg, [value_ptr_reg]
|		add tmp_reg, tmp_reg, #2
|		str tmp_reg, [value_ptr_reg]
|1:
||	}
|.endmacro

|.macro ZVAL_DEREF, reg, info, tmp_reg
||	if (info & MAY_BE_REF) {
|		IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1, tmp_reg
|		GET_Z_PTR reg, reg
|		add reg, reg, #offsetof(zend_reference, val)
|1:
||	}
|.endmacro

|.macro SET_EX_OPLINE, op, tmp_reg
||	if (op == last_valid_opline) {
||		zend_jit_use_last_valid_opline();
|		SAVE_IP
||	} else {
|		ADDR_STORE EX->opline, op, tmp_reg
||		if (!GCC_GLOBAL_REGS) {
||			zend_jit_reset_last_valid_opline();
||		}
||	}
|.endmacro

// arg1 "zval" should be in FCARG1x
|.macro ZVAL_DTOR_FUNC, var_info, opline, tmp_reg
||	do {
||		if (!((var_info) & MAY_BE_GUARD)
||		 && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
||			zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE));
||			if (type == IS_STRING && !ZEND_DEBUG) {
|				EXT_CALL _efree, tmp_reg
||				break;
||			} else if (type == IS_ARRAY) {
||				if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) {
||					if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) {
|						SET_EX_OPLINE opline, tmp_reg
||					}
|					EXT_CALL zend_array_destroy, tmp_reg
||				} else {
|					EXT_CALL zend_jit_array_free, tmp_reg
||				}
||				break;
||			} else if (type == IS_OBJECT) {
||				if (opline) {
|					SET_EX_OPLINE opline, REG0
||				}
|				EXT_CALL zend_objects_store_del, tmp_reg
||				break;
||			}
||		}
||		if (opline) {
|			SET_EX_OPLINE opline, tmp_reg
||		}
|		EXT_CALL rc_dtor_func, tmp_reg
||	} while(0);
|.endmacro

|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline, tmp_reg1, tmp_reg2
||	if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) {
||		if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
|			// if (Z_REFCOUNTED_P(cv)) {
||			if (cold) {
|				IF_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2
|.cold_code
|1:
||			} else {
|				IF_NOT_ZVAL_REFCOUNTED addr, >4, tmp_reg1, tmp_reg2
||			}
||		}
|		// if (!Z_DELREF_P(cv)) {
|		GET_ZVAL_PTR FCARG1x, addr, Rx(tmp_reg2)
|		GC_DELREF FCARG1x, Rw(tmp_reg1)
||		if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) {
||			if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) {
||				if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
|					bne >3
||				} else {
|					bne >4
||				}
||			}
|			// zval_dtor_func(r);
|			ZVAL_DTOR_FUNC op_info, opline, Rx(tmp_reg1)
||			if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
|				b >4
||			}
|3:
||		}
||		if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
||			if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) {
||				zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val));
|				IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1, tmp_reg1
|				IF_NOT_ZVAL_COLLECTABLE ref_addr, >4, tmp_reg1, tmp_reg2
|				GET_ZVAL_PTR FCARG1x, ref_addr, Rx(tmp_reg2)
|1:
||			}
|			IF_GC_MAY_NOT_LEAK FCARG1x, >4, Rw(tmp_reg1), Rw(tmp_reg2)
|			// gc_possible_root(Z_COUNTED_P(z))
|			EXT_CALL gc_possible_root, Rx(tmp_reg1)
||		}
||		if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) {
|			b >4
|.code
||		}
|4:
||	}
|.endmacro

|.macro FREE_OP, op_type, op, op_info, cold, opline, tmp_reg1, tmp_reg2
||	if (op_type & (IS_VAR|IS_TMP_VAR)) {
||		zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var);
|		ZVAL_PTR_DTOR addr, op_info, 0, cold, opline, tmp_reg1, tmp_reg2
||	}
|.endmacro

|.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2
||	if (RC_MAY_BE_N(op_info)) {
||		if (Z_REG(addr) != ZREG_FP) {
|			GET_ZVAL_LVAL ZREG_REG0, addr, Rx(tmp_reg1)
||			if (RC_MAY_BE_1(op_info)) {
|				// if (GC_REFCOUNT() > 1)
|				ldr Rw(tmp_reg1), [REG0]
|				cmp Rw(tmp_reg1), #1
|				bls >2
||			}
||			if (Z_REG(addr) != ZREG_FCARG1 || Z_OFFSET(addr) != 0) {
|				LOAD_ZVAL_ADDR FCARG1x, addr
||			}
|			EXT_CALL zend_jit_zval_array_dup, REG0
|			mov REG0, RETVALx
|2:
|			mov FCARG1x, REG0
||		} else {
|			GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1)
||			if (RC_MAY_BE_1(op_info)) {
|				// if (GC_REFCOUNT() > 1)
|				ldr Rw(tmp_reg1), [FCARG1x]
|				cmp Rw(tmp_reg1), #1
||				if (cold) {
|					bhi >1
|.cold_code
|1:
||				} else {
|					bls >2
||				}
||			}
|			IF_NOT_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2
|			GC_DELREF FCARG1x, Rw(tmp_reg1)
|1:
|			EXT_CALL zend_array_dup, REG0
|			mov REG0, RETVALx
|			SET_ZVAL_PTR addr, REG0, Rx(tmp_reg1)
|			SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX, Rw(tmp_reg1), Rx(tmp_reg2)
|			mov FCARG1x, REG0
||			if (RC_MAY_BE_1(op_info)) {
||				if (cold) {
|					b >2
|.code
||				}
||			}
|2:
||		}
||	} else {
|		GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1)
||	}
|.endmacro

/* argument is passed in FCARG1x */
|.macro EFREE_REFERENCE
||#if ZEND_DEBUG
|		mov FCARG2x, xzr // filename
|		mov CARG3w, wzr  // lineno
|		mov CARG4, xzr
|		mov CARG5, xzr
|		EXT_CALL _efree, REG0
||#else
||#ifdef HAVE_BUILTIN_CONSTANT_P
|		EXT_CALL _efree_32, REG0
||#else
|		EXT_CALL _efree, REG0
||#endif
||#endif
|.endmacro

|.macro EMALLOC, size, op_array, opline
||#if ZEND_DEBUG
||		const char *filename = op_array->filename ? op_array->filename->val : NULL;
|		mov FCARG1x, #size
|		LOAD_ADDR FCARG2x, filename
|		LOAD_32BIT_VAL CARG3w, opline->lineno
|		mov CARG4, xzr
|		mov CARG5, xzr
|		EXT_CALL _emalloc, REG0
|		mov REG0, RETVALx
||#else
||#ifdef HAVE_BUILTIN_CONSTANT_P
||	if (size > 24 && size <= 32) {
|		EXT_CALL _emalloc_32, REG0
|		mov REG0, RETVALx
||	} else {
|		mov FCARG1x, #size
|		EXT_CALL _emalloc, REG0
|		mov REG0, RETVALx
||	}
||#else
|		mov FCARG1x, #size
|		EXT_CALL _emalloc, REG0
|		mov REG0, RETVALx
||#endif
||#endif
|.endmacro

|.macro OBJ_RELEASE, reg, exit_label, tmp_reg1, tmp_reg2
|	GC_DELREF Rx(reg), Rw(tmp_reg1)
|	bne >1
|	// zend_objects_store_del(obj);
||	if (reg != ZREG_FCARG1) {
|		mov FCARG1x, Rx(reg)
||	}
|	EXT_CALL zend_objects_store_del, Rx(tmp_reg1)
|	b exit_label
|1:
|	IF_GC_MAY_NOT_LEAK Rx(reg), >1, Rw(tmp_reg1), Rw(tmp_reg2)
|	// gc_possible_root(obj)
||	if (reg != ZREG_FCARG1) {
|		mov FCARG1x, Rx(reg)
||	}
|	EXT_CALL gc_possible_root, Rx(tmp_reg1)
|1:
|.endmacro

|.macro UNDEFINED_OFFSET, opline
||	if (opline == last_valid_opline) {
||		zend_jit_use_last_valid_opline();
|		bl ->undefined_offset_ex
||	} else {
|		SET_EX_OPLINE  opline, REG0
|		bl ->undefined_offset
||	}
|.endmacro

|.macro UNDEFINED_INDEX, opline
||	if (opline == last_valid_opline) {
||		zend_jit_use_last_valid_opline();
|		bl ->undefined_index_ex
||	} else {
|		SET_EX_OPLINE opline, REG0
|		bl ->undefined_index
||	}
|.endmacro

|.macro CANNOT_ADD_ELEMENT, opline
||	if (opline == last_valid_opline) {
||		zend_jit_use_last_valid_opline();
|		bl ->cannot_add_element_ex
||	} else {
|		SET_EX_OPLINE opline, REG0
|		bl ->cannot_add_element
||	}
|.endmacro

static bool reuse_ip = 0;
static bool delayed_call_chain = 0;
static uint32_t  delayed_call_level = 0;
static const zend_op *last_valid_opline = NULL;
static bool use_last_vald_opline = 0;
static bool track_last_valid_opline = 0;
static int jit_return_label = -1;
static uint32_t current_trace_num = 0;
static uint32_t allowed_opt_flags = 0;

static void zend_jit_track_last_valid_opline(void)
{
	use_last_vald_opline = 0;
	track_last_valid_opline = 1;
}

static void zend_jit_use_last_valid_opline(void)
{
	if (track_last_valid_opline) {
		use_last_vald_opline = 1;
		track_last_valid_opline = 0;
	}
}

static bool zend_jit_trace_uses_initial_ip(void)
{
	return use_last_vald_opline;
}

static void zend_jit_set_last_valid_opline(const zend_op *target_opline)
{
	if (!reuse_ip) {
		track_last_valid_opline = 0;
		last_valid_opline = target_opline;
	}
}

static void zend_jit_reset_last_valid_opline(void)
{
	track_last_valid_opline = 0;
	last_valid_opline = NULL;
}

static void zend_jit_start_reuse_ip(void)
{
	zend_jit_reset_last_valid_opline();
	reuse_ip = 1;
}

static int zend_jit_reuse_ip(dasm_State **Dst)
{
	if (!reuse_ip) {
		zend_jit_start_reuse_ip();
		|	// call = EX(call);
		|	ldr RX, EX->call
	}
	return 1;
}

static void zend_jit_stop_reuse_ip(void)
{
	reuse_ip = 0;
}

static int zend_jit_interrupt_handler_stub(dasm_State **Dst)
{
	|->interrupt_handler:
	|	SAVE_IP
	|	//EG(vm_interrupt) = 0;
	|	MEM_STORE_8_ZTS strb, wzr, executor_globals, vm_interrupt, TMP1
	|	//if (EG(timed_out)) {
	|	MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, timed_out, TMP1
	|	cbz TMP1w, >1
	|	//zend_timeout();
	|	EXT_CALL zend_timeout, TMP1
	|1:
	|	//} else if (zend_interrupt_function) {
	if (zend_interrupt_function) {
		|	//zend_interrupt_function(execute_data);
		|	mov CARG1, FP
		|	EXT_CALL zend_interrupt_function, TMP1
		|	MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
		|	cbz REG0, >1
		|	EXT_CALL zend_jit_exception_in_interrupt_handler_helper, TMP1
		|1:
		|	//ZEND_VM_ENTER();
		|	//execute_data = EG(current_execute_data);
		|	MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1
		|	LOAD_IP
	}
	|	//ZEND_VM_CONTINUE()
	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		|	ADD_HYBRID_SPAD
		|	JMP_IP TMP1
	} else if (GCC_GLOBAL_REGS) {
		|	ldp x29, x30, [sp], # SPAD // stack alignment
		|	JMP_IP TMP1
	} else {
		|	ldp FP, RX, T2                // retore FP and IP
		|	ldp x29, x30, [sp], # NR_SPAD // stack alignment
		|	mov RETVALx, #1               // ZEND_VM_ENTER
		|	ret
	}

	return 1;
}

static int zend_jit_exception_handler_stub(dasm_State **Dst)
{
	|->exception_handler:
	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		const void *handler = zend_get_opcode_handler_func(EG(exception_op));

		|	ADD_HYBRID_SPAD
		|	EXT_CALL handler, REG0
		|	JMP_IP TMP1
	} else {
		const void *handler = EG(exception_op)->handler;

		if (GCC_GLOBAL_REGS) {
			|	ldp x29, x30, [sp], # SPAD    // stack alignment
			|	EXT_JMP handler, REG0
		} else {
			|	mov FCARG1x, FP
			|	EXT_CALL handler, REG0
			|	ldp FP, RX, T2                // retore FP and IP
			|	ldp x29, x30, [sp], # NR_SPAD // stack alignment
			|	tbnz RETVALw, #31, >1
			|	mov RETVALw, #1               // ZEND_VM_ENTER
			|1:
			|	ret
		}
	}

	return 1;
}

static int zend_jit_exception_handler_undef_stub(dasm_State **Dst)
{
	|->exception_handler_undef:
	|	MEM_LOAD_64_ZTS ldr, REG0, executor_globals, opline_before_exception, REG0
	|	ldrb TMP1w, OP:REG0->result_type
	|	TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w
	|	beq >1
	|	ldr REG0w, OP:REG0->result.var
	|	add REG0, REG0, FP
	|	SET_Z_TYPE_INFO REG0, IS_UNDEF, TMP1w
	|1:
	|	b ->exception_handler

	return 1;
}

static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst)
{
	zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);

	|->exception_handler_free_op1_op2:
	|	UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w
	|	ldrb TMP1w, OP:RX->op1_type
	|	TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w
	|	beq >9
	|	ldr REG0w, OP:RX->op1.var
	|	add REG0, REG0, FP
	|	ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2
	|9:
	|	ldrb TMP1w, OP:RX->op2_type
	|	TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w
	|	beq >9
	|	ldr REG0w, OP:RX->op2.var
	|	add REG0, REG0, FP
	|	ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2
	|9:
	|	b ->exception_handler
	return 1;
}

static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst)
{
	zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);

	|->exception_handler_free_op2:
	|	MEM_LOAD_64_ZTS ldr, RX, executor_globals, opline_before_exception, REG0
	|	UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w
	|	ldrb TMP1w, OP:RX->op2_type
	|	TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w
	|	beq >9
	|	ldr REG0w, OP:RX->op2.var
	|	add REG0, REG0, FP
	|	ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2
	|9:
	|	b ->exception_handler
	return 1;
}

static int zend_jit_leave_function_stub(dasm_State **Dst)
{
	|->leave_function_handler:
	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		|	TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w
		|	bne >1
		|	EXT_CALL zend_jit_leave_nested_func_helper, REG0
		|	ADD_HYBRID_SPAD
		|	JMP_IP TMP1
		|1:
		|	EXT_CALL zend_jit_leave_top_func_helper, REG0
		|	ADD_HYBRID_SPAD
		|	JMP_IP TMP1
	} else {
		if (GCC_GLOBAL_REGS) {
			|	ldp x29, x30, [sp], # SPAD    // stack alignment
		} else {
			|	mov FCARG2x, FP
			|	ldp FP, RX, T2                // retore FP and IP
			|	ldp x29, x30, [sp], # NR_SPAD // stack alignment
		}
		|	TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w
		|	bne >1
		|	EXT_JMP zend_jit_leave_nested_func_helper, REG0
		|1:
		|	EXT_JMP zend_jit_leave_top_func_helper, REG0
	}

	return 1;
}

static int zend_jit_leave_throw_stub(dasm_State **Dst)
{
	|->leave_throw_handler:
	|	// if (opline->opcode != ZEND_HANDLE_EXCEPTION) {
	if (GCC_GLOBAL_REGS) {
		|	ldrb TMP1w, OP:IP->opcode
		|	cmp TMP1w, #ZEND_HANDLE_EXCEPTION
		|	beq >5
		|	// EG(opline_before_exception) = opline;
		|	MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2
		|5:
		|	// opline = EG(exception_op);
		|	LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2
		|	// HANDLE_EXCEPTION()
		|	b ->exception_handler
	} else {
		|	GET_IP TMP1
		|	ldrb TMP2w, OP:TMP1->opcode
		|	cmp TMP2w, #ZEND_HANDLE_EXCEPTION
		|	beq >5
		|	// EG(opline_before_exception) = opline;
		|	MEM_STORE_64_ZTS str, TMP1, executor_globals, opline_before_exception, TMP2
		|5:
		|	// opline = EG(exception_op);
		|	LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2
		|	ldp FP, RX, T2                // retore FP and IP
		|	ldp x29, x30, [sp], # NR_SPAD // stack alignment
		|	mov RETVALx, #2               // ZEND_VM_LEAVE
		|	ret
	}

	return 1;
}

static int zend_jit_icall_throw_stub(dasm_State **Dst)
{
	|->icall_throw_handler:
	|	// zend_rethrow_exception(zend_execute_data *execute_data)
	|	ldr IP, EX->opline
	|	// if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) {
	|	ldrb TMP1w, OP:IP->opcode
	|	cmp TMP1w, #ZEND_HANDLE_EXCEPTION
	|	beq >1
	|	// EG(opline_before_exception) = opline;
	|	MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2
	|1:
	|	// opline = EG(exception_op);
	|	LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2
	||	if (GCC_GLOBAL_REGS) {
	|		str IP, EX->opline
	||	}
	|	// HANDLE_EXCEPTION()
	|	b ->exception_handler

	return 1;
}

static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst)
{
	zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);

	|->throw_cannot_pass_by_ref:
	|	ldr REG0, EX->opline
	|	ldr REG1w, OP:REG0->result.var
	|	add REG1, REG1, RX
	|	SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w
	|	// last EX(call) frame may be delayed
	|	ldr TMP1, EX->call
	|	cmp RX, TMP1
	|	beq >1
	|	ldr REG1, EX->call
	|	str REG1, EX:RX->prev_execute_data
	|	str RX, EX->call
	|1:
	|	mov RX, REG0
	|	ldr FCARG1w, OP:REG0->op2.num
	|	EXT_CALL zend_cannot_pass_by_reference, REG0
	|	ldrb TMP1w, OP:RX->op1_type
	|	cmp TMP1w, #IS_TMP_VAR
	|	bne >9
	|	ldr REG0w, OP:RX->op1.var
	|	add REG0, REG0, FP
	|	ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2
	|9:
	|	b ->exception_handler

	return 1;
}

static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst)
{
	|->undefined_offset_ex:
	|	SAVE_IP
	|	b ->undefined_offset

	return 1;
}

static int zend_jit_undefined_offset_stub(dasm_State **Dst)
{
	|->undefined_offset:
#ifdef __APPLE__
	|	stp x29, x30, [sp, # -16]!
	|	mov x29, sp
#endif
	|	//sub r4, 8
	|	ldr REG0, EX->opline
	|	ldr REG1w, OP:REG0->result.var
	|	add REG1, REG1, FP
	|	SET_Z_TYPE_INFO REG1, IS_NULL, TMP1w
	|	ldrb REG1w, OP:REG0->op2_type
	|	cmp REG1w, #IS_CONST
	|	bne >2
	|	ldrsw REG1, OP:REG0->op2.constant
	|	add REG0, REG0, REG1
	|	b >3
	|2:
	|	ldr REG0w, OP:REG0->op2.var
	|	add REG0, REG0, FP
	|3:
	|	mov CARG1, #E_WARNING
	|	LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT
	|	ldr CARG3, [REG0]
#ifdef __APPLE__
	|	str CARG3, [sp, #-16]!
	|	EXT_CALL zend_error, REG0
	|	add sp, sp, #16
	|	ldp x29, x30, [sp], #16
	|	ret
#else
	|	EXT_JMP zend_error, REG0 // tail call
	|	//add r4, 8 // stack alignment
	|	//ret
#endif

	return 1;
}

static int zend_jit_undefined_index_ex_stub(dasm_State **Dst)
{
	|->undefined_index_ex:
	|	SAVE_IP
	|	b ->undefined_index

	return 1;
}

static int zend_jit_undefined_index_stub(dasm_State **Dst)
{
	|->undefined_index:
#ifdef __APPLE__
	|	stp x29, x30, [sp, # -16]!
	|	mov x29, sp
#endif
	|	//sub r4, 8
	|	ldr REG0, EX->opline
	|	ldr REG1w, OP:REG0->result.var
	|	add REG1, REG1, FP
	|	SET_Z_TYPE_INFO REG1, IS_NULL, TMP1w
	|	ldrb REG1w, OP:REG0->op2_type
	|	cmp REG1w, #IS_CONST
	|	bne >2
	|	ldrsw REG1, OP:REG0->op2.constant
	|	add REG0, REG0, REG1
	|	b >3
	|2:
	|	ldr REG0w, OP:REG0->op2.var
	|	add REG0, REG0, FP
	|3:
	|	mov CARG1, #E_WARNING
	|	LOAD_ADDR CARG2, "Undefined array key \"%s\""
	|	ldr CARG3, [REG0]
	|	add CARG3, CARG3, #offsetof(zend_string, val)
#ifdef __APPLE__
	|	str CARG3, [sp, #-16]!
	|	EXT_CALL zend_error, REG0
	|	add sp, sp, #16
	|	ldp x29, x30, [sp], #16
	|	ret
#else
	|	EXT_JMP zend_error, REG0 // tail call
	|	//add r4, 8
	|	//ret
#endif

	return 1;
}

static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst)
{
	|->cannot_add_element_ex:
	|	SAVE_IP
	|	b ->cannot_add_element

	return 1;
}

static int zend_jit_cannot_add_element_stub(dasm_State **Dst)
{
	|->cannot_add_element:
	|	// sub r4, 8
	|	ldr REG0, EX->opline
	|	ldrb TMP1w, OP:REG0->result_type
	|	cmp TMP1w, #IS_UNUSED
	|	beq >1
	|	ldr REG0w, OP:REG0->result.var
	|	add REG0, REG0, FP
	|	SET_Z_TYPE_INFO REG0, IS_NULL, TMP1w
	|1:
	|	mov CARG1, xzr
	|	LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied"
	|	EXT_JMP zend_throw_error, REG0 // tail call
	|	// add r4, 8
	|	//ret

	return 1;
}

static int zend_jit_undefined_function_stub(dasm_State **Dst)
{
	|->undefined_function:
	|	ldr REG0, EX->opline
	|	mov CARG1, xzr
	|	LOAD_ADDR CARG2, "Call to undefined function %s()"
	|	ldrsw CARG3, [REG0, #offsetof(zend_op, op2.constant)]
	|	ldr CARG3, [REG0, CARG3]
	|	add CARG3, CARG3, #offsetof(zend_string, val)
#ifdef __APPLE__
	|	str CARG3, [sp, #-16]!
#endif
	|	EXT_CALL zend_throw_error, REG0
#ifdef __APPLE__
	|	add sp, sp, #16
#endif
	|	b ->exception_handler
	return 1;
}

static int zend_jit_negative_shift_stub(dasm_State **Dst)
{
	|->negative_shift:
	|	ldr RX, EX->opline
	|	LOAD_ADDR CARG1, zend_ce_arithmetic_error
	|	LOAD_ADDR CARG2, "Bit shift by negative number"
	|	EXT_CALL zend_throw_error, REG0
	|	b ->exception_handler_free_op1_op2
	return 1;
}

static int zend_jit_mod_by_zero_stub(dasm_State **Dst)
{
	|->mod_by_zero:
	|	ldr RX, EX->opline
	|	LOAD_ADDR CARG1, zend_ce_division_by_zero_error
	|	LOAD_ADDR CARG2, "Modulo by zero"
	|	EXT_CALL zend_throw_error, REG0
	|	b ->exception_handler_free_op1_op2
	return 1;
}

static int zend_jit_invalid_this_stub(dasm_State **Dst)
{
	|->invalid_this:
	|	UNDEF_OPLINE_RESULT TMP1w
	|	mov CARG1, xzr
	|	LOAD_ADDR CARG2, "Using $this when not in object context"
	|	EXT_CALL zend_throw_error, REG0
	|	b ->exception_handler

	return 1;
}

static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst)
{
	if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
		return 1;
	}

	|->hybrid_runtime_jit:
	|	EXT_CALL zend_runtime_jit, REG0
	|	JMP_IP TMP1
	return 1;
}

static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst)
{
	if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
		return 1;
	}

	|->hybrid_profile_jit:
	|	// ++zend_jit_profile_counter;
	|	LOAD_ADDR REG0, &zend_jit_profile_counter
	|	ldr TMP1, [REG0]
	|	add TMP1, TMP1, #1
	|	str TMP1, [REG0]
	|	// op_array = (zend_op_array*)EX(func);
	|	ldr REG0, EX->func
	|	// run_time_cache = EX(run_time_cache);
	|	ldr REG2, EX->run_time_cache
	|	// jit_extension = (const void*)ZEND_FUNC_INFO(op_array);
	|	ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
	|	// ++ZEND_COUNTER_INFO(op_array)
	||	if ((zend_jit_profile_counter_rid * sizeof(void*)) > LDR_STR_PIMM64) {
	|		LOAD_32BIT_VAL TMP1, (zend_jit_profile_counter_rid * sizeof(void*))
	|		ldr TMP2, [REG2, TMP1]
	|		add TMP2, TMP2, #1
	|		str TMP2, [REG2, TMP1]
	||	} else {
	|		ldr TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))]
	|		add TMP2, TMP2, #1
	|		str TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))]
	||	}
	|	// return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)()
	|	ldr TMP1, [REG0, #offsetof(zend_jit_op_array_extension, orig_handler)]
	|	br TMP1
	return 1;
}

static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst)
{
	if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
		return 1;
	}

	|->hybrid_hot_code:
	||	ZEND_ASSERT(ZEND_JIT_COUNTER_INIT <= MOVZ_IMM);
	|	movz TMP1w, #ZEND_JIT_COUNTER_INIT
	|	strh TMP1w, [REG2]
	|	mov FCARG1x, FP
	|	GET_IP FCARG2x
	|	EXT_CALL zend_jit_hot_func, REG0
	|	JMP_IP TMP1
	return 1;
}

/*
 * This code is based Mike Pall's "Hashed profile counters" idea, implemented
 * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual
 * property disclosure and research opportunities" email
 * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html
 *
 * In addition we use a variation of Knuth's multiplicative hash function
 * described at https://code.i-harness.com/en/q/a21ce
 *
 * uint64_t hash(uint64_t x) {
 *    x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
 *    x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
 *    x = x ^ (x >> 31);
 *    return x;
 * }
 *
 * uint_32_t hash(uint32_t x) {
 *    x = ((x >> 16) ^ x) * 0x45d9f3b;
 *    x = ((x >> 16) ^ x) * 0x45d9f3b;
 *    x = (x >> 16) ^ x;
 *    return x;
 * }
 *
 */
static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost)
{
	|	ldr REG0, EX->func
	|	ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
	|	ldr REG2, [REG1, #offsetof(zend_jit_op_array_hot_extension, counter)]
	|	ldrh TMP2w, [REG2]
	|	ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w
	|	strh TMP2w, [REG2]
	|	ble ->hybrid_hot_code
	|	GET_IP REG2
	|	ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)]
	|	sub REG2, REG2, TMP1
	|	// divide by sizeof(zend_op)
	||	ZEND_ASSERT(sizeof(zend_op) == 32);
	|	add TMP1, REG1, REG2, asr #2
	|	ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_hot_extension, orig_handlers)]
	|	br TMP1
	return 1;
}

static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst)
{
	if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
		return 1;
	}

	|->hybrid_func_hot_counter:

	return zend_jit_hybrid_hot_counter_stub(Dst,
		((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
}

static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst)
{
	if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
		return 1;
	}

	|->hybrid_loop_hot_counter:

	return zend_jit_hybrid_hot_counter_stub(Dst,
		((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}

static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst)
{
	if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
		return 1;
	}

	// On entry from counter stub:
	//   REG2 -> zend_op_trace_info.counter

	|->hybrid_hot_trace:
	|	mov TMP1w, #ZEND_JIT_COUNTER_INIT
	|	strh TMP1w, [REG2]
	|	mov FCARG1x, FP
	|	GET_IP FCARG2x
	|	EXT_CALL zend_jit_trace_hot_root, REG0
	|	tbnz RETVALw, #31, >1  // Result is < 0 on failure.
	|	MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0
	|	LOAD_IP
	|	JMP_IP TMP1
	|1:
	|	EXT_JMP zend_jit_halt_op->handler, REG0

	return 1;
}

static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost)
{
	|	ldr REG0, EX->func
	|	ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
	|	ldr REG1, [REG1, #offsetof(zend_jit_op_array_trace_extension, offset)]
	|	add TMP1, REG1, IP
	|	ldr REG2, [TMP1, #offsetof(zend_op_trace_info, counter)]
	|	ldrh TMP2w, [REG2]
	|	ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w
	|	strh TMP2w, [REG2]
	|	ble ->hybrid_hot_trace
	// Note: "REG1 + IP" is re-calculated as TMP1 is used as temporary register by the prior
	// ADD_SUB_32_WITH_CONST. Will optimize in the future if more temporary registers are available.
	|	add TMP1, REG1, IP
	|	ldr TMP2, [TMP1, #offsetof(zend_op_trace_info, orig_handler)]
	|	br TMP2

	return 1;
}

static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst)
{
	if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
		return 1;
	}

	|->hybrid_func_trace_counter:

	return zend_jit_hybrid_trace_counter_stub(Dst,
		((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1)  / JIT_G(hot_func)));
}

static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst)
{
	if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) {
		return 1;
	}

	|->hybrid_ret_trace_counter:

	return zend_jit_hybrid_trace_counter_stub(Dst,
		((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
}

static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst)
{
	if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
		return 1;
	}

	|->hybrid_loop_trace_counter:

	return zend_jit_hybrid_trace_counter_stub(Dst,
		((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}

static int zend_jit_trace_halt_stub(dasm_State **Dst)
{
	|->trace_halt:
	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		|	ADD_HYBRID_SPAD
		|	EXT_JMP zend_jit_halt_op->handler, REG0
	} else if (GCC_GLOBAL_REGS) {
		|	ldp x29, x30, [sp], # SPAD    // stack alignment
		|	mov IP, xzr                   // PC must be zero
		|	ret
	} else {
		|	ldp FP, RX, T2                // retore FP and IP
		|	ldp x29, x30, [sp], # NR_SPAD // stack alignment
		|	movn RETVALx, #0              // ZEND_VM_RETURN (-1)
		|	ret
	}
	return 1;
}

static int zend_jit_trace_exit_stub(dasm_State **Dst)
{
	|->trace_exit:
	|
	|	// Save CPU registers(32 GP regs + 32 FP regs) on stack in the order of d31 to x0
	|
	|	stp d30, d31, [sp, #-16]!
	|	stp d28, d29, [sp, #-16]!
	|	stp d26, d27, [sp, #-16]!
	|	stp d24, d25, [sp, #-16]!
	|	stp d22, d23, [sp, #-16]!
	|	stp d20, d21, [sp, #-16]!
	|	stp d18, d19, [sp, #-16]!
	|	stp d16, d17, [sp, #-16]!
	|	//stp d14, d15, [sp, #-16]!     // we don't use preserved registers yet
	|	//stp d12, d13, [sp, #-16]!
	|	//stp d10, d11, [sp, #-16]!
	|	//stp d8, d9, [sp, #-16]!
	|	stp d6, d7, [sp, #(-16*5)]!
	|	stp d4, d5, [sp, #-16]!
	|	stp d2, d3, [sp, #-16]!
	|	stp d0, d1, [sp, #-16]!
	|
	|	//str x30, [sp, #-16]!          // we don't use callee-saved registers yet (x31 can be omitted)
	|	stp x28, x29, [sp, #(-16*2)]!   // we have to store RX (x28)
	|	//stp x26, x27, [sp, #-16]!     // we don't use callee-saved registers yet
	|	//stp x24, x25, [sp, #-16]!
	|	//stp x22, x23, [sp, #-16]!
	|	//stp x20, x21, [sp, #-16]!
	|	//stp x18, x19, [sp, #-16]!
	|	//stp x16, x17, [sp, #-16]!     // we don't need temporary registers
	|	stp x14, x15, [sp, #-(16*7)]!
	|	stp x12, x13, [sp, #-16]!
	|	stp x10, x11, [sp, #-16]!
	|	stp x8, x9, [sp, #-16]!
	|	stp x6, x7, [sp, #-16]!
	|	stp x4, x5, [sp, #-16]!
	|	stp x2, x3, [sp, #-16]!
	|	stp x0, x1, [sp, #-16]!
	|
	|	mov FCARG1w, TMP1w              // exit_num
	|	mov FCARG2x, sp
	|
	|	// EX(opline) = opline
	|	SAVE_IP
	|	// zend_jit_trace_exit(trace_num, exit_num)
	|	EXT_CALL zend_jit_trace_exit, REG0
	|
	|	add sp, sp, #(32 * 16)          // restore sp
	|

	|	tst RETVALw, RETVALw
	|	bne >1  // not zero

	|	// execute_data = EG(current_execute_data)
	|	MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0
	|	// opline = EX(opline)
	|	LOAD_IP

	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		|	ADD_HYBRID_SPAD
		|	JMP_IP TMP1
	} else if (GCC_GLOBAL_REGS) {
		|	ldp x29, x30, [sp], # SPAD    // stack alignment
		|	JMP_IP TMP1
	} else {
		|	ldp FP, RX, T2                // retore FP and IP
		|	ldp x29, x30, [sp], # NR_SPAD // stack alignment
		|	mov RETVALx, #1               // ZEND_VM_ENTER
		|	ret
	}

	|1:
	|	blt ->trace_halt

	|	// execute_data = EG(current_execute_data)
	|	MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0
	|	// opline = EX(opline)
	|	LOAD_IP

	|	// check for interrupt (try to avoid this ???)
	|	MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1
	|	cbnz TMP1w, ->interrupt_handler

	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		|	ADD_HYBRID_SPAD
		|	ldr REG0, EX->func
		|	ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
		|	ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)]
		|	ldr REG0, [IP, REG0]
		|	br REG0
	} else if (GCC_GLOBAL_REGS) {
		|	ldp x29, x30, [sp], # SPAD    // stack alignment
		|	ldr REG0, EX->func
		|	ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
		|	ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)]
		|	ldr REG0, [IP, REG0]
		|	br REG0
	} else {
		|	ldr IP, EX->opline
		|	mov FCARG1x, FP
		|	ldr REG0, EX->func
		|	ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
		|	ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)]
		|	ldr REG0, [IP, REG0]
		|	blr REG0
		|
		|	tst RETVALw, RETVALw
		|	blt ->trace_halt
		|
		|	ldp FP, RX, T2                // retore FP and IP
		|	ldp x29, x30, [sp], # NR_SPAD // stack alignment
		|	mov RETVALx, #1               // ZEND_VM_ENTER
		|	ret
	}

	return 1;
}

static int zend_jit_trace_escape_stub(dasm_State **Dst)
{
	|->trace_escape:
	|
	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		|	ADD_HYBRID_SPAD
		|	JMP_IP, TMP1
	} else if (GCC_GLOBAL_REGS) {
		|	ldp x29, x30, [sp], # SPAD    // stack alignment
		|	JMP_IP, TMP1
	} else {
		|	ldp FP, RX, T2                // retore FP and IP
		|	ldp x29, x30, [sp], # NR_SPAD // stack alignment
		|	mov RETVALx, #1               // ZEND_VM_ENTER
		|	ret
	}

	return 1;
}

/* Keep 32 exit points in a single code block */
#define ZEND_JIT_EXIT_POINTS_SPACING    4 // bl = bytes
#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points

static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n)
{
	uint32_t i;

	|	bl >2
	|1:
	for (i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
		|	bl >2
	}
	|2:
	|	adr TMP1, <1
	|	sub TMP1, lr, TMP1
	|	lsr TMP1, TMP1, #2
	if (n) {
		|	ADD_SUB_32_WITH_CONST add, TMP1w, TMP1w, n, TMP2w
	}
	|	b ->trace_exit // pass exit_num in TMP1w

	return 1;
}

#ifdef CONTEXT_THREADED_JIT
static int zend_jit_context_threaded_call_stub(dasm_State **Dst)
{
	|->context_threaded_call:
	|	NIY_STUB	// TODO
	return 1;
}
#endif

static int zend_jit_assign_const_stub(dasm_State **Dst)
{
	zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
	uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;

	|->assign_const:
	|	stp x29, x30, [sp, #-32]!
	|	mov x29, sp
	if (!zend_jit_assign_to_variable(
			Dst, NULL,
			var_addr, var_addr, -1, -1,
			IS_CONST, val_addr, val_info,
			0, 0)) {
		return 0;
	}
	|	ldp x29, x30, [sp], #32
	|	ret
	return 1;
}

static int zend_jit_assign_tmp_stub(dasm_State **Dst)
{
	zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
	uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;

	|->assign_tmp:
	|	stp x29, x30, [sp, #-32]!
	|	mov x29, sp
	if (!zend_jit_assign_to_variable(
			Dst, NULL,
			var_addr, var_addr, -1, -1,
			IS_TMP_VAR, val_addr, val_info,
			0, 0)) {
		return 0;
	}
	|	ldp x29, x30, [sp], #32
	|	ret
	return 1;
}

static int zend_jit_assign_var_stub(dasm_State **Dst)
{
	zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
	uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF;

	|->assign_var:
	|	stp x29, x30, [sp, #-32]!
	|	mov x29, sp
	if (!zend_jit_assign_to_variable(
			Dst, NULL,
			var_addr, var_addr, -1, -1,
			IS_VAR, val_addr, val_info,
			0, 0)) {
		return 0;
	}
	|	ldp x29, x30, [sp], #32
	|	ret
	return 1;
}

static int zend_jit_assign_cv_noref_stub(dasm_State **Dst)
{
	zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
	uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/;

	|->assign_cv_noref:
	|	stp x29, x30, [sp, #-32]!
	|	mov x29, sp
	if (!zend_jit_assign_to_variable(
			Dst, NULL,
			var_addr, var_addr, -1, -1,
			IS_CV, val_addr, val_info,
			0, 0)) {
		return 0;
	}
	|	ldp x29, x30, [sp], #32
	|	ret
	return 1;
}

static int zend_jit_assign_cv_stub(dasm_State **Dst)
{
	zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
	uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/;

	|->assign_cv:
	|	stp x29, x30, [sp, #-32]!
	|	mov x29, sp
	if (!zend_jit_assign_to_variable(
			Dst, NULL,
			var_addr, var_addr, -1, -1,
			IS_CV, val_addr, val_info,
			0, 0)) {
		return 0;
	}
	|	ldp x29, x30, [sp], #32
	|	ret
	return 1;
}

static const zend_jit_stub zend_jit_stubs[] = {
	JIT_STUB(interrupt_handler,         SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(exception_handler,         SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(exception_handler_undef,   SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(exception_handler_free_op1_op2, SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(exception_handler_free_op2,     SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(leave_function,            SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(leave_throw,               SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(icall_throw,               SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(throw_cannot_pass_by_ref,  SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(undefined_offset,          SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(undefined_index,           SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(cannot_add_element,        SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(undefined_offset_ex,       SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(undefined_index_ex,        SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(cannot_add_element_ex,     SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(undefined_function,        SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(negative_shift,            SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(mod_by_zero,               SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(invalid_this,              SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(trace_halt,                SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(trace_exit,                SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(trace_escape,              SP_ADJ_JIT,  SP_ADJ_VM),
	JIT_STUB(hybrid_runtime_jit,        SP_ADJ_VM,   SP_ADJ_NONE),
	JIT_STUB(hybrid_profile_jit,        SP_ADJ_VM,   SP_ADJ_NONE),
	JIT_STUB(hybrid_hot_code,           SP_ADJ_VM,   SP_ADJ_NONE),
	JIT_STUB(hybrid_func_hot_counter,   SP_ADJ_VM,   SP_ADJ_NONE),
	JIT_STUB(hybrid_loop_hot_counter,   SP_ADJ_VM,   SP_ADJ_NONE),
	JIT_STUB(hybrid_hot_trace,          SP_ADJ_VM,   SP_ADJ_NONE),
	JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM,   SP_ADJ_NONE),
	JIT_STUB(hybrid_ret_trace_counter,  SP_ADJ_VM,   SP_ADJ_NONE),
	JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM,   SP_ADJ_NONE),
	JIT_STUB(assign_const,              SP_ADJ_RET,  SP_ADJ_ASSIGN),
	JIT_STUB(assign_tmp,                SP_ADJ_RET,  SP_ADJ_ASSIGN),
	JIT_STUB(assign_var,                SP_ADJ_RET,  SP_ADJ_ASSIGN),
	JIT_STUB(assign_cv_noref,           SP_ADJ_RET,  SP_ADJ_ASSIGN),
	JIT_STUB(assign_cv,                 SP_ADJ_RET,  SP_ADJ_ASSIGN),
#ifdef CONTEXT_THREADED_JIT
	JIT_STUB(context_threaded_call,     SP_ADJ_NONE, SP_ADJ_NONE),
#endif
};

#ifdef HAVE_GDB
# if 0
typedef struct _Unwind_Context _Unwind_Context;
typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *);
extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *);
extern uintptr_t _Unwind_GetCFA(_Unwind_Context *);

typedef struct _zend_jit_unwind_arg {
	int cnt;
	uintptr_t cfa[3];
} zend_jit_unwind_arg;

static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a)
{
	zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a;
	arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx);
	arg->cnt++;
	if (arg->cnt == 3) {
		return 5; // _URC_END_OF_STACK
	}
	return 0;     // _URC_NO_REASON;
}

static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data)
{
	zend_jit_unwind_arg arg;

	memset(&arg, 0, sizeof(arg));
	_Unwind_Backtrace(zend_jit_unwind_cb, &arg);
	if (arg.cnt == 3) {
		sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1];
	}
}
# else
static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data)
{
	uintptr_t ret;

	__asm__ (
		"ldr %0, [x29]\n\t"
		"sub %0 ,%0, x29"
		: "=r"(ret));

	sp_adj[SP_ADJ_VM] = ret;
}
# endif

extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data);

static zend_never_inline void zend_jit_set_sp_adj_vm(void)
{
	void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *);

	orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data;
	zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data;
	execute_ex(NULL);                                        // set sp_adj[SP_ADJ_VM]
	zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data;
}
#endif

static int zend_jit_setup(void)
{
	allowed_opt_flags = 0;

#if ZTS
	tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
	ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0);
#endif

    memset(sp_adj, 0, sizeof(sp_adj));
#ifdef HAVE_GDB
	sp_adj[SP_ADJ_RET] = 0;
	sp_adj[SP_ADJ_ASSIGN] = 32;
	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		zend_jit_set_sp_adj_vm();                                // set sp_adj[SP_ADJ_VM]
#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
		|| sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD
#else
		|| sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM];
#endif
	} else if (GCC_GLOBAL_REGS) {
		|| sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD;       // sub r4, SPAD
	} else {
		|| sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD;    // sub r4, NR_SPAD
	}
#endif

	return SUCCESS;
}

static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst)
{
	|	brk #0
	return 1;
}

static int zend_jit_align_func(dasm_State **Dst)
{
	reuse_ip = 0;
	delayed_call_chain = 0;
	last_valid_opline = NULL;
	use_last_vald_opline = 0;
	track_last_valid_opline = 0;
	jit_return_label = -1;
	|.align 16
	return 1;
}

static int zend_jit_prologue(dasm_State **Dst)
{
	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		|	SUB_HYBRID_SPAD
	} else if (GCC_GLOBAL_REGS) {
		|	stp x29, x30, [sp, # -SPAD]!    // stack alignment
		|//	mov x29, sp
	} else {
		|	stp x29, x30, [sp, # -NR_SPAD]! // stack alignment
		|//	mov	x29, sp
		|	stp FP, RX, T2                  // save FP and IP
		|	mov FP, FCARG1x
	}
	return 1;
}

static int zend_jit_label(dasm_State **Dst, unsigned int label)
{
	|=>label:
	return 1;
}

static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level)
{
	|	// call->prev_execute_data = EX(call);
	if (call_level == 1) {
		|	str xzr, EX:RX->prev_execute_data
	} else {
		|	ldr REG0, EX->call
		|	str REG0, EX:RX->prev_execute_data
	}
	|	// EX(call) = call;
	|	str RX, EX->call

	delayed_call_chain = 0;

	return 1;
}

static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline)
{
	if (last_valid_opline == opline) {
		zend_jit_use_last_valid_opline();
	} else if (GCC_GLOBAL_REGS && last_valid_opline) {
		zend_jit_use_last_valid_opline();
		|	LOAD_64BIT_VAL TMP1, (opline - last_valid_opline) * sizeof(zend_op)
		|	ADD_IP TMP1, TMP2
	} else {
		|	LOAD_IP_ADDR opline
	}
	zend_jit_set_last_valid_opline(opline);

	return 1;
}

static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg)
{
	return zend_jit_set_ip(Dst, opline);
}

static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline)
{
	if (delayed_call_chain) {
		if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
			return 0;
		}
	}
	if (!zend_jit_set_ip(Dst, opline)) {
		return 0;
	}
	reuse_ip = 0;
	return 1;
}

static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr)
{
	|	MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1
	if (exit_addr) {
		|	cbnz TMP1w, &exit_addr
	} else if (last_valid_opline == opline) {
		||	zend_jit_use_last_valid_opline();
		|	cbnz TMP1w, ->interrupt_handler
	} else {
		|	cbnz TMP1w, >1
		|.cold_code
		|1:
		|	LOAD_IP_ADDR opline
		|	b ->interrupt_handler
		|.code
	}
	return 1;
}

static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr)
{
	if (timeout_exit_addr) {
		|	MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1
		|	cbz TMP1w, =>loop_label
		|	b &timeout_exit_addr
	} else {
		|	b =>loop_label
	}
	return 1;
}

static int zend_jit_check_exception(dasm_State **Dst)
{
	|	MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1
	|	cbnz TMP2, ->exception_handler
	return 1;
}

static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline)
{
	if (opline->result_type & (IS_TMP_VAR|IS_VAR)) {
		|	MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1
		|	cbnz TMP2, ->exception_handler_undef
		return 1;
	}
	return zend_jit_check_exception(Dst);
}

static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num)
{

	current_trace_num = trace_num;

	|	// EG(jit_trace_num) = trace_num;
	|	LOAD_32BIT_VAL TMP1w, trace_num
	|	MEM_STORE_32_ZTS str, TMP1w, executor_globals, jit_trace_num, TMP2

	return 1;
}

static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t)
{
	uint32_t i;
	const void *exit_addr;

	/* Emit veneers table for exit points (B instruction for each exit number) */
	|.cold_code
	for (i = 0; i < t->exit_count; i++) {
		exit_addr = zend_jit_trace_get_exit_addr(i);
		if (!exit_addr) {
			return 0;
		}
		|	b &exit_addr
	}
	|=>1: // end of the code
	|.code
	return 1;
}

static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr)
{
	int ret = 0;
	uint8_t *p, *end;
	const void *veneer = NULL;
	ptrdiff_t delta;

	if (jmp_table_size) {
		const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*)));

		do {
			if (*jmp_slot == from_addr) {
				*jmp_slot = to_addr;
				ret++;
			}
			jmp_slot++;
		} while (--jmp_table_size);
	}

	end = (uint8_t*)code;
	p = end + size;
	while (p > end) {
		uint32_t *ins_ptr;
		uint32_t ins;

		p -= 4;
		ins_ptr = (uint32_t*)p;
		ins = *ins_ptr;
		if ((ins & 0xfc000000u) == 0x14000000u) {
			// B (imm26:0..25)
			delta = (uint32_t*)from_addr - ins_ptr;
			if (((ins ^ (uint32_t)delta) & 0x01ffffffu) == 0) {
				delta = (uint32_t*)to_addr - ins_ptr;
				if (((delta + 0x02000000) >> 26) != 0) {
					abort(); // brnach target out of range
				}
				*ins_ptr = (ins & 0xfc000000u) | ((uint32_t)delta & 0x03ffffffu);
				ret++;
				if (!veneer) {
					veneer = p;
				}
			}
		} else if ((ins & 0xff000000u) == 0x54000000u ||
		           (ins & 0x7e000000u) == 0x34000000u) {
			// B.cond, CBZ, CBNZ (imm19:5..23)
			delta = (uint32_t*)from_addr - ins_ptr;
			if (((ins ^ ((uint32_t)delta << 5)) & 0x00ffffe0u) == 0) {
				delta = (uint32_t*)to_addr - ins_ptr;
				if (((delta + 0x40000) >> 19) != 0) {
					if (veneer) {
						delta = (uint32_t*)veneer - ins_ptr;
						if (((delta + 0x40000) >> 19) != 0) {
							abort(); // brnach target out of range
						}
					} else {
						abort(); // brnach target out of range
					}
				}
				*ins_ptr = (ins & 0xff00001fu) | (((uint32_t)delta & 0x7ffffu) << 5);
				ret++;
			}
		} else if ((ins & 0x7e000000u) == 0x36000000u) {
			// TBZ, TBNZ (imm14:5..18)
			delta = (uint32_t*)from_addr - ins_ptr;
			if (((ins ^ ((uint32_t)delta << 5)) & 0x0007ffe0u) == 0) {
				delta = (uint32_t*)to_addr - ins_ptr;
				if (((delta + 0x2000) >> 14) != 0) {
					if (veneer) {
						delta = (uint32_t*)veneer - ins_ptr;
						if (((delta + 0x2000) >> 14) != 0) {
							abort(); // brnach target out of range
						}
					} else {
						abort(); // brnach target out of range
					}
				}
				*ins_ptr = (ins & 0xfff8001fu) | (((uint32_t)delta & 0x3fffu) << 5);
				ret++;
			}
		}
	}

	JIT_CACHE_FLUSH(code, (char*)code + size);

#ifdef HAVE_VALGRIND
	VALGRIND_DISCARD_TRANSLATIONS(code, size);
#endif

	return ret;
}

static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr)
{
	return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
}

static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr)
{
	const void *link_addr;
	size_t prologue_size;

	/* Skip prologue. */
	// TODO: don't hardcode this ???
	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
		prologue_size = 0;
#else
		// sub sp, sp, #0x20
		prologue_size = 4;
#endif
	} else if (GCC_GLOBAL_REGS) {
		// stp x29, x30, [sp, # -SPAD]!
		prologue_size = 4;
	} else {
		// stp x29, x30, [sp, # -NR_SPAD]! // stack alignment
		// stp FP, RX, T2
		// mov FP, FCARG1x
		prologue_size = 12;
	}
	link_addr = (const void*)((const char*)t->code_start + prologue_size);

	if (timeout_exit_addr) {
		/* Check timeout for links to LOOP */
		|	MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1
		|	cbz TMP1w, &link_addr
		|	b &timeout_exit_addr
	} else {
		|	b &link_addr
	}
	return 1;
}

static int zend_jit_trace_return(dasm_State **Dst, bool original_handler, const zend_op *opline)
{
	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		|	ADD_HYBRID_SPAD
		if (!original_handler) {
			|	JMP_IP TMP1
		} else {
			|	ldr REG0, EX->func
			|	ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
			|	ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)]
			|	ldr REG0, [IP, REG0]
			|	br REG0
		}
	} else if (GCC_GLOBAL_REGS) {
		|	ldp x29, x30, [sp], # SPAD // stack alignment
		if (!original_handler) {
			|	JMP_IP TMP1
		} else {
			|	ldr REG0, EX->func
			|	ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
			|	ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)]
			|	ldr REG0, [IP, REG0]
			|	br REG0
		}
	} else {
		if (original_handler) {
			|	mov FCARG1x, FP
			|	ldr REG0, EX->func
			|	ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
			|	ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)]
			|	ldr REG0, [IP, REG0]
			|	blr REG0
		}
		|	ldp FP, RX, T2                // retore FP and IP
		|	ldp x29, x30, [sp], # NR_SPAD // stack alignment
		if (!original_handler || !opline ||
		    (opline->opcode != ZEND_RETURN
		  && opline->opcode != ZEND_RETURN_BY_REF
		  && opline->opcode != ZEND_GENERATOR_RETURN
		  && opline->opcode != ZEND_GENERATOR_CREATE
		  && opline->opcode != ZEND_YIELD
		  && opline->opcode != ZEND_YIELD_FROM)) {
			|	mov RETVALx, #2               // ZEND_VM_LEAVE
		}
		|	ret
	}
	return 1;
}

static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type)
{
	int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
	const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
	zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);

	if (!exit_addr) {
		return 0;
	}

	|	IF_NOT_ZVAL_TYPE var_addr, type, &exit_addr, ZREG_TMP1

	return 1;
}

static int zend_jit_scalar_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var)
{
	int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
	const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

	if (!exit_addr) {
		return 0;
	}
	|	MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, var+offsetof(zval, u1.v.type), TMP1
	|	cmp TMP1w, #IS_STRING
	|	bhs &exit_addr

	return 1;
}
static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info)
{
	int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
	const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
	zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);

	if (!exit_addr) {
		return 0;
	}

	|	GET_ZVAL_LVAL ZREG_FCARG1, var_addr, TMP1
	if (op_info & MAY_BE_ARRAY_PACKED) {
		|	ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
		|	TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
		|	beq &exit_addr
	} else {
		|	ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
		|	TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
		|	bne &exit_addr
	}

	return 1;
}

static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace)
{
	zend_jit_op_array_trace_extension *jit_extension =
		(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
	size_t offset = jit_extension->offset;
	const void *handler =
		(zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;

	if (!zend_jit_set_valid_ip(Dst, opline)) {
		return 0;
	}
	if (!GCC_GLOBAL_REGS) {
		|	mov FCARG1x, FP
	}
	|	EXT_CALL handler, REG0
	if (may_throw
	 && opline->opcode != ZEND_RETURN
	 && opline->opcode != ZEND_RETURN_BY_REF) {
		|	MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
		|	cbnz REG0, ->exception_handler
	}

	while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) {
		trace++;
	}

	if (!GCC_GLOBAL_REGS
	 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) {
		if (opline->opcode == ZEND_RETURN ||
		    opline->opcode == ZEND_RETURN_BY_REF ||
		    opline->opcode == ZEND_DO_UCALL ||
		    opline->opcode == ZEND_DO_FCALL_BY_NAME ||
		    opline->opcode == ZEND_DO_FCALL ||
		    opline->opcode == ZEND_GENERATOR_CREATE) {
			|	MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1
		}
	}

	if (zend_jit_trace_may_exit(op_array, opline)) {
		if (opline->opcode == ZEND_RETURN ||
		    opline->opcode == ZEND_RETURN_BY_REF ||
		    opline->opcode == ZEND_GENERATOR_CREATE) {

			if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#if 0
				/* this check should be handled by the following OPLINE guard or jmp [IP] */
				|	LOAD_ADDR TMP1, zend_jit_halt_op
				|	cmp IP, TMP1
				|	beq ->trace_halt
#endif
			} else if (GCC_GLOBAL_REGS) {
				|	cbz IP, ->trace_halt
			} else {
				|	tst RETVALw, RETVALw
				|	blt ->trace_halt
			}
		} else if (opline->opcode == ZEND_EXIT ||
		           opline->opcode == ZEND_GENERATOR_RETURN ||
		           opline->opcode == ZEND_YIELD ||
		           opline->opcode == ZEND_YIELD_FROM) {
			|	b ->trace_halt
		}
		if (trace->op != ZEND_JIT_TRACE_END ||
		    (trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
		     trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {

			const zend_op *next_opline = trace->opline;
			const zend_op *exit_opline = NULL;
			uint32_t exit_point;
			const void *exit_addr;
			uint32_t old_info = 0;
			uint32_t old_res_info = 0;
			zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;

			if (zend_is_smart_branch(opline)) {
				bool exit_if_true = 0;
				exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true);
			} else {
				switch (opline->opcode) {
					case ZEND_JMPZ:
					case ZEND_JMPNZ:
					case ZEND_JMPZ_EX:
					case ZEND_JMPNZ_EX:
					case ZEND_JMP_SET:
					case ZEND_COALESCE:
					case ZEND_JMP_NULL:
					case ZEND_FE_RESET_R:
					case ZEND_FE_RESET_RW:
						exit_opline = (trace->opline == opline + 1) ?
							OP_JMP_ADDR(opline, opline->op2) :
							opline + 1;
						break;
					case ZEND_JMPZNZ:
						exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ?
							ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
							OP_JMP_ADDR(opline, opline->op2);
						break;
					case ZEND_FE_FETCH_R:
					case ZEND_FE_FETCH_RW:
						exit_opline = (trace->opline == opline + 1) ?
							ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
							opline + 1;
						break;

				}
			}

			switch (opline->opcode) {
				case ZEND_FE_FETCH_R:
				case ZEND_FE_FETCH_RW:
					if (opline->op2_type != IS_UNUSED) {
						old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var));
						SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1);
					}
					break;
			}

			if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
				old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
				SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
			}
			exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
			exit_addr = zend_jit_trace_get_exit_addr(exit_point);

			if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
				SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
			}
			switch (opline->opcode) {
				case ZEND_FE_FETCH_R:
				case ZEND_FE_FETCH_RW:
					if (opline->op2_type != IS_UNUSED) {
						SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info);
					}
					break;
			}

			if (!exit_addr) {
				return 0;
			}
			|	CMP_IP next_opline, TMP1, TMP2
			|	bne &exit_addr
		}
	}

	zend_jit_set_last_valid_opline(trace->opline);

	return 1;
}

static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw)
{
	const void *handler;

	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		handler = zend_get_opcode_handler_func(opline);
	} else {
		handler = opline->handler;
	}

	if (!zend_jit_set_valid_ip(Dst, opline)) {
		return 0;
	}
	if (!GCC_GLOBAL_REGS) {
		|	mov FCARG1x, FP
	}
	|	EXT_CALL handler, REG0
	if (may_throw) {
		zend_jit_check_exception(Dst);
	}

	/* Skip the following OP_DATA */
	switch (opline->opcode) {
		case ZEND_ASSIGN_DIM:
		case ZEND_ASSIGN_OBJ:
		case ZEND_ASSIGN_STATIC_PROP:
		case ZEND_ASSIGN_DIM_OP:
		case ZEND_ASSIGN_OBJ_OP:
		case ZEND_ASSIGN_STATIC_PROP_OP:
		case ZEND_ASSIGN_STATIC_PROP_REF:
		case ZEND_ASSIGN_OBJ_REF:
			zend_jit_set_last_valid_opline(opline + 2);
			break;
		default:
			zend_jit_set_last_valid_opline(opline + 1);
			break;
	}

	return 1;
}

static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline)
{
	if (!zend_jit_set_valid_ip(Dst, opline)) {
		return 0;
	}
	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		if (opline->opcode == ZEND_DO_UCALL ||
		    opline->opcode == ZEND_DO_FCALL_BY_NAME ||
		    opline->opcode == ZEND_DO_FCALL ||
		    opline->opcode == ZEND_RETURN) {

			/* Use inlined HYBRID VM handler */
			const void *handler = opline->handler;

			|	ADD_HYBRID_SPAD
			|	EXT_JMP handler, REG0
		} else {
			const void *handler = zend_get_opcode_handler_func(opline);

			|	EXT_CALL handler, REG0
			|	ADD_HYBRID_SPAD
			|	JMP_IP TMP1
		}
	} else {
		const void *handler = opline->handler;

		if (GCC_GLOBAL_REGS) {
			|	ldp x29, x30, [sp], # SPAD // stack alignment
		} else {
			|	mov FCARG1x, FP
			|	ldp FP, RX, T2                // retore FP and IP
			|	ldp x29, x30, [sp], # NR_SPAD // stack alignment
		}
		|	EXT_JMP handler, REG0
	}
	zend_jit_reset_last_valid_opline();
	return 1;
}

static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline)
{
	uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0);
	const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

	if (!exit_addr) {
		return 0;
	}
	|	CMP_IP opline, TMP1, TMP2
	|	bne &exit_addr

	zend_jit_set_last_valid_opline(opline);

	return 1;
}

static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label)
{
	|	b =>target_label
	return 1;
}

static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label)
{
	|	CMP_IP next_opline, TMP1, TMP2
	|	bne =>target_label

	zend_jit_set_last_valid_opline(next_opline);

	return 1;
}

#ifdef CONTEXT_THREADED_JIT
static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block)
{
	|	NIY	// TODO
	return 1;
}
#endif

static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block)
{
#ifdef CONTEXT_THREADED_JIT
	return zend_jit_context_threaded_call(Dst, opline, next_block);
#else
	return zend_jit_tail_handler(Dst, opline);
#endif
}

static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type)
{
	ZEND_ASSERT(Z_MODE(src) == IS_REG);
	ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);

	if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
		|	SET_ZVAL_LVAL_FROM_REG dst, Rx(Z_REG(src)), TMP1
		if (set_type &&
		    (Z_REG(dst) != ZREG_FP ||
		     !JIT_G(current_frame) ||
		     STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
			|	SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2
		}
	} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
		|	SET_ZVAL_DVAL dst, Z_REG(src), ZREG_TMP1
		if (set_type &&
		    (Z_REG(dst) != ZREG_FP ||
		     !JIT_G(current_frame) ||
		     STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
			|	SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2
		}
	} else {
		ZEND_UNREACHABLE();
	}
	return 1;
}

static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
	ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL);
	ZEND_ASSERT(Z_MODE(dst) == IS_REG);

	if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
		|	GET_ZVAL_LVAL Z_REG(dst), src, TMP1
	} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
		|	GET_ZVAL_DVAL Z_REG(dst), src, ZREG_TMP1
	} else {
		ZEND_UNREACHABLE();
	}
	return 1;
}

static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type)
{
	zend_jit_addr src = ZEND_ADDR_REG(reg);
	zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));

	return zend_jit_spill_store(Dst, src, dst, info, set_type);
}

static int zend_jit_store_var_type(dasm_State **Dst, int var, uint32_t type)
{
	zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));

	|	SET_ZVAL_TYPE_INFO dst, type, TMP1w, TMP2
	return 1;
}

static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info)
{
	if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
		zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
		return zend_jit_spill_store(Dst, src, dst, info, 1);
	}
	return 1;
}

static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info)
{
	if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
		zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
		bool set_type = 1;

		if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) ==
		    (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) {
			if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) {
				set_type = 0;
			}
		}
		return zend_jit_spill_store(Dst, src, dst, info, set_type);
	}
	return 1;
}

static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg)
{
	zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
	zend_jit_addr dst = ZEND_ADDR_REG(reg);

	return zend_jit_load_reg(Dst, src, dst, info);
}

static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, zend_uchar op_type, zend_jit_addr addr, znode_op op)
{
	if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) {
		zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var);
		|	SET_ZVAL_TYPE_INFO dst, IS_UNDEF, TMP1w, TMP2
	}
	return 1;
}

static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
	if (!zend_jit_same_addr(src, dst)) {
		if (Z_MODE(src) == IS_REG) {
			if (Z_MODE(dst) == IS_REG) {
				if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
					|	mov Rx(Z_REG(dst)), Rx(Z_REG(src))
				} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
					|	fmov Rd(Z_REG(dst)-ZREG_V0), Rd(Z_REG(src)-ZREG_V0)
				} else {
					ZEND_UNREACHABLE();
				}
				if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) {
					zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);

					if (!zend_jit_spill_store(Dst, dst, var_addr, info,
							JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
							JIT_G(current_frame) == NULL ||
							STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
							(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
					)) {
						return 0;
					}
				}
			} else if (Z_MODE(dst) == IS_MEM_ZVAL) {
				if (!Z_LOAD(src) && !Z_STORE(src)) {
					if (!zend_jit_spill_store(Dst, src, dst, info,
							JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
							JIT_G(current_frame) == NULL ||
							STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
							(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
					)) {
						return 0;
					}
				}
			} else {
				ZEND_UNREACHABLE();
			}
		} else if (Z_MODE(src) == IS_MEM_ZVAL) {
			if (Z_MODE(dst) == IS_REG) {
				if (!zend_jit_load_reg(Dst, src, dst, info)) {
					return 0;
				}
			} else {
				ZEND_UNREACHABLE();
			}
		} else {
			ZEND_UNREACHABLE();
		}
	} else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) {
		dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
		if (!zend_jit_spill_store(Dst, src, dst, info,
				JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
				JIT_G(current_frame) == NULL ||
				STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
				(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
		)) {
			return 0;
		}
	}
	return 1;
}

static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline)
{
	zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);

	|	IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1

	if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
		if (!zend_jit_save_call_chain(Dst, -1)) {
			return 0;
		}
	}

	ZEND_ASSERT(opline);

	if ((opline-1)->opcode != ZEND_FETCH_CONSTANT
	 && (opline-1)->opcode != ZEND_FETCH_LIST_R
	 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR))
	 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) {
		val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var);

		|	IF_NOT_ZVAL_REFCOUNTED val_addr, >2, ZREG_TMP1, ZREG_TMP2
		|	GET_ZVAL_PTR TMP1, val_addr, TMP2
		|	GC_ADDREF TMP1, TMP2w
		|2:
	}

	|	LOAD_IP_ADDR (opline - 1)
	|	b ->trace_escape
	|1:

	return 1;
}

static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg)
{
	zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));

	if (reg == ZREG_LONG_MIN_MINUS_1) {
		uint64_t val = 0xc3e0000000000000;
		|	SET_ZVAL_LVAL dst, val, TMP1, TMP2
		|	SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2
	} else if (reg == ZREG_LONG_MIN) {
		uint64_t val = 0x8000000000000000;
		|	SET_ZVAL_LVAL dst, val, TMP1, TMP2
		|	SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2
	} else if (reg == ZREG_LONG_MAX) {
		uint64_t val = 0x7fffffffffffffff;
		|	SET_ZVAL_LVAL dst, val, TMP1, TMP2
		|	SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2
	} else if (reg == ZREG_LONG_MAX_PLUS_1) {
		uint64_t val = 0x43e0000000000000;
		|	SET_ZVAL_LVAL dst, val, TMP1, TMP2
		|	SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2
	} else if (reg == ZREG_NULL) {
		|	SET_ZVAL_TYPE_INFO dst, IS_NULL, TMP1w, TMP2
	} else if (reg == ZREG_ZVAL_TRY_ADDREF) {
		|	IF_NOT_ZVAL_REFCOUNTED dst, >1, ZREG_TMP1, ZREG_TMP2
		|	GET_ZVAL_PTR TMP1, dst, TMP2
		|	GC_ADDREF TMP1, TMP2w
		|1:
	} else if (reg == ZREG_ZVAL_COPY_GPR0) {
		zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);

		|	ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
		|	TRY_ADDREF -1, REG1w, REG2, TMP1w
	} else {
		ZEND_UNREACHABLE();
	}
	return 1;
}

static int zend_jit_free_trampoline(dasm_State **Dst)
{
	|	// if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
	|	ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)]
	|	TST_32_WITH_CONST TMP1w, ZEND_ACC_CALL_VIA_TRAMPOLINE, TMP2w
	|	beq >1
	|	mov FCARG1x, REG0
	|	EXT_CALL zend_jit_free_trampoline_helper, REG0
	|1:
	return 1;
}

static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
{
	if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) {
		|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1
	}
	if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
		|	ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
	}
	if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) {
		return 0;
	}
	if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
		|	LONG_ADD_SUB_WITH_IMM adds, op1_def_addr, Z_L(1), TMP1, TMP2
	} else {
		|	LONG_ADD_SUB_WITH_IMM subs, op1_def_addr, Z_L(1), TMP1, TMP2
	}

	if (may_overflow &&
	    (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) ||
	     ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) {
		int32_t exit_point;
		const void *exit_addr;
		zend_jit_trace_stack *stack;
		uint32_t old_op1_info, old_res_info = 0;

		stack = JIT_G(current_frame)->stack;
		old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
		SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0);
		if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
			SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1);
		} else {
			SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1);
		}
		if (opline->result_type != IS_UNUSED) {
			old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
			if (opline->opcode == ZEND_PRE_INC) {
				SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
				SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1);
			} else if (opline->opcode == ZEND_PRE_DEC) {
				SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
				SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1);
			} else if (opline->opcode == ZEND_POST_INC) {
				SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
				SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX);
			} else if (opline->opcode == ZEND_POST_DEC) {
				SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
				SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN);
			}
		}

		exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
		exit_addr = zend_jit_trace_get_exit_addr(exit_point);
		if (!exit_addr) {
			return 0;
		}
		|	bvs &exit_addr

		if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
		    opline->result_type != IS_UNUSED) {
			|	ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
		}

		SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
		if (opline->result_type != IS_UNUSED) {
			SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
		}
	} else if (may_overflow) {
		|	bvs >1
		if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
		    opline->result_type != IS_UNUSED) {
			|	ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
		}
		|.cold_code
		|1:
		if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
			uint64_t val = 0x43e0000000000000;
			if (Z_MODE(op1_def_addr) == IS_REG) {
				|	LOAD_64BIT_VAL TMP1, val
				|	fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1
			} else {
				|	SET_ZVAL_LVAL op1_def_addr, val, TMP2, TMP1
			}
		} else {
			uint64_t val = 0xc3e0000000000000;
			if (Z_MODE(op1_def_addr) == IS_REG) {
				|	LOAD_64BIT_VAL TMP1, val
				|	fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1
			} else {
				|	SET_ZVAL_LVAL op1_def_addr, val, TMP2, TMP1
			}
		}
		if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) {
			|	SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE, TMP1w, TMP2
		}
		if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
		    opline->result_type != IS_UNUSED) {
			|	ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
		}
		|	b >3
		|.code
	} else {
		if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
		    opline->result_type != IS_UNUSED) {
			|	ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
		}
	}
	if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
		|.cold_code
		|2:
		if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
			|	SET_EX_OPLINE opline, REG0
			if (op1_info & MAY_BE_UNDEF) {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2, ZREG_TMP1
				|	// zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
				|	LOAD_32BIT_VAL FCARG1w, opline->op1.var
				|	EXT_CALL zend_jit_undefined_op_helper, REG0
				|	SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2
				op1_info |= MAY_BE_NULL;
			}
			|2:
			|	LOAD_ZVAL_ADDR FCARG1x, op1_addr

			|	// ZVAL_DEREF(var_ptr);
			if (op1_info & MAY_BE_REF) {
				|	IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >2, TMP1w
				|	GET_Z_PTR FCARG1x, FCARG1x
				|	ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
				|	cbz TMP1, >1
				if (RETURN_VALUE_USED(opline)) {
					|	LOAD_ZVAL_ADDR FCARG2x, res_addr
				} else {
					|	mov FCARG2x, xzr
				}
				if (opline->opcode == ZEND_PRE_INC) {
					|	EXT_CALL zend_jit_pre_inc_typed_ref, REG0
				} else if (opline->opcode == ZEND_PRE_DEC) {
					|	EXT_CALL zend_jit_pre_dec_typed_ref, REG0
				} else if (opline->opcode == ZEND_POST_INC) {
					|	EXT_CALL zend_jit_post_inc_typed_ref, REG0
				} else if (opline->opcode == ZEND_POST_DEC) {
					|	EXT_CALL zend_jit_post_dec_typed_ref, REG0
				} else {
					ZEND_UNREACHABLE();
				}
				zend_jit_check_exception(Dst);
				|	b >3
				|1:
				|	add FCARG1x, FCARG1x, #offsetof(zend_reference, val)
				|2:
			}

			if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
				zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);

				|	ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
				|	TRY_ADDREF op1_info, REG0w, REG2, TMP1w
			}
			if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
				if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) {
					|	LOAD_ZVAL_ADDR FCARG2x, res_addr
					|	EXT_CALL zend_jit_pre_inc, REG0
				} else {
					|	EXT_CALL increment_function, REG0
				}
			} else {
				if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) {
					|	LOAD_ZVAL_ADDR FCARG2x, res_addr
					|	EXT_CALL zend_jit_pre_dec, REG0
				} else {
					|	EXT_CALL decrement_function, REG0
				}
			}
			if (may_throw) {
				zend_jit_check_exception(Dst);
			}
		} else {
			zend_reg tmp_reg;

			if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
				|	ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
			}
			if (Z_MODE(op1_def_addr) == IS_REG) {
				tmp_reg = Z_REG(op1_def_addr);
			} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
				tmp_reg = Z_REG(op1_addr);
			} else {
				tmp_reg = ZREG_FPR0;
			}
			|	GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1
			if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
				uint64_t val = 0x3ff0000000000000; // 1.0
				|	LOAD_64BIT_VAL TMP1, val
				|	fmov FPTMP, TMP1
				|	fadd Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP
			} else {
				uint64_t val = 0x3ff0000000000000; // 1.0
				|	LOAD_64BIT_VAL TMP1, val
				|	fmov FPTMP, TMP1
				|	fsub Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP
			}
			|	SET_ZVAL_DVAL op1_def_addr, tmp_reg, ZREG_TMP1
			if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
			    opline->result_type != IS_UNUSED) {
				|	ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
				|	TRY_ADDREF op1_def_info, REG0w, REG1, TMP1w
			}
		}
		|	b >3
		|.code
	}
	|3:
	if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) {
		return 0;
	}
	if (opline->result_type != IS_UNUSED) {
		if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
			return 0;
		}
	}
	return 1;
}

static int zend_jit_opline_uses_reg(const zend_op  *opline, int8_t reg)
{
	if ((opline+1)->opcode == ZEND_OP_DATA
	 && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV))
	 && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) {
		return 1;
	}
	return
		((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) &&
			JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) ||
		((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) &&
			JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) ||
		((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) &&
			JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg);
}

static int zend_jit_math_long_long(dasm_State    **Dst,
                                   const zend_op  *opline,
                                   zend_uchar      opcode,
                                   zend_jit_addr   op1_addr,
                                   zend_jit_addr   op2_addr,
                                   zend_jit_addr   res_addr,
                                   uint32_t        res_info,
                                   uint32_t        res_use_info,
                                   int             may_overflow)
{
	bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
	zend_reg result_reg;
	zend_reg tmp_reg = ZREG_REG0;
	bool use_ovf_flag = 1;

	if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) {
		if (may_overflow && (res_info & MAY_BE_GUARD)
		 && JIT_G(current_frame)
		 && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) {
			result_reg = ZREG_REG0;
		} else {
			result_reg = Z_REG(res_addr);
		}
	} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) {
		result_reg = Z_REG(op1_addr);
	} else if (Z_REG(res_addr) != ZREG_REG0) {
		result_reg = ZREG_REG0;
	} else {
		/* ASSIGN_DIM_OP */
		result_reg = ZREG_FCARG1;
		tmp_reg = ZREG_FCARG1;
	}

	if (opcode == ZEND_MUL &&
			Z_MODE(op2_addr) == IS_CONST_ZVAL &&
			Z_LVAL_P(Z_ZV(op2_addr)) == 2) {
		if (Z_MODE(op1_addr) == IS_REG) {
			|	adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr))
		} else {
			|	GET_ZVAL_LVAL result_reg, op1_addr, TMP1
			|	adds Rx(result_reg), Rx(result_reg), Rx(result_reg)
		}
	} else if (opcode == ZEND_MUL &&
			Z_MODE(op2_addr) == IS_CONST_ZVAL &&
			!may_overflow &&
			zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) {
		|	GET_ZVAL_LVAL result_reg, op1_addr, TMP1
		|	mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr)))
		|	lsl Rx(result_reg), Rx(result_reg), TMP1
	} else if (opcode == ZEND_MUL &&
			Z_MODE(op1_addr) == IS_CONST_ZVAL &&
			Z_LVAL_P(Z_ZV(op1_addr)) == 2) {
		if (Z_MODE(op2_addr) == IS_REG) {
			|	adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr))
		} else {
			|	GET_ZVAL_LVAL result_reg, op2_addr, TMP1
			|	adds Rx(result_reg), Rx(result_reg), Rx(result_reg)
		}
	} else if (opcode == ZEND_MUL &&
			Z_MODE(op1_addr) == IS_CONST_ZVAL &&
			!may_overflow &&
			zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) {
		|	GET_ZVAL_LVAL result_reg, op2_addr, TMP1
		|	mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr)))
		|	lsl Rx(result_reg), Rx(result_reg), TMP1
	} else if (opcode == ZEND_DIV &&
			(Z_MODE(op2_addr) == IS_CONST_ZVAL &&
			zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) {
		|	GET_ZVAL_LVAL result_reg, op1_addr, TMP1
		|	asr Rx(result_reg), Rx(result_reg), #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr)))
#if 0
	/* x86 specific optimizations through LEA instraction are not supported on ARM */
	} else if (opcode == ZEND_ADD &&
			!may_overflow &&
			Z_MODE(op1_addr) == IS_REG &&
			Z_MODE(op2_addr) == IS_CONST_ZVAL) {
		|	NIY	// TODO: test
	} else if (opcode == ZEND_ADD &&
			!may_overflow &&
			Z_MODE(op2_addr) == IS_REG &&
			Z_MODE(op1_addr) == IS_CONST_ZVAL) {
		|	NIY	// TODO: test
	} else if (opcode == ZEND_SUB &&
			!may_overflow &&
			Z_MODE(op1_addr) == IS_REG &&
			Z_MODE(op2_addr) == IS_CONST_ZVAL) {
		|	NIY	// TODO: test
#endif
	} else if (opcode == ZEND_MUL) {
		|	GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1
		|	GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2
		|	mul Rx(result_reg), TMP1, TMP2
		if(may_overflow) {
			/* Use 'smulh' to get the upper 64 bits fo the 128-bit result.
			 * For signed multiplication, the top 65 bits of the result will contain
			 * either all zeros or all ones if no overflow occurred.
			 * Flag: bne -> overflow. beq -> no overflow.
			 */
			use_ovf_flag = 0;
			|	smulh TMP1, TMP1, TMP2
			|	cmp TMP1, Rx(result_reg), asr #63
		}
	} else {
		|	GET_ZVAL_LVAL result_reg, op1_addr, TMP1
		if ((opcode == ZEND_ADD || opcode == ZEND_SUB)
		 && Z_MODE(op2_addr) == IS_CONST_ZVAL
		 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
			/* +/- 0 */
			may_overflow = 0;
		} else if (same_ops && opcode != ZEND_DIV) {
			|	LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg)
		} else {
			|	LONG_MATH opcode, result_reg, op2_addr, TMP1
		}
	}
	if (may_overflow) {
		if (res_info & MAY_BE_GUARD) {
			int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
			const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
			if (!exit_addr) {
				return 0;
			}
			if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) {
				if (use_ovf_flag) {
					|	bvs &exit_addr
				} else {
					|	bne &exit_addr
				}
				if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) {
					|	mov Rx(Z_REG(res_addr)), Rx(result_reg)
				}
			} else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
				if (use_ovf_flag) {
					|	bvc &exit_addr
				} else {
					|	beq &exit_addr
				}
			} else {
				ZEND_UNREACHABLE();
			}
		} else {
			if (res_info & MAY_BE_LONG) {
				if (use_ovf_flag) {
					|	bvs >1
				} else {
					|	bne >1
				}
			} else {
				if (use_ovf_flag) {
					|	bvc >1
				} else {
					|	beq >1
				}
			}
		}
	}

	if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) {
		|	SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1
		if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
			if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
				|	SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
			}
		}
	}

	if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) {
		zend_reg tmp_reg1 = ZREG_FPR0;
		zend_reg tmp_reg2 = ZREG_FPR1;

		if (res_info & MAY_BE_LONG) {
			|.cold_code
			|1:
		}

		do {
			if ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) ||
			    (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)) {
				if (opcode == ZEND_ADD) {
					uint64_t val = 0x43e0000000000000;
					if (Z_MODE(res_addr) == IS_REG) {
						|	LOAD_64BIT_VAL TMP1, val
						|	fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1
					} else {
						|	SET_ZVAL_LVAL res_addr, val, TMP2, TMP1
					}
					break;
				} else if (opcode == ZEND_SUB) {
					uint64_t val = 0xc3e0000000000000;
					if (Z_MODE(res_addr) == IS_REG) {
						|	LOAD_64BIT_VAL TMP1, val
						|	fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1
					} else {
						|	SET_ZVAL_LVAL res_addr, val, TMP2, TMP1
					}
					break;
				}
			}

			|	DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg, ZREG_TMP1
			|	DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg, ZREG_TMP1
			|	DOUBLE_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2
			|	SET_ZVAL_DVAL res_addr, tmp_reg1, ZREG_TMP1
		} while (0);

		if (Z_MODE(res_addr) == IS_MEM_ZVAL
		 && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
			|	SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2
		}
		if (res_info & MAY_BE_LONG) {
			|	b >2
			|.code
		}
		|2:
	}

	return 1;
}

static int zend_jit_math_long_double(dasm_State    **Dst,
                                     zend_uchar      opcode,
                                     zend_jit_addr   op1_addr,
                                     zend_jit_addr   op2_addr,
                                     zend_jit_addr   res_addr,
                                     uint32_t        res_use_info)
{
	zend_reg result_reg =
		(Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_FPR0;
	zend_reg op2_reg;

	|	DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, ZREG_TMP1, ZREG_TMP2

	if (Z_MODE(op2_addr) == IS_REG) {
		op2_reg = Z_REG(op2_addr);
	} else {
		op2_reg = ZREG_FPTMP;
		|	GET_ZVAL_DVAL op2_reg, op2_addr, ZREG_TMP1
	}

	|	DOUBLE_MATH_REG opcode, result_reg, result_reg, op2_reg

	|	SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1

	if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
		if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
			|	SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2
		}
	}

	return 1;
}

static int zend_jit_math_double_long(dasm_State    **Dst,
                                     zend_uchar      opcode,
                                     zend_jit_addr   op1_addr,
                                     zend_jit_addr   op2_addr,
                                     zend_jit_addr   res_addr,
                                     uint32_t        res_use_info)
{
	zend_reg result_reg, op1_reg, op2_reg;

	if (zend_is_commutative(opcode)
	 && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) {
		if (Z_MODE(res_addr) == IS_REG) {
			result_reg = Z_REG(res_addr);
		} else {
			result_reg = ZREG_FPR0;
		}
		|	DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, ZREG_TMP1, ZREG_TMP2
		if (Z_MODE(op1_addr) == IS_REG) {
			op1_reg = Z_REG(op1_addr);
		} else {
			op1_reg = ZREG_FPTMP;
			|	GET_ZVAL_DVAL op1_reg, op1_addr, ZREG_TMP1
		}
		|	DOUBLE_MATH_REG opcode, result_reg, result_reg, op1_reg
	} else {
		if (Z_MODE(res_addr) == IS_REG) {
			result_reg = Z_REG(res_addr);
		} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
			result_reg = Z_REG(op1_addr);
		} else {
			result_reg = ZREG_FPR0;
		}

		if (Z_MODE(op1_addr) == IS_REG) {
			op1_reg = Z_REG(op1_addr);
		} else {
			|	GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1
			op1_reg = result_reg;
		}
		if ((opcode == ZEND_ADD || opcode == ZEND_SUB)
		 && Z_MODE(op2_addr) == IS_CONST_ZVAL
		 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
			/* +/- 0 */
		} else {
			op2_reg = ZREG_FPTMP;
			|	DOUBLE_GET_ZVAL_LVAL op2_reg, op2_addr, ZREG_TMP1, ZREG_TMP2
			|	DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg
		}
	}

	|	SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1

	if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
		if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
			if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
				|	SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2
			}
		}
	}

	return 1;
}

static int zend_jit_math_double_double(dasm_State    **Dst,
                                       zend_uchar      opcode,
                                       zend_jit_addr   op1_addr,
                                       zend_jit_addr   op2_addr,
                                       zend_jit_addr   res_addr,
                                       uint32_t        res_use_info)
{
	bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
	zend_reg result_reg, op1_reg, op2_reg;
	zend_jit_addr val_addr;

	if (Z_MODE(res_addr) == IS_REG) {
		result_reg = Z_REG(res_addr);
	} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
		result_reg = Z_REG(op1_addr);
	} else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) {
		result_reg = Z_REG(op2_addr);
	} else {
		result_reg = ZREG_FPR0;
	}

	if (Z_MODE(op1_addr) == IS_REG) {
		op1_reg = Z_REG(op1_addr);
		val_addr = op2_addr;
	} else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) {
		op1_reg = Z_REG(op2_addr);
		val_addr = op1_addr;
	} else {
		|	GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1
		op1_reg = result_reg;
		val_addr = op2_addr;
	}

	if ((opcode == ZEND_MUL) &&
		Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) {
		|	DOUBLE_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg
	} else {
		if (same_ops) {
			op2_reg = op1_reg;
		} else if (Z_MODE(val_addr) == IS_REG) {
			op2_reg = Z_REG(val_addr);
		} else {
			op2_reg = ZREG_FPTMP;
			|	GET_ZVAL_DVAL op2_reg, val_addr, ZREG_TMP1
		}
		|	DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg
	}

	|	SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1

	if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
		if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
			if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
				|	SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2
			}
		}
	}
	return 1;
}

static int zend_jit_math_helper(dasm_State    **Dst,
                                const zend_op  *opline,
                                zend_uchar      opcode,
                                zend_uchar      op1_type,
                                znode_op        op1,
                                zend_jit_addr   op1_addr,
                                uint32_t        op1_info,
                                zend_uchar      op2_type,
                                znode_op        op2,
                                zend_jit_addr   op2_addr,
                                uint32_t        op2_info,
                                uint32_t        res_var,
                                zend_jit_addr   res_addr,
                                uint32_t        res_info,
                                uint32_t        res_use_info,
                                int             may_overflow,
                                int             may_throw)
/* Labels: 1,2,3,4,5,6 */
{
	bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);

	if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
		if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
			if (op1_info & MAY_BE_DOUBLE) {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1
			} else {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1
			}
		}
		if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
			if (op2_info & MAY_BE_DOUBLE) {
				|	IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, ZREG_TMP1
				|.cold_code
				|1:
				if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
					|	IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1
				}
				if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
					return 0;
				}
				|	b >5
				|.code
			} else {
				|	IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1
			}
		}
		if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
			return 0;
		}
		if (op1_info & MAY_BE_DOUBLE) {
			|.cold_code
			|3:
			if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1
			}
			if (op2_info & MAY_BE_DOUBLE) {
				if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
					if (!same_ops) {
						|	IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1, ZREG_TMP1
					} else {
						|	IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6, ZREG_TMP1
					}
				}
				if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
					return 0;
				}
				|	b >5
			}
			if (!same_ops) {
				|1:
				if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
					|	IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1
				}
				if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
					return 0;
				}
				|	b >5
			}
			|.code
		}
	} else if ((op1_info & MAY_BE_DOUBLE) &&
	           !(op1_info & MAY_BE_LONG) &&
	           (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
	           (res_info & MAY_BE_DOUBLE)) {
		if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
			|	IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1
		}
		if (op2_info & MAY_BE_DOUBLE) {
			if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
				if (!same_ops && (op2_info & MAY_BE_LONG)) {
					|	IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1, ZREG_TMP1
				} else {
					|	IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1
				}
			}
			if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
				return 0;
			}
		}
		if (!same_ops && (op2_info & MAY_BE_LONG)) {
			if (op2_info & MAY_BE_DOUBLE) {
				|.cold_code
			}
		    |1:
			if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
				|	IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1
			}
			if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
				return 0;
			}
			if (op2_info & MAY_BE_DOUBLE) {
				|	b >5
				|.code
			}
		}
	} else if ((op2_info & MAY_BE_DOUBLE) &&
	           !(op2_info & MAY_BE_LONG) &&
	           (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
	           (res_info & MAY_BE_DOUBLE)) {
		if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
			|	IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1
		}
		if (op1_info & MAY_BE_DOUBLE) {
			if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
				if (!same_ops && (op1_info & MAY_BE_LONG)) {
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1, ZREG_TMP1
				} else {
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1
				}
			}
			if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
				return 0;
			}
		}
		if (!same_ops && (op1_info & MAY_BE_LONG)) {
			if (op1_info & MAY_BE_DOUBLE) {
				|.cold_code
			}
			|1:
			if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1
			}
			if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
				return 0;
			}
			if (op1_info & MAY_BE_DOUBLE) {
				|	b >5
				|.code
			}
		}
	}

	|5:

	if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
		(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
		if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
		    (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
		    (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
			|.cold_code
		}
		|6:
		if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
			if (Z_MODE(res_addr) == IS_REG) {
				zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
				|	LOAD_ZVAL_ADDR FCARG1x, real_addr
			} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, res_addr
			}
			if (Z_MODE(op1_addr) == IS_REG) {
				zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
				if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
					return 0;
				}
				op1_addr = real_addr;
			}
			|	LOAD_ZVAL_ADDR FCARG2x, op1_addr
		} else {
			if (Z_MODE(op1_addr) == IS_REG) {
				zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
				if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
					return 0;
				}
				op1_addr = real_addr;
			}
			|	LOAD_ZVAL_ADDR FCARG2x, op1_addr
			if (Z_MODE(res_addr) == IS_REG) {
				zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
				|	LOAD_ZVAL_ADDR FCARG1x, real_addr
			} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, res_addr
			}
		}
		if (Z_MODE(op2_addr) == IS_REG) {
			zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
			if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
				return 0;
			}
			op2_addr = real_addr;
		}
		|	LOAD_ZVAL_ADDR CARG3, op2_addr
		|	SET_EX_OPLINE opline, REG0
		if (opcode == ZEND_ADD) {
			|	EXT_CALL add_function, REG0
		} else if (opcode == ZEND_SUB) {
			|	EXT_CALL sub_function, REG0
		} else if (opcode == ZEND_MUL) {
			|	EXT_CALL mul_function, REG0
		} else if (opcode == ZEND_DIV) {
			|	EXT_CALL div_function, REG0
		} else {
			ZEND_UNREACHABLE();
		}
		|	FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
		|	FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
		if (may_throw) {
			if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
				|	MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1
				|	cbnz TMP2, ->exception_handler_free_op2
			} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
				zend_jit_check_exception_undef_result(Dst, opline);
			} else {
				zend_jit_check_exception(Dst);
			}
		}
		if (Z_MODE(res_addr) == IS_REG) {
			zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
			if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) {
				return 0;
			}
		}
		if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
		    (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
		    (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
			|	b <5
			|.code
		}
	}

	return 1;
}

static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
{
	ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
	ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
	    (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)));

	if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) {
		return 0;
	}
	if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
		return 0;
	}
	return 1;
}

static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr)
{
	if (Z_MODE(op2_addr) != IS_MEM_ZVAL || Z_REG(op2_addr) != ZREG_FCARG1) {
		|	GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1
		|	GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
	} else if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
		|	GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
		|	GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1
	} else {
		|	GET_ZVAL_LVAL ZREG_REG0, op2_addr, TMP1
		|	GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1
		|	mov FCARG2x, REG0
	}
	|	EXT_CALL zend_jit_add_arrays_helper, REG0
	|	SET_ZVAL_PTR res_addr, RETVALx, TMP1
	|	SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX, TMP1w, TMP2
	|	FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2
	|	FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
	return 1;
}

static int zend_jit_long_math_helper(dasm_State    **Dst,
                                     const zend_op  *opline,
                                     zend_uchar      opcode,
                                     zend_uchar      op1_type,
                                     znode_op        op1,
                                     zend_jit_addr   op1_addr,
                                     uint32_t        op1_info,
                                     zend_ssa_range *op1_range,
                                     zend_uchar      op2_type,
                                     znode_op        op2,
                                     zend_jit_addr   op2_addr,
                                     uint32_t        op2_info,
                                     zend_ssa_range *op2_range,
                                     uint32_t        res_var,
                                     zend_jit_addr   res_addr,
                                     uint32_t        res_info,
                                     uint32_t        res_use_info,
                                     int             may_throw)
/* Labels: 6 */
{
	bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
	zend_reg result_reg;

	if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
		|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1
	}
	if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
		|	IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1
	}

	if (Z_MODE(res_addr) == IS_REG) {
		if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR)
		 && opline->op2_type != IS_CONST) {
			result_reg = ZREG_REG0;
		} else {
			result_reg = Z_REG(res_addr);
		}
	} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
		result_reg = Z_REG(op1_addr);
	} else if (Z_REG(res_addr) != ZREG_REG0) {
		result_reg = ZREG_REG0;
	} else {
		/* ASSIGN_DIM_OP */
		result_reg = ZREG_FCARG1;
	}

	if (opcode == ZEND_SL) {
		if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
			zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));

			if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
				if (EXPECTED(op2_lval > 0)) {
					|	mov Rx(result_reg), xzr
				} else {
					zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
					zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
					|	SET_EX_OPLINE opline, REG0
					|	b ->negative_shift
				}
			} else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) {
				|	add Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr))
			} else {
				|	GET_ZVAL_LVAL result_reg, op1_addr, TMP1
				|	lsl Rx(result_reg), Rx(result_reg), #op2_lval
			}
		} else {
			zend_reg op2_reg;

			if (Z_MODE(op2_addr) == IS_REG) {
				op2_reg = Z_REG(op2_addr);
			} else {
				op2_reg = ZREG_TMP2;
				|	GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2
			}
			if (!op2_range ||
			     op2_range->min < 0 ||
			     op2_range->max >= SIZEOF_ZEND_LONG * 8) {

				|	cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8)
				|	bhs >1
				|.cold_code
				|1:
				|	mov Rx(result_reg), xzr
				|	cmp Rx(op2_reg), xzr
				|	bgt >1
				zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
				zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
				|	SET_EX_OPLINE opline, REG0
				|	b ->negative_shift
				|.code
			}
			|	GET_ZVAL_LVAL result_reg, op1_addr, TMP1
			|	lsl Rx(result_reg), Rx(result_reg), Rx(op2_reg)
			|1:
		}
	} else if (opcode == ZEND_SR) {
		|	GET_ZVAL_LVAL result_reg, op1_addr, TMP1
		if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
			zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));

			if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
				if (EXPECTED(op2_lval > 0)) {
					|	asr Rx(result_reg), Rx(result_reg), #((SIZEOF_ZEND_LONG * 8) - 1)
				} else {
					zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
					zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
					|	SET_EX_OPLINE opline, REG0
					|	b ->negative_shift
				}
			} else {
				|	asr Rx(result_reg), Rx(result_reg), #op2_lval
			}
		} else {
			zend_reg op2_reg;

			if (Z_MODE(op2_addr) == IS_REG) {
				op2_reg = Z_REG(op2_addr);
			} else {
				op2_reg = ZREG_TMP2;
				|	GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2
			}
			if (!op2_range ||
			     op2_range->min < 0 ||
			     op2_range->max >= SIZEOF_ZEND_LONG * 8) {
				|	cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8)
				|	bhs >1
				|.cold_code
				|1:
				|	cmp Rx(op2_reg), xzr
				|	mov Rx(op2_reg), #((SIZEOF_ZEND_LONG * 8) - 1)
				|	bgt >1
				zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
				zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
				|	SET_EX_OPLINE opline, REG0
				|	b ->negative_shift
				|.code
			}
			|1:
			|	asr Rx(result_reg), Rx(result_reg), Rx(op2_reg)
		}
	} else if (opcode == ZEND_MOD) {
		if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
			zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));

			if (op2_lval == 0) {
					zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
					zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
				|	SET_EX_OPLINE opline, REG0
				|	b ->mod_by_zero
			} else if (op2_lval == -1) {
				|	mov Rx(result_reg), xzr
			} else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) {
				zval tmp;
				zend_jit_addr tmp_addr;

				/* Optimisation for mod of power of 2 */
				ZVAL_LONG(&tmp, op2_lval - 1);
				tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp);
				|	GET_ZVAL_LVAL result_reg, op1_addr, TMP1
				|	LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, TMP1
			} else {
				|	GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1
				|	GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2
				|	sdiv Rx(result_reg), TMP1, TMP2
				|	msub Rx(result_reg), Rx(result_reg), TMP2, TMP1
			}
		} else {
			zend_reg op2_reg;

			if (Z_MODE(op2_addr) == IS_MEM_ZVAL) {
				|	MEM_ACCESS_64_WITH_UOFFSET ldr, TMP2, Rx(Z_REG(op2_addr)), Z_OFFSET(op2_addr), TMP2
				op2_reg = ZREG_TMP2;
			} else {
				ZEND_ASSERT(Z_MODE(op2_addr) == IS_REG);
				op2_reg = Z_REG(op2_addr);
			}

			if (!op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) {
				|	cbz Rx(op2_reg), >1
				|.cold_code
				|1:
				zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
				zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
				|	SET_EX_OPLINE opline, REG0
				|	b ->mod_by_zero
				|.code
			}

			/* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */
			if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) {
				|	cmn Rx(op2_reg), #1
				|	beq >1
				|.cold_code
				|1:
				|	SET_ZVAL_LVAL_FROM_REG res_addr, xzr, TMP1
				if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
					if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
						if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
							|	SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
						}
					}
				}
				|	b >5
				|.code
			}

			|	GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1
			|	sdiv Rx(result_reg), TMP1, Rx(op2_reg)
			|	msub Rx(result_reg), Rx(result_reg), Rx(op2_reg), TMP1
		}
	} else if (same_ops) {
		|	GET_ZVAL_LVAL result_reg, op1_addr, TMP1
		|	LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg)
	} else {
		|	GET_ZVAL_LVAL result_reg, op1_addr, TMP1
		|	LONG_MATH opcode, result_reg, op2_addr, TMP1
	}

	if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) {
		|	SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1
	}
	if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
		if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
			if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
				|	SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
			}
		}
	}

	if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) ||
		(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
		if ((op1_info & MAY_BE_LONG) &&
		    (op2_info & MAY_BE_LONG)) {
			|.cold_code
		}
		|6:
		if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
			if (Z_MODE(res_addr) == IS_REG) {
				zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
				|	LOAD_ZVAL_ADDR FCARG1x, real_addr
			} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, res_addr
			}
			if (Z_MODE(op1_addr) == IS_REG) {
				zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
				if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
					return 0;
				}
				op1_addr = real_addr;
			}
			|	LOAD_ZVAL_ADDR FCARG2x, op1_addr
		} else {
			if (Z_MODE(op1_addr) == IS_REG) {
				zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
				if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
					return 0;
				}
				op1_addr = real_addr;
			}
			|	LOAD_ZVAL_ADDR FCARG2x, op1_addr
			if (Z_MODE(res_addr) == IS_REG) {
				zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
				|	LOAD_ZVAL_ADDR FCARG1x, real_addr
			} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, res_addr
			}
		}
		if (Z_MODE(op2_addr) == IS_REG) {
			zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
			if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
				return 0;
			}
			op2_addr = real_addr;
		}
		|	LOAD_ZVAL_ADDR CARG3, op2_addr
		|	SET_EX_OPLINE opline, REG0
		if (opcode == ZEND_BW_OR) {
			|	EXT_CALL bitwise_or_function, REG0
		} else if (opcode == ZEND_BW_AND) {
			|	EXT_CALL bitwise_and_function, REG0
		} else if (opcode == ZEND_BW_XOR) {
			|	EXT_CALL bitwise_xor_function, REG0
		} else if (opcode == ZEND_SL) {
			|	EXT_CALL shift_left_function, REG0
		} else if (opcode == ZEND_SR) {
			|	EXT_CALL shift_right_function, REG0
		} else if (opcode == ZEND_MOD) {
			|	EXT_CALL mod_function, REG0
		} else {
			ZEND_UNREACHABLE();
		}
		if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) {
			/* compound assignment may decrement "op2" refcount */
			op2_info |= MAY_BE_RC1;
		}
		|	FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
		|	FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
		if (may_throw) {
			if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
				|	MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1
				|	cbnz TMP2, ->exception_handler_free_op2
			} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
				zend_jit_check_exception_undef_result(Dst, opline);
			} else {
				zend_jit_check_exception(Dst);
			}
		}
		if (Z_MODE(res_addr) == IS_REG) {
			zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
			if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) {
				return 0;
			}
		}
		if ((op1_info & MAY_BE_LONG) &&
		    (op2_info & MAY_BE_LONG)) {
			|	b >5
			|.code
		}
	}
	|5:

	return 1;
}

static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
{
	ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
	ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG));

	if (!zend_jit_long_math_helper(Dst, opline, opline->opcode,
			opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
			opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
			opline->result.var, res_addr, res_info, res_use_info, may_throw)) {
		return 0;
	}
	if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
		return 0;
	}
	return 1;
}

static int zend_jit_concat_helper(dasm_State    **Dst,
                                  const zend_op  *opline,
                                  zend_uchar      op1_type,
                                  znode_op        op1,
                                  zend_jit_addr   op1_addr,
                                  uint32_t        op1_info,
                                  zend_uchar      op2_type,
                                  znode_op        op2,
                                  zend_jit_addr   op2_addr,
                                  uint32_t        op2_info,
                                  zend_jit_addr   res_addr,
                                  int             may_throw)
{
	if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
		if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
			|	IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1
		}
		if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
			|	IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6, ZREG_TMP1
		}
		if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) {
			if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, res_addr
			}
			|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
			|	EXT_CALL zend_jit_fast_assign_concat_helper, REG0
			/* concatination with itself may reduce refcount */
			op2_info |= MAY_BE_RC1;
		} else {
			if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, res_addr
			}
			|	LOAD_ZVAL_ADDR FCARG2x, op1_addr
			|	LOAD_ZVAL_ADDR CARG3, op2_addr
			if (op1_type == IS_CV || op1_type == IS_CONST) {
				|	EXT_CALL zend_jit_fast_concat_helper, REG0
			} else {
				|	EXT_CALL zend_jit_fast_concat_tmp_helper, REG0
			}
		}
		/* concatination with empty string may increase refcount */
		op2_info |= MAY_BE_RCN;
		|	FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
		|5:
	}
	if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) ||
	    (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) {
		if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
			|.cold_code
			|6:
		}
		if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
			if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, res_addr
			}
			|	LOAD_ZVAL_ADDR FCARG2x, op1_addr
		} else {
			|	LOAD_ZVAL_ADDR FCARG2x, op1_addr
			if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, res_addr
			}
		}
		|	LOAD_ZVAL_ADDR CARG3, op2_addr
		|	SET_EX_OPLINE opline, REG0
		|	EXT_CALL concat_function, REG0
		/* concatination with empty string may increase refcount */
		op1_info |= MAY_BE_RCN;
		op2_info |= MAY_BE_RCN;
		|	FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
		|	FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
		if (may_throw) {
			if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
				zend_jit_check_exception_undef_result(Dst, opline);
			} else {
				zend_jit_check_exception(Dst);
			}
		}
		if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
			|	b <5
			|.code
		}
	}

	return 1;
}

static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw)
{
	zend_jit_addr op1_addr, op2_addr;

	ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
	ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING));

	op1_addr = OP1_ADDR();
	op2_addr = OP2_ADDR();

	return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw);
}

static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr)
/* Labels: 1,2,3,4,5 */
{
	zend_jit_addr op2_addr = OP2_ADDR();
	zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);

	if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
	 && type == BP_VAR_R
	 && !exit_addr) {
		int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
		exit_addr = zend_jit_trace_get_exit_addr(exit_point);
		if (!exit_addr) {
			return 0;
		}
	}

	if (op2_info & MAY_BE_LONG) {
		bool op2_loaded = 0;
		bool packed_loaded = 0;
		bool bad_packed_key = 0;

		if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) {
			|	// if (EXPECTED(Z_TYPE_P(dim) == IS_LONG))
			|	IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1
		}
		if (op1_info & MAY_BE_PACKED_GUARD) {
			int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
			const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

			if (!exit_addr) {
				return 0;
			}
			if (op1_info & MAY_BE_ARRAY_PACKED) {
				|	ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
				|	TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
				|	beq &exit_addr
			} else {
				|	ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
				|	TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
				|	bne &exit_addr
			}
		}
		if (type == BP_VAR_W) {
			|	// hval = Z_LVAL_P(dim);
			|	GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
			op2_loaded = 1;
		}
		if (op1_info & MAY_BE_ARRAY_PACKED) {
			zend_long val = -1;

			if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
				val = Z_LVAL_P(Z_ZV(op2_addr));
				if (val >= 0 && val < HT_MAX_SIZE) {
					packed_loaded = 1;
				} else {
					bad_packed_key = 1;
				}
			} else {
				if (!op2_loaded) {
					|	// hval = Z_LVAL_P(dim);
					|	GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
					op2_loaded = 1;
				}
				packed_loaded = 1;
			}

			if (dim_type == IS_UNDEF && type == BP_VAR_W) {
				/* don't generate "fast" code for packed array */
				packed_loaded = 0;
			}

			if (packed_loaded) {
				|	// ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef);
				if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
					|	ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
					|	TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
					|	beq >4 // HASH_FIND
				}
				|	// if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed))

				|	ldr REG0w, [FCARG1x, #offsetof(zend_array, nNumUsed)]
				if (val == 0) {
					|	cmp REG0, xzr
				} else if (val > 0 && !op2_loaded) {
					|	CMP_64_WITH_CONST REG0, val, TMP1
				} else {
					|	cmp REG0, FCARG2x
				}

				if (type == BP_JIT_IS) {
					if (not_found_exit_addr) {
						|	bls &not_found_exit_addr
					} else {
						|	bls >9 // NOT_FOUND
					}
				} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
					|	bls &exit_addr
				} else if (type == BP_VAR_IS && not_found_exit_addr) {
					|	bls &not_found_exit_addr
				} else if (type == BP_VAR_RW && not_found_exit_addr) {
					|	bls &not_found_exit_addr
				} else if (type == BP_VAR_IS && found_exit_addr) {
					|	bls >7 // NOT_FOUND
				} else {
					|	bls >2 // NOT_FOUND
				}
				|	// _ret = &_ht->arData[_h].val;
				if (val >= 0) {
					|	ldr REG0, [FCARG1x, #offsetof(zend_array, arData)]
					if (val != 0) {
						|	ADD_SUB_64_WITH_CONST add, REG0, REG0, (val * sizeof(Bucket)), TMP1
					}
				} else {
					|	ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)]
					|	add REG0, TMP1, FCARG2x, lsl #5
				}
			}
		}
		switch (type) {
			case BP_JIT_IS:
				if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
					if (packed_loaded) {
						|	b >5
					}
					|4:
					if (!op2_loaded) {
						|	// hval = Z_LVAL_P(dim);
						|	GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
					}
					if (packed_loaded) {
						|	EXT_CALL _zend_hash_index_find, REG0
					} else {
						|	EXT_CALL zend_hash_index_find, REG0
					}
					|	mov REG0, RETVALx
					if (not_found_exit_addr) {
						|	cbz REG0, &not_found_exit_addr
					} else {
						|	cbz REG0, >9 // NOT_FOUND
					}
					if (op2_info & MAY_BE_STRING) {
						|	b >5
					}
				} else if (packed_loaded) {
					if (op2_info & MAY_BE_STRING) {
						|	b >5
					}
				} else if (not_found_exit_addr) {
					|	b &not_found_exit_addr
				} else {
					|	b >9 // NOT_FOUND
				}
				break;
			case BP_VAR_R:
			case BP_VAR_IS:
			case BP_VAR_UNSET:
				if (packed_loaded) {
					if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
						|	IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w
					} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
						/* perform IS_UNDEF check only after result type guard (during deoptimization) */
						if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
							|	IF_Z_TYPE REG0, IS_UNDEF, &exit_addr, TMP1w
						}
					} else if (type == BP_VAR_IS && not_found_exit_addr) {
						|	IF_Z_TYPE REG0, IS_UNDEF, &not_found_exit_addr, TMP1w
					} else if (type == BP_VAR_IS && found_exit_addr) {
						|	IF_Z_TYPE REG0, IS_UNDEF, >7, TMP1w // NOT_FOUND
					} else {
						|	IF_Z_TYPE REG0, IS_UNDEF, >2, TMP1w // NOT_FOUND
					}
				}
				if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_NUMERIC_HASH))) {
					if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
						|	b &exit_addr
					} else if (type == BP_VAR_IS && not_found_exit_addr) {
						|	b &not_found_exit_addr
					} else if (type == BP_VAR_IS && found_exit_addr) {
						|	b >7 // NOT_FOUND
					} else {
						|	b >2 // NOT_FOUND
					}
				}
				if (!packed_loaded || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
					|4:
					if (!op2_loaded) {
						|	// hval = Z_LVAL_P(dim);
						|	GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
					}
					if (packed_loaded) {
						|	EXT_CALL _zend_hash_index_find, REG0
					} else {
						|	EXT_CALL zend_hash_index_find, REG0
					}
					|	mov REG0, RETVALx
					if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
						|	cbz REG0, &exit_addr
					} else if (type == BP_VAR_IS && not_found_exit_addr) {
						|	cbz REG0, &not_found_exit_addr
					} else if (type == BP_VAR_IS && found_exit_addr) {
						|	cbz REG0, >7 // NOT_FOUND
					} else {
						|	cbz REG0, >2 // NOT_FOUND
					}
				}
				|.cold_code
				|2:
				switch (type) {
					case BP_VAR_R:
						if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
							|	// zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval);
							|	// retval = &EG(uninitialized_zval);
							|	UNDEFINED_OFFSET opline
							|	b >9
						}
						break;
					case BP_VAR_IS:
					case BP_VAR_UNSET:
						if (!not_found_exit_addr && !found_exit_addr) {
							|	// retval = &EG(uninitialized_zval);
							|	SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
							|	b >9
						}
						break;
					default:
						ZEND_UNREACHABLE();
				}
				|.code
				break;
			case BP_VAR_RW:
				if (packed_loaded && !not_found_exit_addr) {
					|	IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w
				}
				if (!packed_loaded ||
						!not_found_exit_addr ||
						(op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
					if (packed_loaded && not_found_exit_addr) {
						|.cold_code
					}
					|2:
					|4:
					if (!op2_loaded) {
						|	// hval = Z_LVAL_P(dim);
						|	GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
					}
					|	SET_EX_OPLINE opline, REG0
					if (packed_loaded) {
						|	EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, REG0
					} else {
						|	EXT_CALL zend_jit_hash_index_lookup_rw, REG0
					}
					|	mov REG0, RETVALx
					if (not_found_exit_addr) {
						if (packed_loaded) {
							|	cbnz REG0, >8
							|	b &not_found_exit_addr
							|.code
						} else {
							|	cbz REG0, &not_found_exit_addr
						}
					} else {
						|	cbz REG0, >9
					}
				}
				break;
			case BP_VAR_W:
				if (packed_loaded) {
					|	IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w
				}
				if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) {
					|2:
					|4:
					if (!op2_loaded) {
						|	// hval = Z_LVAL_P(dim);
						|	GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
					}
					|	EXT_CALL zend_hash_index_lookup, REG0
					|	mov REG0, RETVALx
				}
				break;
			default:
				ZEND_UNREACHABLE();
		}

		if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) {
			|	b >8
		}
	}

	if (op2_info & MAY_BE_STRING) {
		|3:
		if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
			|	// if (EXPECTED(Z_TYPE_P(dim) == IS_STRING))
			|	IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, ZREG_TMP1
		}
		|	// offset_key = Z_STR_P(dim);
		|	GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
		|	// retval = zend_hash_find(ht, offset_key);
		switch (type) {
			case BP_JIT_IS:
				if (opline->op2_type != IS_CONST) {
					|	ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)]
					|	cmp TMP1w, #((uint8_t) ('9'))
					|	ble >1
					|.cold_code
					|1:
					|	EXT_CALL zend_jit_symtable_find, REG0
					|	b >1
					|.code
					|	EXT_CALL zend_hash_find, REG0
					|1:
				} else {
					|	EXT_CALL zend_hash_find_known_hash, REG0
				}
				|	mov REG0, RETVALx
				if (not_found_exit_addr) {
					|	cbz REG0, &not_found_exit_addr
				} else {
					|	cbz REG0, >9 // NOT_FOUND
				}
				break;
			case BP_VAR_R:
			case BP_VAR_IS:
			case BP_VAR_UNSET:
				if (opline->op2_type != IS_CONST) {
					|	ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)]
					|	cmp TMP1w, #((uint8_t) ('9'))
					|	ble >1
					|.cold_code
					|1:
					|	EXT_CALL zend_jit_symtable_find, REG0
					|	b >1
					|.code
					|	EXT_CALL zend_hash_find, REG0
					|1:
				} else {
					|	EXT_CALL zend_hash_find_known_hash, REG0
				}
				|	mov REG0, RETVALx
				if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
					|	cbz REG0, &exit_addr
				} else if (type == BP_VAR_IS && not_found_exit_addr) {
					|	cbz REG0, &not_found_exit_addr
				} else if (type == BP_VAR_IS && found_exit_addr) {
					|	cbz REG0, >7
				} else {
					|	cbz REG0, >2 // NOT_FOUND
					|.cold_code
					|2:
					switch (type) {
						case BP_VAR_R:
							// zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key));
							|	UNDEFINED_INDEX opline
							|	b >9
							break;
						case BP_VAR_IS:
						case BP_VAR_UNSET:
							|	// retval = &EG(uninitialized_zval);
							|	SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
							|	b >9
							break;
						default:
							ZEND_UNREACHABLE();
					}
					|.code
				}
				break;
			case BP_VAR_RW:
				|	SET_EX_OPLINE opline, REG0
				if (opline->op2_type != IS_CONST) {
					|	EXT_CALL zend_jit_symtable_lookup_rw, REG0
				} else {
					|	EXT_CALL zend_jit_hash_lookup_rw, REG0
				}
				|	mov REG0, RETVALx
				if (not_found_exit_addr) {
					|	cbz REG0, &not_found_exit_addr
				} else {
					|	cbz REG0, >9
				}
				break;
			case BP_VAR_W:
				if (opline->op2_type != IS_CONST) {
					|	EXT_CALL zend_jit_symtable_lookup_w, REG0
				} else {
					|	EXT_CALL zend_hash_lookup, REG0
				}
				|	mov REG0, RETVALx
				break;
			default:
				ZEND_UNREACHABLE();
		}
	}

	if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) {
	    |5:
		if (op1_info & MAY_BE_ARRAY_OF_REF) {
			|	ZVAL_DEREF REG0, MAY_BE_REF, TMP1w
		}
		|	ldrb TMP1w, [REG0,#offsetof(zval, u1.v.type)]
		|	cmp TMP1w, #IS_NULL
		if (not_found_exit_addr) {
			|	ble &not_found_exit_addr
		} else if (found_exit_addr) {
			|	bgt &found_exit_addr
		} else {
			|	ble >9 // NOT FOUND
		}
	}

	if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
		if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
			|.cold_code
			|3:
		}
		|	SET_EX_OPLINE opline, REG0
		|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
		switch (type) {
			case BP_VAR_R:
				|	LOAD_ZVAL_ADDR CARG3, res_addr
				|	EXT_CALL zend_jit_fetch_dim_r_helper, REG0
				|	mov REG0, RETVALx
				|	b >9
				break;
			case BP_JIT_IS:
				|	EXT_CALL zend_jit_fetch_dim_isset_helper, REG0
				|	mov REG0, RETVALx
				if (not_found_exit_addr) {
					|	cbz REG0, &not_found_exit_addr
					if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
						|	b >8
					}
				} else if (found_exit_addr) {
					|	cbnz REG0, &found_exit_addr
					if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
						|	b >9
					}
				} else {
					|	cbnz REG0, >8
					|	b >9
				}
				break;
			case BP_VAR_IS:
			case BP_VAR_UNSET:
				|	LOAD_ZVAL_ADDR CARG3, res_addr
				|	EXT_CALL zend_jit_fetch_dim_is_helper, REG0
				|	mov REG0, RETVALx
				|	b >9
				break;
			case BP_VAR_RW:
				|	EXT_CALL zend_jit_fetch_dim_rw_helper, REG0
				|	mov REG0, RETVALx
				|	cbnz REG0, >8
				|	b >9
				break;
			case BP_VAR_W:
				|	EXT_CALL zend_jit_fetch_dim_w_helper, REG0
				|	mov REG0, RETVALx
				|	cbnz REG0, >8
				|	b >9
				break;
			default:
				ZEND_UNREACHABLE();
		}
		if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
			|.code
		}
	}

	return 1;
}

static int zend_jit_simple_assign(dasm_State    **Dst,
                                  const zend_op  *opline,
                                  zend_jit_addr   var_addr,
                                  uint32_t        var_info,
                                  uint32_t        var_def_info,
                                  zend_uchar      val_type,
                                  zend_jit_addr   val_addr,
                                  uint32_t        val_info,
                                  zend_jit_addr   res_addr,
                                  int             in_cold,
                                  int             save_r1)
/* Labels: 1,2,3 */
{
	zend_reg tmp_reg;

	if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_REG0) {
		tmp_reg = ZREG_REG0;
	} else {
		/* ASSIGN_DIM */
		tmp_reg = ZREG_FCARG1;
	}

	if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
		zval *zv = Z_ZV(val_addr);

		if (!res_addr) {
			|	ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0
		} else {
			|	ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0
		}
		if (Z_REFCOUNTED_P(zv)) {
			if (!res_addr) {
				|	ADDREF_CONST zv, TMP1, TMP2
			} else {
				|	ADDREF_CONST_2 zv, TMP1, TMP2
			}
		}
	} else {
		if (val_info & MAY_BE_UNDEF) {
			if (in_cold) {
				|	IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, ZREG_TMP1
			} else {
				|	IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1
				|.cold_code
				|1:
			}
			|	// zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
			if (save_r1) {
				|	str FCARG1x, T1	// save
			}
			|	SET_ZVAL_TYPE_INFO var_addr, IS_NULL, TMP1w, TMP2
			if (res_addr) {
				|	SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
			}
			if (opline) {
				|	SET_EX_OPLINE opline, Rx(tmp_reg)
			}
			ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP);
			|	LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr)
			|	EXT_CALL zend_jit_undefined_op_helper, REG0
			|	cbz RETVALx, ->exception_handler_undef
			if (save_r1) {
				|	ldr FCARG1x, T1	// restore
			}
			|	b >3
			if (in_cold) {
				|2:
			} else {
				|.code
			}
		}
		if (val_info & MAY_BE_REF) {
			if (val_type == IS_CV) {
				ZEND_ASSERT(Z_REG(var_addr) != ZREG_REG2);
				if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_REG2 || Z_OFFSET(val_addr) != 0) {
					|	LOAD_ZVAL_ADDR REG2, val_addr
				}
				|	ZVAL_DEREF REG2, val_info, TMP1w
				val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0);
			} else {
				zend_jit_addr ref_addr;

				if (in_cold) {
					|	IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1
				} else {
					|	IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1
					|.cold_code
					|1:
				}
				if (Z_REG(val_addr) == ZREG_REG2) {
					|	str REG2, T1 // save
				}
				|	// zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
				|	GET_ZVAL_PTR REG2, val_addr, TMP1
				|	GC_DELREF REG2, TMP1w
				|	// ZVAL_COPY_VALUE(return_value, &ref->val);
				ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, offsetof(zend_reference, val));
				if (!res_addr) {
					|	ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
				} else {
					|	ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
				}
				|	beq >2 // GC_DELREF() reached zero
				|	IF_NOT_REFCOUNTED REG2w, >3, TMP1w
				if (!res_addr) {
					|	GC_ADDREF Rx(tmp_reg), TMP1w
				} else {
					|	GC_ADDREF_2 Rx(tmp_reg), TMP1w
				}
				|	b >3
				|2:
				if (res_addr) {
					|	IF_NOT_REFCOUNTED REG2w, >2, TMP1w
					|	GC_ADDREF Rx(tmp_reg), TMP1w
					|2:
				}
				if (Z_REG(val_addr) == ZREG_REG2) {
					|	ldr REG2, T1 // restore
				}
				if (save_r1) {
					|	str FCARG1x, T1 // save
				}
				|	GET_ZVAL_PTR FCARG1x, val_addr, TMP1
				|	EFREE_REFERENCE
				if (save_r1) {
					|	ldr FCARG1x, T1 // restore
				}
				|	b >3
				if (in_cold) {
					|1:
				} else {
					|.code
				}
			}
		}

		if (!res_addr) {
			|	ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
		} else {
			|	ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
		}

		if (val_type == IS_CV) {
			if (!res_addr) {
				|	TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w
			} else {
				|	TRY_ADDREF_2 val_info, REG2w, Rx(tmp_reg), TMP1w
			}
		} else {
			if (res_addr) {
				|	TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w
			}
		}
		|3:
	}
	return 1;
}

static int zend_jit_assign_to_typed_ref(dasm_State         **Dst,
                                       const zend_op        *opline,
                                       zend_uchar            val_type,
                                       zend_jit_addr         val_addr,
                                       zend_jit_addr         res_addr,
                                       bool                  check_exception)
{
	|	// if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) {
	|	ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
	|	cbnz TMP1, >2
	|.cold_code
	|2:
	if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) {
		|	LOAD_ZVAL_ADDR FCARG2x, val_addr
	}
	if (opline) {
		|	SET_EX_OPLINE opline, REG0
	}
	if (val_type == IS_CONST) {
		|	EXT_CALL zend_jit_assign_const_to_typed_ref, REG0
	} else if (val_type == IS_TMP_VAR) {
		|	EXT_CALL zend_jit_assign_tmp_to_typed_ref, REG0
	} else if (val_type == IS_VAR) {
		|	EXT_CALL zend_jit_assign_var_to_typed_ref, REG0
	} else if (val_type == IS_CV) {
		|	EXT_CALL zend_jit_assign_cv_to_typed_ref, REG0
	} else {
		ZEND_UNREACHABLE();
	}
	if (res_addr) {
		zend_jit_addr ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_X0, 0); // RETVAL

		|	ZVAL_COPY_VALUE res_addr, -1, ret_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
		|	TRY_ADDREF -1, REG1w, REG2, TMP1w
	}
	if (check_exception) {
		|	// if (UNEXPECTED(EG(exception) != NULL)) {
		|	MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
		|	cbz REG0, >8  // END OF zend_jit_assign_to_variable()
		|	b ->exception_handler
	} else {
		|	b >8
	}
	|.code

	return 1;
}

static int zend_jit_assign_to_variable_call(dasm_State    **Dst,
                                            const zend_op  *opline,
                                            zend_jit_addr   __var_use_addr,
                                            zend_jit_addr   var_addr,
                                            uint32_t        __var_info,
                                            uint32_t        __var_def_info,
                                            zend_uchar      val_type,
                                            zend_jit_addr   val_addr,
                                            uint32_t        val_info,
                                            zend_jit_addr   __res_addr,
                                            bool            __check_exception)
{
	if (val_info & MAY_BE_UNDEF) {
		if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
			int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
			const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

			if (!exit_addr) {
				return 0;
			}

			|	IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr, ZREG_TMP1
		} else {
			|	IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1
			|.cold_code
			|1:
			ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP);
			if (Z_REG(var_addr) != ZREG_FP) {
				|	str Rx(Z_REG(var_addr)), T1 // save
			}
			|	SET_EX_OPLINE opline, REG0
			|	LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr)
			|	EXT_CALL zend_jit_undefined_op_helper, REG0
			if (Z_REG(var_addr) != ZREG_FP) {
				|	ldr Rx(Z_REG(var_addr)), T1 // restore
			}
			if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, var_addr
			}
			|	LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
			|	bl ->assign_const
			|	b >9
			|.code
			|1:
		}
	}
	if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {
		|	LOAD_ZVAL_ADDR FCARG1x, var_addr
	}
	if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) {
		|	LOAD_ZVAL_ADDR FCARG2x, val_addr
	}
	if (opline) {
		|	SET_EX_OPLINE opline, REG0
	}
	if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
		|	bl ->assign_tmp
	} else if (val_type == IS_CONST) {
		|	bl ->assign_const
	} else if (val_type == IS_TMP_VAR) {
		|	bl ->assign_tmp
	} else if (val_type == IS_VAR) {
		if (!(val_info & MAY_BE_REF)) {
			|	bl ->assign_tmp
		} else {
			|	bl ->assign_var
		}
	} else if (val_type == IS_CV) {
		if (!(val_info & MAY_BE_REF)) {
			|	bl ->assign_cv_noref
		} else {
			|	bl ->assign_cv
		}
		if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
			|9:
		}
	} else {
		ZEND_UNREACHABLE();
	}

	return 1;
}

static int zend_jit_assign_to_variable(dasm_State    **Dst,
                                       const zend_op  *opline,
                                       zend_jit_addr   var_use_addr,
                                       zend_jit_addr   var_addr,
                                       uint32_t        var_info,
                                       uint32_t        var_def_info,
                                       zend_uchar      val_type,
                                       zend_jit_addr   val_addr,
                                       uint32_t        val_info,
                                       zend_jit_addr   res_addr,
                                       bool            check_exception)
/* Labels: 1,2,3,4,5,8 */
{
	int done = 0;
	zend_reg ref_reg, tmp_reg;

	if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_REG0) {
		ref_reg = ZREG_FCARG1;
		tmp_reg = ZREG_REG0;
	} else {
		/* ASSIGN_DIM */
		ref_reg = ZREG_REG0;
		tmp_reg = ZREG_FCARG1;
	}

	if (var_info & MAY_BE_REF) {
		if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) {
			|	LOAD_ZVAL_ADDR Rx(ref_reg), var_use_addr
			var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0);
		}
		|	// if (Z_ISREF_P(variable_ptr)) {
		|	IF_NOT_Z_TYPE Rx(ref_reg), IS_REFERENCE, >3, TMP1w
		|	// if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) {
		|	GET_Z_PTR FCARG1x, Rx(ref_reg)
		if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) {
			return 0;
		}
		|	add Rx(ref_reg), FCARG1x, #offsetof(zend_reference, val)
		|3:
	}
	if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
		if (RC_MAY_BE_1(var_info)) {
			int in_cold = 0;

			if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
				|	IF_ZVAL_REFCOUNTED var_use_addr, >1, ZREG_TMP1, ZREG_TMP2
				|.cold_code
				|1:
				in_cold = 1;
			}
			if (Z_REG(var_use_addr) == ZREG_FCARG1 || Z_REG(var_use_addr) == ZREG_REG0) {
				bool keep_gc = 0;

				|	GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1
#if 0
				// TODO: This optiization doesn't work on ARM
				if (tmp_reg == ZREG_FCARG1) {
					if (Z_MODE(val_addr) == IS_REG) {
						keep_gc = 1;
					} else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) {
						keep_gc = 1;
					} else if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
						zval *zv = Z_ZV(val_addr);
						if (Z_TYPE_P(zv) == IS_DOUBLE) {
							if (Z_DVAL_P(zv) == 0) {
								keep_gc = 1;
							}
						} else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
							keep_gc = 1;
						}
					} else if (Z_MODE(val_addr) == IS_MEM_ZVAL) {
						if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
							keep_gc = 1;
						}
					}
				}
#endif
				if (!keep_gc) {
					|	str Rx(tmp_reg), T1 // save
				}
				if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0)) {
					return 0;
				}
				if (!keep_gc) {
					|	ldr FCARG1x, T1     // restore
				}
			} else {
				|	GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1
				if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1)) {
					return 0;
				}
			}
			|	GC_DELREF FCARG1x, TMP1w
			if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
				|	bne >4
			} else {
				|	bne >8
			}
			|	ZVAL_DTOR_FUNC var_info, opline, TMP1
			if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) {
				if (check_exception) {
					|	MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
					|	cbz REG0, >8
					|	b ->exception_handler
				} else {
					|	b >8
				}
			}
			if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
				|4:
				|	IF_GC_MAY_NOT_LEAK FCARG1x, >8, TMP1w, TMP2w
				|	EXT_CALL gc_possible_root, REG0
				if (in_cold) {
					|	b >8
				}
			}
			if (in_cold) {
				|.code
			} else {
				done = 1;
			}
		} else /* if (RC_MAY_BE_N(var_info)) */ {
			if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
				|	IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5, ZREG_TMP1, ZREG_TMP2
			}
			if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) {
				if (Z_REG(var_use_addr) != ZREG_FP) {
					|	str Rx(Z_REG(var_use_addr)), T1 // save
				}
				|	GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1
				|	GC_DELREF FCARG1x, TMP1w
				|	IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w
				|	EXT_CALL gc_possible_root, TMP1
				if (Z_REG(var_use_addr) != ZREG_FP) {
					|	ldr Rx(Z_REG(var_use_addr)), T1 // restore
				}
			} else {
				|	GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1
				|	GC_DELREF Rx(tmp_reg), TMP1w
			}
			|5:
	    }
	}

	if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0)) {
		return 0;
	}

	|8:

	return 1;
}

static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw)
{
	zend_jit_addr op2_addr, op3_addr, res_addr;

	op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
	op3_addr = OP1_DATA_ADDR();
	if (opline->result_type == IS_UNUSED) {
		res_addr = 0;
	} else {
		res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
	}

	if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) {
		int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
		const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

		if (!exit_addr) {
			return 0;
		}

		|	IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr, ZREG_TMP1

		val_info &= ~MAY_BE_UNDEF;
	}

	if (op1_info & MAY_BE_REF) {
		|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		|	IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w
		|	GET_Z_PTR FCARG2x, FCARG1x
		|	ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))]
		|	cmp TMP1w, #IS_ARRAY
		|	bne >2
		|	add FCARG1x, FCARG2x, #offsetof(zend_reference, val)
		|	b >3
		|.cold_code
		|2:
		|	SET_EX_OPLINE opline, REG0
		|	EXT_CALL zend_jit_prepare_assign_dim_ref, REG0
		|	mov FCARG1x, RETVALx
		|	cbnz FCARG1x, >1
		|	b ->exception_handler_undef
		|.code
		|1:
		op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	}

	if (op1_info & MAY_BE_ARRAY) {
		if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
			|	IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1
		}
		|3:
		|	SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2
	} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
		if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
			|	CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1
			|	bgt >7
		}
		|	// ZVAL_ARR(container, zend_new_array(8));
		if (Z_REG(op1_addr) != ZREG_FP) {
			|	str Rx(Z_REG(op1_addr)), T1 // save
		}
		|	EXT_CALL _zend_new_array_0, REG0
		|	mov REG0, RETVALx
		if (Z_REG(op1_addr) != ZREG_FP) {
			|	ldr Rx(Z_REG(op1_addr)), T1 // restore
		}
		|	SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1
		|	SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2
		|	mov FCARG1x, REG0
	}

	if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
		|6:
		if (opline->op2_type == IS_UNUSED) {
			uint32_t var_info = MAY_BE_NULL;
			zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);

			|	// var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
			|	LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
			|	EXT_CALL zend_hash_next_index_insert, REG0
			|	// if (UNEXPECTED(!var_ptr)) {
			|	mov REG0, RETVALx
			|	cbz REG0, >1
			|.cold_code
			|1:
			|	// zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
			|	CANNOT_ADD_ELEMENT opline
			|	//ZEND_VM_C_GOTO(assign_dim_op_ret_null);
			|	b >9
			|.code

			if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) {
				return 0;
			}
		} else {
			uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
			zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);

			if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) {
				return 0;
			}

			if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
				var_info |= MAY_BE_REF;
			}
			if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
				var_info |= MAY_BE_RC1;
			}

			|8:
			|	// value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE);
			if (opline->op1_type == IS_VAR) {
				ZEND_ASSERT(opline->result_type == IS_UNUSED);
				if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
					return 0;
				}
			} else {
				if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
					return 0;
				}
			}
		}
	}

	if (((op1_info & MAY_BE_ARRAY) &&
	     (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) ||
	    (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY)))) {
		if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
			|.cold_code
			|7:
		}

		if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) &&
		    (op1_info & MAY_BE_ARRAY)) {
			if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
				|	CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1
				|	bgt >2
			}
			|	// ZVAL_ARR(container, zend_new_array(8));
			if (Z_REG(op1_addr) != ZREG_FP) {
				|	str Rx(Z_REG(op1_addr)), T1 // save
			}
			|	EXT_CALL _zend_new_array_0, REG0
			|	mov REG0, RETVALx
			if (Z_REG(op1_addr) != ZREG_FP) {
				|	ldr Rx(Z_REG(op1_addr)), T1 // restore
			}
			|	SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1
			|	SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2
			|	mov FCARG1x, REG0
			|	// ZEND_VM_C_GOTO(assign_dim_op_new_array);
			|	b <6
			|2:
		}

		if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
			|	SET_EX_OPLINE opline, REG0
		    if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
			}
		    if (opline->op2_type == IS_UNUSED) {
				|	mov FCARG2x, xzr
			} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
				ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
				|	LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1)
			} else {
				|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
			}
			if (opline->result_type == IS_UNUSED) {
				|	mov CARG4, xzr
			} else {
				|	LOAD_ZVAL_ADDR CARG4, res_addr
			}
			|	LOAD_ZVAL_ADDR CARG3, op3_addr
			|	EXT_CALL zend_jit_assign_dim_helper, REG0

#ifdef ZEND_JIT_USE_RC_INFERENCE
			if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) {
				/* ASSIGN_DIM may increase refcount of the value */
				val_info |= MAY_BE_RCN;
			}
#endif

			|	FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
		}

		if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
			if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
				|	b >9 // END
			}
			|.code
		}
	}

#ifdef ZEND_JIT_USE_RC_INFERENCE
	if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
		/* ASSIGN_DIM may increase refcount of the key */
		op2_info |= MAY_BE_RCN;
	}
#endif

	|9:
	|	FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2

	if (may_throw) {
		zend_jit_check_exception(Dst);
	}

	return 1;
}

static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw)
{
	zend_jit_addr op2_addr, op3_addr, var_addr;
	const void *not_found_exit_addr = NULL;
	uint32_t var_info = MAY_BE_NULL;

	ZEND_ASSERT(opline->result_type == IS_UNUSED);

	op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
	op3_addr = OP1_DATA_ADDR();

	if (op1_info & MAY_BE_REF) {
		|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		|	IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w
		|	GET_Z_PTR FCARG2x, FCARG1x
		|	ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))]
		|	cmp TMP1w, #IS_ARRAY
		|	bne >2
		|	add FCARG1x, FCARG2x, #offsetof(zend_reference, val)
		|	b >3
		|.cold_code
		|2:
		|	SET_EX_OPLINE opline, REG0
		|	EXT_CALL zend_jit_prepare_assign_dim_ref, REG0
		|	mov FCARG1x, RETVALx
		|	cbnz RETVALx, >1
		|	b ->exception_handler_undef
		|.code
		|1:
		op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	}

	if (op1_info & MAY_BE_ARRAY) {
		if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
			|	IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1
		}
		|3:
		|	SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2
	}
	if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
		if (op1_info & MAY_BE_ARRAY) {
			|.cold_code
			|7:
		}
		if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
			|	CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1
			|	bgt >7
		}
		if (Z_REG(op1_addr) != ZREG_FP) {
			|	str Rx(Z_REG(op1_addr)), T1 // save
		}
		if (op1_info & MAY_BE_UNDEF) {
			if (op1_info & MAY_BE_NULL) {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
			}
			|	SET_EX_OPLINE opline, REG0
			|	LOAD_32BIT_VAL FCARG1x, opline->op1.var
			|	EXT_CALL zend_jit_undefined_op_helper, REG0
			|1:
		}
		|	// ZVAL_ARR(container, zend_new_array(8));
		|	EXT_CALL _zend_new_array_0, REG0
		|	mov REG0, RETVALx
		if (Z_REG(op1_addr) != ZREG_FP) {
			|	ldr Rx(Z_REG(op1_addr)), T1 // restore
		}
		|	SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1
		|	SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2
		|	mov FCARG1x, REG0
		if (op1_info & MAY_BE_ARRAY) {
			|	b >1
			|.code
			|1:
		}
	}

	if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
		uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0);

		|6:
		if (opline->op2_type == IS_UNUSED) {
			var_info = MAY_BE_NULL;

			|	// var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
			|	LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
			|	EXT_CALL zend_hash_next_index_insert, REG0
			|	mov REG0, RETVALx
			|	// if (UNEXPECTED(!var_ptr)) {
			|	cbz REG0, >1
			|.cold_code
			|1:
			|	// zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
			|	CANNOT_ADD_ELEMENT opline
			|	//ZEND_VM_C_GOTO(assign_dim_op_ret_null);
			|	b >9
			|.code
		} else {
			var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
			if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
				var_info |= MAY_BE_REF;
			}
			if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
				var_info |= MAY_BE_RC1;
			}

			if (dim_type != IS_UNKNOWN
			 && dim_type != IS_UNDEF
			 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
			 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
			 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
				int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
				not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
				if (!not_found_exit_addr) {
					return 0;
				}
			}

			if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL)) {
				return 0;
			}

			|8:
			if (not_found_exit_addr && dim_type != IS_REFERENCE) {
				|	IF_NOT_Z_TYPE, REG0, dim_type, &not_found_exit_addr, TMP1w
				var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
			}
			if (var_info & MAY_BE_REF) {
				binary_op_type binary_op = get_binary_op(opline->extended_value);
				|	IF_NOT_Z_TYPE, REG0, IS_REFERENCE, >1, TMP1w
				|	GET_Z_PTR FCARG1x, REG0
				|	ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
				|	cbnz TMP1, >2
				|	add REG0, FCARG1x, #offsetof(zend_reference, val)
				|.cold_code
				|2:
				|	LOAD_ZVAL_ADDR FCARG2x, op3_addr
				|	LOAD_ADDR CARG3, binary_op
				|	SET_EX_OPLINE opline, REG0
				if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR))
				 && (op1_data_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
					|	EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0
				} else {
					|	EXT_CALL zend_jit_assign_op_to_typed_ref, REG0
				}
				|	b >9
				|.code
				|1:
			}
		}

		var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
		switch (opline->extended_value) {
			case ZEND_ADD:
			case ZEND_SUB:
			case ZEND_MUL:
			case ZEND_DIV:
				if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info,
						1 /* may overflow */, may_throw)) {
					return 0;
				}
				break;
			case ZEND_BW_OR:
			case ZEND_BW_AND:
			case ZEND_BW_XOR:
			case ZEND_SL:
			case ZEND_SR:
			case ZEND_MOD:
				if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value,
						IS_CV, opline->op1, var_addr, var_info, NULL,
						(opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info,
						op1_data_range,
						0, var_addr, var_def_info, var_info, may_throw)) {
					return 0;
				}
				break;
			case ZEND_CONCAT:
				if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr,
						may_throw)) {
					return 0;
				}
				break;
			default:
				ZEND_UNREACHABLE();
		}
		|	FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
	}

	if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
		binary_op_type binary_op;

		if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
			|.cold_code
			|7:
		}

		|	SET_EX_OPLINE opline, REG0
		if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
			|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		}
	    if (opline->op2_type == IS_UNUSED) {
			|	mov FCARG2x, xzr
		} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
			ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
			|	LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1)
		} else {
			|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
		}
		binary_op = get_binary_op(opline->extended_value);
		|	LOAD_ZVAL_ADDR CARG3, op3_addr
		|	LOAD_ADDR CARG4, binary_op
		|	EXT_CALL zend_jit_assign_dim_op_helper, REG0

		|9:
		|	FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
		|	FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
		if (may_throw) {
			zend_jit_check_exception(Dst);
		}

		if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
			|	b >9 // END
			|.code
			|9:
		}
	} else if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY))
			&& (!not_found_exit_addr || (var_info & MAY_BE_REF))) {
		|.cold_code
		|9:
		|	FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, opline, ZREG_TMP1, ZREG_TMP2
		|	FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
		if (may_throw) {
			zend_jit_check_exception(Dst);
		}
		|	b >9
		|.code
		|9:
	}

	return 1;
}

static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw)
{
	zend_jit_addr op1_addr, op2_addr;

	ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED);
	ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));

	op1_addr = OP1_ADDR();
	op2_addr = OP2_ADDR();

	if (op1_info & MAY_BE_REF) {
		binary_op_type binary_op = get_binary_op(opline->extended_value);
		|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		|	IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >1, TMP1w
		|	GET_Z_PTR FCARG1x, FCARG1x
		|	ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
		|	cbnz TMP1, >2
		|	add FCARG1x, FCARG1x, #offsetof(zend_reference, val)
		|.cold_code
		|2:
		|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
		|	LOAD_ADDR CARG3, binary_op
		|	SET_EX_OPLINE opline, REG0
		if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
		 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
			|	EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0
		} else {
			|	EXT_CALL zend_jit_assign_op_to_typed_ref, REG0
		}
		zend_jit_check_exception(Dst);
		|	b >9
		|.code
		|1:
		op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	}

	int result;
	switch (opline->extended_value) {
		case ZEND_ADD:
		case ZEND_SUB:
		case ZEND_MUL:
		case ZEND_DIV:
			result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw);
			break;
		case ZEND_BW_OR:
		case ZEND_BW_AND:
		case ZEND_BW_XOR:
		case ZEND_SL:
		case ZEND_SR:
		case ZEND_MOD:
			result = zend_jit_long_math_helper(Dst, opline, opline->extended_value,
				opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
				opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
				opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw);
			break;
		case ZEND_CONCAT:
			result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw);
			break;
		default:
			ZEND_UNREACHABLE();
	}
	|9:
	return result;
}

static int zend_jit_cmp_long_long(dasm_State    **Dst,
                                  const zend_op  *opline,
                                  zend_ssa_range *op1_range,
                                  zend_jit_addr   op1_addr,
                                  zend_ssa_range *op2_range,
                                  zend_jit_addr   op2_addr,
                                  zend_jit_addr   res_addr,
                                  zend_uchar      smart_branch_opcode,
                                  uint32_t        target_label,
                                  uint32_t        target_label2,
                                  const void     *exit_addr,
                                  bool            skip_comparison)
{
	bool swap = 0;
	bool result;

	if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) {
		if (!smart_branch_opcode ||
		    smart_branch_opcode == ZEND_JMPZ_EX ||
		    smart_branch_opcode == ZEND_JMPNZ_EX) {
			|	SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE), TMP1w, TMP2
		}
		if (smart_branch_opcode && !exit_addr) {
			if (smart_branch_opcode == ZEND_JMPZ ||
			    smart_branch_opcode == ZEND_JMPZ_EX) {
				if (!result) {
					|	b => target_label
				}
			} else if (smart_branch_opcode == ZEND_JMPNZ ||
			           smart_branch_opcode == ZEND_JMPNZ_EX) {
				if (result) {
					|	b => target_label
				}
			} else if (smart_branch_opcode == ZEND_JMPZNZ) {
				if (!result) {
					|	b => target_label
				} else {
					|	b => target_label2
				}
			} else {
				ZEND_UNREACHABLE();
			}
		}
		return 1;
	}

	if (skip_comparison) {
		if (Z_MODE(op1_addr) != IS_REG &&
		    (Z_MODE(op2_addr) == IS_REG ||
		     (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) {
			swap = 1;
		}
	} else if (Z_MODE(op1_addr) == IS_REG) {
		if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
			|	cmp Rx(Z_REG(op1_addr)), xzr
		} else {
			|	LONG_CMP Z_REG(op1_addr), op2_addr, TMP1
		}
	} else if (Z_MODE(op2_addr) == IS_REG) {
		if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) {
			|	cmp Rx(Z_REG(op2_addr)), xzr
		} else {
			|	LONG_CMP Z_REG(op2_addr), op1_addr, TMP1
		}
		swap = 1;
	} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) {
		|	LONG_CMP_WITH_CONST op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2
		swap = 1;
	} else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) {
		|	LONG_CMP_WITH_CONST op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2
	} else {
		|	GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1
		if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
			|	cmp Rx(ZREG_REG0), xzr
		} else {
			|	LONG_CMP ZREG_REG0, op2_addr, TMP1
		}
	}

	if (smart_branch_opcode) {
		if (smart_branch_opcode == ZEND_JMPZ_EX ||
		    smart_branch_opcode == ZEND_JMPNZ_EX) {

			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_IS_IDENTICAL:
				case ZEND_CASE:
				case ZEND_CASE_STRICT:
					|	cset REG0w, eq
					break;
				case ZEND_IS_NOT_EQUAL:
				case ZEND_IS_NOT_IDENTICAL:
					|	cset REG0w, ne
					break;
				case ZEND_IS_SMALLER:
					if (swap) {
						|	cset REG0w, gt
					} else {
						|	cset REG0w, lt
					}
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					if (swap) {
						|	cset REG0w, ge
					} else {
						|	cset REG0w, le
					}
					break;
				default:
					ZEND_UNREACHABLE();
			}
			|	add REG0w, REG0w, #2
			|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
		}
		if (smart_branch_opcode == ZEND_JMPZ ||
		    smart_branch_opcode == ZEND_JMPZ_EX) {
			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_IS_IDENTICAL:
				case ZEND_CASE:
				case ZEND_CASE_STRICT:
					if (exit_addr) {
						|	bne &exit_addr
					} else {
						|	bne => target_label
					}
					break;
				case ZEND_IS_NOT_EQUAL:
					if (exit_addr) {
						|	beq &exit_addr
					} else {
						|	beq => target_label
					}
					break;
				case ZEND_IS_NOT_IDENTICAL:
					if (exit_addr) {
						|	bne &exit_addr
					} else {
						|	beq => target_label
					}
					break;
				case ZEND_IS_SMALLER:
					if (swap) {
						if (exit_addr) {
							|	ble &exit_addr
						} else {
							|	ble => target_label
						}
					} else {
						if (exit_addr) {
							|	bge &exit_addr
						} else {
							|	bge => target_label
						}
					}
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					if (swap) {
						if (exit_addr) {
							|	blt &exit_addr
						} else {
							|	blt => target_label
						}
					} else {
						if (exit_addr) {
							|	bgt &exit_addr
						} else {
							|	bgt => target_label
						}
					}
					break;
				default:
					ZEND_UNREACHABLE();
			}
		} else if (smart_branch_opcode == ZEND_JMPNZ ||
		           smart_branch_opcode == ZEND_JMPNZ_EX) {
			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_IS_IDENTICAL:
				case ZEND_CASE:
				case ZEND_CASE_STRICT:
					if (exit_addr) {
						|	beq &exit_addr
					} else {
						|	beq => target_label
					}
					break;
				case ZEND_IS_NOT_EQUAL:
					if (exit_addr) {
						|	bne &exit_addr
					} else {
						|	bne => target_label
					}
					break;
				case ZEND_IS_NOT_IDENTICAL:
					if (exit_addr) {
						|	beq &exit_addr
					} else {
						|	bne => target_label
					}
					break;
				case ZEND_IS_SMALLER:
					if (swap) {
						if (exit_addr) {
							|	bgt &exit_addr
						} else {
							|	bgt => target_label
						}
					} else {
						if (exit_addr) {
							|	blt &exit_addr
						} else {
							|	blt => target_label
						}
					}
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					if (swap) {
						if (exit_addr) {
							|	bge &exit_addr
						} else {
							|	bge => target_label
						}
					} else {
						if (exit_addr) {
							|	ble &exit_addr
						} else {
							|	ble => target_label
						}
					}
					break;
				default:
					ZEND_UNREACHABLE();
			}
		} else if (smart_branch_opcode == ZEND_JMPZNZ) {
			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_IS_IDENTICAL:
				case ZEND_CASE:
				case ZEND_CASE_STRICT:
					|	bne => target_label
					break;
				case ZEND_IS_NOT_EQUAL:
				case ZEND_IS_NOT_IDENTICAL:
					|	beq => target_label
					break;
				case ZEND_IS_SMALLER:
				    if (swap) {
						|	ble => target_label
				    } else {
						|	bge => target_label
					}
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					if (swap) {
						|	blt => target_label
					} else {
						|	bgt => target_label
					}
					break;
				default:
					ZEND_UNREACHABLE();
			}
			|	b => target_label2
		} else {
			ZEND_UNREACHABLE();
		}
	} else {
		switch (opline->opcode) {
			case ZEND_IS_EQUAL:
			case ZEND_IS_IDENTICAL:
			case ZEND_CASE:
			case ZEND_CASE_STRICT:
				|	cset REG0w, eq
				break;
			case ZEND_IS_NOT_EQUAL:
			case ZEND_IS_NOT_IDENTICAL:
				|	cset REG0w, ne
				break;
			case ZEND_IS_SMALLER:
				if (swap) {
					|	cset REG0w, gt
				} else {
					|	cset REG0w, lt
				}
				break;
			case ZEND_IS_SMALLER_OR_EQUAL:
				if (swap) {
					|	cset REG0w, ge
				} else {
					|	cset REG0w, le
				}
				break;
			default:
				ZEND_UNREACHABLE();
		}
		|	add REG0w, REG0w, #2
		|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
	}

	return 1;
}

static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
	if (smart_branch_opcode) {
		if (smart_branch_opcode == ZEND_JMPZ) {
			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_IS_IDENTICAL:
				case ZEND_CASE:
				case ZEND_CASE_STRICT:
					if (exit_addr) {
						|	bne &exit_addr
					} else {
						|	bne => target_label
					}
					break;
				case ZEND_IS_NOT_EQUAL:
					|	bvs >1
					if (exit_addr) {
						|	beq &exit_addr
					} else {
						|	beq => target_label
					}
					|1:
					break;
				case ZEND_IS_NOT_IDENTICAL:
					if (exit_addr) {
						|	bvs &exit_addr
						|	bne &exit_addr
					} else {
						|	bvs >1
						|	beq => target_label
						|1:
					}
					break;
				case ZEND_IS_SMALLER:
					if (swap) {
						if (exit_addr) {
							|	bvs &exit_addr
							|	bls &exit_addr
						} else {
							|	bvs => target_label
							|	bls => target_label
						}
					} else {
						if (exit_addr) {
							|	bhs &exit_addr
						} else {
							|	bhs => target_label
						}
					}
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					if (swap) {
						if (exit_addr) {
							|	bvs &exit_addr
							|	blo &exit_addr
						} else {
							|	bvs => target_label
							|	blo => target_label
						}
					} else {
						if (exit_addr) {
							|	bhi &exit_addr
						} else {
							|	bhi => target_label
						}
					}
					break;
				default:
					ZEND_UNREACHABLE();
			}
		} else if (smart_branch_opcode == ZEND_JMPNZ) {
			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_IS_IDENTICAL:
				case ZEND_CASE:
				case ZEND_CASE_STRICT:
					|	bvs >1
					if (exit_addr) {
						|	beq &exit_addr
					} else {
						|	beq => target_label
					}
					|1:
					break;
				case ZEND_IS_NOT_EQUAL:
					if (exit_addr) {
						|	bne &exit_addr
					} else {
						|	bne => target_label
					}
					break;
				case ZEND_IS_NOT_IDENTICAL:
					if (exit_addr) {
						|	bvs >1
						|	beq &exit_addr
						|1:
					} else {
						|	bne => target_label
					}
					break;
				case ZEND_IS_SMALLER:
					if (swap) {
						|	bvs >1  // Always False if involving NaN
						if (exit_addr) {
							|	bhi &exit_addr
						} else {
							|	bhi => target_label
						}
						|1:
					} else {
						|	bvs >1
						if (exit_addr) {
							|	blo	&exit_addr
						} else {
							|	blo => target_label
						}
						|1:
					}
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					if (swap) {
						|	bvs >1  // Always False if involving NaN
						if (exit_addr) {
							|	bhs &exit_addr
						} else {
							|	bhs => target_label
						}
						|1:
					} else {
						|	bvs >1
						if (exit_addr) {
							|	bls &exit_addr
						} else {
							|	bls => target_label
						}
						|1:
					}
					break;
				default:
					ZEND_UNREACHABLE();
			}
		} else if (smart_branch_opcode == ZEND_JMPZNZ) {
			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_IS_IDENTICAL:
				case ZEND_CASE:
				case ZEND_CASE_STRICT:
					|	bne => target_label
					break;
				case ZEND_IS_NOT_EQUAL:
				case ZEND_IS_NOT_IDENTICAL:
					|	bvs => target_label2
					|	beq => target_label
					break;
				case ZEND_IS_SMALLER:
					if (swap) {
						|	bvs => target_label
						|	bls => target_label
					} else {
						|	bhs => target_label
					}
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					if (swap) {
						|	bvs => target_label
						|	blo => target_label
					} else {
						|	bhi => target_label
					}
					break;
				default:
					ZEND_UNREACHABLE();
			}
			|	b => target_label2
		} else if (smart_branch_opcode == ZEND_JMPZ_EX) {
			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_IS_IDENTICAL:
				case ZEND_CASE:
				case ZEND_CASE_STRICT:
					|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
					|	bne => target_label
					|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
					break;
				case ZEND_IS_NOT_EQUAL:
				case ZEND_IS_NOT_IDENTICAL:
					|	bvs >1
					|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
					|	beq => target_label
					|1:
					|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
					break;
				case ZEND_IS_SMALLER:
					if (swap) {
						|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
						|	bvs => target_label
						|	bls => target_label
						|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
					} else {
						|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
						|	bhs => target_label
						|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
					}
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					if (swap) {
						|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
						|	bvs => target_label
						|	blo => target_label
						|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
					} else {
						|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
						|	bhi => target_label
						|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
					}
					break;
				default:
					ZEND_UNREACHABLE();
			}
		} else if (smart_branch_opcode == ZEND_JMPNZ_EX) {
			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_IS_IDENTICAL:
				case ZEND_CASE:
				case ZEND_CASE_STRICT:
					|	bvs >1
					|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
					|	beq => target_label
					|1:
					|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
					break;
				case ZEND_IS_NOT_EQUAL:
				case ZEND_IS_NOT_IDENTICAL:
					|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
					|	bvs => target_label
					|	bne => target_label
					|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
					break;
				case ZEND_IS_SMALLER:
					if (swap) {
						|	cset REG0w, hi
						|	add REG0w, REG0w, #2
						|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
						|	bvs >1  // Always False if involving NaN
						|	bhi => target_label
						|1:
					} else {
						|	bvs >1
						|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
						|	blo => target_label
						|1:
						|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
					}
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					if (swap) {
						|	cset REG0w, hs
						|	add REG0w, REG0w, #2
						|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
						|	bvs >1  // Always False if involving NaN
						|	bhs => target_label
						|1:
					} else {
						|	bvs >1
						|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
						|	bls => target_label
						|1:
						|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
					}
					break;
				default:
					ZEND_UNREACHABLE();
			}
		} else {
			ZEND_UNREACHABLE();
		}
	} else {
		switch (opline->opcode) {
			case ZEND_IS_EQUAL:
			case ZEND_IS_IDENTICAL:
			case ZEND_CASE:
			case ZEND_CASE_STRICT:
				|	bvs >1
				|	mov REG0, #IS_TRUE
				|	beq >2
				|1:
				|	mov REG0, #IS_FALSE
				|2:
				break;
			case ZEND_IS_NOT_EQUAL:
			case ZEND_IS_NOT_IDENTICAL:
				|	bvs >1
				|	mov REG0, #IS_FALSE
				|	beq >2
				|1:
				|	mov REG0, #IS_TRUE
				|2:
				break;
			case ZEND_IS_SMALLER:
				|	bvs >1
				|	mov REG0, #IS_TRUE
				||	if (swap) {
				|		bhi >2
				||	} else {
				|		blo >2
				||	}
				|1:
				|	mov REG0, #IS_FALSE
				|2:
				break;
			case ZEND_IS_SMALLER_OR_EQUAL:
				|	bvs >1
				|	mov REG0, #IS_TRUE
				||	if (swap) {
				|		bhs >2
				||	} else {
				|		bls >2
				||	}
				|1:
				|	mov REG0, #IS_FALSE
				|2:
				break;
			default:
				ZEND_UNREACHABLE();
		}
		|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
	}

	return 1;
}

static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
	zend_reg tmp_reg = ZREG_FPR0;

	|	DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_REG0, ZREG_TMP1
	|	DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP

	return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr);
}

static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
	zend_reg tmp_reg = ZREG_FPR0;

	|	DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_REG0, ZREG_TMP1
	|	DOUBLE_CMP tmp_reg, op1_addr, ZREG_TMP1, ZREG_FPTMP

	return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr);
}

static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
	bool swap = 0;

	if (Z_MODE(op1_addr) == IS_REG) {
		|	DOUBLE_CMP Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP
	} else if (Z_MODE(op2_addr) == IS_REG) {
		|	DOUBLE_CMP Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP
		swap = 1;
	} else {
		zend_reg tmp_reg = ZREG_FPR0;

		|	GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1
		|	DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP
	}

	return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr);
}

static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
	|	tst RETVALw, RETVALw
	if (smart_branch_opcode) {
		if (smart_branch_opcode == ZEND_JMPZ_EX ||
		    smart_branch_opcode == ZEND_JMPNZ_EX) {
			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_CASE:
					|	cset REG0w, eq
					break;
				case ZEND_IS_NOT_EQUAL:
					|	cset REG0w, ne
					break;
				case ZEND_IS_SMALLER:
					|	cset REG0w, lt
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					|	cset REG0w, le
					break;
				default:
					ZEND_UNREACHABLE();
			}
			|	add REG0w, REG0w, #2
			|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
		}
		if (smart_branch_opcode == ZEND_JMPZ ||
		    smart_branch_opcode == ZEND_JMPZ_EX) {
			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_CASE:
					if (exit_addr) {
						|	bne &exit_addr
					} else {
						|	bne => target_label
					}
					break;
				case ZEND_IS_NOT_EQUAL:
					if (exit_addr) {
						|	beq &exit_addr
					} else {
						|	beq => target_label
					}
					break;
				case ZEND_IS_SMALLER:
					if (exit_addr) {
						|	bge &exit_addr
					} else {
						|	bge => target_label
					}
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					if (exit_addr) {
						|	bgt &exit_addr
					} else {
						|	bgt => target_label
					}
					break;
				default:
					ZEND_UNREACHABLE();
			}
		} else if (smart_branch_opcode == ZEND_JMPNZ ||
		           smart_branch_opcode == ZEND_JMPNZ_EX) {
			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_CASE:
					if (exit_addr) {
						|	beq &exit_addr
					} else {
						|	beq => target_label
					}
					break;
				case ZEND_IS_NOT_EQUAL:
					if (exit_addr) {
						|	bne &exit_addr
					} else {
						|	bne => target_label
					}
					break;
				case ZEND_IS_SMALLER:
					if (exit_addr) {
						|	blt &exit_addr
					} else {
						|	blt => target_label
					}
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					if (exit_addr) {
						|	ble &exit_addr
					} else {
						|	ble => target_label
					}
					break;
				default:
					ZEND_UNREACHABLE();
			}
		} else if (smart_branch_opcode == ZEND_JMPZNZ) {
			switch (opline->opcode) {
				case ZEND_IS_EQUAL:
				case ZEND_CASE:
					|	bne => target_label
					break;
				case ZEND_IS_NOT_EQUAL:
					|	beq => target_label
					break;
				case ZEND_IS_SMALLER:
					|	bge => target_label
					break;
				case ZEND_IS_SMALLER_OR_EQUAL:
					|	bgt => target_label
					break;
				default:
					ZEND_UNREACHABLE();
			}
			|	b => target_label2
		} else {
			ZEND_UNREACHABLE();
		}
	} else {
		switch (opline->opcode) {
			case ZEND_IS_EQUAL:
			case ZEND_CASE:
				|	cset REG0w, eq
				break;
			case ZEND_IS_NOT_EQUAL:
				|	cset REG0w, ne
				break;
			case ZEND_IS_SMALLER:
				|	cset REG0w, lt
				break;
			case ZEND_IS_SMALLER_OR_EQUAL:
				|	cset REG0w, le
				break;
			default:
				ZEND_UNREACHABLE();
		}
		|	add REG0w, REG0w, #2
		|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
	}

	return 1;
}

static int zend_jit_cmp(dasm_State    **Dst,
                        const zend_op  *opline,
                        uint32_t        op1_info,
                        zend_ssa_range *op1_range,
                        zend_jit_addr   op1_addr,
                        uint32_t        op2_info,
                        zend_ssa_range *op2_range,
                        zend_jit_addr   op2_addr,
                        zend_jit_addr   res_addr,
                        int             may_throw,
                        zend_uchar      smart_branch_opcode,
                        uint32_t        target_label,
                        uint32_t        target_label2,
                        const void     *exit_addr,
                        bool            skip_comparison)
{
	bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var);
	bool has_slow;

	has_slow =
		(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
		(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
		((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
		 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))));

	if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
		if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
			if (op1_info & MAY_BE_DOUBLE) {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, ZREG_TMP1
			} else {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1
			}
		}
		if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
			if (op2_info & MAY_BE_DOUBLE) {
				|	IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1
				|.cold_code
				|3:
				if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
					|	IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1
				}
				if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
					return 0;
				}
				|	b >6
				|.code
			} else {
				|	IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1
			}
		}
		if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) {
			return 0;
		}
		if (op1_info & MAY_BE_DOUBLE) {
			|.cold_code
			|4:
			if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1
			}
			if (op2_info & MAY_BE_DOUBLE) {
				if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
					if (!same_ops) {
						|	IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, ZREG_TMP1
					} else {
						|	IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1
					}
				}
				if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
					return 0;
				}
				|	b >6
			}
			if (!same_ops) {
				|5:
				if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
					|	IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1
				}
				if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
					return 0;
				}
				|	b >6
			}
			|.code
		}
	} else if ((op1_info & MAY_BE_DOUBLE) &&
	           !(op1_info & MAY_BE_LONG) &&
	           (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
		if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
			|	IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1
		}
		if (op2_info & MAY_BE_DOUBLE) {
			if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
				if (!same_ops && (op2_info & MAY_BE_LONG)) {
					|	IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3, ZREG_TMP1
				} else {
					|	IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1
				}
			}
			if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
				return 0;
			}
		}
		if (!same_ops && (op2_info & MAY_BE_LONG)) {
			if (op2_info & MAY_BE_DOUBLE) {
				|.cold_code
			}
		    |3:
			if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
				|	IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1
			}
			if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
				return 0;
			}
			if (op2_info & MAY_BE_DOUBLE) {
				|	b >6
				|.code
			}
		}
	} else if ((op2_info & MAY_BE_DOUBLE) &&
	           !(op2_info & MAY_BE_LONG) &&
	           (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
		if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
			|	IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1
		}
		if (op1_info & MAY_BE_DOUBLE) {
			if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
				if (!same_ops && (op1_info & MAY_BE_LONG)) {
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3, ZREG_TMP1
				} else {
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1
				}
			}
			if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
				return 0;
			}
		}
		if (!same_ops && (op1_info & MAY_BE_LONG)) {
			if (op1_info & MAY_BE_DOUBLE) {
				|.cold_code
			}
			|3:
			if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1
			}
			if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
				return 0;
			}
			if (op1_info & MAY_BE_DOUBLE) {
				|	b >6
				|.code
			}
		}
	}

	if (has_slow ||
	    (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
	    (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
		if (has_slow) {
			|.cold_code
			|9:
		}
		|	SET_EX_OPLINE opline, REG0
		if (Z_MODE(op1_addr) == IS_REG) {
			zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
			if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
				return 0;
			}
			op1_addr = real_addr;
		}
		if (Z_MODE(op2_addr) == IS_REG) {
			zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
			if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
				return 0;
			}
			op2_addr = real_addr;
		}
		|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
			|	IF_NOT_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w
			|	LOAD_32BIT_VAL FCARG1x, opline->op1.var
			|	EXT_CALL zend_jit_undefined_op_helper, REG0
			|	LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval
			|1:
		}
		if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
			|	IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1
			|	str FCARG1x, T1 // save
			|	LOAD_32BIT_VAL FCARG1x, opline->op2.var
			|	EXT_CALL zend_jit_undefined_op_helper, REG0
			|	ldr FCARG1x, T1 // restore
			|	LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
			|	b >2
			|1:
			|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
			|2:
		} else {
			|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
		}
		|	EXT_CALL zend_compare, REG0
		if ((opline->opcode != ZEND_CASE &&
		     (opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
		     (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
		    ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
		     (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
			|	str RETVALw, T1 // save
			if (opline->opcode != ZEND_CASE) {
				|	FREE_OP opline->op1_type, opline->op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
			}
			|	FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
			|	ldr RETVALw, T1 // restore
		}
		if (may_throw) {
			zend_jit_check_exception_undef_result(Dst, opline);
		}
		if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
			return 0;
		}
		if (has_slow) {
			|	b >6
			|.code
		}
	}

	|6:

	return 1;
}

static int zend_jit_identical(dasm_State    **Dst,
                              const zend_op  *opline,
                              uint32_t        op1_info,
                              zend_ssa_range *op1_range,
                              zend_jit_addr   op1_addr,
                              uint32_t        op2_info,
                              zend_ssa_range *op2_range,
                              zend_jit_addr   op2_addr,
                              zend_jit_addr   res_addr,
                              int             may_throw,
                              zend_uchar      smart_branch_opcode,
                              uint32_t        target_label,
                              uint32_t        target_label2,
                              const void     *exit_addr,
                              bool            skip_comparison)
{
	uint32_t identical_label = (uint32_t)-1;
	uint32_t not_identical_label = (uint32_t)-1;

	if (smart_branch_opcode && !exit_addr) {
		if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
			if (smart_branch_opcode == ZEND_JMPZ) {
				not_identical_label = target_label;
			} else if (smart_branch_opcode == ZEND_JMPNZ) {
				identical_label = target_label;
			} else if (smart_branch_opcode == ZEND_JMPZNZ) {
				not_identical_label = target_label;
				identical_label = target_label2;
			} else {
				ZEND_UNREACHABLE();
			}
		} else {
			if (smart_branch_opcode == ZEND_JMPZ) {
				identical_label = target_label;
			} else if (smart_branch_opcode == ZEND_JMPNZ) {
				not_identical_label = target_label;
			} else if (smart_branch_opcode == ZEND_JMPZNZ) {
				identical_label = target_label;
				not_identical_label = target_label2;
			} else {
				ZEND_UNREACHABLE();
			}
		}
	}

	if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG &&
	    (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) {
		if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) {
			return 0;
		}
		return 1;
	} else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE &&
	           (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) {
		if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
			return 0;
		}
		return 1;
	}

	if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) {
		op1_info |= MAY_BE_NULL;
		op2_info |= MAY_BE_NULL;
		|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		|	IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w
		|.cold_code
		|1:
		|	// zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
		|	SET_EX_OPLINE opline, REG0
		|	LOAD_32BIT_VAL FCARG1w, opline->op1.var
		|	EXT_CALL zend_jit_undefined_op_helper, REG0
		if (may_throw) {
			zend_jit_check_exception_undef_result(Dst, opline);
		}
		|	LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval
		|	b >1
		|.code
		|1:
		|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
		|	IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w
		|.cold_code
		|1:
		|	// zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
		|	SET_EX_OPLINE opline, REG0
		|	str FCARG1x, T1 // save
		|	LOAD_32BIT_VAL FCARG1w, opline->op2.var
		|	EXT_CALL zend_jit_undefined_op_helper, REG0
		if (may_throw) {
			zend_jit_check_exception_undef_result(Dst, opline);
		}
		|	ldr FCARG1x, T1 // restore
		|	LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
		|	b >1
		|.code
		|1:
	} else if (op1_info & MAY_BE_UNDEF) {
		op1_info |= MAY_BE_NULL;
		|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		|	IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w
		|.cold_code
		|1:
		|	// zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
		|	SET_EX_OPLINE opline, REG0
		|	LOAD_32BIT_VAL FCARG1w, opline->op1.var
		|	EXT_CALL zend_jit_undefined_op_helper, REG0
		if (may_throw) {
			zend_jit_check_exception_undef_result(Dst, opline);
		}
		|	LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval
		|	b >1
		|.code
		|1:
		if (opline->op2_type != IS_CONST) {
			|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
		}
	} else if (op2_info & MAY_BE_UNDEF) {
		op2_info |= MAY_BE_NULL;
		|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
		|	IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w
		|.cold_code
		|1:
		|	// zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
		|	SET_EX_OPLINE opline, REG0
		|	LOAD_32BIT_VAL FCARG1w, opline->op2.var
		|	EXT_CALL zend_jit_undefined_op_helper, REG0
		if (may_throw) {
			zend_jit_check_exception_undef_result(Dst, opline);
		}
		|	LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
		|	b >1
		|.code
		|1:
		if (opline->op1_type != IS_CONST) {
			|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		}
	} else if ((op1_info & op2_info & MAY_BE_ANY) != 0) {
		if (opline->op1_type != IS_CONST) {
			if (Z_MODE(op1_addr) == IS_REG) {
				zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
				if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
					return 0;
				}
				op1_addr = real_addr;
			}
			|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		}
		if (opline->op2_type != IS_CONST) {
			if (Z_MODE(op2_addr) == IS_REG) {
				zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
				if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
					return 0;
				}
				op2_addr = real_addr;
			}
			|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
		}
	}

	if ((op1_info & op2_info & MAY_BE_ANY) == 0) {
		if ((opline->opcode != ZEND_CASE_STRICT &&
		     (opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
		     (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
		    ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
		     (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
			if (opline->opcode != ZEND_CASE_STRICT) {
				|	FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
			}
			|	FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2
		}
		if (smart_branch_opcode) {
			if (may_throw) {
				zend_jit_check_exception_undef_result(Dst, opline);
			}
			if (exit_addr) {
				if (smart_branch_opcode == ZEND_JMPZ) {
					|	b &exit_addr
				}
			} else if (not_identical_label != (uint32_t)-1) {
				|	b =>not_identical_label
			}
		} else {
			|	SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2
			if (may_throw) {
				zend_jit_check_exception(Dst);
			}
		}
		return 1;
	}

	if (opline->op1_type & (IS_CV|IS_VAR)) {
		|	ZVAL_DEREF FCARG1x, op1_info, TMP1w
	}
	if (opline->op2_type & (IS_CV|IS_VAR)) {
		|	ZVAL_DEREF FCARG2x, op2_info, TMP1w
	}

	if (has_concrete_type(op1_info)
	 && has_concrete_type(op2_info)
	 && concrete_type(op1_info) == concrete_type(op2_info)
	 && concrete_type(op1_info) <= IS_TRUE) {
		if (smart_branch_opcode) {
			if (exit_addr) {
				if (smart_branch_opcode == ZEND_JMPNZ) {
					|	b &exit_addr
				}
			} else if (identical_label != (uint32_t)-1) {
				|	b =>identical_label
			}
		} else {
			|	SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2
		}
	} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) {
		if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) {
			if (smart_branch_opcode) {
				if (exit_addr) {
					if (smart_branch_opcode == ZEND_JMPNZ) {
						|	b &exit_addr
					}
				} else if (identical_label != (uint32_t)-1) {
					|	b =>identical_label
				}
			} else {
				|	SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2
			}
		} else {
			if (smart_branch_opcode) {
				if (exit_addr) {
					if (smart_branch_opcode == ZEND_JMPZ) {
						|	b &exit_addr
					}
				} else if (not_identical_label != (uint32_t)-1) {
					|	b =>not_identical_label
				}
			} else {
				|	SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2
			}
		}
	} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) {
		zval *val = Z_ZV(op1_addr);

		|	ldrb TMP1w, [FCARG2x, #offsetof(zval, u1.v.type)]
		|	cmp TMP1w, #Z_TYPE_P(val)
		if (smart_branch_opcode) {
			if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) {
				|	bne >8
				|	FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2
				if (may_throw) {
					zend_jit_check_exception_undef_result(Dst, opline);
				}
				if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
					|	b &exit_addr
				} else if (identical_label != (uint32_t)-1) {
					|	b =>identical_label
				} else {
					|	b >9
				}
				|8:
			} else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
				|	beq &exit_addr
			} else if (identical_label != (uint32_t)-1) {
				|	beq =>identical_label
			} else {
				|	beq >9
			}
		} else {
			if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
				|	cset REG0w, eq
			} else {
				|	cset REG0w, ne
			}
			|	add REG0w, REG0w, #2
			|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
		}
		if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
		    (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
			|	FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2
			if (may_throw) {
				zend_jit_check_exception_undef_result(Dst, opline);
			}
		}
		if (exit_addr) {
			if (smart_branch_opcode == ZEND_JMPZ) {
				|	b &exit_addr
			}
		} else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) {
			|	b =>not_identical_label
		}
	} else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) {
		zval *val = Z_ZV(op2_addr);

		|	ldrb TMP1w, [FCARG1x, #offsetof(zval, u1.v.type)]
		|	cmp TMP1w, #Z_TYPE_P(val)
		if (smart_branch_opcode) {
			if (opline->opcode != ZEND_CASE_STRICT
			 && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) {
				|	bne >8
				|	FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
				if (may_throw) {
					zend_jit_check_exception_undef_result(Dst, opline);
				}
				if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
					|	b &exit_addr
				} else if (identical_label != (uint32_t)-1) {
					|	b =>identical_label
				} else {
					|	b >9
				}
				|8:
			} else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
				|	beq &exit_addr
			} else if (identical_label != (uint32_t)-1) {
				|	beq =>identical_label
			} else {
				|	beq >9
			}
		} else {
			if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
				|	cset REG0w, eq
			} else {
				|	cset REG0w, ne
			}
			|	add REG0w, REG0w, #2
			|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
		}
		if (opline->opcode != ZEND_CASE_STRICT
		 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
		    (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
			|	FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
			if (may_throw) {
				zend_jit_check_exception_undef_result(Dst, opline);
			}
		}
		if (smart_branch_opcode) {
			if (exit_addr) {
				if (smart_branch_opcode == ZEND_JMPZ) {
					|	b &exit_addr
				}
			} else if (not_identical_label != (uint32_t)-1) {
				|	b =>not_identical_label
			}
		}
	} else {
		if (opline->op1_type == IS_CONST) {
			|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		}
		if (opline->op2_type == IS_CONST) {
			|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
		}
		|	EXT_CALL zend_is_identical, REG0
			if ((opline->opcode != ZEND_CASE_STRICT &&
			     (opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
			     (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
			    ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
			     (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
				|	str RETVALw, T1 // save
				if (opline->opcode != ZEND_CASE_STRICT) {
					|	FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
				}
				|	FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2
				if (may_throw) {
					zend_jit_check_exception_undef_result(Dst, opline);
				}
				|	ldr RETVALw, T1 // restore
			}
		if (smart_branch_opcode) {
			if (exit_addr) {
				if (smart_branch_opcode == ZEND_JMPNZ) {
					|	cbnz RETVALw, &exit_addr
				} else {
					|	cbz RETVALw, &exit_addr
				}
			} else if (not_identical_label != (uint32_t)-1) {
				|	cbz RETVALw, =>not_identical_label
				if (identical_label != (uint32_t)-1) {
					|	b =>identical_label
				}
			} else if (identical_label != (uint32_t)-1) {
				|	cbnz RETVALw, =>identical_label
			}
		} else {
			if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
				|	add RETVALw, RETVALw, #2
			} else {
				|	neg RETVALw, RETVALw
				|	add RETVALw, RETVALw, #3
			}
			|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, RETVALw, TMP1
		}
	}

	|9:
	if (may_throw) {
		zend_jit_check_exception(Dst);
	}
	return 1;
}

static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr)
{
	uint32_t true_label = -1;
	uint32_t false_label = -1;
	bool set_bool = 0;
	bool set_bool_not = 0;
	bool set_delayed = 0;
	bool jmp_done = 0;

	if (branch_opcode == ZEND_BOOL) {
		set_bool = 1;
	} else if (branch_opcode == ZEND_BOOL_NOT) {
		set_bool = 1;
		set_bool_not = 1;
	} else if (branch_opcode == ZEND_JMPZ) {
		false_label = target_label;
	} else if (branch_opcode == ZEND_JMPNZ) {
		true_label = target_label;
	} else if (branch_opcode == ZEND_JMPZNZ) {
		true_label = target_label2;
		false_label = target_label;
	} else if (branch_opcode == ZEND_JMPZ_EX) {
		set_bool = 1;
		false_label = target_label;
	} else if (branch_opcode == ZEND_JMPNZ_EX) {
		set_bool = 1;
		true_label = target_label;
	} else {
		ZEND_UNREACHABLE();
	}

	if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
		if (zend_is_true(Z_ZV(op1_addr))) {
			/* Always TRUE */
			if (set_bool) {
				if (set_bool_not) {
					|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
				} else {
					|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
				}
			}
			if (true_label != (uint32_t)-1) {
				|	b =>true_label
			}
		} else {
			/* Always FALSE */
			if (set_bool) {
				if (set_bool_not) {
					|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
				} else {
					|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
				}
			}
			if (false_label != (uint32_t)-1) {
				|	b =>false_label
			}
		}
		return 1;
	}

	if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) {
		|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		|	ZVAL_DEREF FCARG1x, op1_info, TMP1w
		op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	}

	if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
		if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) {
			/* Always TRUE */
			if (set_bool) {
				if (set_bool_not) {
					|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
				} else {
					|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
				}
			}
			if (true_label != (uint32_t)-1) {
				|	b =>true_label
			}
		} else {
			if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) {
				/* Always FALSE */
				if (set_bool) {
					if (set_bool_not) {
						|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
					} else {
						|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
					}
				}
			} else {
				|	CMP_ZVAL_TYPE op1_addr, IS_TRUE, ZREG_TMP1
				if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
				    if ((op1_info & MAY_BE_LONG) &&
				        !(op1_info & MAY_BE_UNDEF) &&
				        !set_bool) {
						if (exit_addr) {
							if (branch_opcode == ZEND_JMPNZ) {
								|	blt >9
							} else {
								|	blt &exit_addr
							}
						} else if (false_label != (uint32_t)-1) {
							|	blt =>false_label
						} else {
							|	blt >9
						}
						jmp_done = 1;
					} else {
						|	bgt >2
					}
				}
				if (!(op1_info & MAY_BE_TRUE)) {
					/* It's FALSE */
					if (set_bool) {
						if (set_bool_not) {
							|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
						} else {
							|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
						}
					}
				} else {
					if (exit_addr) {
						if (set_bool) {
							|	bne >1
							|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
							if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
								|	b &exit_addr
							} else {
								|	b >9
							}
							|1:
							|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
							if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
								if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
									|	bne &exit_addr
								}
							}
						} else {
							if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
								|	beq &exit_addr
							} else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
								|	bne &exit_addr
							} else {
								|	beq >9
							}
						}
					} else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
						if (set_bool) {
							|	bne >1
							|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
							if (true_label != (uint32_t)-1) {
								|	b =>true_label
							} else {
								|	b >9
							}
							|1:
							|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
						} else {
							if (true_label != (uint32_t)-1) {
								|	beq =>true_label
							} else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
								|	bne =>false_label
								jmp_done = 1;
							} else {
								|	beq >9
							}
						}
					} else if (set_bool) {
						|	cset REG0w, eq
						if (set_bool_not) {
							|	neg REG0w, REG0w
							|	add REG0w, REG0w, #3
						} else {
							|	add REG0w, REG0w, #2
						}
						if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) {
							set_delayed = 1;
						} else {
							|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
						}
					}
				}
			}

			/* It's FALSE, but may be UNDEF */
			if (op1_info & MAY_BE_UNDEF) {
				if (op1_info & MAY_BE_ANY) {
					if (set_delayed) {
						|	CMP_ZVAL_TYPE op1_addr, IS_UNDEF, ZREG_TMP1
						|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
						|	beq >1
					} else {
						|	IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
					}
					|.cold_code
					|1:
				}
				|	LOAD_32BIT_VAL FCARG1w, opline->op1.var
				|	SET_EX_OPLINE opline, REG0
				|	EXT_CALL zend_jit_undefined_op_helper, REG0

				if (may_throw) {
					if (!zend_jit_check_exception_undef_result(Dst, opline)) {
						return 0;
					}
				}

				if (exit_addr) {
					if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
						|	b &exit_addr
					}
				} else if (false_label != (uint32_t)-1) {
					|	b =>false_label
				}
				if (op1_info & MAY_BE_ANY) {
					if (exit_addr) {
						if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
							|	b >9
						}
					} else if (false_label == (uint32_t)-1) {
						|	b >9
					}
					|.code
				}
			}

			if (!jmp_done) {
				if (exit_addr) {
					if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
						if (op1_info & MAY_BE_LONG) {
							|	b >9
						}
					} else if (op1_info & MAY_BE_LONG) {
						|	b &exit_addr
					}
				} else if (false_label != (uint32_t)-1) {
					|	b =>false_label
				} else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
					|	b >9
				}
			}
		}
	}

	if (op1_info & MAY_BE_LONG) {
		|2:
		if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
			|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1
		}
		if (Z_MODE(op1_addr) == IS_REG) {
			|	tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr))
		} else {
			|	LONG_CMP_WITH_CONST op1_addr, Z_L(0), TMP1, TMP2
		}
		if (set_bool) {
			|	cset REG0w, ne
			if (set_bool_not) {
				|	neg REG0w, REG0w
				|	add REG0w, REG0w, #3
			} else {
				|	add REG0w, REG0w, #2
			}
			|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
		}
		if (exit_addr) {
			if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
				|	bne &exit_addr
			} else {
				|	beq &exit_addr
			}
		} else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
			if (true_label != (uint32_t)-1) {
				|	bne =>true_label
				if (false_label != (uint32_t)-1) {
					|	b =>false_label
				}
			} else {
				|	beq =>false_label
			}
		}
	}

	if ((op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) == MAY_BE_DOUBLE) {
		if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
			|.cold_code
		}
	    |2:
		|	fmov FPR0, xzr  // TODO: "movi d0, #0" is not recognized by DynASM/arm64
		|	DOUBLE_CMP ZREG_FPR0, op1_addr, ZREG_TMP1, ZREG_FPTMP

		if (set_bool) {
			if (exit_addr) {
				if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
					|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
					|	bvs &exit_addr
					|	bne &exit_addr
					|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
				} else {
					|	bvs >1
					|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
					|	beq &exit_addr
					|1:
					|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
				}
			} else if (false_label != (uint32_t)-1) { // JMPZ_EX
				|	bvs >1
				|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
				|	beq => false_label
				|1:
				|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
			} else if (true_label != (uint32_t)-1) { // JMPNZ_EX
				|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
				|	bvs => true_label
				|	bne => true_label
				|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
			} else if (set_bool_not) { // BOOL_NOT
				|	mov REG0w, #IS_FALSE
				|	bvs >1
				|	bne >1
				|	mov REG0w, #IS_TRUE
				|1:
				|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
			} else { // BOOL
				|	mov REG0w, #IS_TRUE
				|	bvs >1
				|	bne >1
				|	mov REG0w, #IS_FALSE
				|1:
				|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
			}
			if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
				|	b >9
				|.code
			}
		} else {
			if (exit_addr) {
				if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
					|	bvs &exit_addr
					|	bne &exit_addr
					|1:
				} else {
					|	bvs >1
					|	beq &exit_addr
					|1:
				}
				if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
					|	b >9
				}
			} else {
				ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1);
				if (false_label != (uint32_t)-1 ) {
					|	bvs  >1
					|	beq  => false_label
					|1:
					if (true_label != (uint32_t)-1) {
						|	b =>true_label
					} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
						|	b >9
					}
				} else {
					|	bvs  => true_label
					|	bne  => true_label
					if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
						|	b >9
					}
				}
			}
			if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
				|.code
			}
		}
	} else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
		if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
			|.cold_code
			|2:
		}
		if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
			|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		}
		|	SET_EX_OPLINE opline, REG0
		|	EXT_CALL zend_is_true, REG0
		|	mov REG0, RETVALx

		if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
			(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
			op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);

			if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
				|	IF_NOT_ZVAL_REFCOUNTED op1_addr, >3, ZREG_TMP1, ZREG_TMP2
			}
			|	GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
			|	GC_DELREF FCARG1x, TMP1w
			|	bne >3
			// In x86, r0 is used in macro ZVAL_DTOR_FUNC as temporary register, hence, r0 should be saved/restored
			// before/after this macro. In AArch64, TMP1 is used, but we still have to store REG0,
			// because it's clobbered by function call.
			|	str REG0, T1 // save
			|	ZVAL_DTOR_FUNC op1_info, opline, TMP1
			|	ldr REG0, T1 // restore
			|3:
		}
		if (may_throw) {
			|	MEM_LOAD_64_ZTS ldr, REG1, executor_globals, exception, TMP1
			|	cbnz REG1, ->exception_handler_undef
		}

		if (set_bool) {
			if (set_bool_not) {
				|	neg REG0w, REG0w
				|	add REG0w, REG0w, #3
			} else {
				|	add REG0w, REG0w, #2
			}
			|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
			if (exit_addr) {
				|	CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1
				if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
					|	bne &exit_addr
				} else {
					|	beq &exit_addr
				}
			} else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
				|	CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1
				if (true_label != (uint32_t)-1) {
					|	bne =>true_label
					if (false_label != (uint32_t)-1) {
						|	b =>false_label
					} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
						|	b >9
					}
				} else {
					|	beq =>false_label
				}
			}
			if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
				|	b >9
				|.code
			}
		} else {
			if (exit_addr) {
				if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
					|	cbnz REG0w, &exit_addr
					if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
						|	b >9
					}
				} else {
					|	cbz REG0w, &exit_addr
					if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
						|	b >9
					}
				}
			} else if (true_label != (uint32_t)-1) {
				|	cbnz REG0w, =>true_label
				if (false_label != (uint32_t)-1) {
					|	b =>false_label
				} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
					|	b >9
				}
			} else {
				|	cbz REG0w, =>false_label
				if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
					|	b >9
				}
			}

			if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
				|.code
			}
		}
	}

	|9:

	return 1;
}

static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr)
{
	if (op1_addr != op1_def_addr) {
		if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
			return 0;
		}
		if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
			op1_addr = op1_def_addr;
		}
	}

	if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0)) {
		return 0;
	}
	if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
		return 0;
	}
	if (op1_info & MAY_BE_UNDEF) {
		zend_jit_check_exception(Dst);
	}
	return 1;
}

static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
{
	ZEND_ASSERT(opline->op1_type == IS_CV);

	if (op2_addr != op2_def_addr) {
		if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) {
			return 0;
		}
		if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) {
			op2_addr = op2_def_addr;
		}
	}

	if (Z_MODE(op1_addr) != IS_REG
	 && Z_MODE(op1_use_addr) == IS_REG
	 && !Z_LOAD(op1_use_addr)
	 && !Z_STORE(op1_use_addr)) {
		/* Force type update */
		op1_info |= MAY_BE_UNDEF;
	}
	if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr,
			may_throw)) {
		return 0;
	}
	if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) {
		return 0;
	}
	if (opline->result_type != IS_UNUSED) {
		if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
			return 0;
		}
	}

	return 1;
}

/* copy of hidden zend_closure */
typedef struct _zend_closure {
	zend_object       std;
	zend_function     func;
	zval              this_ptr;
	zend_class_entry *called_scope;
	zif_handler       orig_internal_handler;
} zend_closure;

static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack)
{
	int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
	const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

	if (!exit_addr) {
		return 0;
	}

	|	// Check Stack Overflow
	|	MEM_LOAD_64_ZTS ldr, REG1, executor_globals, vm_stack_end, TMP1
	|	MEM_LOAD_OP_ZTS sub, ldr, REG1, executor_globals, vm_stack_top, TMP1, TMP2
	|	CMP_64_WITH_CONST_32 REG1, used_stack, TMP1
	|	blo &exit_addr

	return 1;
}

static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack)
{
	uint32_t used_stack;
	bool stack_check = 1;

	// REG0   -> zend_function
	// FCARG1 -> used_stack

	if (func) {
		used_stack = zend_vm_calc_used_stack(opline->extended_value, func);
		if ((int)used_stack <= checked_stack) {
			stack_check = 0;
		}
	} else {
		used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval);

		|	// if (EXPECTED(ZEND_USER_CODE(func->type))) {
		if (!is_closure) {
			|	LOAD_32BIT_VAL FCARG1w, used_stack
			|	// Check whether REG0 is an internal function.
			|	ldrb TMP1w, [REG0, #offsetof(zend_function, type)]
			|	TST_32_WITH_CONST TMP1w, 1, TMP2w
			|	bne >1
		} else {
			|	LOAD_32BIT_VAL FCARG1w, used_stack
		}
		|	// used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval);
		|	LOAD_32BIT_VAL REG2w, opline->extended_value
		if (!is_closure) {
			|	ldr TMP1w, [REG0, #offsetof(zend_function, op_array.num_args)]
			|	cmp REG2w, TMP1w
			|	csel REG2w, REG2w, TMP1w, le
			|	ldr TMP1w, [REG0, #offsetof(zend_function, op_array.last_var)]
			|	sub REG2w, REG2w, TMP1w
			|	ldr TMP1w, [REG0, #offsetof(zend_function, op_array.T)]
			|	sub REG2w, REG2w, TMP1w
		} else {
			|	ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.num_args)]
			|	cmp REG2w, TMP1w
			|	csel REG2w, REG2w, TMP1w, le
			|	ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.last_var)]
			|	sub REG2w, REG2w, TMP1w
			|	ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.T)]
			|	sub REG2w, REG2w, TMP1w
		}
		|	sxtw REG2, REG2w
		|	sub FCARG1x, FCARG1x, REG2, lsl #4
		|1:
	}

	zend_jit_start_reuse_ip();

	|	// if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) {
	|	MEM_LOAD_64_ZTS ldr, RX, executor_globals, vm_stack_top, TMP1

	if (stack_check) {
		|	// Check Stack Overflow
		|	MEM_LOAD_64_ZTS ldr, REG2, executor_globals, vm_stack_end, TMP1
		|	sub REG2, REG2, RX
		if (func) {
			|	CMP_64_WITH_CONST_32 REG2, used_stack, TMP1
		} else {
			|	cmp REG2, FCARG1x
		}

		if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
			int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
			const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

			if (!exit_addr) {
				return 0;
			}

			|	blo &exit_addr
		} else {
			|	blo >1
			|	// EG(vm_stack_top) = (zval*)((char*)call + used_stack);
			|.cold_code
			|1:
			if (func) {
				|	LOAD_32BIT_VAL FCARG1w, used_stack
			}
			if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
				|	SET_EX_OPLINE opline, REG0
				|	EXT_CALL zend_jit_int_extend_stack_helper, REG0
			} else {
				if (!is_closure) {
					|	mov FCARG2x, REG0
				} else {
					|	add FCARG2x, REG0, #offsetof(zend_closure, func)
				}
				|	SET_EX_OPLINE opline, REG0
				|	EXT_CALL zend_jit_extend_stack_helper, REG0
			}
			|	mov RX, RETVALx
			|	b >1
			|.code
		}
	}

	if (func) {
		||	if (arm64_may_encode_imm12((int64_t)used_stack)) {
		|		MEM_UPDATE_ZTS add, ldr, str, #used_stack, executor_globals, vm_stack_top, REG2, TMP1
		||	} else {
		|		LOAD_32BIT_VAL TMP1w, used_stack
		|		MEM_UPDATE_ZTS add, ldr, str, TMP1, executor_globals, vm_stack_top, REG2, TMP2
		||	}
	} else {
		|	MEM_UPDATE_ZTS add, ldr, str, FCARG1x, executor_globals, vm_stack_top, REG2, TMP1
	}
	|	// zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object);
	if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) {
		|	// ZEND_SET_CALL_INFO(call, 0, call_info);
		|	LOAD_32BIT_VAL TMP1w, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION)
		|	str TMP1w, EX:RX->This.u1.type_info
	}
	if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
		|	// call->func = func;
		|1:
		|	ADDR_STORE EX:RX->func, func, REG1
	} else {
		if (!is_closure) {
			|	// call->func = func;
			|	str REG0, EX:RX->func
		} else {
			|	// call->func = &closure->func;
			|	add REG1, REG0, #offsetof(zend_closure, func)
			|	str REG1, EX:RX->func
		}
		|1:
	}
	if (opline->opcode == ZEND_INIT_METHOD_CALL) {
		|	// Z_PTR(call->This) = obj;
		|	ldr REG1, T1
		|	str REG1, EX:RX->This.value.ptr
	    if (opline->op1_type == IS_UNUSED || delayed_fetch_this) {
			|	// call->call_info |= ZEND_CALL_HAS_THIS;
			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
				|	LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS
				|	str TMP1w, EX:RX->This.u1.type_info
			} else {
				|	ldr TMP1w, EX:RX->This.u1.type_info
				|	BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_HAS_THIS, TMP2w
				|	str TMP1w, EX:RX->This.u1.type_info
			}
	    } else {
			if (opline->op1_type == IS_CV) {
				|	// GC_ADDREF(obj);
				|	GC_ADDREF REG1, TMP1w
			}
			|	// call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS;
			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
				|	LOAD_32BIT_VAL TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS)
				|	str TMP1w, EX:RX->This.u1.type_info
			} else {
				|	ldr TMP1w, EX:RX->This.u1.type_info
				|	BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS), TMP2w
				|	str TMP1w, EX:RX->This.u1.type_info
			}
	    }
	} else if (!is_closure) {
		|	// Z_CE(call->This) = called_scope;
		|	str xzr, EX:RX->This.value.ptr
	} else {
		if (opline->op2_type == IS_CV) {
			|	// GC_ADDREF(closure);
			|	GC_ADDREF REG0, TMP1w
		}
		|	//	object_or_called_scope = closure->called_scope;
		|	ldr REG1, [REG0, #offsetof(zend_closure, called_scope)]
		|	str REG1, EX:RX->This.value.ptr
		|	// call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE |
		|	//	(closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE);
		|	ldr REG2w, [REG0, #offsetof(zend_closure, func.common.fn_flags)]
		|	BW_OP_32_WITH_CONST and, REG2w, REG2w, ZEND_ACC_FAKE_CLOSURE, TMP1w
		|	BW_OP_32_WITH_CONST orr, REG2w, REG2w, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE), TMP1w
		|	//	if (Z_TYPE(closure->this_ptr) != IS_UNDEF) {
		|	ldrb TMP1w, [REG0, #offsetof(zend_closure, this_ptr.u1.v.type)]
		|	cmp TMP1w, #IS_UNDEF
		|	beq >1
		|	//	call_info |= ZEND_CALL_HAS_THIS;
		|	BW_OP_32_WITH_CONST orr, REG2w, REG2w, ZEND_CALL_HAS_THIS, TMP1w
		|	//	object_or_called_scope = Z_OBJ(closure->this_ptr);
		|	ldr REG1, [REG0, #offsetof(zend_closure, this_ptr.value.ptr)]
	    |1:
		|	// ZEND_SET_CALL_INFO(call, 0, call_info);
		|	ldr TMP1w, EX:RX->This.u1.type_info
		|	orr TMP1w, TMP1w, REG2w
		|	str TMP1w, EX:RX->This.u1.type_info
		|	// Z_PTR(call->This) = object_or_called_scope;
		|	str REG1, EX:RX->This.value.ptr
		|	ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.run_time_cache__ptr)]
		|	cbnz TMP1, >1
		|	add FCARG1x, REG0, #offsetof(zend_closure, func)
		|	EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0
		|1:
	}
	|	// ZEND_CALL_NUM_ARGS(call) = num_args;
	|	LOAD_32BIT_VAL TMP1w, opline->extended_value
	|	str TMP1w, EX:RX->This.u2.num_args

	return 1;
}

static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline)
{
	int32_t exit_point;
	const void *exit_addr;

	if (func->type == ZEND_INTERNAL_FUNCTION) {
#ifdef ZEND_WIN32
		// TODO: ASLR may cause different addresses in different workers ???
		return 0;
#endif
	} else if (func->type == ZEND_USER_FUNCTION) {
		if (!zend_accel_in_shm(func->op_array.opcodes)) {
			/* op_array and op_array->opcodes are not persistent. We can't link. */
			return 0;
		}
	} else {
		ZEND_UNREACHABLE();
		return 0;
	}

	exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM);
	exit_addr = zend_jit_trace_get_exit_addr(exit_point);
	if (!exit_addr) {
		return 0;
	}

	|	// call = EX(call);
	|	ldr REG1, EX->call
	while (level > 0) {
		|	ldr REG1, EX:REG1->prev_execute_data
		level--;
	}

	if (func->type == ZEND_USER_FUNCTION &&
	    (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
	     (func->common.fn_flags & ZEND_ACC_CLOSURE) ||
	     !func->common.function_name)) {
		const zend_op *opcodes = func->op_array.opcodes;

		|	ldr REG1, EX:REG1->func
		|	LOAD_ADDR REG2, ((ptrdiff_t)opcodes)
		|	ldr TMP1, [REG1, #offsetof(zend_op_array, opcodes)]
		|	cmp TMP1, REG2
		|	bne &exit_addr
	} else {
		|	LOAD_ADDR REG2, ((ptrdiff_t)func)
		|	ldr TMP1, EX:REG1->func
		|	cmp TMP1, REG2
		|	bne &exit_addr
	}

	return 1;
}

static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack)
{
	zend_func_info *info = ZEND_FUNC_INFO(op_array);
	zend_call_info *call_info = NULL;
	zend_function *func = NULL;

	if (delayed_call_chain) {
		if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
			return 0;
		}
	}

	if (info) {
		call_info = info->callee_info;
		while (call_info && call_info->caller_init_opline != opline) {
			call_info = call_info->next_callee;
		}
		if (call_info && call_info->callee_func && !call_info->is_prototype) {
			func = call_info->callee_func;
		}
	}

	if (!func
	 && trace
	 && trace->op == ZEND_JIT_TRACE_INIT_CALL) {
		func = (zend_function*)trace->func;
	}

	if (opline->opcode == ZEND_INIT_FCALL
	 && func
	 && func->type == ZEND_INTERNAL_FUNCTION) {
		/* load constant address later */
	} else if (func && op_array == &func->op_array) {
		/* recursive call */
		|	ldr REG0, EX->func
	} else {
		|	// if (CACHED_PTR(opline->result.num))
		|	ldr REG2, EX->run_time_cache
		|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG2, opline->result.num, TMP1
		|	cbz REG0, >1
		|.cold_code
		|1:
		if (opline->opcode == ZEND_INIT_FCALL
		 && func
		 && func->type == ZEND_USER_FUNCTION
		 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) {
			|	LOAD_ADDR FCARG1x, func
			|	MEM_ACCESS_64_WITH_UOFFSET str, FCARG1x, REG2, opline->result.num, TMP1
			|	EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0
			|	mov REG0, RETVALx
			|	b >3
		} else {
			zval *zv = RT_CONSTANT(opline, opline->op2);

			if (opline->opcode == ZEND_INIT_FCALL) {
				|	LOAD_ADDR FCARG1x, Z_STR_P(zv);
				|	ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1
				|	EXT_CALL zend_jit_find_func_helper, REG0
			} else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) {
				|	LOAD_ADDR FCARG1x, Z_STR_P(zv + 1);
				|	ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1
				|	EXT_CALL zend_jit_find_func_helper, REG0
			} else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
				|	LOAD_ADDR FCARG1x, zv;
				|	ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1
				|	EXT_CALL zend_jit_find_ns_func_helper, REG0
			} else {
				ZEND_UNREACHABLE();
			}
			|	// Get the return value of function zend_jit_find_func_helper/zend_jit_find_ns_func_helper
			|	mov REG0, RETVALx
			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
				int32_t exit_point = zend_jit_trace_get_exit_point(opline,
					func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0);
				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

				if (!exit_addr) {
					return 0;
				}

				if (!func || opline->opcode == ZEND_INIT_FCALL) {
					|	cbnz REG0, >3
				} else if (func->type == ZEND_USER_FUNCTION
					 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) {
					const zend_op *opcodes = func->op_array.opcodes;

					|	LOAD_ADDR REG1, ((ptrdiff_t)opcodes)
					|	ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)]
					|	cmp TMP1, REG1
					|	beq >3
				} else {
					|	LOAD_ADDR REG1, ((ptrdiff_t)func)
					|	cmp REG0, REG1
					|	beq >3
				}
				|	b &exit_addr
			} else {
				|	cbnz REG0, >3
				|	// SAVE_OPLINE();
				|	SET_EX_OPLINE opline, REG0
				|	b ->undefined_function
			}
		}
		|.code
		|3:
	}

	if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, checked_stack)) {
		return 0;
	}

	if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
		if (!zend_jit_save_call_chain(Dst, call_level)) {
			return 0;
		}
	} else {
		delayed_call_chain = 1;
		delayed_call_level = call_level;
	}

	return 1;
}

static int zend_jit_init_method_call(dasm_State          **Dst,
                                     const zend_op        *opline,
                                     uint32_t              b,
                                     const zend_op_array  *op_array,
                                     zend_ssa             *ssa,
                                     const zend_ssa_op    *ssa_op,
                                     int                   call_level,
                                     uint32_t              op1_info,
                                     zend_jit_addr         op1_addr,
                                     zend_class_entry     *ce,
                                     bool                  ce_is_instanceof,
                                     bool                  on_this,
                                     bool                  delayed_fetch_this,
                                     zend_class_entry     *trace_ce,
                                     zend_jit_trace_rec   *trace,
                                     int                   checked_stack,
                                     bool                  polymorphic_side_trace)
{
	zend_func_info *info = ZEND_FUNC_INFO(op_array);
	zend_call_info *call_info = NULL;
	zend_function *func = NULL;
	zval *function_name;

	ZEND_ASSERT(opline->op2_type == IS_CONST);
	ZEND_ASSERT(op1_info & MAY_BE_OBJECT);

	function_name = RT_CONSTANT(opline, opline->op2);

	if (info) {
		call_info = info->callee_info;
		while (call_info && call_info->caller_init_opline != opline) {
			call_info = call_info->next_callee;
		}
		if (call_info && call_info->callee_func && !call_info->is_prototype) {
			func = call_info->callee_func;
		}
	}

	if (polymorphic_side_trace) {
		/* function is passed in r0 from parent_trace */
	} else {
		if (on_this) {
			zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));

			|	GET_ZVAL_PTR FCARG1x, this_addr, TMP1
		} else {
		    if (op1_info & MAY_BE_REF) {
				if (opline->op1_type == IS_CV) {
					if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
						|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
					}
					|	ZVAL_DEREF FCARG1x, op1_info, TMP1w
					op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
				} else {
					/* Hack: Convert reference to regular value to simplify JIT code */
					ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP);
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1
					|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
					|	EXT_CALL zend_jit_unref_helper, REG0
					|1:
				}
			}
			if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
				if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
					int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
					const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

					if (!exit_addr) {
						return 0;
					}
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1
				} else {
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1
					|.cold_code
					|1:
					if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
						|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
					}
					|	SET_EX_OPLINE opline, REG0
					if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
						|	EXT_CALL zend_jit_invalid_method_call_tmp, REG0
					} else {
						|	EXT_CALL zend_jit_invalid_method_call, REG0
					}
					|	b ->exception_handler
					|.code
				}
			}
			|	GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
		}

		if (delayed_call_chain) {
			if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
				return 0;
			}
		}

		|	str FCARG1x, T1 // save

		if (func) {
			|	// fbc = CACHED_PTR(opline->result.num + sizeof(void*));
			|	ldr REG0, EX->run_time_cache
			|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1
			|	cbz REG0, >1
		} else {
			|	// if (CACHED_PTR(opline->result.num) == obj->ce)) {
			|	ldr REG0, EX->run_time_cache
			|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->result.num, TMP1
			|	ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)]
			|	cmp REG2, TMP1
			|	bne >1
			|	// fbc = CACHED_PTR(opline->result.num + sizeof(void*));
			|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1
		}

		|.cold_code
		|1:
		|	LOAD_ADDR FCARG2x, function_name
		if (TMP_ZVAL_OFFSET == 0) {
			|	mov CARG3, sp
		} else {
			|	add CARG3, sp, #TMP_ZVAL_OFFSET
		}
		|	SET_EX_OPLINE opline, REG0
		if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
			|	EXT_CALL zend_jit_find_method_tmp_helper, REG0
		} else {
			|	EXT_CALL zend_jit_find_method_helper, REG0
		}
		|	mov REG0, RETVALx
		|	cbnz REG0, >2
		|	b ->exception_handler
		|.code
		|2:
	}

	if ((!func || zend_jit_may_be_modified(func, op_array))
	 && trace
	 && trace->op == ZEND_JIT_TRACE_INIT_CALL
	 && trace->func
	) {
		int32_t exit_point;
		const void *exit_addr;

		exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL);
		exit_addr = zend_jit_trace_get_exit_addr(exit_point);
		if (!exit_addr) {
			return 0;
		}

		func = (zend_function*)trace->func;

		if (func->type == ZEND_USER_FUNCTION &&
		    (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
		     (func->common.fn_flags & ZEND_ACC_CLOSURE) ||
		     !func->common.function_name)) {
			const zend_op *opcodes = func->op_array.opcodes;

			|	LOAD_ADDR TMP1, opcodes
			|	ldr TMP2, [REG0, #offsetof(zend_op_array, opcodes)]
			|	cmp TMP2, TMP1
			|	bne &exit_addr
		} else {
			|	LOAD_ADDR TMP1, func
			|	cmp REG0, TMP1
			|	bne &exit_addr
		}
	}

	if (!func) {
		|	// if (fbc->common.fn_flags & ZEND_ACC_STATIC) {
		|	ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)]
		|	TST_32_WITH_CONST TMP1w, ZEND_ACC_STATIC, TMP2w
		|	bne >1
		|.cold_code
		|1:
	}

	if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) {
		|	ldr FCARG1x, T1 // restore
		|	mov FCARG2x, REG0
		|	LOAD_32BIT_VAL CARG3w, opline->extended_value
		if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
			|	EXT_CALL zend_jit_push_static_metod_call_frame_tmp, REG0
		} else {
			|	EXT_CALL zend_jit_push_static_metod_call_frame, REG0
		}
		if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) {
			|	cbz RETVALx, ->exception_handler
		}
		|	mov RX, RETVALx
	}

	if (!func) {
		|	b >9
		|.code
	}

	if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) {
		if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, delayed_fetch_this, checked_stack)) {
			return 0;
		}
	}

	if (!func) {
		|9:
	}
	zend_jit_start_reuse_ip();

	if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
		if (!zend_jit_save_call_chain(Dst, call_level)) {
			return 0;
		}
	} else {
		delayed_call_chain = 1;
		delayed_call_level = call_level;
	}

	return 1;
}

static int zend_jit_init_closure_call(dasm_State          **Dst,
                                      const zend_op        *opline,
                                      uint32_t              b,
                                      const zend_op_array  *op_array,
                                      zend_ssa             *ssa,
                                      const zend_ssa_op    *ssa_op,
                                      int                   call_level,
                                      zend_jit_trace_rec   *trace,
                                      int                   checked_stack)
{
	zend_function *func = NULL;
	zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);

	|	GET_ZVAL_PTR REG0, op2_addr, TMP1

	if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure
	 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) {
		int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
		const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

		if (!exit_addr) {
			return 0;
		}

		|	LOAD_ADDR FCARG1x, ((ptrdiff_t)zend_ce_closure)
		|	ldr, TMP1, [REG0, #offsetof(zend_object, ce)]
		|	cmp TMP1, FCARG1x
		|	bne &exit_addr
		if (ssa->var_info && ssa_op->op2_use >= 0) {
			ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD;
			ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure;
			ssa->var_info[ssa_op->op2_use].is_instanceof = 0;
		}
	}

	if (trace
	 && trace->op == ZEND_JIT_TRACE_INIT_CALL
	 && trace->func
	 && trace->func->type == ZEND_USER_FUNCTION) {
		const zend_op *opcodes;
		int32_t exit_point;
		const void *exit_addr;

		func = (zend_function*)trace->func;
		opcodes = func->op_array.opcodes;
		exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL);
		exit_addr = zend_jit_trace_get_exit_addr(exit_point);
		if (!exit_addr) {
			return 0;
		}

		|	LOAD_ADDR FCARG1x, ((ptrdiff_t)opcodes)
		|	ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.opcodes)]
		|	cmp TMP1, FCARG1x
		|	bne &exit_addr
	}

	if (delayed_call_chain) {
		if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
			return 0;
		}
	}

	if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, checked_stack)) {
		return 0;
	}

	if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
		if (!zend_jit_save_call_chain(Dst, call_level)) {
			return 0;
		}
	} else {
		delayed_call_chain = 1;
		delayed_call_level = call_level;
	}

	if (trace
	 && trace->op == ZEND_JIT_TRACE_END
	 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
		if (!zend_jit_set_valid_ip(Dst, opline + 1)) {
			return 0;
		}
	}

	return 1;
}

static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace)
{
	zend_func_info *info = ZEND_FUNC_INFO(op_array);
	zend_call_info *call_info = NULL;
	const zend_function *func = NULL;
	uint32_t i;
	zend_jit_addr res_addr;
	uint32_t call_num_args = 0;
	bool unknown_num_args = 0;
	const void *exit_addr = NULL;
	const zend_op *prev_opline;

	if (RETURN_VALUE_USED(opline)) {
		res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
	} else {
		/* CPU stack allocated temporary zval */
		res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RSP, TMP_ZVAL_OFFSET);
	}

	prev_opline = opline - 1;
	while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
		prev_opline--;
	}
	if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY ||
			prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
		unknown_num_args = 1;
	}

	if (info) {
		call_info = info->callee_info;
		while (call_info && call_info->caller_call_opline != opline) {
			call_info = call_info->next_callee;
		}
		if (call_info && call_info->callee_func && !call_info->is_prototype) {
			func = call_info->callee_func;
		}
		if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)
		 && JIT_G(current_frame)
		 && JIT_G(current_frame)->call
		 && !JIT_G(current_frame)->call->func) {
			call_info = NULL; func = NULL; /* megamorphic call from trait */
		}
	}
	if (!func) {
		/* resolve function at run time */
	} else if (func->type == ZEND_USER_FUNCTION) {
		ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL);
		call_num_args = call_info->num_args;
	} else if (func->type == ZEND_INTERNAL_FUNCTION) {
		ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL);
		call_num_args = call_info->num_args;
	} else {
		ZEND_UNREACHABLE();
	}

	if (trace && !func) {
		if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
			ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION);
#ifndef ZEND_WIN32
			// TODO: ASLR may cause different addresses in different workers ???
			func = trace->func;
			if (JIT_G(current_frame) &&
			    JIT_G(current_frame)->call &&
			    TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
				call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
			} else {
				unknown_num_args = 1;
			}
#endif
		} else if (trace->op == ZEND_JIT_TRACE_ENTER) {
			ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION);
			if (zend_accel_in_shm(trace->func->op_array.opcodes)) {
				func = trace->func;
				if (JIT_G(current_frame) &&
				    JIT_G(current_frame)->call &&
				    TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
					call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
				} else {
					unknown_num_args = 1;
				}
			}
		}
	}

	bool may_have_extra_named_params =
		opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS &&
		(!func || func->common.fn_flags & ZEND_ACC_VARIADIC);

	if (!reuse_ip) {
		zend_jit_start_reuse_ip();
		|	// call = EX(call);
		|	ldr RX, EX->call
	}
	zend_jit_stop_reuse_ip();

	|	// fbc = call->func;
	|	// mov r2, EX:RX->func ???
	|	// SAVE_OPLINE();
	|	SET_EX_OPLINE opline, REG0

	if (opline->opcode == ZEND_DO_FCALL) {
		if (!func) {
			if (trace) {
				uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);

				exit_addr = zend_jit_trace_get_exit_addr(exit_point);
				if (!exit_addr) {
					return 0;
				}
				|	ldr REG0, EX:RX->func
				|	ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)]
				|	TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w
				|	bne &exit_addr
			}
		}
	}

	if (!delayed_call_chain) {
		if (call_level == 1) {
			|	str xzr, EX->call
		} else {
			|	//EX(call) = call->prev_execute_data;
			|	ldr REG0, EX:RX->prev_execute_data
			|	str REG0, EX->call
		}
	}
	delayed_call_chain = 0;

	|	//call->prev_execute_data = execute_data;
	|	str EX, EX:RX->prev_execute_data

	if (!func) {
		|	ldr REG0, EX:RX->func
	}

	if (opline->opcode == ZEND_DO_FCALL) {
		if (!func) {
			if (!trace) {
				|	ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)]
				|	TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w
				|	bne >1
				|.cold_code
				|1:
				if (!GCC_GLOBAL_REGS) {
					|	mov FCARG1x, RX
				}
				|	EXT_CALL zend_jit_deprecated_helper, REG0
				|	GET_LOW_8BITS RETVALw, RETVALw
				|	ldr REG0, EX:RX->func // reload
				|	cbnz RETVALw, >1      // Result is 0 on exception
				|	b ->exception_handler
				|.code
				|1:
			}
		} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
			if (!GCC_GLOBAL_REGS) {
				|	mov FCARG1x, RX
			}
			|	EXT_CALL zend_jit_deprecated_helper, REG0
			|	cbz RETVALw, ->exception_handler
		}
	}

	if (!func
	 && opline->opcode != ZEND_DO_UCALL
	 && opline->opcode != ZEND_DO_ICALL) {
		|	ldrb TMP1w, [REG0, #offsetof(zend_function, type)]
		|	cmp TMP1w, #ZEND_USER_FUNCTION
		|	bne >8
	}

	if ((!func || func->type == ZEND_USER_FUNCTION)
	 && opline->opcode != ZEND_DO_ICALL) {
		|	// EX(call) = NULL;
		|	str xzr, EX:RX->call

		if (RETURN_VALUE_USED(opline)) {
			|	// EX(return_value) = EX_VAR(opline->result.var);
			|	LOAD_ZVAL_ADDR REG2, res_addr
			|	str REG2, EX:RX->return_value
		} else {
			|	// EX(return_value) = 0;
			|	str xzr, EX:RX->return_value
		}

		//EX_LOAD_RUN_TIME_CACHE(op_array);
		if (!func || func->op_array.cache_size) {
			if (func && op_array == &func->op_array) {
				/* recursive call */
				if (trace || func->op_array.cache_size > sizeof(void*)) {
					|	ldr REG2, EX->run_time_cache
					|	str REG2, EX:RX->run_time_cache
				}
			} else {
// Always defined as ZEND_MAP_PTR_KIND_PTR_OR_OFFSET. See Zend/zend_map_ptr.h.
#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR
				if (func) {
					|	ldr REG0, EX:RX->func
				}
				|	ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)]
				|	ldr REG2, [REG2]
#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET
				if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) {
					if (ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) {
						|	MEM_LOAD_64_ZTS ldr, REG2, compiler_globals, map_ptr_base, TMP1
						|	ADD_SUB_64_WITH_CONST add, REG2, REG2, (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache), TMP1
						|	ldr REG2, [REG2]
					} else if ((func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)
					        && (!func->op_array.scope || (func->op_array.scope->ce_flags & ZEND_ACC_LINKED))) {
						if (func) {
							|	ldr REG0, EX:RX->func
						}
						|	ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)]
						|	MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1
						|	ldr REG2, [REG2]
					} else {
						/* the called op_array may be not persisted yet */
						if (func) {
							|	ldr REG0, EX:RX->func
						}
						|	ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)]
						|	TST_64_WITH_ONE REG2
						|	beq >1
						|	MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1
						|1:
						|	ldr REG2, [REG2]
					}
				} else {
					if (func) {
						|	ldr REG0, EX:RX->func
					}
					|	ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)]
					|	TST_64_WITH_ONE REG2
					|	beq >1
					|	MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1
					|1:
					|	ldr REG2, [REG2]
				}
#else
# error "Unknown ZEND_MAP_PTR_KIND"
#endif
				|	str REG2, EX:RX->run_time_cache
			}
		}

		|	// EG(current_execute_data) = execute_data;
		|	MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1
		|	mov FP, RX

		|	// opline = op_array->opcodes;
		if (func && !unknown_num_args) {
			|	ADD_SUB_64_WITH_CONST_32 add, TMP1, RX, (EX_NUM_TO_VAR(call_num_args) + offsetof(zval, u1.type_info)), TMP1 // induction variable
			for (i = call_num_args; i < func->op_array.last_var; i++) {
				|	// ZVAL_UNDEF(EX_VAR(n))
				|	str wzr, [TMP1], #16
			}

			if (call_num_args <= func->op_array.num_args) {
				if (!trace || (trace->op == ZEND_JIT_TRACE_END
				 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
					uint32_t num_args;

					if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
						if (trace) {
							num_args = 0;
						} else if (call_info) {
							num_args = skip_valid_arguments(op_array, ssa, call_info);
						} else {
							num_args = call_num_args;
						}
					} else {
						num_args = call_num_args;
					}
					if (zend_accel_in_shm(func->op_array.opcodes)) {
						|	LOAD_IP_ADDR (func->op_array.opcodes + num_args)
					} else {
						|	ldr REG0, EX->func
						||	ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(num_args * sizeof(zend_op))));
						if (GCC_GLOBAL_REGS) {
							|	ldr IP, [REG0, #offsetof(zend_op_array, opcodes)]
							if (num_args) {
								|	add IP, IP, #(num_args * sizeof(zend_op))
							}
						} else {
							|	ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)]
							if (num_args) {
								|	add FCARG1x, FCARG1x, #(num_args * sizeof(zend_op))
							}
							|	str FCARG1x, EX->opline
						}
					}

					if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array
							&& num_args >= op_array->required_num_args) {
						/* recursive call */
						if (ZEND_OBSERVER_ENABLED) {
							|	SAVE_IP
							|	mov FCARG1x, FP
							|	EXT_CALL zend_observer_fcall_begin, REG0
						}
#ifdef CONTEXT_THREADED_JIT
						|	NIY	// TODO
#else
						|	b =>num_args
#endif
						return 1;
					}
				}
			} else {
				if (!trace || (trace->op == ZEND_JIT_TRACE_END
				 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
					if (func && zend_accel_in_shm(func->op_array.opcodes)) {
						|	LOAD_IP_ADDR (func->op_array.opcodes)
					} else if (GCC_GLOBAL_REGS) {
						|	ldr IP, [REG0, #offsetof(zend_op_array, opcodes)]
					} else {
						|	ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)]
						|	str FCARG1x, EX->opline
					}
				}
				if (!GCC_GLOBAL_REGS) {
					|	mov FCARG1x, FP
				}
				|	EXT_CALL zend_jit_copy_extra_args_helper, REG0
			}
		} else {
			|	// opline = op_array->opcodes
			if (func && zend_accel_in_shm(func->op_array.opcodes)) {
				|	LOAD_IP_ADDR (func->op_array.opcodes)
			} else if (GCC_GLOBAL_REGS) {
				|	ldr IP, [REG0, #offsetof(zend_op_array, opcodes)]
			} else {
				|	ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)]
				|	str FCARG1x, EX->opline
			}
			if (func) {
				|	// num_args = EX_NUM_ARGS();
				|	ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)]
				|	// if (UNEXPECTED(num_args > first_extra_arg))
				|	CMP_32_WITH_CONST REG1w, (func->op_array.num_args), TMP1w
			} else {
				|	// first_extra_arg = op_array->num_args;
				|	ldr REG2w, [REG0, #offsetof(zend_op_array, num_args)]
				|	// num_args = EX_NUM_ARGS();
				|	ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)]
				|	// if (UNEXPECTED(num_args > first_extra_arg))
				|	cmp REG1w, REG2w
			}
			|	bgt >1
			|.cold_code
			|1:
			if (!GCC_GLOBAL_REGS) {
				|	mov FCARG1x, FP
			}
			|	EXT_CALL zend_jit_copy_extra_args_helper, REG0
			if (!func) {
				|	ldr REG0, EX->func // reload
			}
			|	ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] // reload
			|	b >1
			|.code
			if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) {
				if (!func) {
					|	// if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0))
					|	ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)]
					|	TST_32_WITH_CONST TMP1w, ZEND_ACC_HAS_TYPE_HINTS, TMP2w
					|	bne >1
				}
				|	// opline += num_args;
				||	ZEND_ASSERT(sizeof(zend_op) == 32);
				|	mov REG2w, REG1w
				|	ADD_IP_SHIFT REG2, lsl #5, TMP1
			}
			|1:
			|	// if (EXPECTED((int)num_args < op_array->last_var)) {
			if (func) {
				|	LOAD_32BIT_VAL REG2w, func->op_array.last_var
			} else {
				|	ldr REG2w, [REG0, #offsetof(zend_op_array, last_var)]
			}
			|	subs REG2w, REG2w, REG1w
			|	ble >3
			|	// zval *var = EX_VAR_NUM(num_args);
			|	add REG1, FP, REG1, lsl #4
			||	ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(ZEND_CALL_FRAME_SLOT * sizeof(zval))));
			|	add REG1, REG1, #(ZEND_CALL_FRAME_SLOT * sizeof(zval))
			|2:
			|	SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w
			|	add REG1, REG1, #16
			|	subs REG2w, REG2w, #1
			|	bne <2
			|3:
		}

		if (ZEND_OBSERVER_ENABLED) {
			|	SAVE_IP
			|	mov FCARG1x, FP
			|	EXT_CALL zend_observer_fcall_begin, REG0
		}

		if (trace) {
			if (!func && (opline->opcode != ZEND_DO_UCALL)) {
				|	b >9
			}
		} else {
#ifdef CONTEXT_THREADED_JIT
			|	NIY	// TODO: CONTEXT_THREADED_JIT is always undefined.
#else
			if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
				|	ADD_HYBRID_SPAD
				|	JMP_IP TMP1
			} else if (GCC_GLOBAL_REGS) {
				|	ldp x29, x30, [sp], # SPAD // stack alignment
				|	JMP_IP TMP1
			} else {
				|	ldp FP, RX, T2                // retore FP and IP
				|	ldp x29, x30, [sp], # NR_SPAD // stack alignment
				|	mov RETVALx, #1     // ZEND_VM_ENTER
				|	ret
			}
		}
#endif
	}

	if ((!func || func->type == ZEND_INTERNAL_FUNCTION)
	 && (opline->opcode != ZEND_DO_UCALL)) {
		if (!func && (opline->opcode != ZEND_DO_ICALL)) {
			|8:
		}
		if (opline->opcode == ZEND_DO_FCALL_BY_NAME) {
			if (!func) {
				if (trace) {
					uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);

					exit_addr = zend_jit_trace_get_exit_addr(exit_point);
					if (!exit_addr) {
						return 0;
					}
					|	ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)]
					|	TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w
					|	bne &exit_addr
				} else {
					|	ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)]
					|	TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w
					|	bne >1
					|.cold_code
					|1:
					if (!GCC_GLOBAL_REGS) {
						|	mov FCARG1x, RX
					}
					|	EXT_CALL zend_jit_deprecated_helper, REG0
					|	GET_LOW_8BITS RETVALw, RETVALw
					|	ldr REG0, EX:RX->func // reload
					|	cbnz RETVALw, >1      // Result is 0 on exception
					|	b ->exception_handler
					|.code
					|1:
				}
			} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
				if (!GCC_GLOBAL_REGS) {
					|	mov FCARG1x, RX
				}
				|	EXT_CALL zend_jit_deprecated_helper, REG0
				|	cbz RETVALw, ->exception_handler
				|	ldr REG0, EX:RX->func // reload
			}
		}

		|	// ZVAL_NULL(EX_VAR(opline->result.var));
		|	LOAD_ZVAL_ADDR FCARG2x, res_addr
		|	SET_Z_TYPE_INFO FCARG2x, IS_NULL, TMP1w

		|	// EG(current_execute_data) = execute_data;
		|	MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1

		zend_jit_reset_last_valid_opline();

		|	// fbc->internal_function.handler(call, ret);
		|	mov FCARG1x, RX
		if (func) {
			|	EXT_CALL func->internal_function.handler, REG0
		} else {
			|	ldr TMP1, [REG0, #offsetof(zend_internal_function, handler)]
			|	blr TMP1
		}

		|	// EG(current_execute_data) = execute_data;
		|	MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0

		|	// zend_vm_stack_free_args(call);
		if (func && !unknown_num_args) {
			for (i = 0; i < call_num_args; i++ ) {
				if (zend_jit_needs_arg_dtor(func, i, call_info)) {
					uint32_t offset = EX_NUM_TO_VAR(i);
					zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset);
					|	ZVAL_PTR_DTOR arg_addr, (MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN), 0, 1, opline, ZREG_TMP1, ZREG_TMP2
				}
			}
		} else {
			|	mov FCARG1x, RX
			|	EXT_CALL zend_jit_vm_stack_free_args_helper, REG0
		}
		if (may_have_extra_named_params) {
		    |	ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 3)]
			|	TST_32_WITH_CONST TMP1w, (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24), TMP2w
			|	bne >1
			|.cold_code
			|1:
			|	ldr FCARG1x, [RX, #offsetof(zend_execute_data, extra_named_params)]
			|	EXT_CALL zend_free_extra_named_params, REG0
			|	b >2
			|.code
			|2:
		}

		|8:
		if (opline->opcode == ZEND_DO_FCALL) {
			// TODO: optimize ???
			|	// if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS))
			|	ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)]
			|	TST_32_WITH_CONST TMP1w, (ZEND_CALL_RELEASE_THIS >> 16), TMP2w
			|	bne >1
			|.cold_code
			|1:
			|	add TMP1, RX, #offsetof(zend_execute_data, This)
			|	GET_Z_PTR FCARG1x, TMP1
			|	// OBJ_RELEASE(object);
			|	OBJ_RELEASE ZREG_FCARG1, >2, ZREG_TMP1, ZREG_TMP2
			|	b >2
			|.code
			|2:
		}

		if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
		    !JIT_G(current_frame) ||
		    !JIT_G(current_frame)->call ||
		    !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) ||
		    prev_opline->opcode == ZEND_SEND_UNPACK ||
		    prev_opline->opcode == ZEND_SEND_ARRAY ||
			prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {

			|	// zend_vm_stack_free_call_frame(call);
			|	ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)]
			|	TST_32_WITH_CONST TMP1w, ((ZEND_CALL_ALLOCATED >> 16) & 0xff), TMP2w
			|	bne >1
			|.cold_code
			|1:
			|	mov FCARG1x, RX
			|	EXT_CALL zend_jit_free_call_frame, REG0
			|	b >1
			|.code
		}
		|	MEM_STORE_64_ZTS str, RX, executor_globals, vm_stack_top, REG0
		|1:

		if (!RETURN_VALUE_USED(opline)) {
			zend_class_entry *ce;
			bool ce_is_instanceof;
			uint32_t func_info = call_info ?
				zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) :
				(MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN);

			/* If an exception is thrown, the return_value may stay at the
			 * original value of null. */
			func_info |= MAY_BE_NULL;

			if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
				|	ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline, ZREG_TMP1, ZREG_TMP2
			}
		}

		|	// if (UNEXPECTED(EG(exception) != NULL)) {
		|	MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
		|	cbnz REG0, ->icall_throw_handler

		// TODO: Can we avoid checking for interrupts after each call ???
		if (trace && last_valid_opline != opline) {
			int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM);

			exit_addr = zend_jit_trace_get_exit_addr(exit_point);
			if (!exit_addr) {
				return 0;
			}
		} else {
			exit_addr = NULL;
		}
		if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) {
			return 0;
		}

		if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) {
			|	LOAD_IP_ADDR (opline + 1)
		} else if (trace
		 && trace->op == ZEND_JIT_TRACE_END
		 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
			|	LOAD_IP_ADDR (opline + 1)
		}
	}

	if (!func) {
		|9:
	}

	return 1;
}

static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr)
{
	uint32_t arg_num = opline->op2.num;
	zend_jit_addr arg_addr;

	ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM);

	if (!zend_jit_reuse_ip(Dst)) {
		return 0;
	}

	if (opline->opcode == ZEND_SEND_VAL_EX) {
		uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2);

		ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM);

		if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
		 && JIT_G(current_frame)
		 && JIT_G(current_frame)->call
		 && JIT_G(current_frame)->call->func) {
			if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
				/* Don't generate code that always throws exception */
				return 0;
			}
		} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
			int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
			const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
			if (!exit_addr) {
				return 0;
			}
			|	ldr REG0, EX:RX->func
			|	ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)]
			|	TST_32_WITH_CONST TMP1w, mask, TMP2w
			|	bne &exit_addr
		} else {
			|	ldr REG0, EX:RX->func
			|	ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)]
			|	TST_32_WITH_CONST TMP1w, mask, TMP2w
			|	bne >1
			|.cold_code
			|1:
			if (Z_MODE(op1_addr) == IS_REG) {
				/* set type to avoid zval_ptr_dtor() on uninitialized value */
				zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
				|	SET_ZVAL_TYPE_INFO addr, IS_UNDEF, TMP1w, TMP2
			}
			|	SET_EX_OPLINE opline, REG0
			|	b ->throw_cannot_pass_by_ref
			|.code
		}
	}

	arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);

	if (opline->op1_type == IS_CONST) {
		zval *zv = RT_CONSTANT(opline, opline->op1);

		|	ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0
		if (Z_REFCOUNTED_P(zv)) {
			|	ADDREF_CONST zv, REG0, TMP1
		}
	} else {
		|	ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
	}

	return 1;
}

static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline)
{
	|	ldr FCARG1x, EX->call
	|	ldrb TMP1w, [FCARG1x, #(offsetof(zend_execute_data, This.u1.type_info) + 3)]
	|	TST_32_WITH_CONST TMP1w, (ZEND_CALL_MAY_HAVE_UNDEF >> 24), TMP2w
	|	bne >1
	|.cold_code
	|1:
	|	SET_EX_OPLINE opline, REG0
	|	EXT_CALL zend_handle_undef_args, REG0
	|	cbz RETVALw, >2
	|	b ->exception_handler
	|.code
	|2:

	return 1;
}

static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold)
{
	zend_jit_addr op1_addr, arg_addr, ref_addr;

	op1_addr = OP1_ADDR();
	arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);

	if (!zend_jit_reuse_ip(Dst)) {
		return 0;
	}

	if (opline->op1_type == IS_VAR) {
		if (op1_info & MAY_BE_INDIRECT) {
			|	LOAD_ZVAL_ADDR REG0, op1_addr
			|	// if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) {
			|	IF_NOT_Z_TYPE REG0, IS_INDIRECT, >1, TMP1w
			|	// ret = Z_INDIRECT_P(ret);
			|	GET_Z_PTR REG0, REG0
			|1:
			op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
		}
	} else if (opline->op1_type == IS_CV) {
		if (op1_info & MAY_BE_UNDEF) {
			if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
				|	SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2
				|	b >2
				|1:
			}
			op1_info &= ~MAY_BE_UNDEF;
			op1_info |= MAY_BE_NULL;
		}
	} else {
		ZEND_UNREACHABLE();
	}

	if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) {
		if (op1_info & MAY_BE_REF) {
			|	IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2, ZREG_TMP1
			|	GET_ZVAL_PTR REG1, op1_addr, TMP1
			|	GC_ADDREF REG1, TMP1w
			|	SET_ZVAL_PTR arg_addr, REG1, TMP1
			|	SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2
			|	b >6
		}
		|2:
		|	// ZVAL_NEW_REF(arg, varptr);
		if (opline->op1_type == IS_VAR) {
			if (Z_REG(op1_addr) != ZREG_REG0 || Z_OFFSET(op1_addr) != 0) {
				|	LOAD_ZVAL_ADDR REG0, op1_addr
			}
			|	str REG0, T1  // save
		}
		|	EMALLOC sizeof(zend_reference), op_array, opline  // Allocate space in REG0
		|	mov TMP1w, #2
		|	str TMP1w, [REG0]
		||	ZEND_ASSERT(GC_REFERENCE <= MOVZ_IMM);
		|	movz TMP1w, #GC_REFERENCE
		|	str TMP1w, [REG0, #offsetof(zend_reference, gc.u.type_info)]
		|	str xzr, [REG0, #offsetof(zend_reference, sources.ptr)]
		ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val));
		if (opline->op1_type == IS_VAR) {
			zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0);

			|	ldr REG1, T1  // restore
			|	ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
			|	SET_ZVAL_PTR val_addr, REG0, TMP1
			|	SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX, TMP1w, TMP2
		} else {
			|	ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
			|	SET_ZVAL_PTR op1_addr, REG0, TMP1
			|	SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2
		}
		|	SET_ZVAL_PTR arg_addr, REG0, TMP1
		|	SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2
	}

	|6:
	|	FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline, ZREG_TMP1, ZREG_TMP2
	|7:

	return 1;
}

static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr)
{
	uint32_t arg_num = opline->op2.num;
	zend_jit_addr arg_addr;

	ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX &&
	     opline->opcode != ZEND_SEND_VAR_NO_REF_EX) ||
	    arg_num <= MAX_ARG_FLAG_NUM);

	arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);

	if (!zend_jit_reuse_ip(Dst)) {
		return 0;
	}

	if (opline->opcode == ZEND_SEND_VAR_EX) {
		if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
		 && JIT_G(current_frame)
		 && JIT_G(current_frame)->call
		 && JIT_G(current_frame)->call->func) {
			if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
				if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) {
					return 0;
				}
				return 1;
			}
		} else {
			uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);

			|	ldr REG0, EX:RX->func
			|	ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)]
			|	TST_32_WITH_CONST TMP1w, mask, TMP2w
			|	bne >1
			|.cold_code
			|1:
			if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) {
				return 0;
			}
			|	b >7
			|.code
		}
	} else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
		if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
		 && JIT_G(current_frame)
		 && JIT_G(current_frame)->call
		 && JIT_G(current_frame)->call->func) {
			if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {

				|	ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0

				if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
					if (!(op1_info & MAY_BE_REF)) {
						/* Don't generate code that always throws exception */
						return 0;
					} else {
						int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
						const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
						if (!exit_addr) {
							return 0;
						}
						|	GET_LOW_8BITS TMP1w, REG1w
						|	cmp TMP1w, #IS_REFERENCE
						|	bne &exit_addr
					}
				}
				return 1;
			}
		} else {
			uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);

			|	ldr REG0, EX:RX->func
			|	ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)]
			|	TST_32_WITH_CONST TMP1w, mask, TMP2w
			|	bne >1
			|.cold_code
			|1:

			mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2);

			|	ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
			if (op1_info & MAY_BE_REF) {
				|	GET_LOW_8BITS TMP1w, REG1w
				|	cmp TMP1w, #IS_REFERENCE
				|	beq >7
			}
			|	ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)]
			|	TST_32_WITH_CONST TMP1w, mask, TMP2w
			|	bne >7
			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
				int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
				if (!exit_addr) {
					return 0;
				}
				|	b &exit_addr
			} else {
				|	SET_EX_OPLINE opline, REG0
				|	LOAD_ZVAL_ADDR FCARG1x, arg_addr
				|	EXT_CALL zend_jit_only_vars_by_reference, REG0
				if (!zend_jit_check_exception(Dst)) {
					return 0;
				}
				|	b >7
			}

			|.code
		}
	} else if (opline->opcode == ZEND_SEND_FUNC_ARG) {
		if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
		 && JIT_G(current_frame)
		 && JIT_G(current_frame)->call
		 && JIT_G(current_frame)->call->func) {
			if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
				if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) {
					return 0;
				}
				return 1;
			}
		} else {
			|	ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
			|	TST_32_WITH_CONST TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w
			|	bne >1
			|.cold_code
			|1:
			if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) {
				return 0;
			}
			|	b >7
			|.code
		}
	}

	if (op1_info & MAY_BE_UNDEF) {
		if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
			|	IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
			|.cold_code
			|1:
		}

		|	SET_EX_OPLINE opline, REG0
		|	LOAD_32BIT_VAL FCARG1w, opline->op1.var
		|	EXT_CALL zend_jit_undefined_op_helper, REG0
		|	SET_ZVAL_TYPE_INFO arg_addr, IS_NULL, TMP1w, TMP2
		|	cbz RETVALx, ->exception_handler

		if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
			|	b >7
			|.code
		} else {
			|7:
			return 1;
		}
	}

	if (opline->opcode == ZEND_SEND_VAR_NO_REF) {
		|	ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
		if (op1_info & MAY_BE_REF) {
			|	GET_LOW_8BITS TMP1w, REG1w
			|	cmp TMP1w, #IS_REFERENCE
			|	beq >7
		}
		if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
			int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
			const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
			if (!exit_addr) {
				return 0;
			}
			|	b &exit_addr
		} else {
			|	SET_EX_OPLINE opline, REG0
			|	LOAD_ZVAL_ADDR FCARG1x, arg_addr
			|	EXT_CALL zend_jit_only_vars_by_reference, REG0
			if (!zend_jit_check_exception(Dst)) {
				return 0;
			}
		}
	} else {
		if (op1_info & MAY_BE_REF) {
			if (opline->op1_type == IS_CV) {
				zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);

				|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
				|	ZVAL_DEREF FCARG1x, op1_info, TMP1w
				|	ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
				|	TRY_ADDREF op1_info, REG0w, REG2, TMP1w
			} else {
				zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 8);

				|	IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1
				|.cold_code
				|1:
				|	// zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
				|	GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
				|	// ZVAL_COPY_VALUE(return_value, &ref->value);
				|	ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
				|	GC_DELREF FCARG1x, TMP1w
				|	beq >1
				|	IF_NOT_REFCOUNTED REG0w, >2, TMP1w
				|	GC_ADDREF REG2, TMP1w
				|	b >2
				|1:
				|	EFREE_REFERENCE
				|	b >2
				|.code
				|	ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
				|2:
			}
		} else {
			if (op1_addr != op1_def_addr) {
				if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
					return 0;
				}
				if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
					op1_addr= op1_def_addr;
				}
			}
			|	ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
			if (opline->op1_type == IS_CV) {
				|	TRY_ADDREF op1_info, REG0w, REG2, TMP1w
			}
		}
	}
	|7:

	return 1;
}

static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline)
{
	uint32_t arg_num = opline->op2.num;

	if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
	 && JIT_G(current_frame)
	 && JIT_G(current_frame)->call
	 && JIT_G(current_frame)->call->func) {
		if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
			if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) {
				TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call);
				|	// ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
				||	if (reuse_ip) {
				|		ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
				|		BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w
				|		str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
				||	} else {
				|		ldr REG0, EX->call
				|		ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)]
				|		BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w
				|		str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)]
				||	}
			}
		} else {
			if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
				TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call);
				|	// ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
				||	if (reuse_ip) {
				|		ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
				|		BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w
				|		str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
				||	} else {
				|		ldr REG0, EX->call
				|		ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)]
				|		BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w
				|		str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)]
				||	}
			}
		}
	} else {
		// if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) {
		uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);

		if (!zend_jit_reuse_ip(Dst)) {
			return 0;
		}

		|	ldr REG0, EX:RX->func
		|	ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)]
		|	TST_32_WITH_CONST TMP1w, mask, TMP2w
		|	bne >1
		|.cold_code
		|1:
		|	// ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
		|	ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
		|	BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w
		|	str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
		|	b >1
		|.code
		|	// ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
		|	ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
		|	BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~(ZEND_CALL_SEND_ARG_BY_REF)), TMP2w
		|	str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
		|1:
	}

	return 1;
}

static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
{
	if (smart_branch_opcode) {
		if (smart_branch_opcode == ZEND_JMPZ) {
			if (jmp) {
				|	b >7
			}
		} else if (smart_branch_opcode == ZEND_JMPNZ) {
			|	b =>target_label
		} else if (smart_branch_opcode == ZEND_JMPZNZ) {
			|	b =>target_label2
		} else {
			ZEND_UNREACHABLE();
		}
	} else {
		zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);

		|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
		if (jmp) {
			|	b >7
		}
	}

	return 1;
}

static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label)
{
	if (smart_branch_opcode) {
		if (smart_branch_opcode == ZEND_JMPZ) {
			|	b =>target_label
		} else if (smart_branch_opcode == ZEND_JMPNZ) {
			if (jmp) {
				|	b >7
			}
		} else if (smart_branch_opcode == ZEND_JMPZNZ) {
			|	b =>target_label
		} else {
			ZEND_UNREACHABLE();
		}
	} else {
		zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);

		|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
		if (jmp) {
			|	b >7
		}
	}

	return 1;
}

static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
	uint32_t defined_label = (uint32_t)-1;
	uint32_t undefined_label = (uint32_t)-1;
	zval *zv = RT_CONSTANT(opline, opline->op1);
	zend_jit_addr res_addr = 0;

	if (smart_branch_opcode && !exit_addr) {
		if (smart_branch_opcode == ZEND_JMPZ) {
			undefined_label = target_label;
		} else if (smart_branch_opcode == ZEND_JMPNZ) {
			defined_label = target_label;
		} else if (smart_branch_opcode == ZEND_JMPZNZ) {
			undefined_label = target_label;
			defined_label = target_label2;
		} else {
			ZEND_UNREACHABLE();
		}
	}

	|	// if (CACHED_PTR(opline->extended_value)) {
	|	ldr REG0, EX->run_time_cache
	|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1
	|	cbz REG0, >1
	|	TST_64_WITH_ONE REG0
	|	bne >4
	|.cold_code
	|4:
	|	MEM_LOAD_64_ZTS ldr, FCARG1x, executor_globals, zend_constants, FCARG1x
	|	ldr TMP1w, [FCARG1x, #offsetof(HashTable, nNumOfElements)]
	|	cmp TMP1, REG0, lsr #1

	if (smart_branch_opcode) {
		if (exit_addr) {
			if (smart_branch_opcode == ZEND_JMPZ) {
				|	beq &exit_addr
			} else {
				|	beq >3
			}
		} else if (undefined_label != (uint32_t)-1) {
			|	beq =>undefined_label
		} else {
			|	beq >3
		}
	} else {
		|	beq >2
	}
	|1:
	|	SET_EX_OPLINE opline, REG0
	|	LOAD_ADDR FCARG1x, zv
	|	EXT_CALL zend_jit_check_constant, REG0
	if (exit_addr) {
		if (smart_branch_opcode == ZEND_JMPNZ) {
			|	cbz RETVALx, >3
		} else {
			|	cbnz RETVALx, >3
		}
		|	b &exit_addr
	} else if (smart_branch_opcode) {
		if (undefined_label != (uint32_t)-1) {
			|	cbz RETVALx, =>undefined_label
		} else {
			|	cbz RETVALx, >3
		}
		if (defined_label != (uint32_t)-1) {
			|	b =>defined_label
		} else {
			|	b >3
		}
	} else {
		res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
		|	cbnz RETVALx, >1
		|2:
		|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
		|	b >3
	}
	|.code
	if (smart_branch_opcode) {
		if (exit_addr) {
			if (smart_branch_opcode == ZEND_JMPNZ) {
				|	b &exit_addr
			}
		} else if (defined_label != (uint32_t)-1) {
			|	b =>defined_label
		}
	} else {
		|1:
		|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
	}
	|3:

	return 1;
}

static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
	uint32_t  mask;
	zend_jit_addr op1_addr = OP1_ADDR();

	// TODO: support for is_resource() ???
	ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);

	if (op1_info & MAY_BE_UNDEF) {
		if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
			|	IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
			|.cold_code
			|1:
		}
		|	SET_EX_OPLINE opline, REG0
		|	LOAD_32BIT_VAL FCARG1w, opline->op1.var
		|	EXT_CALL zend_jit_undefined_op_helper, REG0
		zend_jit_check_exception_undef_result(Dst, opline);
		if (opline->extended_value & MAY_BE_NULL) {
			if (exit_addr) {
				if (smart_branch_opcode == ZEND_JMPNZ) {
					|	b &exit_addr
				} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) {
					|	b >7
				}
			} else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) {
				return 0;
			}
		} else {
			if (exit_addr) {
				if (smart_branch_opcode == ZEND_JMPZ) {
					|	b &exit_addr
				} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) {
					|	b >7
				}
			} else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) {
				return 0;
			}
		}
		if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
			|.code
		}
	}

	if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
		mask = opline->extended_value;
		if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) {
			|	FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
			if (exit_addr) {
				if (smart_branch_opcode == ZEND_JMPNZ) {
					|	b &exit_addr
				}
			} else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) {
				return 0;
			}
	    } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) {
			|	FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
			if (exit_addr) {
				if (smart_branch_opcode == ZEND_JMPZ) {
					|	b &exit_addr
				}
			} else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) {
				return 0;
			}
		} else {
			bool invert = 0;
			zend_uchar type;

			switch (mask) {
				case MAY_BE_NULL:   type = IS_NULL;   break;
				case MAY_BE_FALSE:  type = IS_FALSE;  break;
				case MAY_BE_TRUE:   type = IS_TRUE;   break;
				case MAY_BE_LONG:   type = IS_LONG;   break;
				case MAY_BE_DOUBLE: type = IS_DOUBLE; break;
				case MAY_BE_STRING: type = IS_STRING; break;
				case MAY_BE_ARRAY:  type = IS_ARRAY;  break;
				case MAY_BE_OBJECT: type = IS_OBJECT; break;
				case MAY_BE_ANY - MAY_BE_NULL:     type = IS_NULL;   invert = 1; break;
				case MAY_BE_ANY - MAY_BE_FALSE:    type = IS_FALSE;  invert = 1; break;
				case MAY_BE_ANY - MAY_BE_TRUE:     type = IS_TRUE;   invert = 1; break;
				case MAY_BE_ANY - MAY_BE_LONG:     type = IS_LONG;   invert = 1; break;
				case MAY_BE_ANY - MAY_BE_DOUBLE:   type = IS_DOUBLE; invert = 1; break;
				case MAY_BE_ANY - MAY_BE_STRING:   type = IS_STRING; invert = 1; break;
				case MAY_BE_ANY - MAY_BE_ARRAY:    type = IS_ARRAY;  invert = 1; break;
				case MAY_BE_ANY - MAY_BE_OBJECT:   type = IS_OBJECT; invert = 1; break;
				case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break;
				default:
					type = 0;
			}

			if (op1_info & MAY_BE_REF) {
				|	LOAD_ZVAL_ADDR REG0, op1_addr
				|	ZVAL_DEREF REG0, op1_info, TMP1w
			}
			if (type == 0) {
				if (smart_branch_opcode &&
				    (opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
				    (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
					if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
						|	// if (Z_REFCOUNTED_P(cv)) {
						|	IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2
						|.cold_code
						|1:
					}
					|	// if (!Z_DELREF_P(cv)) {
					|	GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
					|	GC_DELREF FCARG1x, TMP1w
					if (RC_MAY_BE_1(op1_info)) {
						if (RC_MAY_BE_N(op1_info)) {
							|	bne >3
						}
						if (op1_info & MAY_BE_REF) {
							|	ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)]
						} else {
							|	MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1
						}
						|	str REG0w, T1 // save
						|	// zval_dtor_func(r);
						|	ZVAL_DTOR_FUNC op1_info, opline, TMP1
						|	ldr REG1w, T1 // restore
						|	b >2
					}
					if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
						if (!RC_MAY_BE_1(op1_info)) {
							|	b >3
						}
						|.code
					}
					|3:
					if (op1_info & MAY_BE_REF) {
						|	ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)]
					} else {
						|	MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1
					}
					|2:
				} else {
					if (op1_info & MAY_BE_REF) {
						|	ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)]
					} else {
						|	MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1
					}
				}
				|	mov REG0w, #1
				|	lsl REG0w, REG0w, REG1w
				|	TST_32_WITH_CONST REG0w, mask, TMP1w
				if (exit_addr) {
					if (smart_branch_opcode == ZEND_JMPNZ) {
						|	bne &exit_addr
					} else {
						|	beq &exit_addr
					}
				} else if (smart_branch_opcode) {
					if (smart_branch_opcode == ZEND_JMPZ) {
						|	beq =>target_label
					} else if (smart_branch_opcode == ZEND_JMPNZ) {
						|	bne =>target_label
					} else if (smart_branch_opcode == ZEND_JMPZNZ) {
						|	beq =>target_label
						|	b =>target_label2
					} else {
						ZEND_UNREACHABLE();
					}
				} else {
					zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);

					|	cset REG0w, ne
					|	add REG0w, REG0w, #2
					|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
					|	FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
				}
			} else {
				if (smart_branch_opcode &&
				    (opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
				    (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
					if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
						|	// if (Z_REFCOUNTED_P(cv)) {
						|	IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2
						|.cold_code
						|1:
					}
					|	// if (!Z_DELREF_P(cv)) {
					|	GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
					|	GC_DELREF FCARG1x, TMP1w
					if (RC_MAY_BE_1(op1_info)) {
						if (RC_MAY_BE_N(op1_info)) {
							|	bne >3
						}
						if (op1_info & MAY_BE_REF) {
							|	ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)]
						} else {
							|	MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1
						}
						|	str REG0w, T1 // save
						|	// zval_dtor_func(r);
						|	ZVAL_DTOR_FUNC op1_info, opline, TMP1
						|	ldr REG1w, T1 // restore
						|	b >2
					}
					if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
						if (!RC_MAY_BE_1(op1_info)) {
							|	b >3
						}
						|.code
					}
					|3:
					if (op1_info & MAY_BE_REF) {
						|	ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)]
					} else {
						|	MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1
					}
					|2:
					// Note: 'type' is of uchar type and holds a positive value,
					// hence it's safe to directly encode it as the imm field of 'cmp' instruction.
					|	cmp REG1w, #type
				} else {
					if (op1_info & MAY_BE_REF) {
						|	ldrb TMP1w, [REG0, #offsetof(zval,u1.v.type)]
						|	cmp TMP1w, #type
					} else {
						|	MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1
						|	cmp TMP1w, #type
					}
				}
				if (exit_addr) {
					if (invert) {
						if (smart_branch_opcode == ZEND_JMPNZ) {
							|	bne &exit_addr
						} else {
							|	beq &exit_addr
						}
					} else {
						if (smart_branch_opcode == ZEND_JMPNZ) {
							|	beq &exit_addr
						} else {
							|	bne &exit_addr
						}
					}
				} else if (smart_branch_opcode) {
					if (invert) {
						if (smart_branch_opcode == ZEND_JMPZ) {
							|	beq =>target_label
						} else if (smart_branch_opcode == ZEND_JMPNZ) {
							|	bne =>target_label
						} else if (smart_branch_opcode == ZEND_JMPZNZ) {
							|	beq =>target_label
							|	b =>target_label2
						} else {
							ZEND_UNREACHABLE();
						}
					} else {
						if (smart_branch_opcode == ZEND_JMPZ) {
							|	bne =>target_label
						} else if (smart_branch_opcode == ZEND_JMPNZ) {
							|	beq =>target_label
						} else if (smart_branch_opcode == ZEND_JMPZNZ) {
							|	bne =>target_label
							|	b =>target_label2
						} else {
							ZEND_UNREACHABLE();
						}
					}
				} else {
					zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);

					if (invert) {
						|	cset REG0w, ne
					} else {
						|	cset REG0w, eq
					}
					|	add REG0w, REG0w, #2
					|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
					|	FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
				}
			}
	    }
	}

	|7:

	return 1;
}

static int zend_jit_leave_frame(dasm_State **Dst)
{
	|	// EG(current_execute_data) = EX(prev_execute_data);
	|	ldr REG0, EX->prev_execute_data
	|	MEM_STORE_64_ZTS str, REG0, executor_globals, current_execute_data, REG2
	return 1;
}

static int zend_jit_free_cvs(dasm_State **Dst)
{
	|	// EG(current_execute_data) = EX(prev_execute_data);
	|	ldr FCARG1x, EX->prev_execute_data
	|	MEM_STORE_64_ZTS str, FCARG1x, executor_globals, current_execute_data, REG0
	|	// zend_free_compiled_variables(execute_data);
	|	mov FCARG1x, FP
	|	EXT_CALL zend_free_compiled_variables, REG0
	return 1;
}

static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var)
{
	if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
		uint32_t offset = EX_NUM_TO_VAR(var);
		zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset);
		|	ZVAL_PTR_DTOR addr, info, 1, 1, NULL, ZREG_TMP1, ZREG_TMP2
	}
	return 1;
}

static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset)
{
	if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
		zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset);
		|	ZVAL_PTR_DTOR addr, info, 0, 1, opline, ZREG_TMP1, ZREG_TMP2
	}
	return 1;
}

static int zend_jit_leave_func(dasm_State          **Dst,
                               const zend_op_array  *op_array,
                               const zend_op        *opline,
                               uint32_t              op1_info,
                               bool                  left_frame,
                               zend_jit_trace_rec   *trace,
                               zend_jit_trace_info  *trace_info,
                               int                   indirect_var_access,
                               int                   may_throw)
{
	bool may_be_top_frame =
		JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
		!JIT_G(current_frame) ||
		!TRACE_FRAME_IS_NESTED(JIT_G(current_frame));
	bool may_need_call_helper =
		indirect_var_access || /* may have symbol table */
		!op_array->function_name || /* may have symbol table */
		may_be_top_frame ||
		(op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */
		JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
		!JIT_G(current_frame) ||
		TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */
		(uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */
	bool may_need_release_this =
		!(op_array->fn_flags & ZEND_ACC_CLOSURE) &&
		op_array->scope &&
		!(op_array->fn_flags & ZEND_ACC_STATIC) &&
		(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
		 !JIT_G(current_frame) ||
		 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame)));

	if (may_need_call_helper || may_need_release_this) {
		|	ldr FCARG1w, [FP, #offsetof(zend_execute_data, This.u1.type_info)]
	}
	if (may_need_call_helper) {
		if (!left_frame) {
			left_frame = 1;
		    if (!zend_jit_leave_frame(Dst)) {
				return 0;
		    }
		}
		/* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */

		|	TST_32_WITH_CONST FCARG1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE), TMP1w
		if (trace && trace->op != ZEND_JIT_TRACE_END) {
			|	bne >1
			|.cold_code
			|1:
			if (!GCC_GLOBAL_REGS) {
				|	mov FCARG1x, FP
			}
			|	EXT_CALL zend_jit_leave_func_helper, REG0

			if (may_be_top_frame) {
				// TODO: try to avoid this check ???
				if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#if 0
					/* this check should be handled by the following OPLINE guard */
					|	LOAD_ADDR TMP1, zend_jit_halt_op
					|	cmp IP, TMP1
					|	beq ->trace_halt
#endif
				} else if (GCC_GLOBAL_REGS) {
					|	cbz IP, ->trace_halt
				} else {
					|	tst RETVALw, RETVALw
					|	blt ->trace_halt
				}
			}

			if (!GCC_GLOBAL_REGS) {
				|	// execute_data = EG(current_execute_data)
				|	MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1
			}
			|	b >8
			|.code
		} else {
			|	bne ->leave_function_handler
		}
	}

	if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
		if (!left_frame) {
			left_frame = 1;
		    if (!zend_jit_leave_frame(Dst)) {
				return 0;
		    }
		}
		|	// OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
		|	ldr FCARG1x, EX->func
		|	sub FCARG1x, FCARG1x, #sizeof(zend_object)
		|	OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2
		|4:
	} else if (may_need_release_this) {
		if (!left_frame) {
			left_frame = 1;
		    if (!zend_jit_leave_frame(Dst)) {
				return 0;
		    }
		}
		|	// if (call_info & ZEND_CALL_RELEASE_THIS)
		|	TST_32_WITH_CONST FCARG1w, ZEND_CALL_RELEASE_THIS, TMP1w
		|	beq >4
		|	// zend_object *object = Z_OBJ(execute_data->This);
		|	ldr FCARG1x, EX->This.value.obj
		|	// OBJ_RELEASE(object);
		|	OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2
		|4:
		// TODO: avoid EG(excption) check for $this->foo() calls
		may_throw = 1;
	}

	|	// EG(vm_stack_top) = (zval*)execute_data;
	|	MEM_STORE_64_ZTS str, FP, executor_globals, vm_stack_top, REG0
	|	// execute_data = EX(prev_execute_data);
	|	ldr FP, EX->prev_execute_data

	if (!left_frame) {
		|	// EG(current_execute_data) = execute_data;
		|	MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0
	}

	|9:
	if (trace) {
		if (trace->op != ZEND_JIT_TRACE_END
		 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
			zend_jit_reset_last_valid_opline();
		} else {
			|	LOAD_IP
			|	ADD_IP_WITH_CONST sizeof(zend_op), TMP1
		}

		|8:

		if (trace->op == ZEND_JIT_TRACE_BACK
		 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
			const zend_op *next_opline = trace->opline;

			if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
			 && (op1_info & MAY_BE_RC1)
			 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) {
				/* exception might be thrown during destruction of unused return value */
				|	// if (EG(exception))
				|	MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
				|	cbnz REG0, ->leave_throw_handler
			}
			do {
				trace++;
			} while (trace->op == ZEND_JIT_TRACE_INIT_CALL);
			ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
			next_opline = trace->opline;
			ZEND_ASSERT(next_opline != NULL);

			if (trace->op == ZEND_JIT_TRACE_END
			 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
				trace_info->flags |= ZEND_JIT_TRACE_LOOP;
				|	CMP_IP next_opline, TMP1, TMP2
				|	beq =>0 // LOOP
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
				|	JMP_IP TMP1
#else
				|	b ->trace_escape
#endif
			} else {
				|	CMP_IP next_opline, TMP1, TMP2
				|	bne ->trace_escape
			}

			zend_jit_set_last_valid_opline(trace->opline);

			return 1;
		} else if (may_throw ||
				(((opline->op1_type & (IS_VAR|IS_TMP_VAR))
				  && (op1_info & MAY_BE_RC1)
				  && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)))
				 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) {
			|	// if (EG(exception))
			|	MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
			|	cbnz REG0, ->leave_throw_handler
		}

		return 1;
	} else {
		|	// if (EG(exception))
		|	MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
		|	LOAD_IP
		|	cbnz REG0, ->leave_throw_handler
		|	// opline = EX(opline) + 1
		|	ADD_IP_WITH_CONST sizeof(zend_op), TMP1
	}

	if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
		|	ADD_HYBRID_SPAD
#ifdef CONTEXT_THREADED_JIT
		|	NIY	// TODO: CONTEXT_THREADED_JIT is always undefined
#else
		|	JMP_IP TMP1
#endif
	} else if (GCC_GLOBAL_REGS) {
		|	ldp x29, x30, [sp], # SPAD // stack alignment
#ifdef CONTEXT_THREADED_JIT
		|	NIY	// TODO
#else
		|	JMP_IP TMP1
#endif
	} else {
#ifdef CONTEXT_THREADED_JIT
		ZEND_UNREACHABLE();
		// TODO: context threading can't work without GLOBAL REGS because we have to change
		//       the value of execute_data in execute_ex()
		|	NIY	// TODO
#else
		|	ldp FP, RX, T2                // retore FP and IP
		|	ldp x29, x30, [sp], # NR_SPAD // stack alignment
		|	mov RETVALx, #2               // ZEND_VM_LEAVE ????
		|	ret
#endif
	}

	return 1;
}

static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr)
{
	zend_jit_addr ret_addr;
	int8_t return_value_used;

	ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name);
	ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF));

	if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) {
		if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
			return_value_used = 1;
		} else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) {
			return_value_used = 0;
		} else {
			return_value_used = -1;
		}
	} else {
		return_value_used = -1;
	}

	if (ZEND_OBSERVER_ENABLED) {
		if (Z_MODE(op1_addr) == IS_REG) {
			zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);

			if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) {
				return 0;
			}
			op1_addr = dst;
		}
		|	LOAD_ZVAL_ADDR FCARG2x, op1_addr
		|	mov FCARG1x, FP
		|	SET_EX_OPLINE opline, REG0
		|	EXT_CALL zend_observer_fcall_end, REG0
	}

	// if (!EX(return_value))
	if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_REG1) {
		if (return_value_used != 0) {
			|	ldr REG2, EX->return_value
		}
		ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0);
	} else {
		if (return_value_used != 0) {
			|	ldr REG1, EX->return_value
		}
		ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0);
	}
	if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
	    (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
		if (return_value_used == -1) {
			|	cbz Rx(Z_REG(ret_addr)), >1
			|.cold_code
			|1:
		}
		if (return_value_used != 1) {
			if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
				if (jit_return_label >= 0) {
					|	IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label, ZREG_TMP1, ZREG_TMP2
				} else {
					|	IF_NOT_ZVAL_REFCOUNTED op1_addr, >9, ZREG_TMP1, ZREG_TMP2
				}
			}
			|	GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
			|	GC_DELREF FCARG1x, TMP1w
			if (RC_MAY_BE_1(op1_info)) {
				if (RC_MAY_BE_N(op1_info)) {
					if (jit_return_label >= 0) {
						|	bne =>jit_return_label
					} else {
						|	bne >9
					}
				}
				|	//SAVE_OPLINE()
				|	ZVAL_DTOR_FUNC op1_info, opline, TMP1
				|	//????ldr REG1, EX->return_value // reload ???
			}
			if (return_value_used == -1) {
				if (jit_return_label >= 0) {
					|	b =>jit_return_label
				} else {
					|	b >9
				}
				|.code
			}
		}
	} else if (return_value_used == -1) {
		if (jit_return_label >= 0) {
			|	cbz Rx(Z_REG(ret_addr)), =>jit_return_label
		} else {
			|	cbz Rx(Z_REG(ret_addr)), >9
		}
	}

	if (return_value_used == 0) {
		|9:
		return 1;
	}

	if (opline->op1_type == IS_CONST) {
		zval *zv = RT_CONSTANT(opline, opline->op1);
		|	ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0
		if (Z_REFCOUNTED_P(zv)) {
			|	ADDREF_CONST zv, REG0, TMP1
		}
	} else if (opline->op1_type == IS_TMP_VAR) {
		|	ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
	} else if (opline->op1_type == IS_CV) {
		if (op1_info & MAY_BE_REF) {
			|	LOAD_ZVAL_ADDR REG0, op1_addr
			|	ZVAL_DEREF REG0, op1_info, TMP1w
			op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
		}
		|	ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
		if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
			if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
			    (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) ||
			    !op_array->function_name) {
				|	TRY_ADDREF op1_info, REG0w, REG2, TMP1w
			} else if (return_value_used != 1) {
				|	// if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr);
				|	SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2
			}
		}
	} else {
		if (op1_info & MAY_BE_REF) {
			zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val));

			|	IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1
			|.cold_code
			|1:
			|	// zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
			|	GET_ZVAL_PTR REG0, op1_addr, TMP1
			|	// ZVAL_COPY_VALUE(return_value, &ref->value);
			|	ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
			|	GC_DELREF REG0, TMP1w
			|	beq >2
			|	// if (IS_REFCOUNTED())
			if (jit_return_label >= 0) {
				|	IF_NOT_REFCOUNTED REG2w, =>jit_return_label, TMP1w
			} else {
				|	IF_NOT_REFCOUNTED REG2w, >9, TMP1w
			}
			|	// ADDREF
			|	GET_ZVAL_PTR REG2, ret_addr, TMP1 // reload
			|	GC_ADDREF REG2, TMP1w
			if (jit_return_label >= 0) {
				|	b =>jit_return_label
			} else {
				|	b >9
			}
			|2:
			|	mov FCARG1x, REG0
			|	EFREE_REFERENCE
			if (jit_return_label >= 0) {
				|	b =>jit_return_label
			} else {
				|	b >9
			}
			|.code
		}
		|	ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
	}

	|9:
	return 1;
}

static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg)
{
	ZEND_ASSERT(type_reg == ZREG_REG2);

	|	GET_ZVAL_PTR REG1, val_addr, TMP1
	|	IF_NOT_REFCOUNTED REG2w, >2, TMP1w
	|	GET_LOW_8BITS TMP2w, REG2w
	|	IF_NOT_TYPE TMP2w, IS_REFERENCE, >1
	|	add REG1, REG1, #offsetof(zend_reference, val)
	|	GET_Z_TYPE_INFO REG2w, REG1
	|	GET_Z_PTR REG1, REG1
	|	IF_NOT_REFCOUNTED REG2w, >2, TMP1w
	|1:
	|	GC_ADDREF REG1, TMP2w
	|2:
	|	SET_ZVAL_PTR res_addr, REG1, TMP1
	|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1

	return 1;
}

static int zend_jit_fetch_dim_read(dasm_State        **Dst,
                                   const zend_op      *opline,
                                   zend_ssa           *ssa,
                                   const zend_ssa_op  *ssa_op,
                                   uint32_t            op1_info,
                                   zend_jit_addr       op1_addr,
                                   bool                op1_avoid_refcounting,
                                   uint32_t            op2_info,
                                   uint32_t            res_info,
                                   zend_jit_addr       res_addr,
                                   uint8_t             dim_type)
{
	zend_jit_addr orig_op1_addr, op2_addr;
	const void *exit_addr = NULL;
	const void *not_found_exit_addr = NULL;
	const void *res_exit_addr = NULL;
	bool result_avoid_refcounting = 0;
	uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0;
	int may_throw = 0;

	orig_op1_addr = OP1_ADDR();
	op2_addr = OP2_ADDR();

	if (opline->opcode != ZEND_FETCH_DIM_IS
	 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
		int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
		exit_addr = zend_jit_trace_get_exit_addr(exit_point);
		if (!exit_addr) {
			return 0;
		}
	}

	if ((res_info & MAY_BE_GUARD)
	 && JIT_G(current_frame)
	 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
		uint32_t flags = 0;
		uint32_t old_op1_info = 0;
		uint32_t old_info;
		zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
		int32_t exit_point;

		if (opline->opcode != ZEND_FETCH_LIST_R
		 && (opline->op1_type & (IS_VAR|IS_TMP_VAR))
		 && !op1_avoid_refcounting) {
			flags |= ZEND_JIT_EXIT_FREE_OP1;
		}
		if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
		 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
			flags |= ZEND_JIT_EXIT_FREE_OP2;
		}
		if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
		 && !(flags & ZEND_JIT_EXIT_FREE_OP1)
		 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
		 && (ssa_op+1)->op1_use == ssa_op->result_def
		 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))
		 && zend_jit_may_avoid_refcounting(opline+1, res_info)) {
			result_avoid_refcounting = 1;
			ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
		}

		if (op1_avoid_refcounting) {
			old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
			SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
		}

		if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) {
			old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
			SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
			SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0);
			exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
			SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
			res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
			if (!res_exit_addr) {
				return 0;
			}
			res_info &= ~MAY_BE_GUARD;
			ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
		}

		if (opline->opcode == ZEND_FETCH_DIM_IS
		 && !(res_info & MAY_BE_NULL)) {
			old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
			SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0);
			SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL);
			exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
			SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
			not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
			if (!not_found_exit_addr) {
				return 0;
			}
		}

		if (op1_avoid_refcounting) {
			SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
		}
	}

	if (op1_info & MAY_BE_REF) {
		|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		|	ZVAL_DEREF FCARG1x, op1_info, TMP1w
		op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	}

	if (op1_info & MAY_BE_ARRAY) {
		if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
			if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr, ZREG_TMP1
			} else {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1
			}
		}
		|	GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1
		if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) ||
		    (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) {
			may_throw = 1;
		}
		if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, dim_type, res_exit_addr, not_found_exit_addr, exit_addr)) {
			return 0;
		}
	}

	if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
		if (op1_info & MAY_BE_ARRAY) {
			|.cold_code
			|7:
		}

		if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) {
			may_throw = 1;
			if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) {
				if (exit_addr && !(op1_info & MAY_BE_OBJECT)) {
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr, ZREG_TMP1
				} else {
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1
				}
			}
			|	SET_EX_OPLINE opline, REG0
			|	GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1
			if (opline->opcode != ZEND_FETCH_DIM_IS) {
				if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) {
					|	GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
					|	EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, REG0
				} else {
					|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
					|	EXT_CALL zend_jit_fetch_dim_str_r_helper, REG0
				}
				|	SET_ZVAL_PTR res_addr, RETVALx, TMP1
				|	SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2
			} else {
				|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
				|	LOAD_ZVAL_ADDR CARG3, res_addr
				|	EXT_CALL zend_jit_fetch_dim_str_is_helper, REG0
			}
			if ((op1_info & MAY_BE_ARRAY) ||
				(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) {
				|	b >9 // END
			}
			|6:
		}

		if (op1_info & MAY_BE_OBJECT) {
			may_throw = 1;
			if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) {
				if (exit_addr) {
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1
				} else {
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6, ZREG_TMP1
				}
			}
			|	SET_EX_OPLINE opline, REG0
		    if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		    }
			if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
				ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
				|	LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1)
			} else {
				|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
			}
			|	LOAD_ZVAL_ADDR CARG3, res_addr
			if (opline->opcode != ZEND_FETCH_DIM_IS) {
				|	EXT_CALL zend_jit_fetch_dim_obj_r_helper, REG0
			} else {
				|	EXT_CALL zend_jit_fetch_dim_obj_is_helper, REG0
			}
			if ((op1_info & MAY_BE_ARRAY) ||
				(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
				|	b >9 // END
			}
			|6:
		}

		if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))
		 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
			if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) {
				|	SET_EX_OPLINE opline, REG0
				if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) {
					may_throw = 1;
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
					|	// zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
					|	LOAD_32BIT_VAL FCARG1w, opline->op1.var
					|	EXT_CALL zend_jit_undefined_op_helper, REG0
					|1:
				}

				if (op2_info & MAY_BE_UNDEF) {
					may_throw = 1;
					|	IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1
					|	LOAD_32BIT_VAL FCARG1w, opline->op2.var
					|	EXT_CALL zend_jit_undefined_op_helper, REG0
					|1:
				}
			}

			if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) {
				may_throw = 1;
				if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
					|	LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr
				} else {
					|	SET_EX_OPLINE opline, REG0
					if (Z_MODE(op1_addr) != IS_MEM_ZVAL ||
					    Z_REG(op1_addr) != ZREG_FCARG1 ||
					    Z_OFFSET(op1_addr) != 0) {
						|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
					}
				}
				|	EXT_CALL zend_jit_invalid_array_access, REG0
			}
			|	SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
			if (op1_info & MAY_BE_ARRAY) {
				|	b >9 // END
			}
		}

		if (op1_info & MAY_BE_ARRAY) {
			|.code
		}
	}

	if (op1_info & MAY_BE_ARRAY) {
		zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);

		|8:
		if (res_exit_addr) {
			uint32_t type = concrete_type(res_info);
			if ((op1_info & MAY_BE_ARRAY_OF_REF)
			 && dim_type != IS_UNKNOWN
			 && dim_type != IS_REFERENCE) {
				if (type < IS_STRING) {
					|	IF_NOT_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1
					|.cold_code
					|1:
					|	IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, &res_exit_addr, ZREG_TMP1
					|	GET_Z_PTR REG0, REG0
					|	add REG0, REG0, #offsetof(zend_reference, val)
					|	IF_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1
					|	b &res_exit_addr
					|.code
					|1:
				} else {
					|	GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1
					|	GET_LOW_8BITS TMP1w, REG2w
					|	IF_NOT_TYPE TMP1w, type, >1
					|.cold_code
					|1:
					|	IF_NOT_TYPE TMP1w, IS_REFERENCE, &res_exit_addr
					|	GET_Z_PTR REG0, REG0
					|	add REG0, REG0, #offsetof(zend_reference, val)
					|	GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1
					|	GET_LOW_8BITS TMP1w, REG2w
					|	IF_TYPE TMP1w, type, >1
					|	b &res_exit_addr
					|.code
					|1:
				}
			} else {
				if (op1_info & MAY_BE_ARRAY_OF_REF) {
					|	ZVAL_DEREF REG0, MAY_BE_REF, TMP1w
				}
				if (type < IS_STRING) {
					|	IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr, ZREG_TMP1
				} else {
					|	GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1
					|	GET_LOW_8BITS TMP1w, REG2w
					|	IF_NOT_TYPE TMP1w, type, &res_exit_addr
				}
			}
			|	// ZVAL_COPY
			|7:
			|	ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0
			if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
				if (type < IS_STRING) {
					if (Z_REG(res_addr) != ZREG_FP ||
					    JIT_G(current_frame) == NULL ||
					    STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) {
						|	SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2
					}
				} else {
					|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1
					if (!result_avoid_refcounting) {
						|	TRY_ADDREF res_info, REG2w, REG1, TMP1w
					}
				}
			} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
				return 0;
			}
		} else if (op1_info & MAY_BE_ARRAY_OF_REF) {
			|	// ZVAL_COPY_DEREF
			|	GET_ZVAL_TYPE_INFO Rw(ZREG_REG2), val_addr, TMP1
			if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_REG2)) {
				return 0;
			}
		} else  {
			|	// ZVAL_COPY
			|	ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
			|	TRY_ADDREF res_info, REG1w, REG2, TMP1w
		}
	}
	|9: // END

#ifdef ZEND_JIT_USE_RC_INFERENCE
	if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
		/* Magic offsetGet() may increase refcount of the key */
		op2_info |= MAY_BE_RCN;
	}
#endif

    if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) {
		if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) {
			may_throw = 1;
		}
		|	FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
	}
	if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) {
		if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) {
			if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
				may_throw = 1;
			}
			|	FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2
		}
	}

	if (may_throw) {
		if (!zend_jit_check_exception(Dst)) {
			return 0;
		}
	}

	return 1;
}

static int zend_jit_fetch_dim(dasm_State    **Dst,
                              const zend_op  *opline,
                              uint32_t        op1_info,
                              zend_jit_addr   op1_addr,
                              uint32_t        op2_info,
                              zend_jit_addr   res_addr,
                              uint8_t         dim_type)
{
	zend_jit_addr op2_addr;
	int may_throw = 0;

	op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;

	if (op1_info & MAY_BE_REF) {
		may_throw = 1;
		|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		|	IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w
		|	GET_Z_PTR FCARG2x, FCARG1x
		|	ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))]
		|	cmp TMP1w, #IS_ARRAY
		|	bne >2
		|	add FCARG1x, FCARG2x, #offsetof(zend_reference, val)
		|	b >3
		|.cold_code
		|2:
		|	SET_EX_OPLINE opline, REG0
		|	EXT_CALL zend_jit_prepare_assign_dim_ref, REG0
		|	mov FCARG1x, RETVALx
		|	cbnz FCARG1x, >1
		|	b ->exception_handler_undef
		|.code
		|1:
		op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	}

	if (op1_info & MAY_BE_ARRAY) {
		if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
			|	IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1
		}
		|3:
		|	SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2
	}
	if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
		if (op1_info & MAY_BE_ARRAY) {
			|.cold_code
			|7:
		}
		if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
			|	CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1
			|	bgt >7
		}
		if (Z_REG(op1_addr) != ZREG_FP) {
			|	str Rx(Z_REG(op1_addr)), T1 // save
		}
		if ((op1_info & MAY_BE_UNDEF)
		 && opline->opcode == ZEND_FETCH_DIM_RW) {
			may_throw = 1;
			if (op1_info & MAY_BE_NULL) {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
			}
			|	SET_EX_OPLINE opline, REG0
			|	LOAD_32BIT_VAL FCARG1w, opline->op1.var
			|	EXT_CALL zend_jit_undefined_op_helper, REG0
			|1:
		}
		|	// ZVAL_ARR(container, zend_new_array(8));
		|	EXT_CALL _zend_new_array_0, REG0
		|	mov REG0, RETVALx
		if (Z_REG(op1_addr) != ZREG_FP) {
			|	ldr Rx(Z_REG(op1_addr)), T1 // restore
		}
		|	SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1
		|	SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2
		|	mov FCARG1x, REG0
		if (op1_info & MAY_BE_ARRAY) {
			|	b >1
			|.code
			|1:
		}
	}

	if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
		|6:
		if (opline->op2_type == IS_UNUSED) {
			may_throw = 1;
			|	// var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
			|	LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
			|	EXT_CALL zend_hash_next_index_insert, REG0
			|	// if (UNEXPECTED(!var_ptr)) {
			|	cbz RETVALx, >1
			|.cold_code
			|1:
			|	// zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
			|	CANNOT_ADD_ELEMENT opline
			|	SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2
			|	//ZEND_VM_C_GOTO(assign_dim_op_ret_null);
			|	b >8
			|.code
			|	SET_ZVAL_PTR res_addr, RETVALx, TMP1
			|	SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2
		} else {
			uint32_t type;

			switch (opline->opcode) {
				case ZEND_FETCH_DIM_W:
				case ZEND_FETCH_LIST_W:
					type = BP_VAR_W;
					break;
				case ZEND_FETCH_DIM_RW:
					may_throw = 1;
					type = BP_VAR_RW;
					break;
				case ZEND_FETCH_DIM_UNSET:
					type = BP_VAR_UNSET;
					break;
				default:
					ZEND_UNREACHABLE();
			}

			if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
				may_throw = 1;
			}
			if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL)) {
				return 0;
			}

			|8:
			|	SET_ZVAL_PTR res_addr, REG0, TMP1
			|	SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2

			if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
				|.cold_code
				|9:
				|	SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
				|	b >8
				|.code
			}
		}
	}

	if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
		may_throw = 1;
		if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
			|.cold_code
			|7:
		}

		|	SET_EX_OPLINE opline, REG0
		if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
			|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		}
	    if (opline->op2_type == IS_UNUSED) {
			|	mov FCARG2x, xzr
		} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
			ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
			|	LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1)
		} else {
			|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
		}
		|	LOAD_ZVAL_ADDR CARG3, res_addr
		switch (opline->opcode) {
			case ZEND_FETCH_DIM_W:
			case ZEND_FETCH_LIST_W:
				|	EXT_CALL zend_jit_fetch_dim_obj_w_helper, REG0
				break;
			case ZEND_FETCH_DIM_RW:
				|	EXT_CALL zend_jit_fetch_dim_obj_rw_helper, REG0
				break;
//			case ZEND_FETCH_DIM_UNSET:
//				|	EXT_CALL zend_jit_fetch_dim_obj_unset_helper, REG0
//				break;
			default:
				ZEND_UNREACHABLE();
		}

		if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
			|	b >8 // END
			|.code
		}
	}

#ifdef ZEND_JIT_USE_RC_INFERENCE
	if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
		/* ASSIGN_DIM may increase refcount of the key */
		op2_info |= MAY_BE_RCN;
	}
#endif

	if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
	 && (op2_info & MAY_HAVE_DTOR)
	 && (op2_info & MAY_BE_RC1)) {
		may_throw = 1;
	}
	|8:
	|	FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2

	if (may_throw) {
		if (!zend_jit_check_exception(Dst)) {
			return 0;
		}
	}
	return 1;
}

static int zend_jit_isset_isempty_dim(dasm_State    **Dst,
                                      const zend_op  *opline,
                                      uint32_t        op1_info,
                                      zend_jit_addr   op1_addr,
                                      bool            op1_avoid_refcounting,
                                      uint32_t        op2_info,
                                      uint8_t         dim_type,
                                      int             may_throw,
                                      zend_uchar      smart_branch_opcode,
                                      uint32_t        target_label,
                                      uint32_t        target_label2,
                                      const void     *exit_addr)
{
	zend_jit_addr op2_addr, res_addr;

	// TODO: support for empty() ???
	ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));

	op2_addr = OP2_ADDR();
	res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);

	if (op1_info & MAY_BE_REF) {
		|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		|	ZVAL_DEREF FCARG1x, op1_info, TMP1w
		op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	}

	if (op1_info & MAY_BE_ARRAY) {
		const void *found_exit_addr = NULL;
		const void *not_found_exit_addr = NULL;

		if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
			|	IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1
		}
		|	GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1
		if (exit_addr
		 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY))
		 && !may_throw
		 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting)
		 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
			if (smart_branch_opcode == ZEND_JMPNZ) {
				found_exit_addr = exit_addr;
			} else {
				not_found_exit_addr = exit_addr;
			}
		}
		if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL)) {
			return 0;
		}

		if (found_exit_addr) {
			|9:
			return 1;
		} else if (not_found_exit_addr) {
			|8:
			return 1;
		}
	}

	if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
		if (op1_info & MAY_BE_ARRAY) {
			|.cold_code
			|7:
		}

		if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) {
			|	SET_EX_OPLINE opline, REG0
		    if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
			}
			if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
				ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
				|	LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1)
			} else {
				|	LOAD_ZVAL_ADDR FCARG2x, op2_addr
			}
			|	EXT_CALL zend_jit_isset_dim_helper, REG0
			|	cbz RETVALw, >9
			if (op1_info & MAY_BE_ARRAY) {
				|	b >8
				|.code
			}
		} else {
			if (op2_info & MAY_BE_UNDEF) {
				if (op2_info & MAY_BE_ANY) {
					|	IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1
				}
				|	SET_EX_OPLINE opline, REG0
				|	LOAD_32BIT_VAL FCARG1w, opline->op2.var
				|	EXT_CALL zend_jit_undefined_op_helper, REG0
				|1:
			}
			if (op1_info & MAY_BE_ARRAY) {
				|	b >9
				|.code
			}
		}
	}

#ifdef ZEND_JIT_USE_RC_INFERENCE
	if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
		/* Magic offsetExists() may increase refcount of the key */
		op2_info |= MAY_BE_RCN;
	}
#endif

	if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) {
		|8:
		|	FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
		if (!op1_avoid_refcounting) {
			|	FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2
		}
		if (may_throw) {
			if (!zend_jit_check_exception_undef_result(Dst, opline)) {
				return 0;
			}
		}
		if (!(opline->extended_value & ZEND_ISEMPTY)) {
			if (exit_addr) {
				if (smart_branch_opcode == ZEND_JMPNZ) {
					|	b &exit_addr
				} else {
					|	b >8
				}
			} else if (smart_branch_opcode) {
				if (smart_branch_opcode == ZEND_JMPZ) {
					|	b =>target_label2
				} else if (smart_branch_opcode == ZEND_JMPNZ) {
					|	b =>target_label
				} else if (smart_branch_opcode == ZEND_JMPZNZ) {
					|	b =>target_label2
				} else {
					ZEND_UNREACHABLE();
				}
			} else {
				|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
				|	b >8
			}
		} else {
			|	NIY // TODO: support for empty()
		}
	}

	|9: // not found
	|	FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
	if (!op1_avoid_refcounting) {
		|	FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2
	}
	if (may_throw) {
		if (!zend_jit_check_exception_undef_result(Dst, opline)) {
			return 0;
		}
	}
	if (!(opline->extended_value & ZEND_ISEMPTY)) {
		if (exit_addr) {
			if (smart_branch_opcode == ZEND_JMPZ) {
				|	b &exit_addr
			}
		} else if (smart_branch_opcode) {
			if (smart_branch_opcode == ZEND_JMPZ) {
				|	b =>target_label
			} else if (smart_branch_opcode == ZEND_JMPNZ) {
			} else if (smart_branch_opcode == ZEND_JMPZNZ) {
				|	b =>target_label
			} else {
				ZEND_UNREACHABLE();
			}
		} else {
			|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
		}
	} else {
		|	NIY // TODO: support for empty()
	}

	|8:

	return 1;
}

static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info)
{
	zend_jit_addr op1_addr = OP1_ADDR();
	zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2));

	|	// idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1;
	|	ldr FCARG2x, EX->run_time_cache
	|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG2x, opline->extended_value, TMP1
	|	sub REG0, REG0, #1
	|	// if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket)))
	|	MEM_LOAD_32_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1
	|	cmp REG0, REG1, lsl #5
	|	bhs >9
	|	// Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx);
	|	MEM_LOAD_64_ZTS ldr, TMP1, executor_globals, symbol_table.arData, REG1
	|	add REG0, REG0, TMP1
	|	IF_NOT_Z_TYPE REG0, IS_REFERENCE, >9, TMP1w
	|	// (EXPECTED(p->key == varname))
	|	ldr TMP1, [REG0, #offsetof(Bucket, key)]
	|	LOAD_ADDR TMP2, varname
	|	cmp TMP1, TMP2
	|	bne >9
	|	GET_Z_PTR REG0, REG0
	|	GC_ADDREF REG0, TMP1w
	|1:
	if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
		if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
			|	// if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr)))
			|	IF_ZVAL_REFCOUNTED op1_addr, >2, ZREG_TMP1, ZREG_TMP2
			|.cold_code
			|2:
		}
		|	// zend_refcounted *garbage = Z_COUNTED_P(variable_ptr);
		|	GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
		|	// ZVAL_REF(variable_ptr, ref)
		|	SET_ZVAL_PTR op1_addr, REG0, TMP1
		|	SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2
		|	// if (GC_DELREF(garbage) == 0)
		|	GC_DELREF FCARG1x, TMP1w
		if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
			|	bne >3
		} else {
			|	bne >5
		}
		|	ZVAL_DTOR_FUNC op1_info, opline, TMP1
		|	b >5
		if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
			|3:
			|	// GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr)
			|	IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w
			|	EXT_CALL gc_possible_root, REG0
			|	b >5
		}
		if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
			|.code
		}
	}

	if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
		|	// ZVAL_REF(variable_ptr, ref)
		|	SET_ZVAL_PTR op1_addr, REG0, TMP1
		|	SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2
	}
	|5:
	//END of handler

	|.cold_code
	|9:
	|	LOAD_ADDR FCARG1x, (ptrdiff_t)varname
	if (opline->extended_value) {
		|	ADD_SUB_64_WITH_CONST_32 add, FCARG2x, FCARG2x, opline->extended_value, TMP1
	}
	|	EXT_CALL zend_jit_fetch_global_helper, REG0
	|	mov REG0, RETVALx
	|	b <1
	|.code

	return 1;
}

static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception)
{
	zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
	bool in_cold = 0;
	uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
	zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1 : ZREG_REG0;

	if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
	 && JIT_G(current_frame)
	 && JIT_G(current_frame)->prev) {
		zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
		uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));

		if (type != IS_UNKNOWN && (type_mask & (1u << type))) {
			return 1;
		}
	}

	if (ZEND_ARG_SEND_MODE(arg_info)) {
		if (opline->opcode == ZEND_RECV_INIT) {
			|	LOAD_ZVAL_ADDR Rx(tmp_reg), res_addr
			|	ZVAL_DEREF Rx(tmp_reg), MAY_BE_REF, TMP1w
			res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0);
		} else {
			|	GET_ZVAL_PTR Rx(tmp_reg), res_addr, TMP1
			res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val));
		}
	}

	if (type_mask != 0) {
		if (is_power_of_two(type_mask)) {
			uint32_t type_code = concrete_type(type_mask);
			|	IF_NOT_ZVAL_TYPE res_addr, type_code, >1, ZREG_TMP1
		} else {
			|	mov REG2w, #1
			|	MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1
			|	lsl REG2w, REG2w, REG1w
			|	TST_32_WITH_CONST REG2w, type_mask, TMP1w
			|	beq >1
		}

		|.cold_code
		|1:

		in_cold = 1;
	}

	if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
		|	LOAD_ZVAL_ADDR FCARG1x, res_addr
	}
	if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
		|	SET_EX_OPLINE opline, REG0
	} else {
		|	ADDR_STORE EX->opline, opline, REG0
	}
	|	LOAD_ADDR FCARG2x, (ptrdiff_t)arg_info
	|	EXT_CALL zend_jit_verify_arg_slow, REG0

	if (check_exception) {
		|	GET_LOW_8BITS REG0w, RETVALw
		if (in_cold) {
			|	cbnz REG0w, >1
			|	b ->exception_handler
			|.code
			|1:
		} else {
			|	cbz REG0w, ->exception_handler
		}
	} else if (in_cold) {
		|	b >1
		|.code
		|1:
	}

	return 1;
}

static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array)
{
	uint32_t arg_num = opline->op1.num;
	zend_arg_info *arg_info = NULL;

	if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
		if (EXPECTED(arg_num <= op_array->num_args)) {
			arg_info = &op_array->arg_info[arg_num-1];
		} else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) {
			arg_info = &op_array->arg_info[op_array->num_args];
		}
		if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) {
			arg_info = NULL;
		}
	}

	if (arg_info || (opline+1)->opcode != ZEND_RECV) {
		if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
			if (!JIT_G(current_frame) ||
			    TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 ||
			    arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
				int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

				if (!exit_addr) {
					return 0;
				}
				|	ldr TMP1w, EX->This.u2.num_args
				|	CMP_32_WITH_CONST TMP1w, arg_num, TMP2w
				|	blo &exit_addr
			}
		} else {
			|	ldr TMP1w, EX->This.u2.num_args
			|	CMP_32_WITH_CONST TMP1w, arg_num, TMP2w
			|	blo >1
			|.cold_code
			|1:
			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
				|	SET_EX_OPLINE opline, REG0
			} else {
				|	ADDR_STORE EX->opline, opline, REG0
			}
			|	mov FCARG1x, FP
			|	EXT_CALL zend_missing_arg_error, REG0
			|	b ->exception_handler
			|.code
		}
	}

	if (arg_info) {
		if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) {
			return 0;
		}
	}

	if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
		if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) {
			|	LOAD_IP_ADDR (opline + 1)
			zend_jit_set_last_valid_opline(opline + 1);
		}
	}

	return 1;
}

static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw)
{
	uint32_t arg_num = opline->op1.num;
	zval *zv = RT_CONSTANT(opline, opline->op2);
	zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);

	if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
	 && JIT_G(current_frame)
	 && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) {
		if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
			|	ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0
			if (Z_REFCOUNTED_P(zv)) {
				|	ADDREF_CONST zv, REG0, TMP1
			}
		}
	} else {
		if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
		    (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
			|	ldr TMP1w, EX->This.u2.num_args
			|	CMP_32_WITH_CONST TMP1w, arg_num, TMP2w
			|	bhs >5
		}
		|	ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0
		if (Z_REFCOUNTED_P(zv)) {
			|	ADDREF_CONST zv, REG0, TMP1
		}
	}

	if (Z_CONSTANT_P(zv)) {
		if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
			|	SET_EX_OPLINE opline, REG0
		} else {
			|	ADDR_STORE EX->opline, opline, REG0
		}
		|	LOAD_ZVAL_ADDR FCARG1x, res_addr
		|	ldr REG0, EX->func
		|	ldr FCARG2x, [REG0, #offsetof(zend_op_array, scope)]
		|	EXT_CALL zval_update_constant_ex, REG0
		|	cbnz RETVALw, >1
		|.cold_code
		|1:
		|	ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline, ZREG_TMP1, ZREG_TMP2
		|	SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2
		|	b ->exception_handler
		|.code
	}

	|5:

	if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
		do {
			zend_arg_info *arg_info;

			if (arg_num <= op_array->num_args) {
				arg_info = &op_array->arg_info[arg_num-1];
			} else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
				arg_info = &op_array->arg_info[op_array->num_args];
			} else {
				break;
			}
			if (!ZEND_TYPE_IS_SET(arg_info->type)) {
				break;
			}
			if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) {
				return 0;
			}
		} while (0);
	}

	if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
		if (is_last) {
			|	LOAD_IP_ADDR (opline + 1)
			zend_jit_set_last_valid_opline(opline + 1);
		}
	}
	return 1;
}

static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce)
{
	int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
	const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

	if (!exit_addr) {
		return 0;
	}

	|	LOAD_ADDR TMP1, ((ptrdiff_t)ce)
	|	ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)]
	|	cmp TMP2, TMP1
	|	bne &exit_addr

	return 1;
}

static int zend_jit_fetch_obj(dasm_State          **Dst,
                              const zend_op        *opline,
                              const zend_op_array  *op_array,
                              zend_ssa             *ssa,
                              const zend_ssa_op    *ssa_op,
                              uint32_t              op1_info,
                              zend_jit_addr         op1_addr,
                              bool                  op1_indirect,
                              zend_class_entry     *ce,
                              bool                  ce_is_instanceof,
                              bool                  on_this,
                              bool                  delayed_fetch_this,
                              bool                  op1_avoid_refcounting,
                              zend_class_entry     *trace_ce,
                              uint8_t               prop_type,
                              int                   may_throw)
{
	zval *member;
	zend_property_info *prop_info;
	bool may_be_dynamic = 1;
	zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
	zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
	zend_jit_addr prop_addr;
	uint32_t res_info = RES_INFO();
	bool type_loaded = 0;

	ZEND_ASSERT(opline->op2_type == IS_CONST);
	ZEND_ASSERT(op1_info & MAY_BE_OBJECT);

	member = RT_CONSTANT(opline, opline->op2);
	ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
	prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename);

	if (on_this) {
		|	GET_ZVAL_PTR FCARG1x, this_addr, TMP1
	} else {
		if (opline->op1_type == IS_VAR
		 && opline->opcode == ZEND_FETCH_OBJ_W
		 && (op1_info & MAY_BE_INDIRECT)
		 && Z_REG(op1_addr) == ZREG_FP) {
			|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
			|	IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w
			|	GET_Z_PTR FCARG1x, FCARG1x
			|1:
			op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
		}
		if (op1_info & MAY_BE_REF) {
			if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
			}
			|	ZVAL_DEREF FCARG1x, op1_info, TMP1w
			op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
		}
		if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
				int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

				if (!exit_addr) {
					return 0;
				}
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1
			} else {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, ZREG_TMP1
			}
		}
		|	GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
	}

	if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
		prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename);
		if (prop_info) {
			ce = trace_ce;
			ce_is_instanceof = 0;
			if (!(op1_info & MAY_BE_CLASS_GUARD)) {
				if (on_this && JIT_G(current_frame)
				 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
					ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
				} else if (zend_jit_class_guard(Dst, opline, ce)) {
					if (on_this && JIT_G(current_frame)) {
						JIT_G(current_frame)->ce = ce;
						TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
					}
				} else {
					return 0;
				}
				if (ssa->var_info && ssa_op->op1_use >= 0) {
					ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
					ssa->var_info[ssa_op->op1_use].ce = ce;
					ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
				}
			}
		}
	}

	if (!prop_info) {
		|	ldr REG0, EX->run_time_cache
		|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS), TMP1
		|	ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)]
		|	cmp REG2, TMP1
		|	bne >5
		|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)  + sizeof(void*)), TMP1
		may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename);
		if (may_be_dynamic) {
			|	tst REG0, REG0
			if (opline->opcode == ZEND_FETCH_OBJ_W) {
				|	blt >5
			} else {
				|	blt >8 // dynamic property
			}
		}
		|	add TMP1, FCARG1x, REG0
		|	ldr REG2w, [TMP1, #offsetof(zval,u1.type_info)]
		|	IF_UNDEF REG2w, >5
		|	mov FCARG1x, TMP1
		type_loaded = 1;
		prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
		if (opline->opcode == ZEND_FETCH_OBJ_W
		 && (!ce ||	ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS))) {
			uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;

			|	ldr REG0, EX->run_time_cache
			|	MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)  + sizeof(void*) * 2), TMP1
			|	cbnz FCARG2x, >1
			|.cold_code
			|1:
			|	ldr TMP1w, [FCARG2x, #offsetof(zend_property_info, flags)]
			|	tst TMP1w, #ZEND_ACC_READONLY
			if (flags) {
				|	beq >3
			} else {
				|	beq >4
			}
			|	IF_NOT_TYPE REG2w, IS_OBJECT_EX, >2
			|	GET_Z_PTR REG2, FCARG1x
			|	GC_ADDREF REG2, TMP1w
			|	SET_ZVAL_PTR res_addr, REG2, TMP1
			|	SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2
			|	b >9
			|2:
			|	mov FCARG1x, FCARG2x
			|	SET_EX_OPLINE opline, REG0
			|	EXT_CALL zend_readonly_property_modification_error, REG0
			|	SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2
			|	b >9
			|3:
			if (flags == ZEND_FETCH_DIM_WRITE) {
				|	SET_EX_OPLINE opline, REG0
				|	EXT_CALL zend_jit_check_array_promotion, REG0
				|	b >9
			} else if (flags == ZEND_FETCH_REF) {
				|	LOAD_ZVAL_ADDR CARG3, res_addr
				|	EXT_CALL zend_jit_create_typed_ref, REG0
				|	b >9
			} else {
				ZEND_ASSERT(flags == 0);
			}
			|.code
			|4:
		}
	} else {
		prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset);
		if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
			if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) {
				/* perform IS_UNDEF check only after result type guard (during deoptimization) */
				int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

				if (!exit_addr) {
					return 0;
				}
				type_loaded = 1;
				|	MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1
				|	IF_UNDEF REG2w, &exit_addr
			}
		} else {
			type_loaded = 1;
			|	MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1
			|	IF_UNDEF REG2w, >5
		}
		if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) {
			if (!type_loaded) {
				type_loaded = 1;
				|	MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1
			}
			|	IF_NOT_TYPE REG2w, IS_OBJECT_EX, >4
			|	GET_ZVAL_PTR REG2, prop_addr, TMP1
			|	GC_ADDREF REG2, TMP1w
			|	SET_ZVAL_PTR res_addr, REG2, TMP1
			|	SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2
			|	b >9
			|.cold_code
			|4:
			|	LOAD_ADDR FCARG1x, prop_info
			|	SET_EX_OPLINE opline, REG0
			|	EXT_CALL zend_readonly_property_modification_error, REG0
			|	SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2
			|	b >9
			|.code
		}
		if (opline->opcode == ZEND_FETCH_OBJ_W
		 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS)
		 && ZEND_TYPE_IS_SET(prop_info->type)) {
			uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;

			if (flags == ZEND_FETCH_DIM_WRITE) {
				if ((ZEND_TYPE_FULL_MASK(prop_info->type) & (MAY_BE_ITERABLE|MAY_BE_ARRAY)) == 0) {
					if (!type_loaded) {
						type_loaded = 1;
						|	MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1
					}
					|	cmp REG2w, #IS_FALSE
					|	ble >1
					|.cold_code
					|1:
					if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
						|	LOAD_ZVAL_ADDR FCARG1x, prop_addr
					}
					|	LOAD_ADDR FCARG2x, prop_info
					|	SET_EX_OPLINE opline, REG0
					|	EXT_CALL zend_jit_check_array_promotion, REG0
					|	b >9
					|.code
				}
			} else if (flags == ZEND_FETCH_REF) {
				if (!type_loaded) {
					type_loaded = 1;
					|	MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1
				}
				|	GET_LOW_8BITS TMP1w, REG2w
				|	IF_TYPE TMP1w, IS_REFERENCE, >1
				if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
					|	LOAD_ADDR FCARG2x, prop_info
				} else {
					int prop_info_offset =
						(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));

					|	ldr REG0, [FCARG1x, #offsetof(zend_object, ce)]
					|	ldr	REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)]
					|	MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1
				}
				if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
					|	LOAD_ZVAL_ADDR FCARG1x, prop_addr
				}
				|	LOAD_ZVAL_ADDR CARG3, res_addr
				|	EXT_CALL zend_jit_create_typed_ref, REG0
				|	b >9
				|1:
			} else {
				ZEND_UNREACHABLE();
			}
		}
	}
	if (opline->opcode == ZEND_FETCH_OBJ_W) {
		if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
			|	LOAD_ZVAL_ADDR FCARG1x, prop_addr
		}
		|	SET_ZVAL_PTR res_addr, FCARG1x, TMP1
		|	SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2
		if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) {
			ssa->var_info[ssa_op->result_def].indirect_reference = 1;
		}
	} else {
		bool result_avoid_refcounting = 0;

		if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) {
			uint32_t flags = 0;
			uint32_t old_info;
			zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
			int32_t exit_point;
			const void *exit_addr;
			uint32_t type;
			zend_jit_addr val_addr = prop_addr;

			if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
			 && !delayed_fetch_this
			 && !op1_avoid_refcounting) {
				flags = ZEND_JIT_EXIT_FREE_OP1;
			}

			if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
			 && !(flags & ZEND_JIT_EXIT_FREE_OP1)
			 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
			 && (ssa_op+1)->op1_use == ssa_op->result_def
			 && zend_jit_may_avoid_refcounting(opline+1, res_info)) {
				result_avoid_refcounting = 1;
				ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
			}

			type = concrete_type(res_info);

			if (prop_type != IS_UNKNOWN
			 && prop_type != IS_UNDEF
			 && prop_type != IS_REFERENCE
			 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) {
				exit_point = zend_jit_trace_get_exit_point(opline, 0);
				exit_addr = zend_jit_trace_get_exit_addr(exit_point);
				if (!exit_addr) {
					return 0;
				}
			} else {
				val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
				|	LOAD_ZVAL_ADDR REG0, prop_addr
				if (op1_avoid_refcounting) {
					SET_STACK_REG(JIT_G(current_frame)->stack,
						EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
				}
				old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
				SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
				SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0);
				exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
				SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
				exit_addr = zend_jit_trace_get_exit_addr(exit_point);
				if (!exit_addr) {
					return 0;
				}

				if (!type_loaded) {
					type_loaded = 1;
					|	MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1
				}
				|	// ZVAL_DEREF()
				|	GET_LOW_8BITS TMP1w, REG2w
				|	IF_NOT_TYPE TMP1w, IS_REFERENCE, >1
				|	GET_Z_PTR REG0, REG0
				|	add REG0, REG0, #offsetof(zend_reference, val)
				|	GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1
			}
			res_info &= ~MAY_BE_GUARD;
			ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
			if (type < IS_STRING) {
				|1:
				if (type_loaded) {
					|	IF_NOT_TYPE REG2w, type, &exit_addr
				} else {
					|	IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, ZREG_TMP1
				}
			} else {
				if (!type_loaded) {
					type_loaded = 1;
					|	GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1
				}
				|1:
				|	GET_LOW_8BITS TMP1w, REG2w
				|	IF_NOT_TYPE TMP1w, type, &exit_addr
			}
			|	// ZVAL_COPY
			|	ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0
			if (type < IS_STRING) {
				if (Z_REG(res_addr) != ZREG_FP ||
				    JIT_G(current_frame) == NULL ||
				    STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) {
					|	SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2
				}
			} else {
				|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1
				if (!result_avoid_refcounting) {
					|	TRY_ADDREF res_info, REG2w, REG1, TMP1w
				}
			}
		} else {
			if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_REG2)) {
				return 0;
			}
		}
	}

	if (op1_avoid_refcounting) {
		SET_STACK_REG(JIT_G(current_frame)->stack,
			EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
	}

	|.cold_code

	if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) {
		|5:
		|	SET_EX_OPLINE opline, REG0
		if (opline->opcode == ZEND_FETCH_OBJ_W) {
			|	EXT_CALL zend_jit_fetch_obj_w_slow, REG0
		} else if (opline->opcode != ZEND_FETCH_OBJ_IS) {
			|	EXT_CALL zend_jit_fetch_obj_r_slow, REG0
		} else {
			|	EXT_CALL zend_jit_fetch_obj_is_slow, REG0
		}
		|	b >9
	}

	if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
		|7:
		if (opline->opcode != ZEND_FETCH_OBJ_IS) {
			|	SET_EX_OPLINE opline, REG0
			if (opline->opcode != ZEND_FETCH_OBJ_W
			 && (op1_info & MAY_BE_UNDEF)) {
				zend_jit_addr orig_op1_addr = OP1_ADDR();

				if (op1_info & MAY_BE_ANY) {
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
				}
				|	LOAD_32BIT_VAL FCARG1w, opline->op1.var
				|	EXT_CALL zend_jit_undefined_op_helper, REG0
				|1:
				|	LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr
			} else if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
			}
			|	LOAD_ADDR FCARG2x, Z_STRVAL_P(member)
			if (opline->opcode == ZEND_FETCH_OBJ_W) {
				|	EXT_CALL zend_jit_invalid_property_write, REG0
				|	SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2
			} else {
				|	EXT_CALL zend_jit_invalid_property_read, REG0
				|	SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
			}
			|	b >9
		} else {
			|	SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
			|	b >9
		}
	}

	if (!prop_info
	 && may_be_dynamic
	 && opline->opcode != ZEND_FETCH_OBJ_W) {
		|8:
		|	mov FCARG2x, REG0
		|	SET_EX_OPLINE opline, REG0
		if (opline->opcode != ZEND_FETCH_OBJ_IS) {
			|	EXT_CALL zend_jit_fetch_obj_r_dynamic, REG0
		} else {
			|	EXT_CALL zend_jit_fetch_obj_is_dynamic, REG0
		}
		|	b >9
	}

	|.code;
	|9: // END
	if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
		if (opline->op1_type == IS_VAR
		 && opline->opcode == ZEND_FETCH_OBJ_W
		 && (op1_info & MAY_BE_RC1)) {
			zend_jit_addr orig_op1_addr = OP1_ADDR();

			|	IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1, ZREG_TMP1, ZREG_TMP2
			|	GET_ZVAL_PTR FCARG1x, orig_op1_addr, TMP1
			|	GC_DELREF FCARG1x, TMP1w
			|	bne >1
			|	SET_EX_OPLINE opline, REG0
			|	EXT_CALL zend_jit_extract_helper, REG0
			|1:
		} else if (!op1_avoid_refcounting) {
			if (on_this) {
				op1_info &= ~MAY_BE_RC1;
			}
			|	FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
		}
	}

	if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
	 && prop_info
	 && (opline->opcode != ZEND_FETCH_OBJ_W ||
	     !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) ||
	     !ZEND_TYPE_IS_SET(prop_info->type))
	 && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) {
		may_throw = 0;
	}

	if (may_throw) {
		if (!zend_jit_check_exception(Dst)) {
			return 0;
		}
	}

	return 1;
}

static int zend_jit_incdec_obj(dasm_State          **Dst,
                               const zend_op        *opline,
                               const zend_op_array  *op_array,
                               zend_ssa             *ssa,
                               const zend_ssa_op    *ssa_op,
                               uint32_t              op1_info,
                               zend_jit_addr         op1_addr,
                               bool                  op1_indirect,
                               zend_class_entry     *ce,
                               bool                  ce_is_instanceof,
                               bool                  on_this,
                               bool                  delayed_fetch_this,
                               zend_class_entry     *trace_ce,
                               uint8_t               prop_type)
{
	zval *member;
	zend_string *name;
	zend_property_info *prop_info;
	zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
	zend_jit_addr res_addr = 0;
	zend_jit_addr prop_addr;
	bool needs_slow_path = 0;
	bool use_prop_guard = 0;
	bool may_throw = 0;
	uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0;

	ZEND_ASSERT(opline->op2_type == IS_CONST);
	ZEND_ASSERT(op1_info & MAY_BE_OBJECT);

	if (opline->result_type != IS_UNUSED) {
		res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
	}

	member = RT_CONSTANT(opline, opline->op2);
	ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
	name = Z_STR_P(member);
	prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);

	if (on_this) {
		|	GET_ZVAL_PTR FCARG1x, this_addr, TMP1
	} else {
		if (opline->op1_type == IS_VAR
		 && (op1_info & MAY_BE_INDIRECT)
		 && Z_REG(op1_addr) == ZREG_FP) {
			|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
			|	IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w
			|	GET_Z_PTR FCARG1x, FCARG1x
			|1:
			op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
		}
		if (op1_info & MAY_BE_REF) {
			if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
			}
			|	ZVAL_DEREF FCARG1x, op1_info, TMP1w
			op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
		}
		if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
				int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

				if (!exit_addr) {
					return 0;
				}
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1
			} else {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1
				|.cold_code
				|1:
				|	SET_EX_OPLINE opline, REG0
				if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
					|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
				}
				|	LOAD_ADDR FCARG2x, ZSTR_VAL(name)
				|	EXT_CALL zend_jit_invalid_property_incdec, REG0
				|	b ->exception_handler
				|.code
			}
		}
		|	GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
	}

	if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
		prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
		if (prop_info) {
			ce = trace_ce;
			ce_is_instanceof = 0;
			if (!(op1_info & MAY_BE_CLASS_GUARD)) {
				if (on_this && JIT_G(current_frame)
				 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
					ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
				} else if (zend_jit_class_guard(Dst, opline, ce)) {
					if (on_this && JIT_G(current_frame)) {
						JIT_G(current_frame)->ce = ce;
						TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
					}
				} else {
					return 0;
				}
				if (ssa->var_info && ssa_op->op1_use >= 0) {
					ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
					ssa->var_info[ssa_op->op1_use].ce = ce;
					ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
				}
				if (ssa->var_info && ssa_op->op1_def >= 0) {
					ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
					ssa->var_info[ssa_op->op1_def].ce = ce;
					ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
				}
			}
		}
	}

	use_prop_guard = (prop_type != IS_UNKNOWN
		&& prop_type != IS_UNDEF
		&& prop_type != IS_REFERENCE
		&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);

	if (!prop_info) {
		needs_slow_path = 1;

		|	ldr REG0, EX->run_time_cache
		|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1
		|	ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)]
		|	cmp REG2, TMP1
		|	bne >7
		if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
			|	MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1
			|	cbnz TMP1, >7
		}
		|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1
		|	tst REG0, REG0
		|	blt >7
		|	add TMP1, FCARG1x, REG0
		if (!use_prop_guard) {
			|	ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)]
			|	IF_TYPE TMP2w , IS_UNDEF, >7
		}
		|	mov FCARG1x, TMP1
		prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	} else {
		prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset);
		if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
				int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

				if (!exit_addr) {
					return 0;
				}
				|	MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1
				|	IF_TYPE TMP2w, IS_UNDEF, &exit_addr
			} else {
				|	MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1
				|	IF_TYPE TMP2w, IS_UNDEF, >7
				needs_slow_path = 1;
			}
		}
		if (ZEND_TYPE_IS_SET(prop_info->type)) {
			may_throw = 1;
			|	SET_EX_OPLINE opline, REG0
			if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
				|	LOAD_ADDR FCARG2x, prop_info
			} else {
				int prop_info_offset =
					(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));

				|	ldr REG0, [FCARG1x, #offsetof(zend_object, ce)]
				|	ldr	REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)]
				|	MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1
			}
			|	LOAD_ZVAL_ADDR FCARG1x, prop_addr
			if (opline->result_type == IS_UNUSED) {
				switch (opline->opcode) {
					case ZEND_PRE_INC_OBJ:
					case ZEND_POST_INC_OBJ:
						|	EXT_CALL zend_jit_inc_typed_prop, REG0
						break;
					case ZEND_PRE_DEC_OBJ:
					case ZEND_POST_DEC_OBJ:
						|	EXT_CALL zend_jit_dec_typed_prop, REG0
						break;
					default:
						ZEND_UNREACHABLE();
				}
			} else {
				|	LOAD_ZVAL_ADDR CARG3, res_addr
				switch (opline->opcode) {
					case ZEND_PRE_INC_OBJ:
						|	EXT_CALL zend_jit_pre_inc_typed_prop, REG0
						break;
					case ZEND_PRE_DEC_OBJ:
						|	EXT_CALL zend_jit_pre_dec_typed_prop, REG0
						break;
					case ZEND_POST_INC_OBJ:
						|	EXT_CALL zend_jit_post_inc_typed_prop, REG0
						break;
					case ZEND_POST_DEC_OBJ:
						|	EXT_CALL zend_jit_post_dec_typed_prop, REG0
						break;
					default:
						ZEND_UNREACHABLE();
				}
			}
		}
	}

	if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
		uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
		zend_jit_addr var_addr = prop_addr;

		if (use_prop_guard) {
			int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
			const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
			if (!exit_addr) {
				return 0;
			}

			|	IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1
			var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
		}

		if (var_info & MAY_BE_REF) {
			may_throw = 1;
			var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
			if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, prop_addr
			}
			|	IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1
			|	GET_ZVAL_PTR FCARG1x, var_addr, TMP1
			|	ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
			|	cbnz TMP1, >1
			|	add FCARG1x, FCARG1x, #offsetof(zend_reference, val)
			|.cold_code
			|1:
			if (opline) {
				|	SET_EX_OPLINE opline, REG0
			}
			if (opline->result_type == IS_UNUSED) {
				|	mov FCARG2x, xzr
			} else {
				|	LOAD_ZVAL_ADDR FCARG2x, res_addr
			}
			switch (opline->opcode) {
				case ZEND_PRE_INC_OBJ:
					|	EXT_CALL zend_jit_pre_inc_typed_ref, REG0
					break;
				case ZEND_PRE_DEC_OBJ:
					|	EXT_CALL zend_jit_pre_dec_typed_ref, REG0
					break;
				case ZEND_POST_INC_OBJ:
					|	EXT_CALL zend_jit_post_inc_typed_ref, REG0
					break;
				case ZEND_POST_DEC_OBJ:
					|	EXT_CALL zend_jit_post_dec_typed_ref, REG0
					break;
				default:
					ZEND_UNREACHABLE();
			}
			|	b >9
			|.code
			|2:
		}

		if (var_info & MAY_BE_LONG) {
			if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
				|	IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2, ZREG_TMP1
			}
			if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
				if (opline->result_type != IS_UNUSED) {
					|	ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
				}
			}
			if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
				|	LONG_ADD_SUB_WITH_IMM adds, var_addr, Z_L(1), TMP1, TMP2
			} else {
				|	LONG_ADD_SUB_WITH_IMM subs, var_addr, Z_L(1), TMP1, TMP2
			}
			|	bvs	>3
			if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) {
				if (opline->result_type != IS_UNUSED) {
					|	ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
				}
			}
			|.cold_code
		}
		if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
			if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
				may_throw = 1;
			}
			if (var_info & MAY_BE_LONG) {
				|2:
			}
			if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {
				var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
				|	LOAD_ZVAL_ADDR FCARG1x, prop_addr
			}
			if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
				|	ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
				|	TRY_ADDREF MAY_BE_ANY, REG0w, REG2, TMP1w
			}
			if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
				if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
					|	LOAD_ZVAL_ADDR FCARG2x, res_addr
					|	EXT_CALL zend_jit_pre_inc, REG0
				} else {
					|	EXT_CALL increment_function, REG0
				}
			} else {
				if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
					|	LOAD_ZVAL_ADDR FCARG2x, res_addr
					|	EXT_CALL zend_jit_pre_dec, REG0
				} else {
					|	EXT_CALL decrement_function, REG0
				}
			}
			if (var_info & MAY_BE_LONG) {
				|	b >4
			}
		}

		if (var_info & MAY_BE_LONG) {
			|3:
			if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
				uint64_t val = 0x43e0000000000000;
				|	LOAD_64BIT_VAL TMP2, val
				|	SET_ZVAL_LVAL_FROM_REG var_addr, TMP2, TMP1
				|	SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2
				if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
					|	SET_ZVAL_LVAL_FROM_REG res_addr, TMP2, TMP1
					|	SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2
				}
			} else {
				uint64_t val = 0xc3e0000000000000;
				|	LOAD_64BIT_VAL TMP2, val
				|	SET_ZVAL_LVAL_FROM_REG var_addr, TMP2, TMP1
				|	SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2
				if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
					|	SET_ZVAL_LVAL_FROM_REG res_addr, TMP2, TMP1
					|	SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2
				}
			}
			if (opline->result_type != IS_UNUSED
			 && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ)
			 && prop_info
			 && !ZEND_TYPE_IS_SET(prop_info->type)
			 && (res_info & MAY_BE_GUARD)
			 && (res_info & MAY_BE_LONG)) {
				zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
				uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
				int32_t exit_point;
				const void *exit_addr;

				SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
				exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
				exit_addr = zend_jit_trace_get_exit_addr(exit_point);
				if (!exit_addr) {
					return 0;
				}
				SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
				ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD;
				|	b &exit_addr
				|.code
			} else {
				|	b >4
				|.code
				|4:
			}
		}
	}

	if (needs_slow_path) {
		may_throw = 1;
		|.cold_code
		|7:
		|	SET_EX_OPLINE opline, REG0
		|	// value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
		|	LOAD_ADDR FCARG2x, name
		|	ldr CARG3, EX->run_time_cache
		|	ADD_SUB_64_WITH_CONST_32 add, CARG3, CARG3, opline->extended_value, TMP1
		if (opline->result_type == IS_UNUSED) {
			|	mov CARG4, xzr
		} else {
			|	LOAD_ZVAL_ADDR CARG4, res_addr
		}

		switch (opline->opcode) {
			case ZEND_PRE_INC_OBJ:
				|	EXT_CALL zend_jit_pre_inc_obj_helper, REG0
				break;
			case ZEND_PRE_DEC_OBJ:
				|	EXT_CALL zend_jit_pre_dec_obj_helper, REG0
				break;
			case ZEND_POST_INC_OBJ:
				|	EXT_CALL zend_jit_post_inc_obj_helper, REG0
				break;
			case ZEND_POST_DEC_OBJ:
				|	EXT_CALL zend_jit_post_dec_obj_helper, REG0
				break;
			default:
				ZEND_UNREACHABLE();
		}

		|	b >9
		|.code
	}

	|9:
	if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
		if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
			may_throw = 1;
		}
		|	FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
	}

	if (may_throw) {
		if (!zend_jit_check_exception(Dst)) {
			return 0;
		}
	}

	return 1;
}

static int zend_jit_assign_obj_op(dasm_State          **Dst,
                                  const zend_op        *opline,
                                  const zend_op_array  *op_array,
                                  zend_ssa             *ssa,
                                  const zend_ssa_op    *ssa_op,
                                  uint32_t              op1_info,
                                  zend_jit_addr         op1_addr,
                                  uint32_t              val_info,
                                  zend_ssa_range       *val_range,
                                  bool                  op1_indirect,
                                  zend_class_entry     *ce,
                                  bool                  ce_is_instanceof,
                                  bool                  on_this,
                                  bool                  delayed_fetch_this,
                                  zend_class_entry     *trace_ce,
                                  uint8_t               prop_type)
{
	zval *member;
	zend_string *name;
	zend_property_info *prop_info;
	zend_jit_addr val_addr = OP1_DATA_ADDR();
	zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
	zend_jit_addr prop_addr;
	bool needs_slow_path = 0;
	bool use_prop_guard = 0;
	bool may_throw = 0;
	binary_op_type binary_op = get_binary_op(opline->extended_value);

	ZEND_ASSERT(opline->op2_type == IS_CONST);
	ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
	ZEND_ASSERT(opline->result_type == IS_UNUSED);

	member = RT_CONSTANT(opline, opline->op2);
	ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
	name = Z_STR_P(member);
	prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);

	if (on_this) {
		|	GET_ZVAL_PTR FCARG1x, this_addr, TMP1
	} else {
		if (opline->op1_type == IS_VAR
		 && (op1_info & MAY_BE_INDIRECT)
		 && Z_REG(op1_addr) == ZREG_FP) {
			|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
			|	IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w
			|	GET_Z_PTR FCARG1x, FCARG1x
			|1:
			op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
		}
		if (op1_info & MAY_BE_REF) {
			if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
			}
			|	ZVAL_DEREF FCARG1x, op1_info, TMP1w
			op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
		}
		if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
				int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

				if (!exit_addr) {
					return 0;
				}
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1
			} else {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1
				|.cold_code
				|1:
				|	SET_EX_OPLINE opline, REG0
				if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
					|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
				}
				|	LOAD_ADDR FCARG2x, ZSTR_VAL(name)
				if (op1_info & MAY_BE_UNDEF) {
					|	EXT_CALL zend_jit_invalid_property_assign_op, REG0
				} else {
					|	EXT_CALL zend_jit_invalid_property_assign, REG0
				}
				may_throw = 1;
				if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
				 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
					|	b >8
				} else {
					|	b >9
				}
				|.code
			}
		}
		|	GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
	}

	if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
		prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
		if (prop_info) {
			ce = trace_ce;
			ce_is_instanceof = 0;
			if (!(op1_info & MAY_BE_CLASS_GUARD)) {
				if (on_this && JIT_G(current_frame)
				 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
					ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
				} else if (zend_jit_class_guard(Dst, opline, ce)) {
					if (on_this && JIT_G(current_frame)) {
						JIT_G(current_frame)->ce = ce;
						TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
					}
				} else {
					return 0;
				}
				if (ssa->var_info && ssa_op->op1_use >= 0) {
					ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
					ssa->var_info[ssa_op->op1_use].ce = ce;
					ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
				}
				if (ssa->var_info && ssa_op->op1_def >= 0) {
					ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
					ssa->var_info[ssa_op->op1_def].ce = ce;
					ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
				}
			}
		}
	}

	use_prop_guard = (prop_type != IS_UNKNOWN
		&& prop_type != IS_UNDEF
		&& prop_type != IS_REFERENCE
		&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);

	if (!prop_info) {
		needs_slow_path = 1;

		|	ldr REG0, EX->run_time_cache
		|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, ((opline+1)->extended_value), TMP1
		|	ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)]
		|	cmp REG2, TMP2
		|	bne >7
		if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
			|	MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, ((opline+1)->extended_value + sizeof(void*) * 2), TMP1
			|	cbnz TMP1, >7
		}
		|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline+1)->extended_value + sizeof(void*)), TMP1
		|	tst REG0, REG0
		|	blt >7
		|	add TMP1, FCARG1x, REG0
		if (!use_prop_guard) {
			|	ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)]
			|	IF_TYPE TMP2w, IS_UNDEF, >7
		}
		|	mov FCARG1x, TMP1
		prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	} else {
		prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset);
		if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
				int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

				if (!exit_addr) {
					return 0;
				}
				|	MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1
				|	IF_TYPE TMP2w, IS_UNDEF, &exit_addr
			} else {
				|	MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1
				|	IF_TYPE TMP2w, IS_UNDEF, >7
				needs_slow_path = 1;
			}
		}
		if (ZEND_TYPE_IS_SET(prop_info->type)) {
			uint32_t info = val_info;

			may_throw = 1;

			if (opline) {
				|	SET_EX_OPLINE opline, REG0
			}

			|	IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1, ZREG_TMP1
			|.cold_code
			|1:
			|	GET_ZVAL_PTR FCARG1x, prop_addr, TMP1
			if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG2x, val_addr
			}
			|	LOAD_ADDR CARG3, binary_op
			if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
			 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
				|	EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0
			} else {
				|	EXT_CALL zend_jit_assign_op_to_typed_ref, REG0
			}
			|	b >9
			|.code

			|	// value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);

			if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
				|	LOAD_ADDR FCARG2x, prop_info
			} else {
				int prop_info_offset =
					(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));

				|	ldr REG0, [FCARG1x, #offsetof(zend_object, ce)]
				|	ldr	REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)]
				|	MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1
			}
			|	LOAD_ZVAL_ADDR FCARG1x, prop_addr
			|	LOAD_ZVAL_ADDR CARG3, val_addr
			|	LOAD_ADDR CARG4, binary_op

			|	EXT_CALL zend_jit_assign_op_to_typed_prop, REG0

			if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
				info |= MAY_BE_RC1|MAY_BE_RCN;
			}

			|	FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2
		}
	}

	if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
		zend_jit_addr var_addr = prop_addr;
		uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
		uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;

		if (use_prop_guard) {
			int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
			const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
			if (!exit_addr) {
				return 0;
			}

			|	IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1
			var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
		}

		if (var_info & MAY_BE_REF) {
			may_throw = 1;
			var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
			|	LOAD_ZVAL_ADDR REG0, prop_addr
			|	IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1
			|	GET_ZVAL_PTR FCARG1x, var_addr, TMP1
			|	ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
			|	cbnz TMP1, >1
			|	add REG0, FCARG1x, #offsetof(zend_reference, val)
			|.cold_code
			|1:
			if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG2x, val_addr
			}
			if (opline) {
				|	SET_EX_OPLINE opline, REG0
			}
			|	LOAD_ADDR CARG3, binary_op
			if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR))
			 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
				|	EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0
			} else {
				|	EXT_CALL zend_jit_assign_op_to_typed_ref, REG0
			}
			|	b >9
			|.code
			|2:
		}

		switch (opline->extended_value) {
			case ZEND_ADD:
			case ZEND_SUB:
			case ZEND_MUL:
				if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
			    (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
					if (opline->extended_value != ZEND_ADD ||
					    (var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
					    (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
						may_throw = 1;
					}
				}
				if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info,
						1 /* may overflow */, 0)) {
					return 0;
				}
				break;
			case ZEND_BW_OR:
			case ZEND_BW_AND:
			case ZEND_BW_XOR:
				may_throw = 1;
				if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
				    (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
					if ((var_info & MAY_BE_ANY) != MAY_BE_STRING ||
					    (val_info & MAY_BE_ANY) != MAY_BE_STRING) {
						may_throw = 1;
					}
				}
				goto long_math;
			case ZEND_SL:
			case ZEND_SR:
				if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
				    (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
					may_throw = 1;
				}
				if ((opline+1)->op1_type != IS_CONST ||
				    Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
				    Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) {
					may_throw = 1;
				}
				goto long_math;
			case ZEND_MOD:
				if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
				    (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
					if (opline->extended_value != ZEND_ADD ||
					    (var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
					    (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
						may_throw = 1;
					}
				}
				if ((opline+1)->op1_type != IS_CONST ||
				    Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
				    Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) {
					may_throw = 1;
				}
long_math:
				if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value,
						IS_CV, opline->op1, var_addr, var_info, NULL,
						(opline+1)->op1_type, (opline+1)->op1, val_addr, val_info,
						val_range,
						0, var_addr, var_def_info, var_info, 0)) {
					return 0;
				}
				break;
			case ZEND_CONCAT:
				may_throw = 1;
				if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr,
						0)) {
					return 0;
				}
				break;
			default:
				ZEND_UNREACHABLE();
		}
	}

	if (needs_slow_path) {
		may_throw = 1;
		|.cold_code
		|7:
		|	SET_EX_OPLINE opline, REG0
		|	// value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
		|	LOAD_ADDR FCARG2x, name
		|	LOAD_ZVAL_ADDR CARG3, val_addr
		|	ldr CARG4, EX->run_time_cache
		|	ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, (opline+1)->extended_value, TMP1
		|	LOAD_ADDR CARG5, binary_op
		|	EXT_CALL zend_jit_assign_obj_op_helper, REG0

		if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
			val_info |= MAY_BE_RC1|MAY_BE_RCN;
		}

		|8:
		|	// FREE_OP_DATA();
		|	FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2
		|	b >9
		|.code
	}

	|9:
	if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
		if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
			may_throw = 1;
		}
		|	FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
	}

	if (may_throw) {
		if (!zend_jit_check_exception(Dst)) {
			return 0;
		}
	}

	return 1;
}

static int zend_jit_assign_obj(dasm_State          **Dst,
                               const zend_op        *opline,
                               const zend_op_array  *op_array,
                               zend_ssa             *ssa,
                               const zend_ssa_op    *ssa_op,
                               uint32_t              op1_info,
                               zend_jit_addr         op1_addr,
                               uint32_t              val_info,
                               bool                  op1_indirect,
                               zend_class_entry     *ce,
                               bool                  ce_is_instanceof,
                               bool                  on_this,
                               bool                  delayed_fetch_this,
                               zend_class_entry     *trace_ce,
                               uint8_t               prop_type,
                               int                   may_throw)
{
	zval *member;
	zend_string *name;
	zend_property_info *prop_info;
	zend_jit_addr val_addr = OP1_DATA_ADDR();
	zend_jit_addr res_addr = 0;
	zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
	zend_jit_addr prop_addr;
	bool needs_slow_path = 0;
	bool needs_val_dtor = 0;

	if (RETURN_VALUE_USED(opline)) {
		res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
	}

	ZEND_ASSERT(opline->op2_type == IS_CONST);
	ZEND_ASSERT(op1_info & MAY_BE_OBJECT);

	member = RT_CONSTANT(opline, opline->op2);
	ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
	name = Z_STR_P(member);
	prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);

	if (on_this) {
		|	GET_ZVAL_PTR FCARG1x, this_addr, TMP1
	} else {
		if (opline->op1_type == IS_VAR
		 && (op1_info & MAY_BE_INDIRECT)
		 && Z_REG(op1_addr) == ZREG_FP) {
			|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
			|	IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w
			|	GET_Z_PTR FCARG1x, FCARG1x
			|1:
			op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
		}
		if (op1_info & MAY_BE_REF) {
			if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
				|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
			}
			|	ZVAL_DEREF FCARG1x, op1_info, TMP1w
			op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
		}
		if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
				int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

				if (!exit_addr) {
					return 0;
				}
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1
			} else {
				|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1
				|.cold_code
				|1:
				|	SET_EX_OPLINE opline, REG0
				if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
					|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
				}
				|	LOAD_ADDR FCARG2x, ZSTR_VAL(name)
				|	EXT_CALL zend_jit_invalid_property_assign, REG0
				if (RETURN_VALUE_USED(opline)) {
					|	SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
				}
				if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
				 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
					needs_val_dtor = 1;
					|	b >7
				} else {
					|	b >9
				}
				|.code
			}
		}
		|	GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
	}

	if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
		prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
		if (prop_info) {
			ce = trace_ce;
			ce_is_instanceof = 0;
			if (!(op1_info & MAY_BE_CLASS_GUARD)) {
				if (on_this && JIT_G(current_frame)
				 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
					ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
				} else if (zend_jit_class_guard(Dst, opline, ce)) {
					if (on_this && JIT_G(current_frame)) {
						JIT_G(current_frame)->ce = ce;
						TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
					}
				} else {
					return 0;
				}
				if (ssa->var_info && ssa_op->op1_use >= 0) {
					ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
					ssa->var_info[ssa_op->op1_use].ce = ce;
					ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
				}
				if (ssa->var_info && ssa_op->op1_def >= 0) {
					ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
					ssa->var_info[ssa_op->op1_def].ce = ce;
					ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
				}
			}
		}
	}

	if (!prop_info) {
		needs_slow_path = 1;

		|	ldr REG0, EX->run_time_cache
		|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1
		|	ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)]
		|	cmp REG2, TMP1
		|	bne >5
		if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
			|	MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1
		}
		|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1
		|	tst REG0, REG0
		|	blt >5
		|	add TMP2, FCARG1x, REG0
		|	ldrb TMP1w, [TMP2, #offsetof(zval,u1.v.type)]
		|	IF_TYPE TMP1w, IS_UNDEF, >5
		|	mov FCARG1x, TMP2
		prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
		if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
			|	cbnz FCARG2x, >1
			|.cold_code
			|1:
			|	// value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
			|	SET_EX_OPLINE opline, REG0
			|	LOAD_ZVAL_ADDR CARG3, val_addr
			if (RETURN_VALUE_USED(opline)) {
				|	LOAD_ZVAL_ADDR CARG4, res_addr
			} else {
				|	mov CARG4, xzr
			}

			|	EXT_CALL zend_jit_assign_to_typed_prop, REG0

			if ((opline+1)->op1_type == IS_CONST) {
				|	// TODO: ???
				|	// if (Z_TYPE_P(value) == orig_type) {
				|	// CACHE_PTR_EX(cache_slot + 2, NULL);
			}

			if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
			 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
				|	b >7
			} else {
				|	b >9
			}
			|.code
		}
	} else {
		prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset);
		if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) {
			// Undefined property with magic __get()/__set()
			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
				int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

				if (!exit_addr) {
					return 0;
				}
				|	MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1
				|	IF_TYPE TMP2w, IS_UNDEF, &exit_addr
			} else {
				|	MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1
				|	IF_TYPE TMP2w, IS_UNDEF, >5
				needs_slow_path = 1;
			}
		}
		if (ZEND_TYPE_IS_SET(prop_info->type)) {
			uint32_t info = val_info;

			|	// value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
			|	SET_EX_OPLINE opline, REG0
			if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
				|	LOAD_ADDR FCARG2x, prop_info
			} else {
				int prop_info_offset =
					(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));

				|	ldr REG0, [FCARG1x, #offsetof(zend_object, ce)]
				|	ldr	REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)]
				|	MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1
			}
			|	LOAD_ZVAL_ADDR FCARG1x, prop_addr
			|	LOAD_ZVAL_ADDR CARG3, val_addr
			if (RETURN_VALUE_USED(opline)) {
				|	LOAD_ZVAL_ADDR CARG4, res_addr
			} else {
				|	mov CARG4, xzr
			}

			|	EXT_CALL zend_jit_assign_to_typed_prop, REG0

			if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
				info |= MAY_BE_RC1|MAY_BE_RCN;
			}

			|	FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2
		}
	}

	if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
		// value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES());
		if (opline->result_type == IS_UNUSED) {
			if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
				return 0;
			}
		} else {
			if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
				return 0;
			}
		}
	}

	if (needs_slow_path) {
		|.cold_code
		|5:
		|	SET_EX_OPLINE opline, REG0
		|	// value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
		|	LOAD_ADDR FCARG2x, name

		|	LOAD_ZVAL_ADDR CARG3, val_addr
		|	ldr CARG4, EX->run_time_cache
		|	ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, opline->extended_value, TMP1
		if (RETURN_VALUE_USED(opline)) {
			|	LOAD_ZVAL_ADDR CARG5, res_addr
		} else {
			|	mov CARG5, xzr
		}

		|	EXT_CALL zend_jit_assign_obj_helper, REG0

		if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
			val_info |= MAY_BE_RC1|MAY_BE_RCN;
		}

		|7:
		|	// FREE_OP_DATA();
		|	FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2
		|	b >9
		|.code
	} else if (needs_val_dtor) {
		|.cold_code
		|7:
		|	// FREE_OP_DATA();
		|	FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2
		|	b >9
		|.code
	}

	|9:
	if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
		|	FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
	}

	if (may_throw) {
		if (!zend_jit_check_exception(Dst)) {
			return 0;
		}
	}

	return 1;
}

static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw)
{
	zend_jit_addr op1_addr = OP1_ADDR();

	if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
		if (may_throw) {
			|	SET_EX_OPLINE opline, REG0
		}
		if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) {
			if (op1_info & MAY_BE_ARRAY) {
				|	IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1
			}
			|	MEM_ACCESS_32_WITH_UOFFSET ldr, FCARG1w, FP, (opline->op1.var + offsetof(zval, u2.fe_iter_idx)), TMP1
			|	mvn TMP1w, wzr // TODO: DynAsm fails loading #-1
			|	cmp FCARG1w, TMP1w
			|	beq >7
			|	EXT_CALL zend_hash_iterator_del, REG0
			|7:
		}
		|	ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2
		if (may_throw) {
			if (!zend_jit_check_exception(Dst)) {
				return 0;
			}
		}
	}
	return 1;
}

static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info)
{
	if (opline->op1_type == IS_CONST) {
		zval *zv;
		size_t len;

		zv = RT_CONSTANT(opline, opline->op1);
		ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
		len = Z_STRLEN_P(zv);

		if (len > 0) {
			const char *str = Z_STRVAL_P(zv);

			|	SET_EX_OPLINE opline, REG0
			|	LOAD_ADDR CARG1, str
			|	LOAD_64BIT_VAL CARG2, len
			|	EXT_CALL zend_write, REG0
			if (!zend_jit_check_exception(Dst)) {
				return 0;
			}
		}
	} else {
		zend_jit_addr op1_addr = OP1_ADDR();

		ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);

		|	SET_EX_OPLINE opline, REG0
		|	GET_ZVAL_PTR REG0, op1_addr, TMP1
		|	add CARG1, REG0, #offsetof(zend_string, val)
		|	ldr CARG2, [REG0, #offsetof(zend_string, len)]
		|	EXT_CALL zend_write, REG0
		if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) {
			|	ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2
		}
		if (!zend_jit_check_exception(Dst)) {
			return 0;
		}
	}
	return 1;
}

static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr)
{
	if (opline->op1_type == IS_CONST) {
		zval *zv;
		size_t len;

		zv = RT_CONSTANT(opline, opline->op1);
		ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
		len = Z_STRLEN_P(zv);

		|	SET_ZVAL_LVAL res_addr, len, TMP1, TMP2
		if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
			|	SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
		} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) {
			return 0;
		}
	} else {
		ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);

		if (Z_MODE(res_addr) == IS_REG) {
			|	GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1
			|	ldr Rx(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(zend_string, len)]
			if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) {
				return 0;
			}
		} else {
			|	GET_ZVAL_PTR REG0, op1_addr, TMP1
			|	ldr REG0, [REG0, #offsetof(zend_string, len)]
			|	SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1
			|	SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
		}
		|	FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2
	}
	return 1;
}

static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw)
{
	if (opline->op1_type == IS_CONST) {
		zval *zv;
		zend_long count;

		zv = RT_CONSTANT(opline, opline->op1);
		ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY);
		count = zend_hash_num_elements(Z_ARRVAL_P(zv));

		|	SET_ZVAL_LVAL res_addr, count, TMP1, TMP2
		if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
			|	SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
		} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) {
			return 0;
		}
	} else {
		ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY);
		// Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+.

		if (Z_MODE(res_addr) == IS_REG) {
			|	GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1
			// Sign-extend the 32-bit value to a potentially 64-bit zend_long
			|	ldr Rw(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(HashTable, nNumOfElements)]
			if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) {
				return 0;
			}
		} else {
			|	GET_ZVAL_PTR REG0, op1_addr, TMP1
			// Sign-extend the 32-bit value to a potentially 64-bit zend_long
			|	ldr REG0w, [REG0, #offsetof(HashTable, nNumOfElements)]
			|	SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1
			|	SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
		}
		|	FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2
	}

	if (may_throw) {
		return zend_jit_check_exception(Dst);
	}
	return 1;
}

static int zend_jit_load_this(dasm_State **Dst, uint32_t var)
{
	zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);

	|	ldr FCARG1x, EX->This.value.ptr
	|	SET_ZVAL_PTR var_addr, FCARG1x, TMP1
	|	SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX, TMP1w, TMP2
	|	GC_ADDREF FCARG1x, TMP1w
	return 1;
}

static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only)
{
	if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) {
		if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
			if (!JIT_G(current_frame) ||
			    !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) {

				int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

				if (!exit_addr) {
					return 0;
				}

				|	ldrb TMP1w, EX->This.u1.v.type
				|	cmp TMP1w, #IS_OBJECT
				|	bne &exit_addr

				if (JIT_G(current_frame)) {
					TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame));
				}
			}
		} else {

			|	ldrb TMP1w, EX->This.u1.v.type
			|	cmp TMP1w, #IS_OBJECT
			|	bne >1
			|.cold_code
			|1:
			|	SET_EX_OPLINE opline, REG0
			|	b ->invalid_this
			|.code
		}
	}

	if (!check_only) {
		if (!zend_jit_load_this(Dst, opline->result.var)) {
			return 0;
		}
	}
	return 1;
}

static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info)
{
	uint32_t count;
	Bucket *p;
	const zend_op *target;
	int b;
	int32_t exit_point;
	const void *exit_addr;

	if (default_label) {
		|	cbz REG0, &default_label
	} else if (next_opline) {
		|	cbz REG0, >3
	} else {
		|	cbz REG0, =>default_b
	}
	|	LOAD_ADDR FCARG1x, jumptable
	|	ldr TMP1, [FCARG1x, #offsetof(HashTable, arData)]
	|	sub REG0, REG0, TMP1
	|	mov FCARG1x, #(sizeof(Bucket) / sizeof(void*))
	|	sdiv REG0, REG0, FCARG1x
	|	adr FCARG1x, >4
	|	ldr TMP1, [FCARG1x, REG0]
	|	br TMP1

	|.jmp_table
	|.align 8
	|4:
	if (trace_info) {
		trace_info->jmp_table_size += zend_hash_num_elements(jumptable);
	}

	count = jumptable->nNumUsed;
	p = jumptable->arData;
	do {
		if (Z_TYPE(p->val) == IS_UNDEF) {
			if (default_label) {
				|	.addr &default_label
			} else if (next_opline) {
				|	.addr >3
			} else {
				|	.addr =>default_b
			}
		} else {
			target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val));
			if (!next_opline) {
				b = ssa->cfg.map[target - op_array->opcodes];
				|	.addr =>b
			} else if (next_opline == target) {
				|	.addr >3
			} else {
				exit_point = zend_jit_trace_get_exit_point(target, 0);
				exit_addr = zend_jit_trace_get_exit_addr(exit_point);
				if (!exit_addr) {
					return 0;
				}
				|	.addr &exit_addr
			}
		}
		p++;
		count--;
	} while (count);
	|.code

	return 1;
}

static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info)
{
	HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
	const zend_op *next_opline = NULL;

	if (trace) {
		ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
		ZEND_ASSERT(trace->opline != NULL);
		next_opline = trace->opline;
	}

	if (opline->op1_type == IS_CONST) {
		zval *zv = RT_CONSTANT(opline, opline->op1);
		zval *jump_zv = NULL;
		int b;

		if (opline->opcode == ZEND_SWITCH_LONG) {
			if (Z_TYPE_P(zv) == IS_LONG) {
				jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
			}
		} else if (opline->opcode == ZEND_SWITCH_STRING) {
			if (Z_TYPE_P(zv) == IS_STRING) {
				jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
			}
		} else if (opline->opcode == ZEND_MATCH) {
			if (Z_TYPE_P(zv) == IS_LONG) {
				jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
			} else if (Z_TYPE_P(zv) == IS_STRING) {
				jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
			}
		} else {
			ZEND_UNREACHABLE();
		}
		if (next_opline) {
			const zend_op *target;

			if (jump_zv != NULL) {
				target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv));
			} else {
				target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
			}
			ZEND_ASSERT(target == next_opline);
		} else {
			if (jump_zv != NULL) {
				b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
			} else {
				b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
			}
			|	b =>b
		}
	} else {
		zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
		uint32_t op1_info = OP1_INFO();
		zend_jit_addr op1_addr = OP1_ADDR();
		const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
		const zend_op *target;
		int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes];
		int b;
		int32_t exit_point;
		const void *fallback_label = NULL;
		const void *default_label = NULL;
		const void *exit_addr;

		if (next_opline) {
			if (next_opline != opline + 1) {
				exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
				fallback_label = zend_jit_trace_get_exit_addr(exit_point);
				if (!fallback_label) {
					return 0;
				}
			}
			if (next_opline != default_opline) {
				exit_point = zend_jit_trace_get_exit_point(default_opline, 0);
				default_label = zend_jit_trace_get_exit_addr(exit_point);
				if (!default_label) {
					return 0;
				}
			}
		}

		if (opline->opcode == ZEND_SWITCH_LONG) {
			if (op1_info & MAY_BE_LONG) {
				if (op1_info & MAY_BE_REF) {
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1, ZREG_TMP1
					|	GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1
					|.cold_code
					|1:
					|	// ZVAL_DEREF(op)
					if (fallback_label) {
						|	IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1
					} else {
						|	IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1
					}
					|	GET_ZVAL_PTR FCARG2x, op1_addr, TMP1
					if (fallback_label) {
						|	add TMP1, FCARG2x, #offsetof(zend_reference, val)
						|	IF_NOT_Z_TYPE TMP1, IS_LONG, &fallback_label, TMP2w
					} else {
						|	add TMP1, FCARG2x, #offsetof(zend_reference, val)
						|	IF_NOT_Z_TYPE TMP1, IS_LONG, >3, TMP2w
					}
					|	ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.lval)]
					|	b >2
					|.code
					|2:
				} else {
					if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
						if (fallback_label) {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label, ZREG_TMP1
						} else {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1
						}
					}
					|	GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1
				}
				if (HT_IS_PACKED(jumptable)) {
					uint32_t count = jumptable->nNumUsed;
					Bucket *p = jumptable->arData;

					|	CMP_64_WITH_CONST_32 FCARG2x, jumptable->nNumUsed, TMP1
					if (default_label) {
						|	bhs &default_label
					} else if (next_opline) {
						|	bhs >3
					} else {
						|	bhs =>default_b
					}
					|	adr REG0, >4
					|	ldr TMP1, [REG0, FCARG2x, lsl #3]
					|	br TMP1

					|.jmp_table
					|.align 8
					|4:
					if (trace_info) {
						trace_info->jmp_table_size += count;
					}
					p = jumptable->arData;
					do {
						if (Z_TYPE(p->val) == IS_UNDEF) {
							if (default_label) {
								|	.addr &default_label
							} else if (next_opline) {
								|	.addr >3
							} else {
								|	.addr =>default_b
							}
						} else {
							target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val));
							if (!next_opline) {
								b = ssa->cfg.map[target - op_array->opcodes];
								|	.addr =>b
							} else if (next_opline == target) {
								|	.addr >3
							} else {
								exit_point = zend_jit_trace_get_exit_point(target, 0);
								exit_addr = zend_jit_trace_get_exit_addr(exit_point);
								if (!exit_addr) {
									return 0;
								}
								|	.addr &exit_addr
							}
						}
						p++;
						count--;
					} while (count);
					|.code
					|3:
				} else {
					|	LOAD_ADDR FCARG1x, jumptable
					|	EXT_CALL zend_hash_index_find, REG0
					|	mov REG0, RETVALx
					if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) {
						return 0;
					}
					|3:
				}
			}
		} else if (opline->opcode == ZEND_SWITCH_STRING) {
			if (op1_info & MAY_BE_STRING) {
				if (op1_info & MAY_BE_REF) {
					|	IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1, ZREG_TMP1
					|	GET_ZVAL_PTR FCARG2x, op1_addr, TMP1
					|.cold_code
					|1:
					|	// ZVAL_DEREF(op)
					if (fallback_label) {
						|	IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1
					} else {
						|	IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1
					}
					|	GET_ZVAL_PTR FCARG2x, op1_addr, TMP1
					if (fallback_label) {
						|	add TMP1, FCARG2x, #offsetof(zend_reference, val)
						|	IF_NOT_Z_TYPE TMP1, IS_STRING, &fallback_label, TMP2w
					} else {
						|	add TMP1, FCARG2x, #offsetof(zend_reference, val)
						|	IF_NOT_Z_TYPE TMP1, IS_STRING, >3, TMP2w
					}
					|	ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.ptr)]
					|	b >2
					|.code
					|2:
				} else {
					if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) {
						if (fallback_label) {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label, ZREG_TMP1
						} else {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1
						}
					}
					|	GET_ZVAL_PTR FCARG2x, op1_addr, TMP1
				}
				|	LOAD_ADDR FCARG1x, jumptable
				|	EXT_CALL zend_hash_find, REG0
				|	mov REG0, RETVALx
				if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) {
					return 0;
				}
				|3:
			}
		} else if (opline->opcode == ZEND_MATCH) {
			if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) {
				if (op1_info & MAY_BE_REF) {
					|	LOAD_ZVAL_ADDR FCARG2x, op1_addr
					|	ZVAL_DEREF FCARG2x, op1_info, TMP1w
					op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
				}
				|	LOAD_ADDR FCARG1x, jumptable
				if (op1_info & MAY_BE_LONG) {
					if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
						if (op1_info & MAY_BE_STRING) {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5, ZREG_TMP1
						} else if (op1_info & MAY_BE_UNDEF) {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1
						} else if (default_label) {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label, ZREG_TMP1
						} else if (next_opline) {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1
						} else {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b, ZREG_TMP1
						}
					}
					|	GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1
					|	EXT_CALL zend_hash_index_find, REG0
					|	mov REG0, RETVALx
					if (op1_info & MAY_BE_STRING) {
						|	b >2
					}
				}
				if (op1_info & MAY_BE_STRING) {
					|5:
					if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) {
						if (op1_info & MAY_BE_UNDEF) {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1
						} else if (default_label) {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label, ZREG_TMP1
						} else if (next_opline) {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1
						} else {
							|	IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b, ZREG_TMP1
						}
					}
					|	GET_ZVAL_PTR FCARG2x, op1_addr, TMP1
					|	EXT_CALL zend_hash_find, REG0
					|	mov REG0, RETVALx
				}
				|2:
				if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) {
					return 0;
				}
			}
			if (op1_info & MAY_BE_UNDEF) {
				|6:
				if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) {
					if (default_label) {
						|	IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label, ZREG_TMP1
					} else if (next_opline) {
						|	IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3, ZREG_TMP1
					} else {
						|	IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b, ZREG_TMP1
					}
				}
				|	// zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
				|	SET_EX_OPLINE opline, REG0
				|	LOAD_32BIT_VAL FCARG1w, opline->op1.var
				|	EXT_CALL zend_jit_undefined_op_helper, REG0
				if (!zend_jit_check_exception_undef_result(Dst, opline)) {
					return 0;
				}
			}
			if (default_label) {
				|	b &default_label
			} else if (next_opline) {
				|	b >3
			} else {
				|	b =>default_b
			}
			|3:
		} else {
			ZEND_UNREACHABLE();
		}
	}
	return 1;
}

static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info)
{
	zend_arg_info *arg_info = &op_array->arg_info[-1];
	ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type));
	zend_jit_addr op1_addr = OP1_ADDR();
	bool needs_slow_check = 1;
	bool slow_check_in_cold = 1;
	uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;

	if (type_mask == 0) {
		slow_check_in_cold = 0;
	} else {
		if (((op1_info & MAY_BE_ANY) & type_mask) == 0) {
			slow_check_in_cold = 0;
		} else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) {
			needs_slow_check = 0;
		} else if (is_power_of_two(type_mask)) {
			uint32_t type_code = concrete_type(type_mask);
			|	IF_NOT_ZVAL_TYPE op1_addr, type_code, >6, ZREG_TMP1
		} else {
			|	mov REG2w, #1
			|	GET_ZVAL_TYPE REG1w, op1_addr, TMP1
			|	lsl REG2w, REG2w, REG1w
			|	TST_32_WITH_CONST REG2w, type_mask, TMP1w
			|	beq >6
		}
	}
	if (needs_slow_check) {
		if (slow_check_in_cold) {
			|.cold_code
			|6:
		}
		|	SET_EX_OPLINE opline, REG1
		if (op1_info & MAY_BE_UNDEF) {
			|	IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7, ZREG_TMP1
			|	LOAD_32BIT_VAL FCARG1x, opline->op1.var
			|	EXT_CALL zend_jit_undefined_op_helper, REG0
			|	cbz RETVALx, ->exception_handler
			|	LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval
			|	b >8
		}
		|7:
		|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
		|8:
		|	ldr FCARG2x, EX->func
		|	LOAD_ADDR CARG3, (ptrdiff_t)arg_info
		|	ldr REG0, EX->run_time_cache
		|	ADD_SUB_64_WITH_CONST_32 add, CARG4, REG0, opline->op2.num, TMP1
		|	EXT_CALL zend_jit_verify_return_slow, REG0
		if (!zend_jit_check_exception(Dst)) {
			return 0;
		}
		if (slow_check_in_cold) {
			|	b >9
			|.code
		}
	}
	|9:
	return 1;
}

static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr,  zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
	zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);

	// TODO: support for empty() ???
	ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));

	if (op1_info & MAY_BE_REF) {
		if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
			|	LOAD_ZVAL_ADDR FCARG1x, op1_addr
			op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
		}
		|	ZVAL_DEREF FCARG1x, op1_info, TMP1w
		|1:
	}

	if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
		if (exit_addr) {
			ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ);
		} else if (smart_branch_opcode) {
			if (smart_branch_opcode == ZEND_JMPNZ) {
				|	b =>target_label
			} else if (smart_branch_opcode == ZEND_JMPZNZ) {
				|	b =>target_label2
			}
		} else {
			|	SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
		}
	} else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) {
		if (exit_addr) {
			ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ);
		} else if (smart_branch_opcode) {
			if (smart_branch_opcode != ZEND_JMPNZ) {
				|	b =>target_label
			}
		} else {
			|	SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
		}
	} else {
		ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL);
		|	MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, Rx(Z_REG(op1_addr)), (Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)), TMP1
		|	cmp TMP1w, #IS_NULL
		if (exit_addr) {
			if (smart_branch_opcode == ZEND_JMPNZ) {
				|	bgt &exit_addr
			} else {
				|	ble &exit_addr
			}
		} else if (smart_branch_opcode) {
			if (smart_branch_opcode == ZEND_JMPZ) {
				|	ble =>target_label
			} else if (smart_branch_opcode == ZEND_JMPNZ) {
				|	bgt =>target_label
			} else if (smart_branch_opcode == ZEND_JMPZNZ) {
				|	ble =>target_label
				|	b =>target_label2
			} else {
				ZEND_UNREACHABLE();
			}
		} else {
			|	cset REG0w, gt
			|	add REG0w, REG0w, #IS_FALSE
			|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
		}
	}

	return 1;
}

static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info)
{
	zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);

	if (opline->op1_type == IS_CONST) {
		zval *zv = RT_CONSTANT(opline, opline->op1);

		|	ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0
		if (Z_REFCOUNTED_P(zv)) {
			|	ADDREF_CONST zv, REG0, TMP1
		}
	} else {
		zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);

		|	// ZVAL_COPY(res, value);
		|	ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
		if (opline->op1_type == IS_CV) {
			|	TRY_ADDREF op1_info, REG0w, FCARG1x, TMP1w
		}
	}
	|	// Z_FE_POS_P(res) = 0;
	|	MEM_ACCESS_32_WITH_UOFFSET str, wzr, FP, (opline->result.var + offsetof(zval, u2.fe_pos)), TMP1

	return 1;
}

static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr)
{
	zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);

	|	// array = EX_VAR(opline->op1.var);
	|	// fe_ht = Z_ARRVAL_P(array);
	|	GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
	|	// pos = Z_FE_POS_P(array);
	|	MEM_ACCESS_32_WITH_UOFFSET ldr, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1
	|	// p = fe_ht->arData + pos;
	||	ZEND_ASSERT(sizeof(Bucket) == 32);
	|	mov FCARG2w, REG0w
	|	ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)]
	|	add FCARG2x, TMP1, FCARG2x, lsl #5
	|1:
	|	// if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
	|	ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)]
	|	cmp TMP1w, REG0w
	|	// ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
	|	// ZEND_VM_CONTINUE();
	if (exit_addr) {
		if (exit_opcode == ZEND_JMP) {
			|	bls &exit_addr
		} else {
			|	bls >3
		}
	} else {
		|	bls =>target_label
	}
	|	// pos++;
	|	add REG0w, REG0w, #1
	|	// value_type = Z_TYPE_INFO_P(value);
	|	// if (EXPECTED(value_type != IS_UNDEF)) {
	if (!exit_addr || exit_opcode == ZEND_JMP) {
		|	IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w
	} else {
		|	IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr, TMP1w
	}
	|	// p++;
	|	add FCARG2x, FCARG2x, #sizeof(Bucket)
	|	b <1
	|3:

	if (!exit_addr || exit_opcode == ZEND_JMP) {
		zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
		zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
		uint32_t val_info;

		|	// Z_FE_POS_P(array) = pos + 1;
		|	MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1

		if (RETURN_VALUE_USED(opline)) {
			zend_jit_addr res_addr = RES_ADDR();

			if ((op1_info & MAY_BE_ARRAY_KEY_LONG)
			 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) {
				|	// if (!p->key) {
				|	ldr REG0, [FCARG2x, #offsetof(Bucket, key)]
				|	cbz REG0, >2
			}
			if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
				|	// ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key);
				|	ldr REG0, [FCARG2x, #offsetof(Bucket, key)]
				|	SET_ZVAL_PTR res_addr, REG0, TMP1
				|	ldr TMP1w, [REG0, #offsetof(zend_refcounted, gc.u.type_info)]
				|	TST_32_WITH_CONST TMP1w, IS_STR_INTERNED, TMP2w
				|	beq >1
				|	SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2
				|	b >3
				|1:
				|	GC_ADDREF REG0, TMP1w
				|	SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2

				if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
				    |	b >3
					|2:
				}
			}
			if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
				|	// ZVAL_LONG(EX_VAR(opline->result.var), p->h);
				|	ldr REG0, [FCARG2x, #offsetof(Bucket, h)]
				|	SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1
				|	SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
			}
			|3:
		}

		val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT);
		if (val_info & MAY_BE_ARRAY) {
			val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
		}
		if (op1_info & MAY_BE_ARRAY_OF_REF) {
			val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY |
				MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
		} else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
			val_info |= MAY_BE_RC1 | MAY_BE_RCN;
		}

		if (opline->op2_type == IS_CV) {
			|	// zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES());
			if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) {
				return 0;
			}
		} else {
			|	// ZVAL_COPY(res, value);
			|	ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
			|	TRY_ADDREF val_info, REG0w, FCARG1x, TMP1w
		}
	}

	return 1;
}

static int zend_jit_fetch_constant(dasm_State          **Dst,
                                   const zend_op        *opline,
                                   const zend_op_array  *op_array,
                                   zend_ssa             *ssa,
                                   const zend_ssa_op    *ssa_op,
                                   zend_jit_addr         res_addr)
{
	zval *zv = RT_CONSTANT(opline, opline->op2) + 1;
	zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
	uint32_t res_info = RES_INFO();

	|	// c = CACHED_PTR(opline->extended_value);
	|	ldr FCARG1x, EX->run_time_cache
	|	MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG1x, opline->extended_value, TMP1
	|	// if (c != NULL)
	|	cbz REG0, >9
	if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) {
		|	// if (!IS_SPECIAL_CACHE_VAL(c))
		||	ZEND_ASSERT(CACHE_SPECIAL == 1);
		|	TST_64_WITH_ONE REG0
		|	bne >9
	}
	|8:

	if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
		zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
		uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
		int32_t exit_point;
		const void *exit_addr = NULL;

		SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
		SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0);
		exit_point = zend_jit_trace_get_exit_point(opline+1, 0);
		SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
		exit_addr = zend_jit_trace_get_exit_addr(exit_point);
		if (!exit_addr) {
			return 0;
		}
		res_info &= ~MAY_BE_GUARD;
		ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;

		uint32_t type = concrete_type(res_info);

		if (type < IS_STRING) {
			|	IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr, ZREG_TMP1
		} else {
			|	GET_ZVAL_TYPE_INFO REG2w, const_addr, TMP1
			|	IF_NOT_TYPE REG2w, type, &exit_addr
		}
		|	ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0
		if (type < IS_STRING) {
			if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
				|	SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2
			} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
				return 0;
			}
		} else {
			|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1
			|	TRY_ADDREF res_info, REG2w, REG1, TMP1w
		}
	} else {
		|	ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
		|	TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1w
	}

	|.cold_code
	|9:
	|	// SAVE_OPLINE();
	|	SET_EX_OPLINE opline, REG0
	|	// zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC);
	|	LOAD_ADDR FCARG1x, zv
	|	LOAD_32BIT_VAL FCARG2w, opline->op1.num
	|	EXT_CALL zend_jit_get_constant, REG0
	|	mov REG0, RETVALx
	|	// ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION();
	|	cbnz REG0, <8
	|	b ->exception_handler
	|.code
	return 1;
}

static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr,  zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
	HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
	zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);

	ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR);
	ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING);

	|	// result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST);
	|	LOAD_ADDR FCARG1x, ht
	if (opline->op1_type != IS_CONST) {
		|	GET_ZVAL_PTR FCARG2x, op1_addr, TMP1
		|	EXT_CALL zend_hash_find, REG0
	} else {
		zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1));
		|	LOAD_ADDR FCARG2x, str
		|	EXT_CALL zend_hash_find_known_hash, REG0
	}
	if (exit_addr) {
		if (smart_branch_opcode == ZEND_JMPZ) {
			|	cbz RETVALx, &exit_addr
		} else {
			|	cbnz RETVALx, &exit_addr
		}
	} else if (smart_branch_opcode) {
		if (smart_branch_opcode == ZEND_JMPZ) {
			|	cbz RETVALx, =>target_label
		} else if (smart_branch_opcode == ZEND_JMPNZ) {
			|	cbnz RETVALx, =>target_label
		} else if (smart_branch_opcode == ZEND_JMPZNZ) {
			|	cbz RETVALx, =>target_label
			|	b =>target_label2
		} else {
			ZEND_UNREACHABLE();
		}
	} else {
		|	tst RETVALx, RETVALx
		|	cset REG0w, ne
		|	add REG0w, REG0w, #IS_FALSE
		|	SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
	}

	return 1;
}

static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info)
{
	uint32_t offset;

	offset = (opline->opcode == ZEND_ROPE_INIT) ?
		opline->result.var :
		opline->op1.var + opline->extended_value * sizeof(zend_string*);

	if (opline->op2_type == IS_CONST) {
		zval *zv = RT_CONSTANT(opline, opline->op2);
		zend_string *str;

		ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
		str = Z_STR_P(zv);
		|	LOAD_ADDR REG0, str
		|	MEM_ACCESS_64_WITH_UOFFSET str, REG0, FP, offset, TMP1
	} else {
		zend_jit_addr op2_addr = OP2_ADDR();

		ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);

		|	GET_ZVAL_PTR REG1, op2_addr, TMP1
		|	MEM_ACCESS_64_WITH_UOFFSET str, REG1, FP, offset, TMP1
		if (opline->op2_type == IS_CV) {
			|	GET_ZVAL_TYPE_INFO REG0w, op2_addr, TMP1
			|	TRY_ADDREF op2_info, REG0w, REG1, TMP1w
		}
	}

	if (opline->opcode == ZEND_ROPE_END) {
		zend_jit_addr res_addr = RES_ADDR();

		|	ADD_SUB_64_WITH_CONST add, FCARG1x, FP, opline->op1.var, TMP1
		|	LOAD_32BIT_VAL FCARG2w, opline->extended_value
		|	EXT_CALL zend_jit_rope_end, TMP1
		|	SET_ZVAL_PTR res_addr, RETVALx, TMP1
		|	SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2
	}

	return 1;
}

static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr)
{
	int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
	const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

	if (!exit_addr) {
		return 0;
	}
	|	IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1

	return 1;
}

static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard)
{
	zend_jit_addr var_addr = *var_addr_ptr;
	uint32_t var_info = *var_info_ptr;
	const void *exit_addr = NULL;

	if (add_ref_guard || add_type_guard) {
		int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);

		exit_addr = zend_jit_trace_get_exit_addr(exit_point);
		if (!exit_addr) {
			return 0;
		}
	}

	if (add_ref_guard) {
		|	IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1
	}
	if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) {
		/* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */
		if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {
			|	LOAD_ZVAL_ADDR FCARG1x, var_addr
		}
		|	EXT_CALL zend_jit_unref_helper, REG0
	} else {
		|	GET_ZVAL_PTR FCARG1x, var_addr, TMP1
		var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val));
		*var_addr_ptr = var_addr;
	}

	if (var_type != IS_UNKNOWN) {
		var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED);
	}
	if (add_type_guard
	 && var_type != IS_UNKNOWN
	 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
		|	IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr, ZREG_TMP1

		ZEND_ASSERT(var_info & (1 << var_type));
		if (var_type < IS_STRING) {
			var_info = (1 << var_type);
		} else if (var_type != IS_ARRAY) {
			var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
		} else {
			var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
		}

		*var_info_ptr = var_info;
	} else {
		var_info &= ~MAY_BE_REF;
		*var_info_ptr = var_info;
	}
	*var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */

	return 1;
}

static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard)
{
	zend_jit_addr var_addr = *var_addr_ptr;
	uint32_t var_info = *var_info_ptr;
	int32_t exit_point;
	const void *exit_addr;

	if (add_indirect_guard) {
		int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
		const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

		if (!exit_addr) {
			return 0;
		}
		|	IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr, ZREG_TMP1
		|	GET_ZVAL_PTR FCARG1x, var_addr, TMP1
	} else {
		/* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */
		if (opline->op1_type != IS_VAR ||
				(opline-1)->result_type != IS_VAR  ||
				(opline-1)->result.var != opline->op1.var ||
				(opline-1)->op2_type == IS_VAR ||
				(opline-1)->op2_type == IS_TMP_VAR) {
			|	GET_ZVAL_PTR FCARG1x, var_addr, TMP1
		} else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) {
			|	mov FCARG1x, REG0
		}
	}
	*var_info_ptr &= ~MAY_BE_INDIRECT;
	var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
	*var_addr_ptr = var_addr;

	if (var_type != IS_UNKNOWN) {
		var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED);
	}
	if (!(var_type & IS_TRACE_REFERENCE)
	 && var_type != IS_UNKNOWN
	 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
		exit_point = zend_jit_trace_get_exit_point(opline, 0);
		exit_addr = zend_jit_trace_get_exit_addr(exit_point);

		if (!exit_addr) {
			return 0;
		}

		|	IF_NOT_Z_TYPE FCARG1x, var_type, &exit_addr, TMP1w

		//var_info = zend_jit_trace_type_to_info_ex(var_type, var_info);
		ZEND_ASSERT(var_info & (1 << var_type));
		if (var_type < IS_STRING) {
			var_info = (1 << var_type);
		} else if (var_type != IS_ARRAY) {
			var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
		} else {
			var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
		}

		*var_info_ptr = var_info;
	}

	return 1;
}

static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var)
{
	if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) {
		return 0;
	}

	switch (opline->opcode) {
		case ZEND_QM_ASSIGN:
		case ZEND_SEND_VAR:
		case ZEND_ASSIGN:
		case ZEND_PRE_INC:
		case ZEND_PRE_DEC:
		case ZEND_POST_INC:
		case ZEND_POST_DEC:
			return 1;
		case ZEND_ADD:
		case ZEND_SUB:
		case ZEND_MUL:
		case ZEND_BW_OR:
		case ZEND_BW_AND:
		case ZEND_BW_XOR:
		case ZEND_SL:
		case ZEND_SR:
			if (def_var == ssa_op->result_def &&
			    use_var == ssa_op->op1_use) {
				return 1;
			}
			break;
		default:
			break;
	}
	return 0;
}

static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace)
{
	uint32_t op1_info, op2_info;

	switch (opline->opcode) {
		case ZEND_SEND_VAR:
		case ZEND_SEND_VAL:
		case ZEND_SEND_VAL_EX:
			return (opline->op2_type != IS_CONST);
		case ZEND_QM_ASSIGN:
		case ZEND_IS_SMALLER:
		case ZEND_IS_SMALLER_OR_EQUAL:
		case ZEND_IS_EQUAL:
		case ZEND_IS_NOT_EQUAL:
		case ZEND_IS_IDENTICAL:
		case ZEND_IS_NOT_IDENTICAL:
		case ZEND_CASE:
			return 1;
		case ZEND_RETURN:
			return (op_array->type != ZEND_EVAL_CODE && op_array->function_name);
		case ZEND_ASSIGN:
			op1_info = OP1_INFO();
			op2_info = OP2_INFO();
			return
				opline->op1_type == IS_CV &&
				!(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) &&
				!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)));
		case ZEND_ADD:
		case ZEND_SUB:
		case ZEND_MUL:
			op1_info = OP1_INFO();
			op2_info = OP2_INFO();
			return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE)));
		case ZEND_BW_OR:
		case ZEND_BW_AND:
		case ZEND_BW_XOR:
		case ZEND_SL:
		case ZEND_SR:
		case ZEND_MOD:
			op1_info = OP1_INFO();
			op2_info = OP2_INFO();
			return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG));
		case ZEND_PRE_INC:
		case ZEND_PRE_DEC:
		case ZEND_POST_INC:
		case ZEND_POST_DEC:
			op1_info = OP1_INFO();
			op2_info = OP1_DEF_INFO();
			return opline->op1_type == IS_CV
				&& !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG))
				&& (op2_info & MAY_BE_LONG);
		case ZEND_STRLEN:
			op1_info = OP1_INFO();
			return (opline->op1_type & (IS_CV|IS_CONST))
				&& (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING;
		case ZEND_COUNT:
			op1_info = OP1_INFO();
			return (opline->op1_type & (IS_CV|IS_CONST))
				&& (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY;
		case ZEND_JMPZ:
		case ZEND_JMPNZ:
			if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
				if (!ssa->cfg.map) {
					return 0;
				}
				if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start &&
				    ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
					return 0;
				}
			}
			ZEND_FALLTHROUGH;
		case ZEND_BOOL:
		case ZEND_BOOL_NOT:
		case ZEND_JMPZNZ:
		case ZEND_JMPZ_EX:
		case ZEND_JMPNZ_EX:
			return 1;
		case ZEND_FETCH_CONSTANT:
			return 1;
		case ZEND_FETCH_DIM_R:
			op1_info = OP1_INFO();
			op2_info = OP2_INFO();
			if (trace
			 && trace->op1_type != IS_UNKNOWN
			 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
				op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
			}
			return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
				(!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) &&
					(((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
					 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) &&
						 (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1))));
	}
	return 0;
}

static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var)
{
	if (ssa->vars[var].no_val) {
		/* we don't need the value */
		return 0;
	}

	if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) {
		/* Disable global register allocation,
		 * register allocation for SSA variables connected through Phi functions
		 */
		if (ssa->vars[var].definition_phi) {
			return 0;
		}
		if (ssa->vars[var].phi_use_chain) {
			zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
			do {
				if (!ssa->vars[phi->ssa_var].no_val) {
					return 0;
				}
				phi = zend_ssa_next_use_phi(ssa, var, phi);
			} while (phi);
		}
	}

	if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) &&
	    ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) {
	    /* bad type */
		return 0;
	}

	return 1;
}

static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var)
{
	if (!zend_jit_var_supports_reg(ssa, var)) {
		return 0;
	}

	if (ssa->vars[var].definition >= 0) {
		uint32_t def = ssa->vars[var].definition;
		if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) {
			return 0;
		}
	}

	if (ssa->vars[var].use_chain >= 0) {
		int use = ssa->vars[var].use_chain;

		do {
			if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) &&
			    !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) {
				return 0;
			}
			use = zend_ssa_next_use(ssa->ops, var, use);
		} while (use >= 0);
	}

	return 1;
}

static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use)
{
	uint32_t op1_info, op2_info;

	switch (opline->opcode) {
		case ZEND_FETCH_DIM_R:
			op1_info = OP1_INFO();
			op2_info = OP2_INFO();
			if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) &&
			     (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) ||
			    ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) &&
			     (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) {
				return ZEND_REGSET(ZREG_FCARG1);
			}
			break;
		default:
			break;
	}

	return ZEND_REGSET_EMPTY;
}

static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use)
{
	uint32_t op1_info, op2_info, res_info;
	zend_regset regset = ZEND_REGSET_SCRATCH;

	switch (opline->opcode) {
		case ZEND_NOP:
		case ZEND_OP_DATA:
		case ZEND_JMP:
		case ZEND_RETURN:
			regset = ZEND_REGSET_EMPTY;
			break;
		case ZEND_QM_ASSIGN:
			if (ssa_op->op1_def == current_var ||
			    ssa_op->result_def == current_var) {
				regset = ZEND_REGSET_EMPTY;
				break;
			}
			/* break missing intentionally */
		case ZEND_SEND_VAL:
		case ZEND_SEND_VAL_EX:
			if (opline->op2_type == IS_CONST) {
				break;
			}
			if (ssa_op->op1_use == current_var) {
				regset = ZEND_REGSET(ZREG_REG0);
				break;
			}
			op1_info = OP1_INFO();
			if (!(op1_info & MAY_BE_UNDEF)) {
				if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
					regset = ZEND_REGSET(ZREG_FPR0);
				} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
					regset = ZEND_REGSET(ZREG_REG0);
				} else {
					regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2));
				}
			}
			break;
		case ZEND_SEND_VAR:
			if (opline->op2_type == IS_CONST) {
				break;
			}
			if (ssa_op->op1_use == current_var ||
			    ssa_op->op1_def == current_var) {
				regset = ZEND_REGSET_EMPTY;
				break;
			}
			op1_info = OP1_INFO();
			if (!(op1_info & MAY_BE_UNDEF)) {
				if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
					regset = ZEND_REGSET(ZREG_FPR0);
				} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
				} else {
					regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2));
					if (op1_info & MAY_BE_REF) {
						ZEND_REGSET_INCL(regset, ZREG_REG1);
					}
				}
			}
			break;
		case ZEND_ASSIGN:
			if (ssa_op->op2_use == current_var ||
			    ssa_op->op2_def == current_var ||
			    ssa_op->op1_def == current_var ||
			    ssa_op->result_def == current_var) {
				regset = ZEND_REGSET_EMPTY;
				break;
			}
			op1_info = OP1_INFO();
			op2_info = OP2_INFO();
			if (opline->op1_type == IS_CV
			 && !(op2_info & MAY_BE_UNDEF)
			 && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
				if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
					regset = ZEND_REGSET(ZREG_FPR0);
				} else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
					regset = ZEND_REGSET(ZREG_REG0);
				} else {
					regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2));
				}
			}
			break;
		case ZEND_PRE_INC:
		case ZEND_PRE_DEC:
		case ZEND_POST_INC:
		case ZEND_POST_DEC:
			if (ssa_op->op1_use == current_var ||
			    ssa_op->op1_def == current_var ||
			    ssa_op->result_def == current_var) {
				regset = ZEND_REGSET_EMPTY;
				break;
			}
			op1_info = OP1_INFO();
			if (opline->op1_type == IS_CV
			 && (op1_info & MAY_BE_LONG)
			 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
				regset = ZEND_REGSET_EMPTY;
				if (op1_info & MAY_BE_DOUBLE) {
					regset = ZEND_REGSET(ZREG_FPR0);
				}
				if (opline->result_type != IS_UNUSED && (op1_info & MAY_BE_LONG)) {
					ZEND_REGSET_INCL(regset, ZREG_REG1);
				}
			}
			break;
		case ZEND_ADD:
		case ZEND_SUB:
		case ZEND_MUL:
			op1_info = OP1_INFO();
			op2_info = OP2_INFO();
			if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) &&
			    !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {

				regset = ZEND_REGSET_EMPTY;
				if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
					if (ssa_op->result_def != current_var &&
					    (ssa_op->op1_use != current_var || !last_use)) {
						ZEND_REGSET_INCL(regset, ZREG_REG0);
					}
					res_info = RES_INFO();
					if (res_info & MAY_BE_DOUBLE) {
						ZEND_REGSET_INCL(regset, ZREG_REG0);
						ZEND_REGSET_INCL(regset, ZREG_FPR0);
						ZEND_REGSET_INCL(regset, ZREG_FPR1);
					} else if (res_info & MAY_BE_GUARD) {
						ZEND_REGSET_INCL(regset, ZREG_REG0);
					}
				}
				if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) {
					if (ssa_op->result_def != current_var) {
						ZEND_REGSET_INCL(regset, ZREG_FPR0);
					}
				}
				if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) {
					if (zend_is_commutative(opline->opcode)) {
						if (ssa_op->result_def != current_var) {
							ZEND_REGSET_INCL(regset, ZREG_FPR0);
						}
					} else {
						ZEND_REGSET_INCL(regset, ZREG_FPR0);
						if (ssa_op->result_def != current_var &&
						    (ssa_op->op1_use != current_var || !last_use)) {
							ZEND_REGSET_INCL(regset, ZREG_FPR1);
						}
					}
				}
				if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) {
					if (ssa_op->result_def != current_var &&
					    (ssa_op->op1_use != current_var || !last_use) &&
					    (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) {
						ZEND_REGSET_INCL(regset, ZREG_FPR0);
					}
				}
			}
			break;
		case ZEND_BW_OR:
		case ZEND_BW_AND:
		case ZEND_BW_XOR:
		case ZEND_SL:
		case ZEND_SR:
		case ZEND_MOD:
			op1_info = OP1_INFO();
			op2_info = OP2_INFO();
			if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) &&
			    !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) {
				regset = ZEND_REGSET_EMPTY;
				if (ssa_op->result_def != current_var &&
				    (ssa_op->op1_use != current_var || !last_use)) {
					ZEND_REGSET_INCL(regset, ZREG_REG0);
				}
			}
			break;
		case ZEND_IS_SMALLER:
		case ZEND_IS_SMALLER_OR_EQUAL:
		case ZEND_IS_EQUAL:
		case ZEND_IS_NOT_EQUAL:
		case ZEND_IS_IDENTICAL:
		case ZEND_IS_NOT_IDENTICAL:
		case ZEND_CASE:
			op1_info = OP1_INFO();
			op2_info = OP2_INFO();
			if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) &&
			    !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
				regset = ZEND_REGSET_EMPTY;
				if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) {
					ZEND_REGSET_INCL(regset, ZREG_REG0);
				}
				if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) &&
				    opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) {
					if (ssa_op->op1_use != current_var &&
					    ssa_op->op2_use != current_var) {
						ZEND_REGSET_INCL(regset, ZREG_REG0);
					}
				}
				if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) {
					ZEND_REGSET_INCL(regset, ZREG_FPR0);
				}
				if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) {
					ZEND_REGSET_INCL(regset, ZREG_FPR0);
				}
				if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) {
					if (ssa_op->op1_use != current_var &&
					    ssa_op->op2_use != current_var) {
						ZEND_REGSET_INCL(regset, ZREG_FPR0);
					}
				}
			}
			break;
		case ZEND_BOOL:
		case ZEND_BOOL_NOT:
		case ZEND_JMPZ:
		case ZEND_JMPNZ:
		case ZEND_JMPZNZ:
		case ZEND_JMPZ_EX:
		case ZEND_JMPNZ_EX:
			op1_info = OP1_INFO();
			if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) {
				regset = ZEND_REGSET_EMPTY;
				if (op1_info & MAY_BE_DOUBLE) {
					ZEND_REGSET_INCL(regset, ZREG_FPR0);
				}
				if (opline->opcode == ZEND_BOOL ||
				    opline->opcode == ZEND_BOOL_NOT ||
				    opline->opcode == ZEND_JMPZ_EX ||
				    opline->opcode == ZEND_JMPNZ_EX) {
					ZEND_REGSET_INCL(regset, ZREG_REG0);
				}
			}
			break;
		case ZEND_DO_UCALL:
		case ZEND_DO_FCALL:
		case ZEND_DO_FCALL_BY_NAME:
		case ZEND_INCLUDE_OR_EVAL:
		case ZEND_GENERATOR_CREATE:
		case ZEND_YIELD:
		case ZEND_YIELD_FROM:
			regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP);
			break;
		default:
			break;
	}

	if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
		if (ssa_op == ssa->ops
		 && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL
		 && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
			ZEND_REGSET_INCL(regset, ZREG_REG0);
			ZEND_REGSET_INCL(regset, ZREG_REG1);
		}
	}

	return regset;
}

static size_t dasm_venners_size = 0;
void **dasm_labels_veneers = NULL;

static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset)
{
	void *veneer;
	ptrdiff_t na;
	int n, m;

	/* try to reuse veneers for global labels */
	if ((ins >> 16) == DASM_REL_LG
	 && *(b-1) < 0
	 && dasm_labels_veneers[-*(b-1)]) {

		veneer = dasm_labels_veneers[-*(b-1)];
		na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4;
		n = (int)na;

		/* check if we can jump to veneer */
		if ((ptrdiff_t)n != na) {
			/* pass */
		} else if (!(ins & 0xf800)) {  /* B, BL */
			if ((n & 3) == 0 && ((n+0x08000000) >> 28) == 0) {
				return n;
			}
		} else if ((ins & 0x800)) {  /* B.cond, CBZ, CBNZ, LDR* literal */
			if ((n & 3) == 0 && ((n+0x00100000) >> 21) == 0) {
				return n;
			}
		} else if ((ins & 0x3000) == 0x2000) {  /* ADR */
			/* pass */
		} else if ((ins & 0x3000) == 0x3000) {  /* ADRP */
			/* pass */
		} else if ((ins & 0x1000)) {  /* TBZ, TBNZ */
			if ((n & 3) == 0 && ((n+0x00008000) >> 16) == 0) {
				return n;
			}
		}
	} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
	 && (ins >> 16) == DASM_REL_A) {
		ptrdiff_t addr = (((ptrdiff_t)(*(b-1))) << 32) | (unsigned int)(*(b-2));

		if ((void*)addr >= dasm_buf && (void*)addr < dasm_end) {
			uint32_t exit_point = zend_jit_trace_find_exit_point((void*)addr);
			zend_jit_trace_info *t = zend_jit_get_current_trace_info();

			if (exit_point != (uint32_t)-1) {
				/* Use exit points table */

				ZEND_ASSERT(exit_point < t->exit_count);

				veneer = (char*)buffer + dasm_getpclabel(&Dst, 1) - (t->exit_count - exit_point) * 4;
				na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4;
				n = (int)na;

				/* check if we can jump to veneer */
				if ((ptrdiff_t)n != na) {
					ZEND_ASSERT(0);
					return 0;
				} else if (!(ins & 0xf800)) {  /* B, BL */
					if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) {
						ZEND_ASSERT(0);
						return 0;
					}
				} else if ((ins & 0x800)) {  /* B.cond, CBZ, CBNZ, LDR* literal */
					if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) {
						ZEND_ASSERT(0);
						return 0;
					}
				} else if ((ins & 0x3000) == 0x2000) {  /* ADR */
					ZEND_ASSERT(0);
					return 0;
				} else if ((ins & 0x3000) == 0x3000) {  /* ADRP */
					ZEND_ASSERT(0);
					return 0;
				} else if ((ins & 0x1000)) {  /* TBZ, TBNZ */
					if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) {
						ZEND_ASSERT(0);
						return 0;
					}
				} else {
					ZEND_ASSERT(0);
					return 0;
				}
				return n;
			}
		}
	}

	veneer = (char*)buffer + (Dst->codesize + dasm_venners_size);

	if (veneer > dasm_end) {
		return 0; /* jit_buffer_size overflow */
	}

	na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4;
	n = (int)na;

	/* check if we can jump to veneer */
	if ((ptrdiff_t)n != na) {
		ZEND_ASSERT(0);
		return 0;
	} else if (!(ins & 0xf800)) {  /* B, BL */
		if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) {
			ZEND_ASSERT(0);
			return 0;
		}
	} else if ((ins & 0x800)) {  /* B.cond, CBZ, CBNZ, LDR* literal */
		if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) {
			ZEND_ASSERT(0);
			return 0;
		}
	} else if ((ins & 0x3000) == 0x2000) {  /* ADR */
		ZEND_ASSERT(0);
		return 0;
	} else if ((ins & 0x3000) == 0x3000) {  /* ADRP */
		ZEND_ASSERT(0);
		return 0;
	} else if ((ins & 0x1000)) {  /* TBZ, TBNZ */
		if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) {
			ZEND_ASSERT(0);
			return 0;
		}
	} else if ((ins & 0x8000)) {  /* absolute */
		ZEND_ASSERT(0);
		return 0;
	} else {
		ZEND_ASSERT(0);
		return 0;
	}

	// TODO: support for long veneers (above 128MB) ???

	/* check if we can use B to jump from veneer */
	na = (ptrdiff_t)cp + offset - (ptrdiff_t)veneer - 4;
	m = (int)na;
	if ((ptrdiff_t)m != na) {
		ZEND_ASSERT(0);
		return 0;
	} else if ((m & 3) != 0 || ((m+0x08000000) >> 28) != 0) {
		ZEND_ASSERT(0);
		return 0;
	}

	/* generate B instruction */
	*(uint32_t*)veneer = 0x14000000 | ((m >> 2) & 0x03ffffff);
	dasm_venners_size += 4;

	if ((ins >> 16) == DASM_REL_LG
	 && *(b-1) < 0) {
		/* reuse this veneer for the future jumps to global label */
		dasm_labels_veneers[-*(b-1)] = veneer;
		/* Dst->globals[*(b-1)] = veneer; */

#ifdef HAVE_DISASM
	    if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
			const char *name = zend_jit_disasm_find_symbol((ptrdiff_t)cp + offset - 4, (int64_t *)(&offset));

			if (name && !offset) {
				if (strstr(name, "@veneer") == NULL) {
					char *new_name;

					zend_spprintf(&new_name, 0, "%s@veneer", name);
					zend_jit_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4);
					efree(new_name);
				} else {
					zend_jit_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4);
				}
			}
		}
#endif
	}

	return n;
}

/*
 * Local variables:
 * tab-width: 4
 * c-basic-offset: 4
 * indent-tabs-mode: t
 * End:
 */