diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle index 36af4ba75cc5..d1a862f76675 100644 --- a/cranelift/codegen/src/isa/s390x/inst.isle +++ b/cranelift/codegen/src/isa/s390x/inst.isle @@ -2082,6 +2082,12 @@ (_ Unit (emit (MInst.AluRSImm16 op dst src imm)))) dst)) +(decl alu_rsimm16_with_flags_paired (Type ALUOp Reg i16) ProducesFlags) +(rule (alu_rsimm16_with_flags_paired ty op src imm) + (let ((dst WritableReg (temp_writable_reg ty))) + (ProducesFlags.ProducesFlagsReturnsResultWithConsumer + (MInst.AluRSImm16 op dst src imm) dst))) + ;; Helper for emitting `MInst.AluRSImm32` instructions. (decl alu_rsimm32 (Type ALUOp Reg i32) Reg) (rule (alu_rsimm32 ty op src imm) @@ -2089,6 +2095,12 @@ (_ Unit (emit (MInst.AluRSImm32 op dst src imm)))) dst)) +(decl alu_rsimm32_with_flags_paired (Type ALUOp Reg i32) ProducesFlags) +(rule (alu_rsimm32_with_flags_paired ty op src imm) + (let ((dst WritableReg (temp_writable_reg ty))) + (ProducesFlags.ProducesFlagsReturnsResultWithConsumer + (MInst.AluRSImm32 op dst src imm) dst))) + ;; Helper for emitting `MInst.AluRUImm32` instructions. (decl alu_ruimm32 (Type ALUOp Reg u32) Reg) (rule (alu_ruimm32 ty op src imm) @@ -3371,6 +3383,7 @@ (bool producer (invert_cond cond))) ;; Use a boolean condition to select between two registers. +; important (decl select_bool_reg (Type ProducesBool Reg Reg) Reg) (rule (select_bool_reg ty (ProducesBool.ProducesBool producer cond) reg_true reg_false) (with_flags_reg producer (cmov_reg_reg ty cond reg_true reg_false))) @@ -4060,24 +4073,52 @@ (decl mul_reg (Type Reg Reg) Reg) (rule (mul_reg ty x y) (alu_rrr ty (aluop_mul ty) x y)) +(decl mul_reg_with_flags_paired (Type Reg Reg) ProducesFlags) +(rule (mul_reg_with_flags_paired ty x y) + (alu_rrr_with_flags_paired ty (aluop_mul ty) x y)) + (decl mul_reg_sext32 (Type Reg Reg) Reg) (rule (mul_reg_sext32 ty x y) (alu_rr ty (aluop_mul_sext32 ty) x y)) +(decl mul_reg_sext32_with_flags_paired (Type Reg Reg) ProducesFlags) +(rule (mul_reg_sext32_with_flags_paired ty x y) + (alu_rrr_with_flags_paired ty (aluop_mul_sext32 ty) x y)) + (decl mul_simm16 (Type Reg i16) Reg) (rule (mul_simm16 ty x y) (alu_rsimm16 ty (aluop_mul ty) x y)) +(decl mul_simm16_with_flags_paired (Type Reg i16) ProducesFlags) +(rule (mul_simm16_with_flags_paired ty x y) + (alu_rsimm16_with_flags_paired ty (aluop_mul ty) x y)) + (decl mul_simm32 (Type Reg i32) Reg) (rule (mul_simm32 ty x y) (alu_rsimm32 ty (aluop_mul ty) x y)) +(decl mul_simm32_with_flags_paired (Type Reg i32) ProducesFlags) +(rule (mul_simm32_with_flags_paired ty x y) + (alu_rsimm32_with_flags_paired ty (aluop_mul ty) x y)) + (decl mul_mem (Type Reg MemArg) Reg) (rule (mul_mem ty x y) (alu_rx ty (aluop_mul ty) x y)) +(decl mul_mem_with_flags_paired (Type Reg MemArg) ProducesFlags) +(rule (mul_mem_with_flags_paired ty x y) + (alu_rx_with_flags_paired ty (aluop_mul ty) x y)) + (decl mul_mem_sext16 (Type Reg MemArg) Reg) (rule (mul_mem_sext16 ty x y) (alu_rx ty (aluop_mul_sext16 ty) x y)) +(decl mul_mem_sext16_with_flags_paired (Type Reg MemArg) ProducesFlags) +(rule (mul_mem_sext16_with_flags_paired ty x y) + (alu_rx_with_flags_paired ty (aluop_mul_sext16 ty) x y)) + (decl mul_mem_sext32 (Type Reg MemArg) Reg) (rule (mul_mem_sext32 ty x y) (alu_rx ty (aluop_mul_sext32 ty) x y)) +(decl mul_mem_sext32_with_flags_paired (Type Reg MemArg) ProducesFlags) +(rule (mul_mem_sext32_with_flags_paired ty x y) + (alu_rx_with_flags_paired ty (aluop_mul_sext32 ty) x y)) + (decl vecop_mul (Type) VecBinaryOp) (rule (vecop_mul $I8X16) (VecBinaryOp.Mul8x16)) (rule (vecop_mul $I16X8) (VecBinaryOp.Mul16x8)) diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle index 330cb3910cfc..0d54ff0a78b3 100644 --- a/cranelift/codegen/src/isa/s390x/lower.isle +++ b/cranelift/codegen/src/isa/s390x/lower.isle @@ -409,6 +409,83 @@ (res_hi Reg (add_reg $I64 res_hi_3 (add_reg $I64 res_hi_2 res_hi_1)))) (mov_to_vec128 $I64X2 res_hi res_lo))) +;;;; Rules for `umul_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule 0 (lower (has_type (fits_in_64 ty) (smul_overflow x y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_reg_with_flags_paired ty x y) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 1 (lower (has_type (fits_in_64 ty) (smul_overflow x (sext32_value y)))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_reg_sext32_with_flags_paired ty x y) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 2 (lower (has_type (fits_in_64 ty) (smul_overflow (sext32_value x) y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_reg_sext32_with_flags_paired ty y x) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 3 (lower (has_type (fits_in_64 ty) (smul_overflow x (i16_from_value y)))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_simm16_with_flags_paired ty x y) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 4 (lower (has_type (fits_in_64 ty) (smul_overflow (i16_from_value x) y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_simm16_with_flags_paired ty y x) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 5 (lower (has_type (fits_in_64 ty) (smul_overflow x (i32_from_value y)))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_simm32_with_flags_paired ty x y) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 6 (lower (has_type (fits_in_64 ty) (smul_overflow (i32_from_value x) y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_simm32_with_flags_paired ty y x) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 7 (lower (has_type (fits_in_64 ty) (smul_overflow x (sinkable_load_32_64 y)))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_mem_with_flags_paired ty x (sink_load y)) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 8 (lower (has_type (fits_in_64 ty) (smul_overflow (sinkable_load_32_64 x) y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_mem_with_flags_paired ty y (sink_load x)) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 9 (lower (has_type (fits_in_64 ty) (smul_overflow x (sinkable_load_16 y)))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_mem_sext16_with_flags_paired ty x (sink_load y)) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 10 (lower (has_type (fits_in_64 ty) (smul_overflow (sinkable_load_16 x) y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_mem_sext16_with_flags_paired ty y (sink_load x)) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 11 (lower (has_type (fits_in_64 ty) (smul_overflow x (sinkable_sload16 y)))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_mem_sext16_with_flags_paired ty x (sink_sload16 y)) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 12 (lower (has_type (fits_in_64 ty) (smul_overflow (sinkable_sload16 x) y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_mem_sext16_with_flags_paired ty y (sink_sload16 x)) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 13 (lower (has_type (fits_in_64 ty) (smul_overflow x (sinkable_sload32 y)))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_mem_sext32_with_flags_paired ty x (sink_sload32 y)) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 14 (lower (has_type (fits_in_64 ty) (smul_overflow (sinkable_sload32 x) y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (mul_mem_sext32_with_flags_paired ty y (sink_sload32 x)) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + ;;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -3844,6 +3921,53 @@ (add_logical_mem_zext32_with_flags_paired ty y (sink_uload32 x)) (trap_if_impl (mask_as_cond 3) tc))) +;;;; Rules for `uadd_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule 0 (lower (has_type (fits_in_64 ty) (uadd_overflow x y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (add_logical_reg_with_flags_paired ty x y) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 4 (lower (has_type (fits_in_64 ty) (uadd_overflow x (zext32_value y)))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (add_logical_reg_zext32_with_flags_paired ty x y) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 8 (lower (has_type (fits_in_64 ty) (uadd_overflow (zext32_value x) y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (add_logical_reg_zext32_with_flags_paired ty y x) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 3 (lower (has_type (fits_in_64 ty) (uadd_overflow x (u32_from_value y)))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (add_logical_zimm32_with_flags_paired ty x y) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 7 (lower (has_type (fits_in_64 ty) (uadd_overflow (u32_from_value x) y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (add_logical_zimm32_with_flags_paired ty y x) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 2 (lower (has_type (fits_in_64 ty) (uadd_overflow x (sinkable_load_32_64 y)))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (add_logical_mem_with_flags_paired ty x (sink_load y)) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 6 (lower (has_type (fits_in_64 ty) (uadd_overflow (sinkable_load_32_64 x) y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (add_logical_mem_with_flags_paired ty y (sink_load x)) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 1 (lower (has_type (fits_in_64 ty) (uadd_overflow x (sinkable_uload32 y)))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (add_logical_mem_zext32_with_flags_paired ty x (sink_uload32 y)) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + +(rule 5 (lower (has_type (fits_in_64 ty) (uadd_overflow (sinkable_uload32 x) y))) + (let ((one Reg (imm $I8 1)) + (overflow Reg (select_bool_reg ty (bool (add_logical_mem_zext32_with_flags_paired ty y (sink_uload32 x)) (mask_as_cond 3)) one (zero_reg)))) + (value_reg overflow))) + ;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (return args))