Skip to content

Commit

Permalink
8274242: Implement fast-path for ASCII-compatible CharsetEncoders on x86
Browse files Browse the repository at this point in the history 
Reviewed-by: naoto, thartmann
  • Loading branch information
@cl4es
cl4es committed Sep 29, 2021
1 parent c4d1157 commit aaa36cc
Show file tree
Hide file tree
Showing 28 changed files with 428 additions and 391 deletions.
1 change: 1 addition & 0 deletions src/hotspot/cpu/aarch64/aarch64.ad
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16864,6 +16864,7 @@ instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
iRegI_R0 result, rFlagsReg cr)
%{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(USE_KILL src, USE_KILL dst, USE_KILL len,
KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/cpu/aarch64/matcher_aarch64.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,4 +163,7 @@
return true;
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = false;

#endif // CPU_AARCH64_MATCHER_AARCH64_HPP
3 changes: 3 additions & 0 deletions src/hotspot/cpu/arm/matcher_arm.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,4 +155,7 @@
return false;
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = false;

#endif // CPU_ARM_MATCHER_ARM_HPP
2 changes: 2 additions & 0 deletions src/hotspot/cpu/ppc/matcher_ppc.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,5 +164,7 @@
return VM_Version::has_fcfids();
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = false;

#endif // CPU_PPC_MATCHER_PPC_HPP
1 change: 1 addition & 0 deletions src/hotspot/cpu/ppc/ppc.ad
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12789,6 +12789,7 @@ instruct has_negatives(rarg1RegP ary1, iRegIsrc len, iRegIdst result, iRegLdst t
// encode char[] to byte[] in ISO_8859_1
instruct encode_iso_array(rarg1RegP src, rarg2RegP dst, iRegIsrc len, iRegIdst result, iRegLdst tmp1,
iRegLdst tmp2, iRegLdst tmp3, iRegLdst tmp4, iRegLdst tmp5, regCTR ctr, flagsRegCR0 cr0) %{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP_DEF result, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
USE_KILL src, USE_KILL dst, KILL ctr, KILL cr0);
Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/cpu/s390/matcher_s390.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,4 +152,7 @@
return true;
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = false;

#endif // CPU_S390_MATCHER_S390_HPP
1 change: 1 addition & 0 deletions src/hotspot/cpu/s390/s390.ad
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10282,6 +10282,7 @@ instruct has_negatives(rarg5RegP ary1, iRegI len, iRegI result, roddRegI oddReg,

// encode char[] to byte[] in ISO_8859_1
instruct encode_iso_array(iRegP src, iRegP dst, iRegI result, iRegI len, iRegI tmp, flagsReg cr) %{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP_DEF result, TEMP tmp, KILL cr); // R0, R1 are killed, too.
ins_cost(300);
Expand Down
30 changes: 23 additions & 7 deletions src/hotspot/cpu/x86/macroAssembler_x86.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5423,7 +5423,7 @@ void MacroAssembler::generate_fill(BasicType t, bool aligned,
BIND(L_exit);
}

// encode char[] to byte[] in ISO_8859_1
// encode char[] to byte[] in ISO_8859_1 or ASCII
//@IntrinsicCandidate
//private static int implEncodeISOArray(byte[] sa, int sp,
//byte[] da, int dp, int len) {
Expand All @@ -5436,10 +5436,23 @@ void MacroAssembler::generate_fill(BasicType t, bool aligned,
// }
// return i;
//}
//
//@IntrinsicCandidate
//private static int implEncodeAsciiArray(char[] sa, int sp,
// byte[] da, int dp, int len) {
// int i = 0;
// for (; i < len; i++) {
// char c = sa[sp++];
// if (c >= '\u0080')
// break;
// da[dp++] = (byte)c;
// }
// return i;
//}
void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
XMMRegister tmp1Reg, XMMRegister tmp2Reg,
XMMRegister tmp3Reg, XMMRegister tmp4Reg,
Register tmp5, Register result) {
Register tmp5, Register result, bool ascii) {

// rsi: src
// rdi: dst
Expand All @@ -5450,6 +5463,9 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
assert_different_registers(src, dst, len, tmp5, result);
Label L_done, L_copy_1_char, L_copy_1_char_exit;

int mask = ascii ? 0xff80ff80 : 0xff00ff00;
int short_mask = ascii ? 0xff80 : 0xff00;

// set result
xorl(result, result);
// check for zero length
Expand All @@ -5469,7 +5485,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,

if (UseAVX >= 2) {
Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vector
movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector
movdl(tmp1Reg, tmp5);
vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
jmp(L_chars_32_check);
Expand All @@ -5478,7 +5494,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
vptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector
vptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector
jccb(Assembler::notZero, L_copy_32_chars_exit);
vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
Expand All @@ -5493,7 +5509,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
jccb(Assembler::greater, L_copy_16_chars_exit);

} else if (UseSSE42Intrinsics) {
movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vector
movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector
movdl(tmp1Reg, tmp5);
pshufd(tmp1Reg, tmp1Reg, 0);
jmpb(L_chars_16_check);
Expand All @@ -5517,7 +5533,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
por(tmp2Reg, tmp4Reg);
}
ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector
ptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector
jccb(Assembler::notZero, L_copy_16_chars_exit);
packuswb(tmp3Reg, tmp4Reg);
}
Expand Down Expand Up @@ -5555,7 +5571,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,

bind(L_copy_1_char);
load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
testl(tmp5, 0xff00); // check if Unicode char
testl(tmp5, short_mask); // check if Unicode or non-ASCII char
jccb(Assembler::notZero, L_copy_1_char_exit);
movb(Address(dst, len, Address::times_1, 0), tmp5);
addptr(len, 1);
Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/cpu/x86/macroAssembler_x86.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1725,7 +1725,7 @@ class MacroAssembler: public Assembler {

void encode_iso_array(Register src, Register dst, Register len,
XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
XMMRegister tmp4, Register tmp5, Register result);
XMMRegister tmp4, Register tmp5, Register result, bool ascii);

#ifdef _LP64
void add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2);
Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/cpu/x86/matcher_x86.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,4 +195,7 @@
return true;
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = true;

#endif // CPU_X86_MATCHER_X86_HPP
21 changes: 19 additions & 2 deletions src/hotspot/cpu/x86/x86_32.ad
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12199,18 +12199,35 @@ instruct string_inflate_evex(Universe dummy, eSIRegP src, eDIRegP dst, eDXRegI l
instruct encode_iso_array(eSIRegP src, eDIRegP dst, eDXRegI len,
regD tmp1, regD tmp2, regD tmp3, regD tmp4,
eCXRegI tmp5, eAXRegI result, eFlagsReg cr) %{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);

format %{ "Encode array $src,$dst,$len -> $result // KILL ECX, EDX, $tmp1, $tmp2, $tmp3, $tmp4, ESI, EDI " %}
format %{ "Encode iso array $src,$dst,$len -> $result // KILL ECX, EDX, $tmp1, $tmp2, $tmp3, $tmp4, ESI, EDI " %}
ins_encode %{
__ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
$tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
%}
ins_pipe( pipe_slow );
%}

// encode char[] to byte[] in ASCII
instruct encode_ascii_array(eSIRegP src, eDIRegP dst, eDXRegI len,
regD tmp1, regD tmp2, regD tmp3, regD tmp4,
eCXRegI tmp5, eAXRegI result, eFlagsReg cr) %{
predicate(((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);

format %{ "Encode ascii array $src,$dst,$len -> $result // KILL ECX, EDX, $tmp1, $tmp2, $tmp3, $tmp4, ESI, EDI " %}
ins_encode %{
__ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
$tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
%}
ins_pipe( pipe_slow );
%}

//----------Control Flow Instructions------------------------------------------
// Signed compare Instructions
Expand Down
22 changes: 20 additions & 2 deletions src/hotspot/cpu/x86/x86_64.ad
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11770,14 +11770,32 @@ instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_Reg
instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);

format %{ "Encode array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
format %{ "Encode iso array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
ins_encode %{
__ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
$tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
%}
ins_pipe( pipe_slow );
%}

// encode char[] to byte[] in ASCII
instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
predicate(((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);

format %{ "Encode ascii array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
ins_encode %{
__ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
$tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
%}
ins_pipe( pipe_slow );
%}
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/share/classfile/vmIntrinsics.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,7 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) {
if (!SpecialArraysEquals) return true;
break;
case vmIntrinsics::_encodeISOArray:
case vmIntrinsics::_encodeAsciiArray:
case vmIntrinsics::_encodeByteISOArray:
if (!SpecialEncodeISOArray) return true;
break;
Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/share/classfile/vmIntrinsics.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,9 @@ class methodHandle;
\
do_intrinsic(_encodeByteISOArray, java_lang_StringCoding, encodeISOArray_name, indexOfI_signature, F_S) \
\
do_intrinsic(_encodeAsciiArray, java_lang_StringCoding, encodeAsciiArray_name, encodeISOArray_signature, F_S) \
do_name( encodeAsciiArray_name, "implEncodeAsciiArray") \
\
do_class(java_math_BigInteger, "java/math/BigInteger") \
do_intrinsic(_multiplyToLen, java_math_BigInteger, multiplyToLen_name, multiplyToLen_signature, F_S) \
do_name( multiplyToLen_name, "implMultiplyToLen") \
Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/share/opto/c2compiler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,9 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt
case vmIntrinsics::_copyMemory:
if (StubRoutines::unsafe_arraycopy() == NULL) return false;
break;
case vmIntrinsics::_encodeAsciiArray:
if (!Matcher::match_rule_supported(Op_EncodeISOArray) || !Matcher::supports_encode_ascii_array) return false;
break;
case vmIntrinsics::_encodeISOArray:
case vmIntrinsics::_encodeByteISOArray:
if (!Matcher::match_rule_supported(Op_EncodeISOArray)) return false;
Expand Down
8 changes: 6 additions & 2 deletions src/hotspot/share/opto/intrinsicnode.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,10 +168,14 @@ class HasNegativesNode: public StrIntrinsicNode {


//------------------------------EncodeISOArray--------------------------------
// encode char[] to byte[] in ISO_8859_1
// encode char[] to byte[] in ISO_8859_1 or ASCII
class EncodeISOArrayNode: public Node {
bool ascii;
public:
EncodeISOArrayNode(Node* control, Node* arymem, Node* s1, Node* s2, Node* c): Node(control, arymem, s1, s2, c) {};
EncodeISOArrayNode(Node* control, Node* arymem, Node* s1, Node* s2, Node* c, bool ascii)
: Node(control, arymem, s1, s2, c), ascii(ascii) {}

bool is_ascii() { return ascii; }
virtual int Opcode() const;
virtual bool depends_only_on_test() const { return false; }
virtual const Type* bottom_type() const { return TypeInt::INT; }
Expand Down
10 changes: 6 additions & 4 deletions src/hotspot/share/opto/library_call.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -591,7 +591,9 @@ bool LibraryCallKit::try_to_inline(int predicate) {

case vmIntrinsics::_encodeISOArray:
case vmIntrinsics::_encodeByteISOArray:
return inline_encodeISOArray();
return inline_encodeISOArray(false);
case vmIntrinsics::_encodeAsciiArray:
return inline_encodeISOArray(true);

case vmIntrinsics::_updateCRC32:
return inline_updateCRC32();
Expand Down Expand Up @@ -4882,8 +4884,8 @@ LibraryCallKit::tightly_coupled_allocation(Node* ptr) {
}

//-------------inline_encodeISOArray-----------------------------------
// encode char[] to byte[] in ISO_8859_1
bool LibraryCallKit::inline_encodeISOArray() {
// encode char[] to byte[] in ISO_8859_1 or ASCII
bool LibraryCallKit::inline_encodeISOArray(bool ascii) {
assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters");
// no receiver since it is static method
Node *src = argument(0);
Expand Down Expand Up @@ -4918,7 +4920,7 @@ bool LibraryCallKit::inline_encodeISOArray() {
// 'dst_start' points to dst array + scaled offset

const TypeAryPtr* mtype = TypeAryPtr::BYTES;
Node* enc = new EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
Node* enc = new EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length, ascii);
enc = _gvn.transform(enc);
Node* res_mem = _gvn.transform(new SCMemProjNode(enc));
set_memory(res_mem, mtype);
Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/share/opto/library_call.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ class LibraryCallKit : public GraphKit {
Node* get_state_from_digest_object(Node *digestBase_object, const char* state_type);
Node* get_digest_length_from_digest_object(Node *digestBase_object);
Node* inline_digestBase_implCompressMB_predicate(int predicate);
bool inline_encodeISOArray();
bool inline_encodeISOArray(bool ascii);
bool inline_updateCRC32();
bool inline_updateBytesCRC32();
bool inline_updateByteBufferCRC32();
Expand Down
16 changes: 15 additions & 1 deletion src/java.base/share/classes/java/lang/StringCoding.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public static boolean hasNegatives(byte[] ba, int off, int len) {

@IntrinsicCandidate
public static int implEncodeISOArray(byte[] sa, int sp,
byte[] da, int dp, int len) {
byte[] da, int dp, int len) {
int i = 0;
for (; i < len; i++) {
char c = StringUTF16.getChar(sa, sp++);
Expand All @@ -57,4 +57,18 @@ public static int implEncodeISOArray(byte[] sa, int sp,
return i;
}

@IntrinsicCandidate
public static int implEncodeAsciiArray(char[] sa, int sp,
byte[] da, int dp, int len)
{
int i = 0;
for (; i < len; i++) {
char c = sa[sp++];
if (c >= '\u0080')
break;
da[dp++] = (byte)c;
}
return i;
}

}
4 changes: 4 additions & 0 deletions src/java.base/share/classes/java/lang/System.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2419,6 +2419,10 @@ public int decodeASCII(byte[] src, int srcOff, char[] dst, int dstOff, int len)
return String.decodeASCII(src, srcOff, dst, dstOff, len);
}

public int encodeASCII(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
return StringCoding.implEncodeAsciiArray(src, srcOff, dst, dstOff, len);
}

public void setCause(Throwable t, Throwable cause) {
t.setCause(cause);
}
Expand Down
9 changes: 9 additions & 0 deletions src/java.base/share/classes/jdk/internal/access/JavaLangAccess.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,15 @@ public interface JavaLangAccess {
*/
int decodeASCII(byte[] src, int srcOff, char[] dst, int dstOff, int len);

/**
* Encodes ASCII codepoints as possible from the source array into
* the destination byte array, assuming that the encoding is ASCII
* compatible
*
* @return the number of bytes successfully encoded, or 0 if none
*/
int encodeASCII(char[] src, int srcOff, byte[] dst, int dstOff, int len);

/**
* Set the cause of Throwable
* @param cause set t's cause to new value
Expand Down
Loading

1 comment on commit aaa36cc

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Review

Issues

Please sign in to comment.