Skip to content

Improve x86 arrayset #7763

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Improve x86 arrayset #7763

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ab2a852
Add x86 vpbroadcast{b,w,d,q} instructions
ymanton May 22, 2025
8e5212a
Add vpbroadcast{b,w,d,q} binary encoder unit tests
ymanton May 30, 2025
32a90a2
x86: Create vector broadcast helper
BradleyWood Apr 9, 2025
761c586
Add vpbroadcast{b,w} paths to broadcast helper
ymanton Jun 2, 2025
27ae963
Improve x86 arrayset
ymanton Apr 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 179 additions & 1 deletion compiler/x/codegen/OMRTreeEvaluator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3854,6 +3854,176 @@ static void arraySetDefault(TR::Node* node, uint8_t elementSize, TR::Register* a
cg->stopUsingRegister(EAX);
}

// Assumes valueReg is a GP register containing a single element, and that sizeReg is within the range 1..3.
static void arraySet1to3Bytes(TR::Node* node, uint8_t elementSize, TR::Register* addressReg, TR::Register* valueReg, TR::Register* sizeReg, TR::CodeGenerator* cg, TR::LabelSymbol *doneLabel = NULL)
{
bool localDoneLabel = doneLabel == NULL;
generateMemRegInstruction(TR::InstOpCode::S1MemReg, node, generateX86MemoryReference(addressReg, 0, cg), valueReg, cg);
generateMemRegInstruction(TR::InstOpCode::S1MemReg, node, generateX86MemoryReference(addressReg, sizeReg, 0, -1, cg), valueReg, cg);
generateRegImmInstruction(TR::InstOpCode::CMPRegImm4(), node, sizeReg, 2, cg);
if (localDoneLabel)
doneLabel = generateLabelSymbol(cg);
generateLabelInstruction(TR::InstOpCode::JBE4, node, doneLabel, cg);
generateMemRegInstruction(TR::InstOpCode::S1MemReg, node, generateX86MemoryReference(addressReg, 1, cg), valueReg, cg);
if (localDoneLabel)
generateLabelInstruction(TR::InstOpCode::label, node, doneLabel, cg);
}

// Assumes valueReg is a GP register that contains a packed set of elements, and that sizeReg is within the range 4..15.
static void arraySet4to15Bytes(TR::Node* node, uint8_t elementSize, TR::Register* addressReg, TR::Register* valueReg, TR::Register* sizeReg, TR::Register *lastWordAddressReg, TR::CodeGenerator* cg)
{
generateRegMemInstruction(TR::InstOpCode::LEARegMem(), node, lastWordAddressReg, generateX86MemoryReference(addressReg, sizeReg, 0, -4, cg), cg);
generateRegImmInstruction(TR::InstOpCode::ANDRegImms(), node, sizeReg, 8, cg);
generateMemRegInstruction(TR::InstOpCode::S4MemReg, node, generateX86MemoryReference(addressReg, 0, cg), valueReg, cg);
generateRegImmInstruction(TR::InstOpCode::SHRRegImm1(), node, sizeReg, 1, cg);
generateMemRegInstruction(TR::InstOpCode::S4MemReg, node, generateX86MemoryReference(lastWordAddressReg, 0, cg), valueReg, cg);
generateMemRegInstruction(TR::InstOpCode::S4MemReg, node, generateX86MemoryReference(addressReg, sizeReg, 0, cg), valueReg, cg);
generateRegInstruction(TR::InstOpCode::NEGReg(), node, sizeReg, cg);
generateMemRegInstruction(TR::InstOpCode::S4MemReg, node, generateX86MemoryReference(lastWordAddressReg, sizeReg, 0, cg), valueReg, cg);
}

// Assumes valueReg is an XMM register that contains a vector of elements, and that sizeReg is within the range 16..31.
static void arraySet16to31Bytes(TR::Node* node, uint8_t elementSize, TR::Register* addressReg, TR::Register* xmmValueReg, TR::Register* sizeReg, TR::CodeGenerator* cg)
{
generateMemRegInstruction(TR::InstOpCode::MOVUPSMemReg, node, generateX86MemoryReference(addressReg, 0, cg), xmmValueReg, cg);
generateMemRegInstruction(TR::InstOpCode::MOVUPSMemReg, node, generateX86MemoryReference(addressReg, sizeReg, 0, -16, cg), xmmValueReg, cg);
}

// Assumes valueReg is an XMM register that contains a vector of elements, and that sizeReg is within the range 32..63.
static void arraySet32to63Bytes(TR::Node* node, uint8_t elementSize, TR::Register* addressReg, TR::Register* xmmValueReg, TR::Register* sizeReg, TR::CodeGenerator* cg)
{
generateMemRegInstruction(TR::InstOpCode::MOVUPSMemReg, node, generateX86MemoryReference(addressReg, 0, cg), xmmValueReg, cg);
generateMemRegInstruction(TR::InstOpCode::MOVUPSMemReg, node, generateX86MemoryReference(addressReg, 16, cg), xmmValueReg, cg);
generateMemRegInstruction(TR::InstOpCode::MOVUPSMemReg, node, generateX86MemoryReference(addressReg, sizeReg, 0, -32, cg), xmmValueReg, cg);
generateMemRegInstruction(TR::InstOpCode::MOVUPSMemReg, node, generateX86MemoryReference(addressReg, sizeReg, 0, -16, cg), xmmValueReg, cg);
}

// Assumes valueReg is an XMM register that contains a vector of elements, and that sizeReg is >= 64.
static void arraySet64ByteLoop(TR::Node* node, uint8_t elementSize, TR::Register* addressReg, TR::Register* xmmValueReg, TR::Register* sizeReg, TR::Register *scratch1Reg, TR::Register *scratch2Reg, TR::CodeGenerator* cg)
{
TR::LabelSymbol *loopLabel = generateLabelSymbol(cg);
TR::LabelSymbol *residueLabel = generateLabelSymbol(cg);

// Unaligned store to the first 16 bytes
generateMemRegInstruction(TR::InstOpCode::MOVUPSMemReg, node, generateX86MemoryReference(addressReg, 0, cg), xmmValueReg, cg);
generateRegRegInstruction(TR::InstOpCode::MOVRegReg(), node, scratch1Reg, addressReg, cg);
// Advance the address reg to the next 16-byte aligned address
generateRegImmInstruction(TR::InstOpCode::ADDRegImms(), node, addressReg, 16, cg);
generateRegImmInstruction(TR::InstOpCode::ANDRegImms(), node, addressReg, -16, cg);
// Calculate how many unaligned bytes were stored and subtract from the size
generateRegRegInstruction(TR::InstOpCode::SUBRegReg(), node, scratch1Reg, addressReg, cg);
generateRegRegInstruction(TR::InstOpCode::ADDRegReg(), node, sizeReg, scratch1Reg, cg);
// Check if we have at least 64 bytes left to store
// Enter the 64-byte per iteration aligned store loop if so
// Otherwise go to the residue loop
generateRegImmInstruction(TR::InstOpCode::CMPRegImm4(), node, sizeReg, 64, cg);
generateLabelInstruction(TR::InstOpCode::JB4, node, residueLabel, cg);
generateLabelInstruction(TR::InstOpCode::label, node, loopLabel, cg);
// 64-byte per iteration aligned store loop
generateMemRegInstruction(TR::InstOpCode::MOVAPSMemReg, node, generateX86MemoryReference(addressReg, 0, cg), xmmValueReg, cg);
generateMemRegInstruction(TR::InstOpCode::MOVAPSMemReg, node, generateX86MemoryReference(addressReg, 16, cg), xmmValueReg, cg);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

movaps requires alignment (to the size of the vector length), is the memory reference always aligned?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, there's a preceding movups that will set any unaligned bytes, and then the pointer is bumped to the next 16-byte aligned address.

(Previous patch left the pointer at the same address if it was already aligned, so the same 16 bytes would be set with movups and then again by the first movaps in the loop. I've fixed that to move the pointer to the next aligned address regardless of whether it was previously aligned or not to improve that.)

generateMemRegInstruction(TR::InstOpCode::MOVAPSMemReg, node, generateX86MemoryReference(addressReg, 32, cg), xmmValueReg, cg);
generateMemRegInstruction(TR::InstOpCode::MOVAPSMemReg, node, generateX86MemoryReference(addressReg, 48, cg), xmmValueReg, cg);
generateRegImmInstruction(TR::InstOpCode::ADDRegImms(), node, addressReg, 64, cg);
generateRegImmInstruction(TR::InstOpCode::SUBRegImms(), node, sizeReg, 64, cg);
generateRegImmInstruction(TR::InstOpCode::CMPRegImm4(), node, sizeReg, 64, cg);
generateLabelInstruction(TR::InstOpCode::JAE4, node, loopLabel, cg);
// Residue
generateLabelInstruction(TR::InstOpCode::label, node, residueLabel, cg);
// At this point we have less than 64 bytes to store
// Calculate the address of the last 3 aligned stores
generateRegMemInstruction(TR::InstOpCode::LEARegMem(), node, scratch1Reg, generateX86MemoryReference(addressReg, sizeReg, 0, -48, cg), cg);
generateRegImmInstruction(TR::InstOpCode::ANDRegImms(), node, scratch1Reg, -16, cg);
// Calculate the address of the last (unaligned) store
generateRegMemInstruction(TR::InstOpCode::LEARegMem(), node, scratch2Reg, generateX86MemoryReference(addressReg, sizeReg, 0, -16, cg), cg);
// Do the stores
generateMemRegInstruction(TR::InstOpCode::MOVAPSMemReg, node, generateX86MemoryReference(scratch1Reg, 0, cg), xmmValueReg, cg);
generateMemRegInstruction(TR::InstOpCode::MOVAPSMemReg, node, generateX86MemoryReference(scratch1Reg, 16, cg), xmmValueReg, cg);
generateMemRegInstruction(TR::InstOpCode::MOVAPSMemReg, node, generateX86MemoryReference(scratch1Reg, 32, cg), xmmValueReg, cg);
generateMemRegInstruction(TR::InstOpCode::MOVUPSMemReg, node, generateX86MemoryReference(scratch2Reg, 0, cg), xmmValueReg, cg);
}

static void arraySetXMM(TR::Node* node, uint8_t elementSize, TR::Register* addressReg, TR::Register* valueReg, TR::Register* sizeReg, const uintptr_t *size, TR::CodeGenerator* cg)
{
// This code is loosely based on the approach discussed in
// "Building Faster AMD64 Memset Routines" by Joe Bialek (January 11, 2021).
// https://msrc.microsoft.com/blog/2021/01/building-faster-amd64-memset-routines/
// We reduce path length and handle unaligned bytes more efficiently
// by setting some bytes multiple times, on the assumption
// that stores to overlapping memory ranges are cheaper than executing
// extra comparisons and branches to set each byte exactly once.
TR::LabelSymbol *doneLabel = generateLabelSymbol(cg);

TR::Register *scratch1Reg = cg->allocateRegister(TR_GPR);
TR::Register *scratch2Reg = cg->allocateRegister(TR_GPR);
TR::Register *xmmValueReg = cg->allocateRegister(TR_VRF);

generateRegRegInstruction(TR::InstOpCode::MOVDRegReg4, node, xmmValueReg, valueReg, cg);
TR::TreeEvaluator::broadcastHelper(node, xmmValueReg, TR::VectorLength128, TR::Int8, cg);

// If we don't know the size at compile-time or it's known to be less than 64 bytes, generate
// a series of tests for various sizes and branch to short, branch free sequences of stores.
// Currently `size` is expected to be NULL or >=64, since short compile-time sizes are handled
// in the main evaluator, otherwise we could avoid generating some of these tests for certain
// sizes.
if (size == NULL || *size < 64)
{
TR::LabelSymbol *ge64Label = generateLabelSymbol(cg);
TR::LabelSymbol *lt32Label = generateLabelSymbol(cg);
TR::LabelSymbol *lt16Label = generateLabelSymbol(cg);
TR::LabelSymbol *ge4lt16Label = generateLabelSymbol(cg);

generateRegImmInstruction(TR::InstOpCode::CMPRegImm4(), node, sizeReg, 64, cg);
generateLabelInstruction(TR::InstOpCode::JAE4, node, ge64Label, cg);
generateRegImmInstruction(TR::InstOpCode::CMPRegImm4(), node, sizeReg, 32, cg);
generateLabelInstruction(TR::InstOpCode::JB4, node, lt32Label, cg);

arraySet32to63Bytes(node, elementSize, addressReg, xmmValueReg, sizeReg, cg);
generateLabelInstruction(TR::InstOpCode::JMP4, node, doneLabel, cg);

generateLabelInstruction(TR::InstOpCode::label, node, lt32Label, cg);
generateRegImmInstruction(TR::InstOpCode::CMPRegImm4(), node, sizeReg, 16, cg);
generateLabelInstruction(TR::InstOpCode::JB4, node, lt16Label, cg);

arraySet16to31Bytes(node, elementSize, addressReg, xmmValueReg, sizeReg, cg);
generateLabelInstruction(TR::InstOpCode::JMP4, node, doneLabel, cg);

generateLabelInstruction(TR::InstOpCode::label, node, lt16Label, cg);
generateRegImmInstruction(TR::InstOpCode::CMPRegImm4(), node, sizeReg, 4, cg);
generateLabelInstruction(TR::InstOpCode::JAE4, node, ge4lt16Label, cg);
generateRegRegInstruction(TR::InstOpCode::TEST4RegReg, node, sizeReg, sizeReg, cg);
generateLabelInstruction(TR::InstOpCode::JE4, node, doneLabel, cg);

arraySet1to3Bytes(node, elementSize, addressReg, valueReg, sizeReg, cg, doneLabel);
generateLabelInstruction(TR::InstOpCode::JMP4, node, doneLabel, cg);

generateLabelInstruction(TR::InstOpCode::label, node, ge4lt16Label, cg);
// The value is packed into the XMM reg, but arraySet4to15Bytes() needs a packed GPR. The original valueReg
// can't be clobbered, so we use scratch2Reg to hold the packed value.
generateRegRegInstruction(TR::InstOpCode::MOVDReg4Reg, node, scratch2Reg, xmmValueReg, cg);
arraySet4to15Bytes(node, elementSize, addressReg, scratch2Reg, sizeReg, scratch1Reg, cg);
generateLabelInstruction(TR::InstOpCode::JMP4, node, doneLabel, cg);

generateLabelInstruction(TR::InstOpCode::label, node, ge64Label, cg);
}

arraySet64ByteLoop(node, elementSize, addressReg, xmmValueReg, sizeReg, scratch1Reg, scratch2Reg, cg);

TR::RegisterDependencyConditions *deps = generateRegisterDependencyConditions(0, 6, cg);

deps->addPostCondition(addressReg, TR::RealRegister::NoReg, cg);
deps->addPostCondition(valueReg, TR::RealRegister::NoReg, cg);
deps->addPostCondition(sizeReg, TR::RealRegister::NoReg, cg);

deps->addPostCondition(scratch1Reg, TR::RealRegister::NoReg, cg);
deps->addPostCondition(scratch2Reg, TR::RealRegister::NoReg, cg);
deps->addPostCondition(xmmValueReg, TR::RealRegister::NoReg, cg);

deps->stopAddingConditions();

generateLabelInstruction(TR::InstOpCode::label, node, doneLabel, deps, cg);
}

TR::Register *OMR::X86::TreeEvaluator::arraysetEvaluator(TR::Node *node, TR::CodeGenerator *cg)
{
// arrayset
Expand Down Expand Up @@ -3946,7 +4116,15 @@ TR::Register *OMR::X86::TreeEvaluator::arraysetEvaluator(TR::Node *node, TR::Cod
}
else
{
arraySetDefault(node, elementSize, addressReg, valueReg, sizeReg, cg);
static bool disableArraySetXMM = feGetEnv("TR_disableArraySetXMM") != NULL;
if (cg->comp()->target().cpu.supportsFeature(OMR_FEATURE_X86_AVX2) && elementSize == 1 && !disableArraySetXMM)
{
arraySetXMM(node, elementSize, addressReg, valueReg, sizeReg, isSizeConst ? &size : NULL, cg);
}
else
{
arraySetDefault(node, elementSize, addressReg, valueReg, sizeReg, cg);
}
}
cg->decReferenceCount(sizeNode);
cg->decReferenceCount(valueNode);
Expand Down
15 changes: 15 additions & 0 deletions compiler/x/codegen/OMRTreeEvaluator.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,21 @@ class OMR_EXTENSIBLE TreeEvaluator: public OMR::TreeEvaluator
static TR::Register *maskLoadEvaluator(TR::Node *node, TR::CodeGenerator *cg);
static TR::Register *maskStoreEvaluator(TR::Node *node, TR::CodeGenerator *cg);
static TR::Register *SIMDstoreEvaluator(TR::Node *node, TR::CodeGenerator *cg);

/**
* @brief Helper function to broadcast a single element from a vector register across all lanes.
*
* This function emits the appropriate instructions to broadcast the value from a specified
* vector register across all elements of a vector, based on the given vector length and element type.
*
* @param node The IR node associated with the broadcast operation.
* @param vectorReg The vector register containing the value to broadcast.
* @param vl The vector length (e.g., 128-bit, 256-bit) that determines the width of the target vector.
* @param et The element type (e.g., Int32, Int64, Float32) that defines the lane size for broadcasting.
* @param cg The code generator used to emit the broadcast instructions.
* @return vectorReg
*/
static TR::Register *broadcastHelper(TR::Node *node, TR::Register *vectorReg, TR::VectorLength vl, TR::DataType et, TR::CodeGenerator *cg);
static TR::Register *SIMDsplatsEvaluator(TR::Node *node, TR::CodeGenerator *cg);
static TR::Register *SIMDvgetelemEvaluator(TR::Node *node, TR::CodeGenerator *cg);

Expand Down
97 changes: 63 additions & 34 deletions compiler/x/codegen/SIMDTreeEvaluator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,68 @@ TR::Register* OMR::X86::TreeEvaluator::SIMDstoreEvaluator(TR::Node* node, TR::Co
return NULL;
}

TR::Register* OMR::X86::TreeEvaluator::broadcastHelper(TR::Node *node, TR::Register *vectorReg, TR::VectorLength vl, TR::DataType et, TR::CodeGenerator *cg)
{
bool broadcast64 = et.isInt64() || et.isDouble();

if (!cg->comp()->target().cpu.supportsFeature(OMR_FEATURE_X86_AVX2) ||
vl != TR::VectorLength128)
{
// Expand byte & word to 32-bits
switch (et)
{
case TR::Int8:
generateRegRegInstruction(TR::InstOpCode::PUNPCKLBWRegReg, node, vectorReg, vectorReg, cg);
// fallthrough
case TR::Int16:
generateRegRegImmInstruction(TR::InstOpCode::PSHUFLWRegRegImm1, node, vectorReg, vectorReg, 0x0, cg);
et = TR::Int32;
break;
default:
break;
}
}

switch (vl)
{
case TR::VectorLength128:
switch (et)
{
case TR::Int8:
TR_ASSERT_FATAL(cg->comp()->target().cpu.supportsFeature(OMR_FEATURE_X86_AVX2), "8-bit to 128-bit vsplats requires AVX2");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So I expect Int8 and Int16 would crash if AVX2 is not supported. This original implementation was a little biased towards older micro archtechures by using the punpcklbw/pshuflw sequence to expand byte and word to 32-bits before using pshufd to expand broadcast to all lanes.

I think its best to write this in two sections, one for AVX2+ hardware and keep the older logic for SSE/AVX1. Some rough pseudocode below.

if (avx2) {
  switch (et) {
    case Int8:
      opcode = VPBROADCASTB; break;
    case Int16:
      opcode = VPBROADCASTW; break;
    case Int32:
    case Float:
      opcode = VPBROADCASTD; break;
    case Int64:
    case Double:
      opcode = VPBROADCASTQ; break;
    default:
      // assert (unexpected type)
  }

  encoding = opcode.getSIMDEncoding(&cg->comp()->target().cpu, vl);
  TR_ASSERT_FATAL(encoding != OMR::X86::Bad, "Broadcast instruction is not supported");

  generateRegRegInstruction(opcode, node, vectorReg, vectorReg, cg, encoding);
} else {
  // Expand byte and word to 32-bits.
  // assert if VL > 128; operation not supported
  // PSHUFD
}

Try to run omr tests locally and set TR_Options=disableAVX2 or TR_Options=disableAVX

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So I expect Int8 and Int16 would crash if AVX2 is not supported.

No, it works because Int8 and Int16 don't reach there. As written now if AVX2 isn't supported we'll do the 8->16->32 expansion first and et would be updated to Int32. Anyhow I can try it as suggested, I'm not particularly fond of the nested switches anyway.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, I see you set et = TR::Int32;

I'm not particularly fond of the nested switches anyway.

Either ways is OK. I will let you decide.

generateRegRegInstruction(TR::InstOpCode::VPBROADCASTBRegReg, node, vectorReg, vectorReg, cg);
break;
case TR::Int16:
TR_ASSERT_FATAL(cg->comp()->target().cpu.supportsFeature(OMR_FEATURE_X86_AVX2), "16-bit to 128-bit vsplats requires AVX2");
generateRegRegInstruction(TR::InstOpCode::VPBROADCASTWRegReg, node, vectorReg, vectorReg, cg);
break;
default:
generateRegRegImmInstruction(TR::InstOpCode::PSHUFDRegRegImm1, node, vectorReg, vectorReg, broadcast64 ? 0x44 : 0, cg);
break;
}
break;
case TR::VectorLength256:
{
TR_ASSERT_FATAL(cg->comp()->target().cpu.supportsFeature(OMR_FEATURE_X86_AVX2), "256-bit vsplats requires AVX2");
TR::InstOpCode opcode = broadcast64 ? TR::InstOpCode::VBROADCASTSDYmmYmm : TR::InstOpCode::VBROADCASTSSRegReg;
generateRegRegInstruction(opcode.getMnemonic(), node, vectorReg, vectorReg, cg, opcode.getSIMDEncoding(&cg->comp()->target().cpu, TR::VectorLength256));
break;
}
case TR::VectorLength512:
{
TR_ASSERT_FATAL(cg->comp()->target().cpu.supportsFeature(OMR_FEATURE_X86_AVX512F), "512-bit vsplats requires AVX-512");
TR::InstOpCode opcode = broadcast64 ? TR::InstOpCode::VBROADCASTSDZmmXmm : TR::InstOpCode::VBROADCASTSSRegReg;
generateRegRegInstruction(opcode.getMnemonic(), node, vectorReg, vectorReg, cg, OMR::X86::EVEX_L512);
break;
}
default:
TR_ASSERT_FATAL(0, "Unsupported vector length");
break;
}

return vectorReg;
}

TR::Register* OMR::X86::TreeEvaluator::SIMDsplatsEvaluator(TR::Node* node, TR::CodeGenerator* cg)
{
TR::Node* childNode = node->getChild(0);
Expand Down Expand Up @@ -269,40 +331,7 @@ TR::Register* OMR::X86::TreeEvaluator::SIMDsplatsEvaluator(TR::Node* node, TR::C
break;
}

// Expand byte & word to 32-bits
switch (et)
{
case TR::Int8:
generateRegRegInstruction(TR::InstOpCode::PUNPCKLBWRegReg, node, resultReg, resultReg, cg);
case TR::Int16:
generateRegRegImmInstruction(TR::InstOpCode::PSHUFLWRegRegImm1, node, resultReg, resultReg, 0x0, cg);
default:
break;
}

switch (vl)
{
case TR::VectorLength128:
generateRegRegImmInstruction(TR::InstOpCode::PSHUFDRegRegImm1, node, resultReg, resultReg, broadcast64 ? 0x44 : 0, cg);
break;
case TR::VectorLength256:
{
TR_ASSERT_FATAL(cg->comp()->target().cpu.supportsFeature(OMR_FEATURE_X86_AVX2), "256-bit vsplats requires AVX2");
TR::InstOpCode opcode = broadcast64 ? TR::InstOpCode::VBROADCASTSDYmmYmm : TR::InstOpCode::VBROADCASTSSRegReg;
generateRegRegInstruction(opcode.getMnemonic(), node, resultReg, resultReg, cg, opcode.getSIMDEncoding(&cg->comp()->target().cpu, TR::VectorLength256));
break;
}
case TR::VectorLength512:
{
TR_ASSERT_FATAL(cg->comp()->target().cpu.supportsFeature(OMR_FEATURE_X86_AVX512F), "512-bit vsplats requires AVX-512");
TR::InstOpCode opcode = broadcast64 ? TR::InstOpCode::VBROADCASTSDZmmXmm : TR::InstOpCode::VBROADCASTSSRegReg;
generateRegRegInstruction(opcode.getMnemonic(), node, resultReg, resultReg, cg, OMR::X86::EVEX_L512);
break;
}
default:
TR_ASSERT_FATAL(0, "Unsupported vector length");
break;
}
TR::TreeEvaluator::broadcastHelper(node, resultReg, vl, et, cg);

node->setRegister(resultReg);
cg->decReferenceCount(childNode);
Expand Down
Loading