blob: 8a73d131ce5f8942b029eac7a6bab85495734228 [file] [log] [blame]
// Copyright (c) 2017-2019 ARM Limited
// All rights reserved
//
// The license below extends only to copyright in the software and shall
// not be construed as granting a license to any other intellectual
// property including but not limited to intellectual property relating
// to a hardware implementation of the functionality of the software
// licensed hereunder. You may use the software subject to the license
// terms below provided that you ensure that this notice is replicated
// unmodified and in its entirety in all distributions of the software,
// modified or unmodified, in source code or in binary form.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met: redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer;
// redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution;
// neither the name of the copyright holders nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// @file Definition of SVE memory access instructions.
output header {{
// Decodes SVE contiguous load instructions, scalar plus scalar form.
template <template <typename T1, typename T2> class Base>
StaticInstPtr
decodeSveContigLoadSSInsts(uint8_t dtype, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex rn,
RegIndex rm, bool firstFaulting)
{
const char* mn = firstFaulting ? "ldff1" : "ld1";
switch (dtype) {
case 0x0:
return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
case 0x1:
return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
case 0x2:
return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
case 0x3:
return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
case 0x4:
return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, rm);
case 0x5:
return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
case 0x6:
return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
case 0x7:
return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
case 0x8:
return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, rm);
case 0x9:
return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, rm);
case 0xa:
return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
case 0xb:
return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
case 0xc:
return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, rm);
case 0xd:
return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, rm);
case 0xe:
return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, rm);
case 0xf:
return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm);
}
return new Unknown64(machInst);
}
// Decodes SVE contiguous load instructions, scalar plus immediate form.
template <template <typename T1, typename T2> class Base>
StaticInstPtr
decodeSveContigLoadSIInsts(uint8_t dtype, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex rn,
uint64_t imm, bool nonFaulting,
bool replicate = false)
{
assert(!(nonFaulting && replicate));
const char* mn = replicate ? "ld1r" : (nonFaulting ? "ldnf1" : "ld1");
switch (dtype) {
case 0x0:
return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
case 0x1:
return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
case 0x2:
return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
case 0x3:
return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
case 0x4:
return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, imm);
case 0x5:
return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
case 0x6:
return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
case 0x7:
return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
case 0x8:
return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, imm);
case 0x9:
return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, imm);
case 0xa:
return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
case 0xb:
return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
case 0xc:
return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, imm);
case 0xd:
return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, imm);
case 0xe:
return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, imm);
case 0xf:
return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm);
}
return new Unknown64(machInst);
}
// Decodes SVE contiguous store instructions, scalar plus scalar form.
template <template <typename T1, typename T2> class Base>
StaticInstPtr
decodeSveContigStoreSSInsts(uint8_t dtype, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex rn,
RegIndex rm)
{
const char* mn = "st1";
switch (dtype) {
case 0x0:
return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
case 0x1:
return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
case 0x2:
return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
case 0x3:
return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
case 0x5:
return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
case 0x6:
return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
case 0x7:
return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
case 0xa:
return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
case 0xb:
return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
case 0xf:
return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm);
}
return new Unknown64(machInst);
}
// Decodes SVE contiguous store instructions, scalar plus immediate form.
template <template <typename T1, typename T2> class Base>
StaticInstPtr
decodeSveContigStoreSIInsts(uint8_t dtype, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex rn,
int8_t imm)
{
const char* mn = "st1";
switch (dtype) {
case 0x0:
return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
case 0x1:
return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
case 0x2:
return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
case 0x3:
return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
case 0x5:
return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
case 0x6:
return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
case 0x7:
return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
case 0xa:
return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
case 0xb:
return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
case 0xf:
return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm);
}
return new Unknown64(machInst);
}
// NOTE: SVE load-and-replicate instructions are decoded with
// decodeSveContigLoadSIInsts(...).
}};
output decoder {{
template <class etype>
StaticInstPtr
decodeSveStructLoadSIInstsByNReg(uint8_t esize, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex xn,
int64_t imm, int numregs)
{
static const char* nm[5][4] = {
{ nullptr, nullptr, nullptr, nullptr},
{ nullptr, nullptr, nullptr, nullptr},
{ "ld2b", "ld2h", "ld2w", "ld2d" },
{ "ld3b", "ld3h", "ld3w", "ld3d" },
{ "ld4b", "ld4h", "ld4w", "ld4d" } };
switch (numregs) {
case 2:
return new SveLdStructSI<etype,
SveLoadRegImmMicroop,
SveDeIntrlv2Microop>(
nm[numregs][esize], machInst, MemReadOp,
zt, pg, xn, imm, numregs);
case 3:
return new SveLdStructSI<etype,
SveLoadRegImmMicroop,
SveDeIntrlv3Microop>(
nm[numregs][esize], machInst, MemReadOp,
zt, pg, xn, imm, numregs);
case 4:
return new SveLdStructSI<etype,
SveLoadRegImmMicroop,
SveDeIntrlv4Microop>(
nm[numregs][esize], machInst, MemReadOp,
zt, pg, xn, imm, numregs);
}
return new Unknown64(machInst);
}
StaticInstPtr
decodeSveStructLoadSIInsts(uint8_t esize, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex xn,
int64_t imm, int numregs)
{
switch (esize) {
case 0:
return decodeSveStructLoadSIInstsByNReg<uint8_t>(esize,
machInst, zt, pg, xn, imm, numregs);
case 1:
return decodeSveStructLoadSIInstsByNReg<uint16_t>(esize,
machInst, zt, pg, xn, imm, numregs);
case 2:
return decodeSveStructLoadSIInstsByNReg<uint32_t>(esize,
machInst, zt, pg, xn, imm, numregs);
case 3:
return decodeSveStructLoadSIInstsByNReg<uint64_t>(esize,
machInst, zt, pg, xn, imm, numregs);
}
return new Unknown64(machInst);
}
template <class etype>
StaticInstPtr
decodeSveStructStoreSIInstsByNReg(uint8_t esize, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex xn,
int64_t imm, int numregs)
{
static const char* nm[5][4] = {
{ nullptr, nullptr, nullptr, nullptr},
{ nullptr, nullptr, nullptr, nullptr},
{ "st2b", "st2h", "st2w", "st2d" },
{ "st3b", "st3h", "st3w", "st3d" },
{ "st4b", "st4h", "st4w", "st4d" } };
switch (numregs) {
case 2:
return new SveStStructSI<etype,
SveStoreRegImmMicroop,
SveIntrlv2Microop>(
nm[numregs][esize], machInst, MemWriteOp,
zt, pg, xn, imm, numregs);
case 3:
return new SveStStructSI<etype,
SveStoreRegImmMicroop,
SveIntrlv3Microop>(
nm[numregs][esize], machInst, MemWriteOp,
zt, pg, xn, imm, numregs);
case 4:
return new SveStStructSI<etype,
SveStoreRegImmMicroop,
SveIntrlv4Microop>(
nm[numregs][esize], machInst, MemWriteOp,
zt, pg, xn, imm, numregs);
}
return new Unknown64(machInst);
}
StaticInstPtr
decodeSveStructStoreSIInsts(uint8_t esize, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex xn,
int64_t imm, int numregs)
{
switch (esize) {
case 0:
return decodeSveStructStoreSIInstsByNReg<uint8_t>(esize,
machInst, zt, pg, xn, imm, numregs);
case 1:
return decodeSveStructStoreSIInstsByNReg<uint16_t>(esize,
machInst, zt, pg, xn, imm, numregs);
case 2:
return decodeSveStructStoreSIInstsByNReg<uint32_t>(esize,
machInst, zt, pg, xn, imm, numregs);
case 3:
return decodeSveStructStoreSIInstsByNReg<uint64_t>(esize,
machInst, zt, pg, xn, imm, numregs);
}
return new Unknown64(machInst);
}
template <class etype>
StaticInstPtr
decodeSveStructLoadSSInstsByNReg(uint8_t esize, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex xn,
RegIndex xm, int numregs)
{
static const char* nm[5][4] = {
{ nullptr, nullptr, nullptr, nullptr},
{ nullptr, nullptr, nullptr, nullptr},
{ "ld2b", "ld2h", "ld2w", "ld2d" },
{ "ld3b", "ld3h", "ld3w", "ld3d" },
{ "ld4b", "ld4h", "ld4w", "ld4d" } };
switch (numregs) {
case 2:
return new SveLdStructSS<etype,
SveLoadRegRegMicroop,
SveDeIntrlv2Microop>(
nm[numregs][esize], machInst, MemReadOp,
zt, pg, xn, xm, numregs);
case 3:
return new SveLdStructSS<etype,
SveLoadRegRegMicroop,
SveDeIntrlv3Microop>(
nm[numregs][esize], machInst, MemReadOp,
zt, pg, xn, xm, numregs);
case 4:
return new SveLdStructSS<etype,
SveLoadRegRegMicroop,
SveDeIntrlv4Microop>(
nm[numregs][esize], machInst, MemReadOp,
zt, pg, xn, xm, numregs);
}
return new Unknown64(machInst);
}
StaticInstPtr
decodeSveStructLoadSSInsts(uint8_t esize, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex xn,
RegIndex xm, int numregs)
{
switch (esize) {
case 0:
return decodeSveStructLoadSSInstsByNReg<uint8_t>(esize,
machInst, zt, pg, xn, xm, numregs);
case 1:
return decodeSveStructLoadSSInstsByNReg<uint16_t>(esize,
machInst, zt, pg, xn, xm, numregs);
case 2:
return decodeSveStructLoadSSInstsByNReg<uint32_t>(esize,
machInst, zt, pg, xn, xm, numregs);
case 3:
return decodeSveStructLoadSSInstsByNReg<uint64_t>(esize,
machInst, zt, pg, xn, xm, numregs);
}
return new Unknown64(machInst);
}
template <class etype>
StaticInstPtr
decodeSveStructStoreSSInstsByNReg(uint8_t esize, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex xn,
RegIndex xm, int numregs)
{
static const char* nm[5][4] = {
{ nullptr, nullptr, nullptr, nullptr},
{ nullptr, nullptr, nullptr, nullptr},
{ "st2b", "st2h", "st2w", "st2d" },
{ "st3b", "st3h", "st3w", "st3d" },
{ "st4b", "st4h", "st4w", "st4d" } };
switch (numregs) {
case 2:
return new SveStStructSS<etype,
SveStoreRegRegMicroop,
SveIntrlv2Microop>(
nm[numregs][esize], machInst, MemWriteOp,
zt, pg, xn, xm, numregs);
case 3:
return new SveStStructSS<etype,
SveStoreRegRegMicroop,
SveIntrlv3Microop>(
nm[numregs][esize], machInst, MemWriteOp,
zt, pg, xn, xm, numregs);
case 4:
return new SveStStructSS<etype,
SveStoreRegRegMicroop,
SveIntrlv4Microop>(
nm[numregs][esize], machInst, MemWriteOp,
zt, pg, xn, xm, numregs);
}
return new Unknown64(machInst);
}
StaticInstPtr
decodeSveStructStoreSSInsts(uint8_t esize, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex xn,
RegIndex xm, int numregs)
{
switch (esize) {
case 0:
return decodeSveStructStoreSSInstsByNReg<uint8_t>(esize,
machInst, zt, pg, xn, xm, numregs);
case 1:
return decodeSveStructStoreSSInstsByNReg<uint16_t>(esize,
machInst, zt, pg, xn, xm, numregs);
case 2:
return decodeSveStructStoreSSInstsByNReg<uint32_t>(esize,
machInst, zt, pg, xn, xm, numregs);
case 3:
return decodeSveStructStoreSSInstsByNReg<uint64_t>(esize,
machInst, zt, pg, xn, xm, numregs);
}
return new Unknown64(machInst);
}
StaticInstPtr
decodeSveGatherLoadVIInsts(uint8_t dtype, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex zn,
uint64_t imm, bool esizeIs32,
bool firstFault)
{
const char* mn = firstFault ? "ldff1" : "ld1";
switch (dtype) {
case 0x0:
if (esizeIs32) {
return new SveIndexedMemVI<int32_t, int8_t,
SveGatherLoadVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
} else {
return new SveIndexedMemVI<int64_t, int8_t,
SveGatherLoadVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
}
case 0x1:
if (esizeIs32) {
return new SveIndexedMemVI<uint32_t, uint8_t,
SveGatherLoadVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
} else {
return new SveIndexedMemVI<uint64_t, uint8_t,
SveGatherLoadVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
}
case 0x2:
if (esizeIs32) {
return new SveIndexedMemVI<int32_t, int16_t,
SveGatherLoadVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
} else {
return new SveIndexedMemVI<int64_t, int16_t,
SveGatherLoadVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
}
case 0x3:
if (esizeIs32) {
return new SveIndexedMemVI<uint32_t, uint16_t,
SveGatherLoadVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
} else {
return new SveIndexedMemVI<uint64_t, uint16_t,
SveGatherLoadVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
}
case 0x4:
if (esizeIs32) {
break;
} else {
return new SveIndexedMemVI<int64_t, int32_t,
SveGatherLoadVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
}
case 0x5:
if (esizeIs32) {
return new SveIndexedMemVI<uint32_t, uint32_t,
SveGatherLoadVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
} else {
return new SveIndexedMemVI<uint64_t, uint32_t,
SveGatherLoadVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
}
case 0x7:
if (esizeIs32) {
break;
} else {
return new SveIndexedMemVI<uint64_t, uint64_t,
SveGatherLoadVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
}
}
return new Unknown64(machInst);
}
StaticInstPtr
decodeSveGatherLoadSVInsts(uint8_t dtype, ExtMachInst machInst,
RegIndex zt, RegIndex pg, RegIndex rn,
RegIndex zm, bool esizeIs32, bool offsetIs32,
bool offsetIsSigned, bool offsetIsScaled,
bool firstFault)
{
const char* mn = firstFault ? "ldff1" : "ld1";
switch (dtype) {
case 0x0:
if (esizeIs32) {
return new SveIndexedMemSV<int32_t, int8_t,
SveGatherLoadSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
} else {
return new SveIndexedMemSV<int64_t, int8_t,
SveGatherLoadSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
}
case 0x1:
if (esizeIs32) {
return new SveIndexedMemSV<uint32_t, uint8_t,
SveGatherLoadSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
} else {
return new SveIndexedMemSV<uint64_t, uint8_t,
SveGatherLoadSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
}
case 0x2:
if (esizeIs32) {
return new SveIndexedMemSV<int32_t, int16_t,
SveGatherLoadSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
} else {
return new SveIndexedMemSV<int64_t, int16_t,
SveGatherLoadSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
}
case 0x3:
if (esizeIs32) {
return new SveIndexedMemSV<uint32_t, uint16_t,
SveGatherLoadSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
} else {
return new SveIndexedMemSV<uint64_t, uint16_t,
SveGatherLoadSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
}
case 0x4:
if (esizeIs32) {
break;
} else {
return new SveIndexedMemSV<int64_t, int32_t,
SveGatherLoadSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
}
case 0x5:
if (esizeIs32) {
return new SveIndexedMemSV<uint32_t, uint32_t,
SveGatherLoadSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
} else {
return new SveIndexedMemSV<uint64_t, uint32_t,
SveGatherLoadSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
}
case 0x7:
if (esizeIs32) {
break;
} else {
return new SveIndexedMemSV<uint64_t, uint64_t,
SveGatherLoadSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemReadOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
}
}
return new Unknown64(machInst);
}
StaticInstPtr
decodeSveScatterStoreVIInsts(uint8_t msz, ExtMachInst machInst,
RegIndex zt, RegIndex pg,
RegIndex zn, uint64_t imm,
bool esizeIs32)
{
const char* mn = "st1";
switch (msz) {
case 0x0:
if (esizeIs32) {
return new SveIndexedMemVI<uint32_t, uint8_t,
SveScatterStoreVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
} else {
return new SveIndexedMemVI<uint64_t, uint8_t,
SveScatterStoreVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
}
case 0x1:
if (esizeIs32) {
return new SveIndexedMemVI<uint32_t, uint16_t,
SveScatterStoreVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
} else {
return new SveIndexedMemVI<uint64_t, uint16_t,
SveScatterStoreVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
}
case 0x2:
if (esizeIs32) {
return new SveIndexedMemVI<uint32_t, uint32_t,
SveScatterStoreVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
} else {
return new SveIndexedMemVI<uint64_t, uint32_t,
SveScatterStoreVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
}
case 0x3:
if (esizeIs32) {
break;
} else {
return new SveIndexedMemVI<uint64_t, uint64_t,
SveScatterStoreVIMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
}
}
return new Unknown64(machInst);
}
StaticInstPtr
decodeSveScatterStoreSVInsts(uint8_t msz, ExtMachInst machInst,
RegIndex zt, RegIndex pg,
RegIndex rn, RegIndex zm,
bool esizeIs32, bool offsetIs32,
bool offsetIsSigned, bool offsetIsScaled)
{
const char* mn = "st1";
switch (msz) {
case 0x0:
if (esizeIs32) {
return new SveIndexedMemSV<uint32_t, uint8_t,
SveScatterStoreSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, false);
} else {
return new SveIndexedMemSV<uint64_t, uint8_t,
SveScatterStoreSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, false);
}
case 0x1:
if (esizeIs32) {
return new SveIndexedMemSV<uint32_t, uint16_t,
SveScatterStoreSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, false);
} else {
return new SveIndexedMemSV<uint64_t, uint16_t,
SveScatterStoreSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, false);
}
case 0x2:
if (esizeIs32) {
return new SveIndexedMemSV<uint32_t, uint32_t,
SveScatterStoreSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, false);
} else {
return new SveIndexedMemSV<uint64_t, uint32_t,
SveScatterStoreSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, false);
}
case 0x3:
if (esizeIs32) {
break;
} else {
return new SveIndexedMemSV<uint64_t, uint64_t,
SveScatterStoreSVMicroop,
SveFirstFaultWritebackMicroop>(
mn, machInst, MemWriteOp, zt, pg, rn, zm,
offsetIs32, offsetIsSigned, offsetIsScaled, false);
}
}
return new Unknown64(machInst);
}
}};
let {{
header_output = ''
exec_output = ''
decoders = { 'Generic': {} }
SPAlignmentCheckCode = '''
if (this->baseIsSP && bits(XBase, 3, 0) &&
SPAlignmentCheckEnabled(xc->tcBase())) {
return std::make_shared<SPAlignmentFault>();
}
'''
def emitSveMemFillSpill(isPred):
global header_output, exec_output, decoders
eaCode = SPAlignmentCheckCode + '''
int memAccessSize = %(memacc_size)s;
EA = XBase + ((int64_t) imm * %(memacc_size)s)''' % {
'memacc_size': 'eCount / 8' if isPred else 'eCount'}
loadRdEnableCode = '''
auto rdEn = std::vector<bool>(memAccessSize, true);
'''
if isPred:
loadMemAccCode = '''
int index = 0;
uint8_t byte;
for (int i = 0; i < eCount / 8; i++) {
byte = memDataView[i];
for (int j = 0; j < 8; j++, index++) {
PDest_x[index] = (byte >> j) & 1;
}
}
'''
storeMemAccCode = '''
int index = 0;
uint8_t byte;
for (int i = 0; i < eCount / 8; i++) {
byte = 0;
for (int j = 0; j < 8; j++, index++) {
byte |= PDest_x[index] << j;
}
memDataView[i] = byte;
}
'''
storeWrEnableCode = '''
auto wrEn = std::vector<bool>(eCount / 8, true);
'''
else:
loadMemAccCode = '''
for (int i = 0; i < eCount; i++) {
AA64FpDest_x[i] = memDataView[i];
}
'''
storeMemAccCode = '''
for (int i = 0; i < eCount; i++) {
memDataView[i] = AA64FpDest_x[i];
}
'''
storeWrEnableCode = '''
auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
'''
loadIop = ArmInstObjParams('ldr',
'SveLdrPred' if isPred else 'SveLdrVec',
'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill',
{'tpl_header': '',
'tpl_args': '',
'memacc_code': loadMemAccCode,
'ea_code' : sveEnabledCheckCode + eaCode,
'rden_code' : loadRdEnableCode,
'fault_code' : '',
'fa_code' : ''},
['IsLoad'])
storeIop = ArmInstObjParams('str',
'SveStrPred' if isPred else 'SveStrVec',
'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill',
{'tpl_header': '',
'tpl_args': '',
'wren_code': storeWrEnableCode,
'memacc_code': storeMemAccCode,
'ea_code' : sveEnabledCheckCode + eaCode,
'fa_code' : ''},
['IsStore'])
header_output += SveMemFillSpillOpDeclare.subst(loadIop)
header_output += SveMemFillSpillOpDeclare.subst(storeIop)
exec_output += (
SveContigLoadExecute.subst(loadIop) +
SveContigLoadInitiateAcc.subst(loadIop) +
SveContigLoadCompleteAcc.subst(loadIop) +
SveContigStoreExecute.subst(storeIop) +
SveContigStoreInitiateAcc.subst(storeIop) +
SveContigStoreCompleteAcc.subst(storeIop))
loadTplArgs = (
('uint8_t', 'uint8_t'),
('uint16_t', 'uint8_t'),
('uint32_t', 'uint8_t'),
('uint64_t', 'uint8_t'),
('int64_t', 'int32_t'),
('uint16_t', 'uint16_t'),
('uint32_t', 'uint16_t'),
('uint64_t', 'uint16_t'),
('int64_t', 'int16_t'),
('int32_t', 'int16_t'),
('uint32_t', 'uint32_t'),
('uint64_t', 'uint32_t'),
('int64_t', 'int8_t'),
('int32_t', 'int8_t'),
('int16_t', 'int8_t'),
('uint64_t', 'uint64_t'),
)
storeTplArgs = (
('uint8_t', 'uint8_t'),
('uint16_t', 'uint8_t'),
('uint32_t', 'uint8_t'),
('uint64_t', 'uint8_t'),
('uint16_t', 'uint16_t'),
('uint32_t', 'uint16_t'),
('uint64_t', 'uint16_t'),
('uint32_t', 'uint32_t'),
('uint64_t', 'uint32_t'),
('uint64_t', 'uint64_t'),
)
gatherLoadTplArgs = (
('int32_t', 'int8_t'),
('int64_t', 'int8_t'),
('uint32_t', 'uint8_t'),
('uint64_t', 'uint8_t'),
('int32_t', 'int16_t'),
('int64_t', 'int16_t'),
('uint32_t', 'uint16_t'),
('uint64_t', 'uint16_t'),
('int64_t', 'int32_t'),
('uint32_t', 'uint32_t'),
('uint64_t', 'uint32_t'),
('uint64_t', 'uint64_t'),
)
scatterStoreTplArgs = (
('uint32_t', 'uint8_t'),
('uint64_t', 'uint8_t'),
('uint32_t', 'uint16_t'),
('uint64_t', 'uint16_t'),
('uint32_t', 'uint32_t'),
('uint64_t', 'uint32_t'),
('uint64_t', 'uint64_t'),
)
# Generates definitions for SVE contiguous loads
def emitSveContigMemInsts(offsetIsImm):
global header_output, exec_output, decoders
# First-faulting instructions only have a scalar plus scalar form,
# while non-faulting instructions only a scalar plus immediate form, so
# `offsetIsImm` is used to determine which class of instructions is
# generated
firstFaulting = not offsetIsImm
tplHeader = 'template <class RegElemType, class MemElemType>'
tplArgs = '<RegElemType, MemElemType>'
eaCode = SPAlignmentCheckCode + '''
int memAccessSize = eCount * sizeof(MemElemType);
EA = XBase + '''
if offsetIsImm:
eaCode += '((int64_t) this->imm * eCount * sizeof(MemElemType))'
else:
eaCode += '(XOffset * sizeof(MemElemType));'
loadRdEnableCode = '''
auto rdEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
for (int i = 0; i < eCount; i++) {
if (!GpOp_x[i]) {
for (int j = 0; j < sizeof(MemElemType); j++) {
rdEn[sizeof(MemElemType) * i + j] = false;
}
}
}
'''
loadMemAccCode = '''
for (int i = 0; i < eCount; i++) {
if (GpOp_x[i]) {
AA64FpDest_x[i] = memDataView[i];
} else {
AA64FpDest_x[i] = 0;
}
}
'''
storeMemAccCode = '''
for (int i = 0; i < eCount; i++) {
if (GpOp_x[i]) {
memDataView[i] = AA64FpDest_x[i];
} else {
memDataView[i] = 0;
for (int j = 0; j < sizeof(MemElemType); j++) {
wrEn[sizeof(MemElemType) * i + j] = false;
}
}
}
'''
storeWrEnableCode = '''
auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
'''
ffrReadBackCode = '''
auto& firstFaultReg = Ffr;'''
fautlingLoadmemAccCode = '''
for (int i = 0; i < eCount; i++) {
if (GpOp_x[i] && firstFaultReg[i * sizeof(RegElemType)]) {
AA64FpDest_x[i] = memDataView[i];
} else {
AA64FpDest_x[i] = 0;
}
}
'''
nonFaultingCode = 'true ||'
faultCode = '''
Addr fault_addr;
if (fault == NoFault || getFaultVAddr(fault, fault_addr)) {
unsigned fault_elem_index;
if (fault != NoFault) {
assert(fault_addr >= EA);
fault_elem_index = (fault_addr - EA) / sizeof(MemElemType);
} else {
fault_elem_index = eCount + 1;
}
int first_active_index;
for (first_active_index = 0;
first_active_index < eCount && !(GpOp_x[first_active_index]);
first_active_index++);
if (%s first_active_index < fault_elem_index) {
for (int i = 0; i < eCount; i++) {
for (int j = 0; j < sizeof(RegElemType); j++) {
if (i < fault_elem_index) {
Ffr_ub[i * sizeof(RegElemType) + j] = FfrAux_x[i];
} else {
Ffr_ub[i * sizeof(RegElemType) + j] = 0;
}
}
}
fault = NoFault;
if (first_active_index >= fault_elem_index) {
// non-faulting load needs this
xc->setMemAccPredicate(false);
}
}
}
''' % ('' if firstFaulting else nonFaultingCode)
loadIop = ArmInstObjParams('ld1',
'SveContigLoadSI' if offsetIsImm else 'SveContigLoadSS',
'SveContigMemSI' if offsetIsImm else 'SveContigMemSS',
{'tpl_header': tplHeader,
'tpl_args': tplArgs,
'rden_code' : loadRdEnableCode,
'memacc_code': loadMemAccCode,
'ea_code' : sveEnabledCheckCode + eaCode,
'fault_code' : '',
'fa_code' : ''},
['IsLoad'])
storeIop = ArmInstObjParams('st1',
'SveContigStoreSI' if offsetIsImm else 'SveContigStoreSS',
'SveContigMemSI' if offsetIsImm else 'SveContigMemSS',
{'tpl_header': tplHeader,
'tpl_args': tplArgs,
'wren_code': storeWrEnableCode,
'memacc_code': storeMemAccCode,
'ea_code' : sveEnabledCheckCode + eaCode,
'fa_code' : ''},
['IsStore'])
faultIop = ArmInstObjParams('ldff1' if firstFaulting else 'ldnf1',
'SveContigFFLoadSS' if firstFaulting else 'SveContigNFLoadSI',
'SveContigMemSS' if firstFaulting else 'SveContigMemSI',
{'tpl_header': tplHeader,
'tpl_args': tplArgs,
'rden_code' : loadRdEnableCode,
'memacc_code': fautlingLoadmemAccCode,
'ea_code' : sveEnabledCheckCode + eaCode,
'fault_code' : faultCode,
'fa_code' : ''},
['IsLoad'])
faultIop.snippets['memacc_code'] = (ffrReadBackCode +
faultIop.snippets['memacc_code'])
if offsetIsImm:
header_output += SveContigMemSIOpDeclare.subst(loadIop)
header_output += SveContigMemSIOpDeclare.subst(storeIop)
header_output += SveContigMemSIOpDeclare.subst(faultIop)
else:
header_output += SveContigMemSSOpDeclare.subst(loadIop)
header_output += SveContigMemSSOpDeclare.subst(storeIop)
header_output += SveContigMemSSOpDeclare.subst(faultIop)
exec_output += (
SveContigLoadExecute.subst(loadIop) +
SveContigLoadInitiateAcc.subst(loadIop) +
SveContigLoadCompleteAcc.subst(loadIop) +
SveContigStoreExecute.subst(storeIop) +
SveContigStoreInitiateAcc.subst(storeIop) +
SveContigStoreCompleteAcc.subst(storeIop) +
SveContigLoadExecute.subst(faultIop) +
SveContigLoadInitiateAcc.subst(faultIop) +
SveContigLoadCompleteAcc.subst(faultIop))
for args in loadTplArgs:
substDict = {'tpl_args': '<%s>' % ', '.join(args),
'class_name': 'SveContigLoadSI' if offsetIsImm
else 'SveContigLoadSS'}
exec_output += SveContigMemExecDeclare.subst(substDict)
for args in storeTplArgs:
substDict = {'tpl_args': '<%s>' % ', '.join(args),
'class_name': 'SveContigStoreSI' if offsetIsImm
else 'SveContigStoreSS'}
exec_output += SveContigMemExecDeclare.subst(substDict)
for args in loadTplArgs:
substDict = {'tpl_args': '<%s>' % ', '.join(args),
'class_name': 'SveContigFFLoadSS' if firstFaulting
else 'SveContigNFLoadSI'}
exec_output += SveContigMemExecDeclare.subst(substDict)
# Generates definitions for SVE load-and-replicate instructions
def emitSveLoadAndRepl():
global header_output, exec_output, decoders
tplHeader = 'template <class RegElemType, class MemElemType>'
tplArgs = '<RegElemType, MemElemType>'
eaCode = SPAlignmentCheckCode + '''
EA = XBase + imm * sizeof(MemElemType);'''
memAccCode = '''
for (int i = 0; i < eCount; i++) {
if (GpOp_x[i]) {
AA64FpDest_x[i] = memData;
} else {
AA64FpDest_x[i] = 0;
}
}
'''
iop = ArmInstObjParams('ld1r',
'SveLoadAndRepl',
'SveContigMemSI',
{'tpl_header': tplHeader,
'tpl_args': tplArgs,
'memacc_code': memAccCode,
'ea_code' : sveEnabledCheckCode + eaCode,
'fa_code' : ''},
['IsLoad'])
header_output += SveContigMemSIOpDeclare.subst(iop)
exec_output += (
SveLoadAndReplExecute.subst(iop) +
SveLoadAndReplInitiateAcc.subst(iop) +
SveLoadAndReplCompleteAcc.subst(iop))
for args in loadTplArgs:
substDict = {'tpl_args': '<%s>' % ', '.join(args),
'class_name': 'SveLoadAndRepl'}
exec_output += SveContigMemExecDeclare.subst(substDict)
class IndexedAddrForm:
VEC_PLUS_IMM = 0
SCA_PLUS_VEC = 1
# Generates definitions for the transfer microops of SVE indexed memory
# operations (gather loads, scatter stores)
def emitSveIndexedMemMicroops(indexed_addr_form):
assert indexed_addr_form in (IndexedAddrForm.VEC_PLUS_IMM,
IndexedAddrForm.SCA_PLUS_VEC)
global header_output, exec_output, decoders
tplHeader = 'template <class RegElemType, class MemElemType>'
tplArgs = '<RegElemType, MemElemType>'
if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM:
eaCode_store = '''
EA = AA64FpBase_x[elemIndex] + imm * sizeof(MemElemType)'''
eaCode_load = '''
EA = AA64FpUreg0_x[elemIndex] + imm * sizeof(MemElemType)'''
else:
offset_code = '''
if (offsetIs32) {
offset &= (1ULL << 32) - 1;
}
if (offsetIsSigned) {
offset = sext<32>(offset);
}
if (offsetIsScaled) {
offset *= sizeof(MemElemType);
}
EA = XBase + offset'''
eaCode_store = '''
uint64_t offset = AA64FpOffset_x[elemIndex];''' + offset_code
eaCode_load = '''
uint64_t offset = AA64FpUreg0_x[elemIndex];''' + offset_code
loadMemAccCode = '''
AA64FpDest_x[elemIndex] = memData;
'''
storeMemAccCode = '''
memData = AA64FpDest_x[elemIndex];
'''
predCheckCode = 'GpOp_x[index]'
faultStatusSetCode = 'PUreg0_x[elemIndex] = 1;'
faultStatusResetCode = 'PUreg0_x[elemIndex] = 0;'
loadIop = ArmInstObjParams('ld1',
('SveGatherLoadVIMicroop'
if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM
else 'SveGatherLoadSVMicroop'),
'MicroOp',
{'tpl_header': tplHeader,
'tpl_args': tplArgs,
'memacc_code': loadMemAccCode,
'ea_code' : sveEnabledCheckCode + eaCode_load,
'fault_status_set_code' : faultStatusSetCode,
'fault_status_reset_code' : faultStatusResetCode,
'pred_check_code' : predCheckCode,
'fa_code' : ''},
['IsMicroop', 'IsLoad'])
storeIop = ArmInstObjParams('st1',
('SveScatterStoreVIMicroop'
if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM
else 'SveScatterStoreSVMicroop'),
'MicroOp',
{'tpl_header': tplHeader,
'tpl_args': tplArgs,
'memacc_code': storeMemAccCode,
'ea_code' : sveEnabledCheckCode + eaCode_store,
'pred_check_code' : predCheckCode,
'fa_code' : ''},
['IsMicroop', 'IsStore'])
if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM:
header_output += SveIndexedMemVIMicroopDeclare.subst(loadIop)
header_output += SveIndexedMemVIMicroopDeclare.subst(storeIop)
else:
header_output += SveIndexedMemSVMicroopDeclare.subst(loadIop)
header_output += SveIndexedMemSVMicroopDeclare.subst(storeIop)
exec_output += (
SveGatherLoadMicroopExecute.subst(loadIop) +
SveGatherLoadMicroopInitiateAcc.subst(loadIop) +
SveGatherLoadMicroopCompleteAcc.subst(loadIop) +
SveScatterStoreMicroopExecute.subst(storeIop) +
SveScatterStoreMicroopInitiateAcc.subst(storeIop) +
SveScatterStoreMicroopCompleteAcc.subst(storeIop))
for args in gatherLoadTplArgs:
substDict = {'tpl_args': '<%s>' % ', '.join(args),
'class_name': (
'SveGatherLoadVIMicroop'
if indexed_addr_form == \
IndexedAddrForm.VEC_PLUS_IMM
else 'SveGatherLoadSVMicroop')}
# TODO: this should become SveMemExecDeclare
exec_output += SveContigMemExecDeclare.subst(substDict)
for args in scatterStoreTplArgs:
substDict = {'tpl_args': '<%s>' % ', '.join(args),
'class_name': (
'SveScatterStoreVIMicroop'
if indexed_addr_form == \
IndexedAddrForm.VEC_PLUS_IMM
else 'SveScatterStoreSVMicroop')}
# TODO: this should become SveMemExecDeclare
exec_output += SveContigMemExecDeclare.subst(substDict)
firstFaultTplArgs = ('int32_t', 'int64_t', 'uint32_t', 'uint64_t')
def emitSveFirstFaultWritebackMicroop():
global header_output, exec_output, decoders
tplHeader = 'template <class RegElemType>'
tplArgs = '<RegElemType>'
faultStatusCheckCode = 'PUreg0_x[index]'
firstFaultResetCode = '''
for(int j = 0; j < sizeof(RegElemType); j++) {
Ffr_ub[index * sizeof(RegElemType) + j] = 0;
}
'''
firstFaultForwardCode = '''
for(int j = 0; j < sizeof(RegElemType); j++) {
Ffr_ub[index * sizeof(RegElemType) + j] = FfrAux_x[index];
}
'''
iop = ArmInstObjParams('ldff1',
'SveFirstFaultWritebackMicroop',
'MicroOp',
{'tpl_header': tplHeader,
'tpl_args': tplArgs,
'fault_status_check_code' : faultStatusCheckCode,
'first_fault_reset_code' : firstFaultResetCode,
'first_fault_forward_code' : firstFaultForwardCode},
['IsMicroop'])
header_output += SveFirstFaultWritebackMicroopDeclare.subst(iop)
exec_output += SveFirstFaultWritebackMicroopExecute.subst(iop)
for args in firstFaultTplArgs:
substDict = {'targs': args,
'class_name' : 'SveFirstFaultWritebackMicroop' }
exec_output += SveOpExecDeclare.subst(substDict)
# Generates definitions for the first microop of SVE gather loads, required
# to propagate the source vector register to the transfer microops
def emitSveGatherLoadCpySrcVecMicroop():
global header_output, exec_output, decoders
code = sveEnabledCheckCode + '''
unsigned eCount = ArmStaticInst::getCurSveVecLen<uint8_t>(
xc->tcBase());
for (unsigned i = 0; i < eCount; i++) {
AA64FpUreg0_ub[i] = AA64FpOp1_ub[i];
}'''
iop = ArmInstObjParams('ld1',
'SveGatherLoadCpySrcVecMicroop',
'MicroOp',
{'code': code},
['IsMicroop'])
header_output += SveGatherLoadCpySrcVecMicroopDeclare.subst(iop)
exec_output += SveGatherLoadCpySrcVecMicroopExecute.subst(iop)
def emitSveInterleaveMicroop():
global header_output, exec_output, decoders
code2 = sveEnabledCheckCode + '''
unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
xc->tcBase());
for (unsigned int i = 0; i < eCount; ++i) {
unsigned int absIdx = regIndex * eCount + i;
unsigned int srcIdx = absIdx / numRegs;
unsigned int srcVec = absIdx % numRegs;
if (srcVec == 0)
AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx];
else if (srcVec == 1)
AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx];
}'''
code3 = sveEnabledCheckCode + '''
unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
xc->tcBase());
for (unsigned int i = 0; i < eCount; ++i) {
unsigned int absIdx = regIndex * eCount + i;
unsigned int srcIdx = absIdx / numRegs;
unsigned int srcVec = absIdx % numRegs;
if (srcVec == 0)
AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx];
else if (srcVec == 1)
AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx];
else if (srcVec == 2)
AA64FpDest_x[i] = AA64FpOp1V2S_x[srcIdx];
}'''
code4 = sveEnabledCheckCode + '''
unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
xc->tcBase());
for (unsigned int i = 0; i < eCount; ++i) {
unsigned int absIdx = regIndex * eCount + i;
unsigned int srcIdx = absIdx / numRegs;
unsigned int srcVec = absIdx % numRegs;
if (srcVec == 0)
AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx];
else if (srcVec == 1)
AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx];
else if (srcVec == 2)
AA64FpDest_x[i] = AA64FpOp1V2S_x[srcIdx];
else if (srcVec == 3)
AA64FpDest_x[i] = AA64FpOp1V3S_x[srcIdx];
}'''
iop2 = ArmInstObjParams('intrlv',
'SveIntrlv2Microop',
'MicroOp',
{'code': code2},
['IsMicroop'])
iop3 = ArmInstObjParams('intrlv',
'SveIntrlv3Microop',
'MicroOp',
{'code': code3},
['IsMicroop'])
iop4 = ArmInstObjParams('intrlv',
'SveIntrlv4Microop',
'MicroOp',
{'code': code4},
['IsMicroop'])
header_output += SveIntrlvMicroopDeclare.subst(iop2);
header_output += SveIntrlvMicroopDeclare.subst(iop3);
header_output += SveIntrlvMicroopDeclare.subst(iop4);
exec_output += SveIntrlvMicroopExecute.subst(iop2);
exec_output += SveIntrlvMicroopExecute.subst(iop3);
exec_output += SveIntrlvMicroopExecute.subst(iop4);
for type in ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'):
for nreg in range(2,5):
substDict = {'targs' : type,
'class_name' : 'SveIntrlv' + str(nreg) + 'Microop'}
exec_output += SveIntrlvMicroopExecDeclare.subst(substDict)
def emitSveDeInterleaveMicroop():
global header_output, exec_output, decoders
code2 = sveEnabledCheckCode + '''
unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
xc->tcBase());
for (unsigned int i = 0; i < eCount; ++i) {
unsigned int absIdx = (regIndex + numRegs * i);
unsigned int srcIdx = absIdx % eCount;
unsigned int srcVec = absIdx / eCount;
if (srcVec == 0)
AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx];
else if(srcVec == 1)
AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx];
}'''
code3 = sveEnabledCheckCode + '''
unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
xc->tcBase());
for (unsigned int i = 0; i < eCount; ++i) {
unsigned int absIdx = (regIndex + numRegs * i);
unsigned int srcIdx = absIdx % eCount;
unsigned int srcVec = absIdx / eCount;
if (srcVec == 0)
AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx];
else if(srcVec == 1)
AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx];
else if(srcVec == 2)
AA64FpDest_x[i] = AA64IntrlvReg2_x[srcIdx];
}'''
code4 = sveEnabledCheckCode + '''
unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
xc->tcBase());
for (unsigned int i = 0; i < eCount; ++i) {
unsigned int absIdx = (regIndex + numRegs * i);
unsigned int srcIdx = absIdx % eCount;
unsigned int srcVec = absIdx / eCount;
if (srcVec == 0)
AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx];
else if(srcVec == 1)
AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx];
else if(srcVec == 2)
AA64FpDest_x[i] = AA64IntrlvReg2_x[srcIdx];
else if(srcVec == 3)
AA64FpDest_x[i] = AA64IntrlvReg3_x[srcIdx];
}'''
iop2 = ArmInstObjParams('deintrlv',
'SveDeIntrlv2Microop',
'MicroOp',
{'code': code2},
['IsMicroop'])
iop3 = ArmInstObjParams('deintrlv',
'SveDeIntrlv3Microop',
'MicroOp',
{'code': code3},
['IsMicroop'])
iop4 = ArmInstObjParams('deintrlv',
'SveDeIntrlv4Microop',
'MicroOp',
{'code': code4},
['IsMicroop'])
header_output += SveDeIntrlvMicroopDeclare.subst(iop2);
header_output += SveDeIntrlvMicroopDeclare.subst(iop3);
header_output += SveDeIntrlvMicroopDeclare.subst(iop4);
exec_output += SveIntrlvMicroopExecute.subst(iop2);
exec_output += SveIntrlvMicroopExecute.subst(iop3);
exec_output += SveIntrlvMicroopExecute.subst(iop4);
for type in ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'):
for nreg in range(2,5):
substDict = {'targs' : type,
'class_name' : 'SveDeIntrlv' + str(nreg) + 'Microop'}
exec_output += SveIntrlvMicroopExecDeclare.subst(substDict)
# Generates definitions for SVE struct load/store microops
def emitSveStructMemInsts(offsetIsImm):
global header_output, exec_output, decoders
eaCode = SPAlignmentCheckCode + '''
int memAccessSize = eCount * sizeof(Element);
EA = memAccessSize * regIndex + XBase + '''
if offsetIsImm:
eaCode += '((int64_t) this->imm * eCount * sizeof(Element))'
else:
eaCode += '(XOffset * sizeof(Element));'
loadMemAccCode = '''
for (int i = 0; i < eCount; i++) {
int gpIdx = (regIndex * eCount + i) / numRegs;
if (GpOp_x[gpIdx]) {
AA64FpDest_x[i] = memDataView[i];
} else {
AA64FpDest_x[i] = 0;
}
}
'''
storeMemAccCode = '''
for (int i = 0; i < eCount; i++) {
int gpIdx = (regIndex * eCount + i) / numRegs;
if (GpOp_x[gpIdx]) {
memDataView[i] = AA64FpDest_x[i];
} else {
memDataView[i] = 0;
for (int j = 0; j < sizeof(Element); j++) {
wrEn[sizeof(Element) * i + j] = false;
}
}
}
'''
storeWrEnableCode = '''
auto wrEn = std::vector<bool>(sizeof(Element) * eCount, true);
'''
loadIop = ArmInstObjParams('ldxx',
'SveLoadRegImmMicroop' if offsetIsImm else 'SveLoadRegRegMicroop',
'MicroOp',
{'targs': 'Element',
'memacc_code': loadMemAccCode,
'ea_code' : sveEnabledCheckCode + eaCode,
'fa_code' : ''},
['IsLoad', 'IsMicroop'])
storeIop = ArmInstObjParams('stxx',
'SveStoreRegImmMicroop' if offsetIsImm
else 'SveStoreRegRegMicroop',
'MicroOp',
{'targs': 'Element',
'wren_code': storeWrEnableCode,
'memacc_code': storeMemAccCode,
'ea_code' : sveEnabledCheckCode + eaCode,
'fa_code' : ''},
['IsStore', 'IsMicroop'])
if offsetIsImm:
header_output += SveStructMemSIMicroopDeclare.subst(loadIop)
header_output += SveStructMemSIMicroopDeclare.subst(storeIop)
else:
header_output += SveStructMemSSMicroopDeclare.subst(loadIop)
header_output += SveStructMemSSMicroopDeclare.subst(storeIop)
exec_output += (
SveStructLoadExecute.subst(loadIop) +
SveStructLoadInitiateAcc.subst(loadIop) +
SveStructLoadCompleteAcc.subst(loadIop) +
SveStructStoreExecute.subst(storeIop) +
SveStructStoreInitiateAcc.subst(storeIop) +
SveStructStoreCompleteAcc.subst(storeIop))
tplArgs = ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t')
for type in tplArgs:
substDict = {'targs': type,
'class_name': 'SveLoadRegImmMicroop' if offsetIsImm
else 'SveLoadRegRegMicroop'}
exec_output += SveStructMemExecDeclare.subst(substDict)
substDict['class_name'] = ('SveStoreRegImmMicroop' if offsetIsImm
else 'SveStoreRegRegMicroop')
exec_output += SveStructMemExecDeclare.subst(substDict)
# Generates definitions for SVE load-and-replicate quadword instructions
def emitSveLoadAndReplQuad(offsetIsImm):
global header_output, exec_output, decoders
tplHeader = 'template <class RegElemType, class MemElemType>'
tplArgs = '<RegElemType, MemElemType>'
eaCode = SPAlignmentCheckCode + '''
int memAccessSize = 16;
EA = XBase + '''
if offsetIsImm:
eaCode += '(((int64_t) this->imm) * 16);'
else:
eaCode += '(XOffset * sizeof(MemElemType));'
loadRdEnableCode = '''
eCount = 16/sizeof(RegElemType);
auto rdEn = std::vector<bool>(16, true);
for (int i = 0; i < eCount; ++i) {
if (!GpOp_x[i]) {
for (int j = 0; j < sizeof(RegElemType); ++j) {
rdEn[sizeof(RegElemType) * i + j] = false;
}
}
}
'''
memAccCode = '''
__uint128_t qword;
RegElemType* qp = reinterpret_cast<RegElemType*>(&qword);
for (int i = 0; i < 16/sizeof(RegElemType); ++i) {
if (GpOp_x[i]) {
qp[i] = memDataView[i];
} else {
qp[i] = 0;
}
}
eCount = ArmStaticInst::getCurSveVecLen<__uint128_t>(
xc->tcBase());
for (int i = 0; i < eCount; ++i) {
AA64FpDest_uq[i] = qword;
}
'''
iop = ArmInstObjParams('ld1rq',
'SveLd1RqSI' if offsetIsImm else 'SveLd1RqSS',
'SveContigMemSI' if offsetIsImm else 'SveContigMemSS',
{'tpl_header': tplHeader,
'tpl_args': tplArgs,
'rden_code': loadRdEnableCode,
'memacc_code': memAccCode,
'ea_code': sveEnabledCheckCode + eaCode,
'fault_code': '',
'fa_code': ''},
['IsLoad'])
if offsetIsImm:
header_output += SveContigMemSIOpDeclare.subst(iop)
else:
header_output += SveContigMemSSOpDeclare.subst(iop)
exec_output += (
SveContigLoadExecute.subst(iop) +
SveContigLoadInitiateAcc.subst(iop) +
SveContigLoadCompleteAcc.subst(iop))
for ttype in ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'):
substDict = {'tpl_args': '<%s, %s>' % (ttype, ttype),
'class_name': 'SveLd1RqSI' if offsetIsImm
else 'SveLd1RqSS'}
exec_output += SveContigMemExecDeclare.subst(substDict)
# LD1[S]{B,H,W,D} (scalar plus immediate)
# ST1[S]{B,H,W,D} (scalar plus immediate)
# LDNF1[S]{B,H,W,D} (scalar plus immediate)
emitSveContigMemInsts(True)
# LD1[S]{B,H,W,D} (scalar plus scalar)
# ST1[S]{B,H,W,D} (scalar plus scalar)
# LDFF1[S]{B,H,W,D} (scalar plus vector)
emitSveContigMemInsts(False)
# LD1R[S]{B,H,W,D}
emitSveLoadAndRepl()
# LD1RQ{B,H,W,D} (scalar plus immediate)
emitSveLoadAndReplQuad(offsetIsImm = True)
# LD1RQ{B,H,W,D} (scalar plus scalar)
emitSveLoadAndReplQuad(offsetIsImm = False)
# LD{2,3,4}{B,H,W,D} (scalar plus immediate)
# ST{2,3,4}{B,H,W,D} (scalar plus immediate)
emitSveStructMemInsts(offsetIsImm = True)
# LD{2,3,4}{B,H,W,D} (scalar plus scalar)
# ST{2,3,4}{B,H,W,D} (scalar plus scalar)
emitSveStructMemInsts(offsetIsImm = False)
# LDR (predicate), STR (predicate)
emitSveMemFillSpill(True)
# LDR (vector), STR (vector)
emitSveMemFillSpill(False)
# LD1[S]{B,H,W,D} (vector plus immediate)
# ST1[S]{B,H,W,D} (vector plus immediate)
# LDFF1[S]{B,H,W,D} (scalar plus immediate)
emitSveIndexedMemMicroops(IndexedAddrForm.VEC_PLUS_IMM)
# LD1[S]{B,H,W,D} (scalar plus vector)
# ST1[S]{B,H,W,D} (scalar plus vector)
# LDFF1[S]{B,H,W,D} (scalar plus vector)
emitSveIndexedMemMicroops(IndexedAddrForm.SCA_PLUS_VEC)
# FFR writeback microop for gather loads
emitSveFirstFaultWritebackMicroop()
# Source vector copy microop for gather loads
emitSveGatherLoadCpySrcVecMicroop()
# ST/LD struct de/interleave microops
emitSveInterleaveMicroop()
emitSveDeInterleaveMicroop()
}};