| // Copyright (c) 2022 ARM Limited |
| // All rights reserved |
| // |
| // The license below extends only to copyright in the software and shall |
| // not be construed as granting a license to any other intellectual |
| // property including but not limited to intellectual property relating |
| // to a hardware implementation of the functionality of the software |
| // licensed hereunder. You may use the software subject to the license |
| // terms below provided that you ensure that this notice is replicated |
| // unmodified and in its entirety in all distributions of the software, |
| // modified or unmodified, in source code or in binary form. |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions are |
| // met: redistributions of source code must retain the above copyright |
| // notice, this list of conditions and the following disclaimer; |
| // redistributions in binary form must reproduce the above copyright |
| // notice, this list of conditions and the following disclaimer in the |
| // documentation and/or other materials provided with the distribution; |
| // neither the name of the copyright holders nor the names of its |
| // contributors may be used to endorse or promote products derived from |
| // this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| // @file Definition of SME instructions. |
| |
| let {{ |
| |
| header_output = "" |
| decoder_output = "" |
| exec_output = "" |
| |
| def smeAddInst(name, Name, opClass, types, op): |
| global header_output, decoder_output, exec_output |
| code = smEnCheckCode + smeZaWrite + ''' |
| // imm stores the tile index |
| // op1 is the source SVE vector register |
| // gp1 is the row predecate register |
| // gp2 is the column predecate register |
| |
| unsigned eCount = ArmStaticInst::getCurSmeVecLen<TPElem>( |
| xc->tcBase()); |
| |
| uint8_t tile_index = imm & 0x7; |
| |
| // View the tile as the correct data type, extract the sub-tile |
| auto tile = getTile<TPElem>(ZA, tile_index); |
| ''' |
| code += op |
| |
| iop = InstObjParams(name, "Sme" + Name, "SmeAddOp", |
| {'code': code, 'op_class': opClass}, |
| ['IsNonSpeculative']) |
| header_output += SmeAddDeclare.subst(iop) |
| exec_output += SmeTemplatedExecute.subst(iop) |
| |
| for type in types: |
| substDict = {'targs' : type, |
| 'class_name' : 'Sme' + Name} |
| exec_output += SmeOpExecDeclare.subst(substDict) |
| |
| def smeAddVlInst(name, Name, opClass, op): |
| global header_output, decoder_output, exec_output |
| code = smEnCheckCodeNoPstate + ''' |
| // dest is the 64-bit destination register |
| // op1 is the 64-bit source register |
| // imm is a signed multiplier |
| ''' |
| code += op |
| |
| iop = InstObjParams(name, "Sme" + Name, "SmeAddVlOp", |
| {'code': code, 'op_class': opClass}, |
| ['IsNonSpeculative']) |
| header_output += SmeAddVlDeclare.subst(iop) |
| exec_output += SmeExecute.subst(iop) |
| |
| def smeLd1xInst(name, Name, opClass, types): |
| global header_output, decoder_output, exec_output |
| code = smEnCheckCode + smeZaWrite + ''' |
| // imm stores the tile number as well as the vector offset. The |
| // size of the fields changes based on the data type being used. |
| // XOp1 stores Rn |
| // GpOp stores the governing predicate register |
| // WOp2 stores Rs - the vector index register |
| // XOp3 stores Rm - the offset register (applied to Rn) |
| |
| |
| unsigned eCount = ArmStaticInst::getCurSmeVecLen<TPElem>( |
| xc->tcBase()); |
| |
| uint8_t offset = imm & (0xf >> (findMsbSet(sizeof(TPElem)))); |
| M5_VAR_USED uint8_t tile_idx = |
| imm >> (4 - findMsbSet(sizeof(TPElem))); |
| M5_VAR_USED uint8_t vec_idx = (WOp2 + offset) % eCount; |
| |
| // Calculate the address |
| M5_VAR_USED Addr EA = XOp1 + XOp3 * sizeof(TPElem); |
| |
| // Calculate the read predicate. One boolean per byte, |
| // initialised to all true. |
| auto rdEn = std::vector<bool>(eCount * sizeof(TPElem), true); |
| for (int i = 0; i < eCount; ++i) { |
| if (GpOp_x[i]) { |
| continue; |
| } |
| |
| // Mark each byte of the corresponding elem as false |
| for (int j = 0; j < sizeof(TPElem); ++j) { |
| rdEn[i * sizeof(TPElem) + j] = false; |
| } |
| } |
| ''' |
| |
| zaWriteCode = ''' |
| // Here we write the data we just got from memory to the tile: |
| if (V) { |
| auto col = getTileVSlice<TPElem>(ZA, tile_idx, vec_idx); |
| for(int i = 0; i < eCount; ++i) { |
| col[i] = GpOp_x[i] ? data[i] : 0; |
| } |
| } else { |
| auto row = getTileHSlice<TPElem>(ZA, tile_idx, vec_idx); |
| for(int i = 0; i < eCount; ++i) { |
| row[i] = GpOp_x[i] ? data[i] : 0; |
| } |
| } |
| ''' |
| |
| iop = InstObjParams(name, "Sme" + Name, "SmeLd1xSt1xOp", |
| {'code': code, 'za_write': zaWriteCode, |
| 'op_class': opClass}, ['IsLoad', |
| 'IsNonSpeculative']) |
| header_output += SmeLd1xDeclare.subst(iop) |
| exec_output += SmeLd1xExecute.subst(iop) |
| exec_output += SmeLd1xInitiateAcc.subst(iop) |
| exec_output += SmeLd1xCompleteAcc.subst(iop) |
| for type in types: |
| substDict = {'targs' : type, |
| 'class_name' : 'Sme' + Name} |
| exec_output += SmeLd1xExecDeclare.subst(substDict) |
| |
| def smeLdrInst(name, Name, opClass): |
| global header_output, decoder_output, exec_output |
| code = smEnCheckCodeNoSM + smeZaWrite + ''' |
| // imm stores the vector offset. We do not have a tile number as |
| // we target the whole accumulator array. |
| // imm also stores the offset applied to the base memory access |
| // register. |
| // Op1 stores Rn, which is the base memory access register |
| // Op2 stores Rv, which is the vector select register |
| |
| |
| unsigned eCount = ArmStaticInst::getCurSmeVecLen<uint8_t>( |
| xc->tcBase()); |
| |
| M5_VAR_USED uint8_t vec_index = (WOp2 + imm) % eCount; |
| |
| // Calculate the address |
| M5_VAR_USED Addr EA = XOp1 + imm; |
| ''' |
| |
| iop = InstObjParams(name, "Sme" + Name, "SmeLdrStrOp", |
| {'code': code, 'op_class': opClass}, |
| ['IsLoad', 'IsNonSpeculative']) |
| header_output += SmeLdrDeclare.subst(iop) |
| exec_output += SmeLdrExecute.subst(iop) |
| exec_output += SmeLdrInitiateAcc.subst(iop) |
| exec_output += SmeLdrCompleteAcc.subst(iop) |
| |
| def smeMovaExtractInst(name, Name, opClass, types): |
| global header_output, decoder_output, exec_output |
| code = smEnCheckCode + ''' |
| // imm stores the tile index |
| // op1 is the source SVE vector register |
| // gp is the governing predecate register |
| // op2 is the slice index register |
| // v is the row/col select immediate - true for column accesses |
| |
| unsigned eCount = ArmStaticInst::getCurSmeVecLen<TPElem>( |
| xc->tcBase()); |
| |
| uint8_t offset = imm & (0xf >> (findMsbSet(sizeof(TPElem)))); |
| uint8_t tile_idx = imm >> (4 - findMsbSet(sizeof(TPElem))); |
| |
| uint32_t vec_idx = (WOp2 + offset) % eCount; |
| |
| if (!v) { // Horizontal (row) access |
| auto row = getTileHSlice<TPElem>(ZA, tile_idx, vec_idx); |
| for (int i = 0; i < eCount; ++i) { |
| if (!GpOp_x[i]) { |
| continue; |
| } |
| |
| AA64FpOp1_x[i] = row[i]; |
| } |
| } else { // Vertical (column) access |
| auto col = getTileVSlice<TPElem>(ZA, tile_idx, vec_idx); |
| for (int i = 0; i < eCount; ++i) { |
| if (!GpOp_x[i]) { |
| continue; |
| } |
| |
| AA64FpOp1_x[i] = col[i]; |
| } |
| } |
| ''' |
| |
| iop = InstObjParams(name, "Sme" + Name, "SmeMovExtractOp", |
| {'code': code, 'op_class': opClass}, |
| ['IsNonSpeculative']) |
| header_output += SmeMovaExtractDeclare.subst(iop) |
| exec_output += SmeTemplatedExecute.subst(iop) |
| |
| for type in types: |
| substDict = {'targs' : type, |
| 'class_name' : 'Sme' + Name} |
| exec_output += SmeOpExecDeclare.subst(substDict) |
| |
| def smeMovaInsertInst(name, Name, opClass, types): |
| global header_output, decoder_output, exec_output |
| code = smEnCheckCode + smeZaWrite + ''' |
| // imm stores the tile index |
| // op1 is the source SVE vector register |
| // gp is the governing predecate register |
| // op2 is the slice index register |
| // v is the row/col select immediate - true for column accesses |
| |
| unsigned eCount = ArmStaticInst::getCurSmeVecLen<TPElem>( |
| xc->tcBase()); |
| |
| uint8_t offset = imm & (0xf >> (findMsbSet(sizeof(TPElem)))); |
| uint8_t tile_idx = imm >> (4 - findMsbSet(sizeof(TPElem))); |
| |
| uint32_t vec_idx = (WOp2 + offset) % eCount; |
| |
| if (!v) { // Horizontal (row) access |
| auto row = getTileHSlice<TPElem>(ZA, tile_idx, vec_idx); |
| for (int i = 0; i < eCount; ++i) { |
| if (!GpOp_x[i]) { |
| continue; |
| } |
| |
| row[i] = AA64FpOp1_x[i]; |
| } |
| } else { // Vertical (column) access |
| auto col = getTileVSlice<TPElem>(ZA, tile_idx, vec_idx); |
| for (int i = 0; i < eCount; ++i) { |
| if (!GpOp_x[i]) { |
| continue; |
| } |
| |
| col[i] = AA64FpOp1_x[i]; |
| } |
| } |
| ''' |
| |
| iop = InstObjParams(name, "Sme" + Name, "SmeMovInsertOp", |
| {'code': code, 'op_class': opClass}, |
| ['IsNonSpeculative']) |
| header_output += SmeMovaInsertDeclare.subst(iop) |
| exec_output += SmeTemplatedExecute.subst(iop) |
| |
| for type in types: |
| substDict = {'targs' : type, |
| 'class_name' : 'Sme' + Name} |
| exec_output += SmeOpExecDeclare.subst(substDict) |
| |
| def smeMsrInst(name, Name, opClass, op): |
| global header_output, decoder_output, exec_output |
| code = ''' |
| if (FullSystem) { |
| fault = this->checkSmeAccess(xc->tcBase(), Cpsr, Cpacr64); |
| if (fault != NoFault) { |
| return fault; |
| } |
| } |
| ''' + op |
| |
| iop = InstObjParams(name, "Sme" + Name, "ImmOp64", |
| {'code': code, 'op_class': opClass}, |
| ['IsNonSpeculative', 'IsSerializeAfter']) |
| header_output += SMEMgmtDeclare.subst(iop) |
| exec_output += SmeExecute.subst(iop) |
| |
| def smeFPOPInst(name, Name, opClass, srcTypes, dstTypes, op): |
| global header_output, decoder_output, exec_output |
| code = smEnCheckCode + smeZaWrite + ''' |
| // imm stores the tile index |
| // op1 is the first SVE vector register |
| // gp1 is the predecate register corresponding to the first |
| // SVE vector register |
| // gp2 is the predecate register corresponding to the second |
| // SVE vector register |
| // op2 is the second SVE vector register |
| |
| unsigned eCount = ArmStaticInst::getCurSmeVecLen<TPDElem>( |
| xc->tcBase()); |
| ''' |
| code += op |
| |
| iop = InstObjParams(name, "Sme" + Name, "SmeOPOp", |
| {'code': code, 'op_class': opClass}, |
| ['IsNonSpeculative']) |
| header_output += SmeFPOPDeclare.subst(iop) |
| exec_output += SmeDualTemplatedExecute.subst(iop) |
| for src, dst in zip(srcTypes, dstTypes): |
| substDict = {'targs' : "{}, {}".format(src, dst), |
| 'class_name' : 'Sme' + Name} |
| exec_output += SmeOpExecDeclare.subst(substDict) |
| |
| def smeIntOPInst(name, Name, opClass, src1Types, src2Types, dstTypes, op): |
| global header_output, decoder_output, exec_output |
| code = smEnCheckCode + smeZaWrite + ''' |
| // imm stores the tile index |
| // op1 is the first SVE vector register |
| // gp1 is the predecate register corresponding to the first |
| // SVE vector register |
| // gp2 is the predecate register corresponding to the second |
| // SVE vector register |
| // op2 is the second SVE vector register |
| |
| unsigned eCount = ArmStaticInst::getCurSmeVecLen<TPDElem>( |
| xc->tcBase()); |
| ''' |
| code += op |
| |
| iop = InstObjParams(name, "Sme" + Name, "SmeOPOp", |
| {'code': code, 'op_class': opClass}, |
| ['IsNonSpeculative']) |
| header_output += SmeIntOPDeclare.subst(iop) |
| exec_output += SmeTripleTemplatedExecute.subst(iop) |
| for src1, src2, dst in zip(src1Types, src2Types, dstTypes): |
| substDict = {'targs' : "{}, {}, {}".format(src1, src2, dst), |
| 'class_name' : 'Sme' + Name} |
| exec_output += SmeOpExecDeclare.subst(substDict) |
| |
| def smeRdsvlInst(name, Name, opClass): |
| global header_output, decoder_output, exec_output |
| code = smEnCheckCodeNoPstate + ''' |
| // dest is the 64-bit destination register |
| // imm is a signed multiplier |
| |
| unsigned eCount = ArmStaticInst::getCurSmeVecLen<uint8_t>( |
| xc->tcBase()); |
| |
| Dest64 = eCount * imm; |
| ''' |
| |
| iop = InstObjParams(name, "Sme" + Name, "SmeRdsvlOp", |
| {'code': code, 'op_class': opClass}, |
| ['IsNonSpeculative']) |
| header_output += SmeRdsvlDeclare.subst(iop) |
| exec_output += SmeExecute.subst(iop) |
| |
| def smeSt1xInst(name, Name, opClass, types): |
| global header_output, decoder_output, exec_output |
| code = smEnCheckCode + ''' |
| // imm stores the tile number as well as the vector offset. The |
| // size of the fields changes based on the data type being used. |
| // XOp1 stores Rn |
| // GpOp stores the governing predicate register |
| // WOp2 stores Rs - the vector index register |
| // XOp3 stores Rm - the offset register (applied to Rn) |
| |
| |
| unsigned eCount = ArmStaticInst::getCurSmeVecLen<TPElem>( |
| xc->tcBase()); |
| |
| uint8_t offset = imm & (0xf >> (findMsbSet(sizeof(TPElem)))); |
| M5_VAR_USED uint8_t tile_idx = |
| imm >> (4 - findMsbSet(sizeof(TPElem))); |
| M5_VAR_USED uint8_t vec_idx = (WOp2 + offset) % eCount; |
| |
| // Calculate the address |
| M5_VAR_USED Addr EA = XOp1 + XOp3 * sizeof(TPElem); |
| |
| // Calculate the write predicate. One boolean per byte, |
| // initialised to all true. |
| auto wrEn = std::vector<bool>(eCount * sizeof(TPElem), true); |
| for (int i = 0; i < eCount; ++i) { |
| if (GpOp_x[i]) { |
| continue; |
| } |
| |
| // Mark each byte of the corresponding elem as false |
| for (int j = 0; j < sizeof(TPElem); ++j) { |
| wrEn[i * sizeof(TPElem) + j] = false; |
| } |
| } |
| |
| // Extract the data to be stored from the tile. We don't worry |
| // about the predicate here as that's already handled by wrEn. |
| TPElem data[MaxSmeVecLenInBytes / sizeof(TPElem)]; |
| if(V) { |
| auto col = getTileVSlice<TPElem>(ZA, tile_idx, vec_idx); |
| for (int i = 0; i < eCount; ++i) { |
| data[i] = col[i]; |
| } |
| } else { |
| auto row = getTileHSlice<TPElem>(ZA, tile_idx, vec_idx); |
| for (int i = 0; i < eCount; ++i) { |
| data[i] = row[i]; |
| } |
| } |
| ''' |
| |
| iop = InstObjParams(name, "Sme" + Name, "SmeLd1xSt1xOp", |
| {'code': code, 'op_class': opClass}, |
| ['IsStore', 'IsNonSpeculative']) |
| header_output += SmeSt1xDeclare.subst(iop) |
| exec_output += SmeSt1xExecute.subst(iop) |
| exec_output += SmeSt1xInitiateAcc.subst(iop) |
| exec_output += SmeSt1xCompleteAcc.subst(iop) |
| for type in types: |
| substDict = {'targs' : type, |
| 'class_name' : 'Sme' + Name} |
| exec_output += SmeSt1xExecDeclare.subst(substDict) |
| |
| def smeStrInst(name, Name, opClass): |
| global header_output, decoder_output, exec_output |
| code = smEnCheckCodeNoSM + ''' |
| // imm stores the vector offset. We do not have a tile number |
| // as we target the whole accumulator array. |
| // imm also stores the offset applied to the base memory access |
| // register. |
| // Op1 stores Rn, which is the base memory access register |
| // Op2 stores Rv, which is the vector select register |
| |
| |
| unsigned eCount = ArmStaticInst::getCurSmeVecLen<uint8_t>( |
| xc->tcBase()); |
| |
| uint8_t vec_index = (WOp2 + imm) % eCount; |
| |
| auto row = getTileHSlice<uint8_t>(ZA, 0, vec_index); |
| |
| // Calculate the address |
| M5_VAR_USED Addr EA = XOp1 + imm; |
| |
| uint8_t data[MaxSmeVecLenInBytes]; |
| |
| // Update data which will then by used to store the row to memory |
| for (int i = 0; i < eCount; ++i) { |
| data[i] = row[i]; |
| } |
| ''' |
| |
| iop = InstObjParams(name, "Sme" + Name, "SmeLdrStrOp", |
| {'code': code, 'op_class': opClass}, |
| ['IsStore', 'IsNonSpeculative']) |
| header_output += SmeStrDeclare.subst(iop) |
| exec_output += SmeStrExecute.subst(iop) |
| exec_output += SmeStrInitiateAcc.subst(iop) |
| exec_output += SmeStrCompleteAcc.subst(iop) |
| |
| def smeZeroInst(name, Name, opClass, types): |
| global header_output, decoder_output, exec_output |
| code = smEnCheckCodeNoSM + smeZaWrite + ''' |
| // When zeroing tiles, we use 64-bit elements. This means |
| // that we have up to eight subtiles to clear in the ZA tile. |
| |
| ZA = ZA; |
| |
| for (int i = 0; i < 8; ++i) { |
| if (((imm >> i) & 0x1) == 0x1) { |
| getTile<TPElem>(ZA, i).zero(); |
| } |
| }''' |
| |
| iop = InstObjParams(name, "Sme" + Name, "SmeZeroOp", |
| {'code': code, 'op_class': opClass}, |
| ['IsNonSpeculative']) |
| header_output += SmeZeroDeclare.subst(iop) |
| exec_output += SmeTemplatedExecute.subst(iop) |
| |
| for type in types: |
| substDict = {'targs' : type, |
| 'class_name' : 'Sme' + Name} |
| exec_output += SmeOpExecDeclare.subst(substDict) |
| |
| # ADDHA |
| addCode = ''' |
| for (int col = 0; col < eCount; ++col) { |
| TPElem val = AA64FpOp1_x[col]; |
| |
| for (int row = 0; row < eCount; ++row) { |
| if (!(GpOp1_x[row] && GpOp2_x[col])) { |
| continue; |
| } |
| |
| tile[col][row] += val; |
| } |
| } |
| ''' |
| smeAddInst('addha', "Addha", "SimdAddOp", ['int32_t', 'int64_t'], addCode) |
| |
| # ADDSPL |
| addSplCode = ''' |
| Dest64 = imm * ArmStaticInst::getCurSmeVecLen<uint8_t>(xc->tcBase()); |
| // Divide down to get the predicate length in bytes |
| Dest64 /= 8; |
| Dest64 += XOp1; |
| ''' |
| smeAddVlInst('addspl', "Addspl", "SimdAddOp", addSplCode) |
| |
| # ADDSVL |
| addSvlCode = ''' |
| Dest64 = imm * ArmStaticInst::getCurSmeVecLen<uint8_t>(xc->tcBase()); |
| Dest64 += XOp1; |
| ''' |
| smeAddVlInst('addsvl', "Addsvl", "SimdAddOp", addSvlCode) |
| |
| # ADDVA |
| addCode = ''' |
| for (int row = 0; row < eCount; ++row) { |
| TPElem val = AA64FpOp1_x[row]; |
| |
| for (int col = 0; col < eCount; ++col) { |
| if (!(GpOp1_x[row] && GpOp2_x[col])) { |
| continue; |
| } |
| |
| tile[col][row] += val; |
| } |
| } |
| ''' |
| smeAddInst('addva', "Addva", "SimdAddOp", ['int32_t', 'int64_t'], addCode) |
| |
| # BFMOPA |
| # BFMOPS |
| |
| # FMOPA (non-widening) |
| fmopxCode = ''' |
| auto tile = getTile<TPDElem>(ZA, imm); |
| FPSCR fpscr = (FPSCR) Fpscr; |
| |
| for (int j = 0; j < eCount; ++j) { |
| if (!GpOp1_xd[j]) { |
| continue; |
| } |
| |
| TPDElem val1 = AA64FpOp1_xd[j]; |
| |
| for (int i = 0; i < eCount; ++i) { |
| if (!GpOp2_xd[i]) { |
| continue; |
| } |
| |
| TPDElem val2 = AA64FpOp2_xd[i]; |
| |
| #if %s |
| val2 = fplibNeg(val2); |
| #endif |
| |
| TPDElem res = fplibMul(val1, val2, fpscr); |
| |
| tile[j][i] = fplibAdd(tile[j][i], |
| res, fpscr); |
| } |
| } |
| ''' |
| smeFPOPInst('fmopa', 'Fmopa', 'MatrixOPOp', ['uint32_t', 'uint64_t'], |
| ['uint32_t', 'uint64_t'], fmopxCode % "0") |
| |
| # FMOPA (widening) |
| wideningFmopxCode = ''' |
| auto tile = getTile<TPDElem>(ZA, imm); |
| FPSCR fpscr = (FPSCR) Fpscr; |
| |
| for (int j = 0; j < eCount; ++j) { |
| if (!GpOp1_xd[j]) { |
| continue; |
| } |
| for (int i = 0; i < eCount; ++i) { |
| if (!GpOp2_xd[i]) { |
| continue; |
| } |
| |
| for (int k = 0; k < 2; ++k) { |
| TPSElem temp1 = (AA64FpOp1_xd[j] >> (16 * k)) & 0xFFFF; |
| TPSElem temp2 = (AA64FpOp2_xd[j] >> (16 * k)) & 0xFFFF; |
| TPDElem val1 = fplibConvert<TPSElem, TPDElem>(temp1, |
| FPCRRounding(fpscr), fpscr); |
| TPDElem val2 = fplibConvert<TPSElem, TPDElem>(temp2, |
| FPCRRounding(fpscr), fpscr); |
| |
| #if %s |
| val2 = fplibNeg(val2); |
| #endif |
| |
| TPDElem res = fplibMul(val1, val2, fpscr); |
| tile[j][i] = fplibAdd(tile[j][i], res, fpscr); |
| } |
| } |
| } |
| ''' |
| smeFPOPInst('fmopa', 'FmopaWidening', 'MatrixOPOp', |
| ['uint16_t'], ['uint32_t'], wideningFmopxCode % "0") |
| |
| # FMOPS (non-widening) |
| smeFPOPInst('fmops', 'Fmops', 'MatrixOPOp', ['uint32_t', 'uint64_t'], |
| ['uint32_t', 'uint64_t'], fmopxCode % "1") |
| |
| # FMOPS (widening) |
| smeFPOPInst('fmops', 'FmopsWidening', 'MatrixOPOp', |
| ['uint16_t'], ['uint32_t'], wideningFmopxCode % "1") |
| |
| # LD1B |
| smeLd1xInst('ld1b', 'Ld1b', 'MemReadOp', ['uint8_t']) |
| |
| # LD1D |
| smeLd1xInst('ld1d', 'Ld1d', 'MemReadOp', ['uint64_t']) |
| |
| # LD1H |
| smeLd1xInst('ld1h', 'Ld1h', 'MemReadOp', ['uint16_t']) |
| |
| # LD1Q |
| smeLd1xInst('ld1q', 'Ld1q', 'MemReadOp', ['__uint128_t']) |
| |
| # LD1W |
| smeLd1xInst('ld1w', 'Ld1w', 'MemReadOp', ['uint32_t']) |
| |
| # LDR |
| smeLdrInst("ldr", "Ldr", 'MemReadOp') |
| |
| # MOV (tile to vector) - ALIAS; see MOVA |
| # MOV (vector to tile) - ALIAS; see MOVA |
| # MOVA (tile to vector) |
| smeMovaExtractInst("mova", "MovaExtract", 'MatrixMovOp', |
| ["uint8_t", "uint16_t", "uint32_t", "uint64_t", |
| "__uint128_t"]) |
| |
| # MOVA (vector to tile) |
| smeMovaInsertInst("mova", "MovaInsert", 'MatrixMovOp', |
| ["uint8_t", "uint16_t", "uint32_t", "uint64_t", |
| "__uint128_t"]) |
| |
| # RDSVL |
| smeRdsvlInst('rdsvl', 'Rdsvl', 'SimdAddOp') |
| |
| # SMOPA |
| intMopxCode = ''' |
| auto tile = getTile<TPDElem>(ZA, imm); |
| |
| size_t shift = 8 * sizeof(TPS1Elem); |
| size_t mask = (1 << shift) - 1; |
| |
| for (int j = 0; j < eCount; ++j) { |
| for (int i = 0; i < eCount; ++i) { |
| for (int k = 0; k < 4; ++k) { |
| if (!GpOp1_xs1[4 * j + k]) { |
| continue; |
| } |
| |
| if (!GpOp2_xs2[4 * i + k]) { |
| continue; |
| } |
| |
| TPS1Elem temp1 = |
| (TPS1Elem)(AA64FpOp1_xd[j] >> (shift * k)) & mask; |
| TPS2Elem temp2 = |
| (TPS2Elem)(AA64FpOp2_xd[i] >> (shift * k)) & mask; |
| |
| tile[j][i] %s= (TPDElem)temp1 * (TPDElem)temp2; |
| } |
| } |
| } |
| ''' |
| smeIntOPInst('smopa', 'Smopa', 'MatrixOPOp', ['int8_t', 'int16_t'], |
| ['int8_t', 'int16_t'], ['int32_t', 'int64_t'], |
| intMopxCode % "+") |
| |
| # SMOPS |
| smeIntOPInst('smops', 'Smops', 'MatrixOPOp', ['int8_t', 'int16_t'], |
| ['int8_t', 'int16_t'], ['int32_t', 'int64_t'], |
| intMopxCode % "-") |
| |
| # SMSTART |
| smstartSmstopCode = ''' |
| // Bit 0 of imm determines if we are setting or clearing |
| // (smstart vs smstop) |
| // Bit 1 means that we are applying this to SM |
| // Bit 2 means that we are applying this to ZA |
| bool new_state = imm & 0x1; |
| bool sm_affected = imm & 0x2; |
| bool za_affected = imm & 0x4; |
| bool old_sm_state = Svcr & 0x1; |
| bool old_za_state = Svcr & 0x2; |
| |
| bool sm_changed = sm_affected && old_sm_state != new_state; |
| bool za_changed = za_affected && old_za_state != new_state; |
| |
| if (sm_changed) { |
| // We need to zero the SVE Z, P, FFR registers on SM change. Also, |
| // set FPSR to a default value. Note that we use the max SVE len |
| // instead of the actual vector length. |
| // |
| // For the Z, P registers we are directly setting these to zero |
| // without going through the ISA parser (which generates the |
| // dependencies) as otherwise the O3 CPU can deadlock when there |
| // are too few free physical registers. We therefore rely on this |
| // instruction being a barrier (IsSerialiseAfter). |
| |
| // Z Registers, including special and interleave registers |
| ArmISA::VecRegContainer zeroed_z_reg; |
| zeroed_z_reg.zero(); |
| |
| for (int reg_idx = 0; reg_idx < NumVecRegs; ++reg_idx) { |
| auto reg_id = ArmISA::vecRegClass[reg_idx]; |
| xc->tcBase()->setReg(reg_id, &zeroed_z_reg); |
| } |
| |
| // P Registers, including the FFR |
| ArmISA::VecPredRegContainer zeroed_p_reg; |
| zeroed_p_reg.reset(); |
| |
| for (int reg_idx = 0; reg_idx < NumVecPredRegs; ++reg_idx) { |
| auto reg_id = ArmISA::vecPredRegClass[reg_idx]; |
| xc->tcBase()->setReg(reg_id, &zeroed_p_reg); |
| } |
| |
| // FPSR |
| Fpsr = 0x0800009f; |
| } |
| |
| if (za_changed) { |
| // ZA write |
| ZA = ZA; |
| ZA.zero(); |
| } |
| |
| // Now that we've handled the zeroing of the appropriate registers, |
| // we update the pstate accordingly. |
| |
| if (sm_changed) { |
| if (new_state == 1) { |
| Svcr = Svcr | 0x1; // Set SM |
| } else { |
| Svcr = Svcr & ~(uint64_t)0x1; // Clear SM |
| } |
| } |
| |
| if (za_changed) { |
| if (new_state == 1) { |
| Svcr = Svcr | 0x2; // Set ZA |
| } else { |
| Svcr = Svcr & ~(uint64_t)0x2; // Clear ZA |
| } |
| } |
| ''' |
| |
| smeMsrInst('smstart', 'Smstart', 'IntAluOp', |
| smstartSmstopCode) |
| |
| # SMSTOP |
| smeMsrInst('smstop', 'Smstop', 'IntAluOp', |
| smstartSmstopCode) |
| |
| # ST1B |
| smeSt1xInst('st1b', 'St1b', 'MemWriteOp', ['uint8_t']) |
| |
| # ST1D |
| smeSt1xInst('st1d', 'St1d', 'MemWriteOp', ['uint64_t']) |
| |
| # ST1H |
| smeSt1xInst('st1h', 'St1h', 'MemWriteOp', ['uint16_t']) |
| |
| # ST1Q |
| smeSt1xInst('st1q', 'St1q', 'MemWriteOp', ['__uint128_t']) |
| |
| # ST1W |
| smeSt1xInst('st1w', 'St1w', 'MemWriteOp', ['uint32_t']) |
| |
| # STR |
| smeStrInst("str", "Str", "MemWriteOp") |
| |
| # SUMOPA |
| smeIntOPInst('sumopa', 'Sumopa', 'MatrixOPOp', ['int8_t', 'int16_t'], |
| ['uint8_t', 'uint16_t'], ['int32_t', 'int64_t'], |
| intMopxCode % "+") |
| |
| # SUMOPS |
| smeIntOPInst('sumops', 'Sumops', 'MatrixOPOp', ['int8_t', 'int16_t'], |
| ['uint8_t', 'uint16_t'], ['int32_t', 'int64_t'], |
| intMopxCode % "-") |
| |
| # UMOPA |
| smeIntOPInst('umopa', 'Umopa', 'MatrixOPOp', ['uint8_t', 'uint16_t'], |
| ['uint8_t', 'uint16_t'], ['int32_t', 'int64_t'], |
| intMopxCode % "+") |
| |
| # UMOPS |
| smeIntOPInst('umops', 'Umops', 'MatrixOPOp', ['uint8_t', 'uint16_t'], |
| ['uint8_t', 'uint16_t'], ['int32_t', 'int64_t'], |
| intMopxCode % "-") |
| |
| # USMOPA |
| smeIntOPInst('usmopa', 'Usmopa', 'MatrixOPOp', ['uint8_t', 'uint16_t'], |
| ['int8_t', 'int16_t'], ['int32_t', 'int64_t'], |
| intMopxCode % "+") |
| |
| # USMOPS |
| smeIntOPInst('usmops', 'Usmops', 'MatrixOPOp', ['uint8_t', 'uint16_t'], |
| ['int8_t', 'int16_t'], ['int32_t', 'int64_t'], |
| intMopxCode % "-") |
| |
| # ZERO |
| smeZeroInst("zero", "Zero", "MatrixOp", ["uint64_t"]) |
| |
| }}; |
