/*========================== begin_copyright_notice ============================

Copyright (C) 2017-2021 Intel Corporation

SPDX-License-Identifier: MIT

============================= end_copyright_notice ===========================*/

//
/// GenXLowering
/// ------------
///
/// GenXLowering is a function pass that lowers certain LLVM IR instructions
/// that the rest of the GenX backend cannot deal with, or to implement peephole
/// optimizations.
///
/// It also performs a few other tasks:
///
/// 1. It implements add sinking for a variable index in a region/element
///    access. This ensures that, in a sequence of operations to calculate a
///    variable index for a region/element access, any add constant is sunk to
///    the end, such that it can become a constant offset in an indirect
///    operand, and give GenXAddressCommoning more chance to common up address
///    calculations.
///
/// 2. It splits struct values where possible, by splitting all struct phi nodes
///    before running the main pass, then removing an extractvalue by using the
///    corresponding insertvalue's input instead. Any struct value used as an
///    arg or return value still remains, and needs to be dealt with by register
///    allocation.
///
/// 3. It widens some byte vector operations to short vector.
///
///    Gen has restrictions on byte operands. The jitter copes with that, but
///    sometimes it needs to do even-odd splitting, which can lead to suboptimal
///    code if cmps and predicates are involved.
///    Here we attempt to pick up the common cases by converting a byte
///    operation to short.
///
///    Note that we might end up with the extends being baled into the
///    instruction anyway, resulting in a byte operation in vISA.
///
/// 4. Certain uses of shufflevector are lowered:
///
///    a. a splat (copy of one element across a vector);
///    b. a boolean slice (extract of a subvector) becomes rdpredregion;
///    c. a boolean unslice (insert subvector) becomes wrpredregion.
///    d. non-boolean shufflevector is lowered to sequence of rd/wrregions
///
///    The only one case of shufflevector allowed is shufflevector of
///    predicate and undef with replicated mask.
///
/// 5. A Trunc is lowered to a bitcast then a region/element read with a stride.
///    GenXCoalescing will coalesce the bitcast, and possibly bale in the region
///    read, so this will hopefully save an instruction or two.
///
/// 6. Certain floating point comparison instructions are lowered.
///
/// **IR restriction**: LLVM IR instructions not supported after this pass:
///
/// * insertelement
/// * extractelement
/// * trunc
/// * zext/sext/uitofp from (vector of) i1
/// * select on vector of i1
/// * ``llvm.uadd.with.overflow`` (the other
///   overflowing arithmetic intrinsics are not allowed by the GenX backend
///   anyway.)
/// * ``llvm.genx.*imad``
///
///
/// **IR restriction**: all gather/scatter/atomic must have the width supported
/// by the hardware target.
///
/// **IR restriction**: rdpredregion intrinsic (which is generated by this pass
/// from certain cases of shufflevector, and represents a use of part of a
/// predicate) can only be used in select, wrregion, wrpredpredregion.
///
/// **IR restriction**: wrpredregion intrinsic (which is generated by this pass
/// from certain cases of shufflevector, and represents the write of part of a
/// predicate) must have a compare as its "new value" input.
///
/// **IR restriction**: No phi node of struct type after this pass. This is only
/// a general rule; subsequent passes have been known to reintroduce them so
/// GenXLiveness has another go at splitting them up.
///
//===----------------------------------------------------------------------===//
#define DEBUG_TYPE "GENX_LOWERING"

#include "GenX.h"
#include "GenXGotoJoin.h"
#include "GenXIntrinsics.h"
#include "GenXModule.h"
#include "GenXSubtarget.h"
#include "GenXTargetMachine.h"
#include "GenXUtil.h"
#include "GenXVisa.h"

#include "IGC/common/debug/DebugMacros.hpp"
#include "visa_igc_common_header.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"

#include "llvmWrapper/IR/DerivedTypes.h"
#include "llvmWrapper/IR/Instructions.h"
#include "llvmWrapper/Support/TypeSize.h"

#include "vc/Support/GenXDiagnostic.h"
#include "vc/Utils/GenX/GlobalVariable.h"

#include <algorithm>
#include <iterator>
#include <numeric>
#include "Probe/Assertion.h"

using namespace llvm;
using namespace genx;

static cl::opt<bool>
    EnableGenXByteWidening("enable-genx-byte-widening", cl::init(true),
                           cl::Hidden, cl::desc("Enable GenX byte widening."));
STATISTIC(NumLegacy, "Number of translated legacy messages");
STATISTIC(NumOword, "Number of translated legacy oword.ld/oword.st messages");
STATISTIC(NumQuad, "Number of translated legacy gather4/scatter4 messages");
STATISTIC(NumMedia,
          "Number of translated legacy media block read/write messages");
namespace {

// Diagnostic for lowering problems
class DiagnosticInfoLowering : public DiagnosticInfo {
private:
  std::string Description;
  StringRef Filename;
  unsigned Line;
  unsigned Col;

  static const int KindID;

  static int getKindID() { return KindID; }

public:
  DiagnosticInfoLowering(const Instruction *Inst, const Twine &Desc,
                         DiagnosticSeverity Severity = DS_Error);
  void print(DiagnosticPrinter &DP) const override;
  static bool classof(const DiagnosticInfo *DI) {
    return DI->getKind() == getKindID();
  }
};

const int DiagnosticInfoLowering::KindID =
    llvm::getNextAvailablePluginDiagnosticKind();

using IRChecker = vc::IRChecker<DiagnosticInfoLowering>;

DiagnosticInfoLowering::DiagnosticInfoLowering(const Instruction *Inst,
                                               const Twine &Desc,
                                               DiagnosticSeverity Severity)
    : DiagnosticInfo(getKindID(), Severity), Line(0), Col(0) {
  auto DL = Inst->getDebugLoc();
  if (DL) {
    Filename = DL->getFilename();
    Line = DL.getLine();
    Col = DL.getCol();
  }
  std::string str;
  vc::printToString(str, *Inst);

  Description =
      (Twine("GenXLowering failed for instruction ") + str + ": " + Desc).str();
}

void DiagnosticInfoLowering::print(DiagnosticPrinter &DP) const {
  std::string Loc((Twine(!Filename.empty() ? Filename : "<unknown>") + ":" +
                   Twine(Line) + (!Col ? Twine() : Twine(":") + Twine(Col)) +
                   ": ")
                      .str());
  DP << Loc << Description;
}

// GenXLowering : legalize execution widths and GRF crossing
class GenXLowering : public FunctionPass {
  DominatorTree *DT = nullptr;
  const GenXSubtarget *ST = nullptr;
  SmallVector<Instruction *, 8> ToErase;
  const DataLayout *DL = nullptr;

public:
  static char ID;
  explicit GenXLowering() : FunctionPass(ID), DT(nullptr) {}
  StringRef getPassName() const override { return "GenX lowering"; }
  void getAnalysisUsage(AnalysisUsage &AU) const override;
  bool runOnFunction(Function &F) override;

private:
  bool widenSIMD8GatherScatter(CallInst *CI, unsigned IID);
  IGCLLVM::FixedVectorType *
  adjustVTForTransposedMessage(IGCLLVM::FixedVectorType *VT);
  bool translateLegacyToLSC(CallInst *CI, unsigned IID);
  bool lowerMediaWalkerAPIs(CallInst *CI, unsigned IID);
  bool translateSLMOWord(CallInst* CI, unsigned IID);
  bool splitGatherScatter(CallInst *CI, unsigned IID);
  bool processTwoAddressOpnd(CallInst *CI);
  bool processInst(Instruction *Inst);
  bool lowerRdRegion(Instruction *Inst);
  bool lowerWrRegion(Instruction *Inst);
  bool lowerRdPredRegion(Instruction *Inst);
  bool lowerWrPredRegion(Instruction *Inst);
  bool lowerInsertElement(Instruction *Inst);
  bool lowerExtractElement(Instruction *Inst);
  Value *scaleInsertExtractElementIndex(Value *IdxVal, Type *ElTy,
                                        Instruction *InsertBefore);
  bool lowerTrunc(Instruction *Inst);
  bool lowerCast(Instruction *Inst);
  bool lowerBoolScalarSelect(SelectInst *SI);
  bool lowerBoolVectorSelect(SelectInst *SI);
  bool lowerBoolShuffle(ShuffleVectorInst *Inst);
  bool lowerBoolSplat(ShuffleVectorInst *SI, Value *In, unsigned Idx);
  bool lowerShuffle(ShuffleVectorInst *Inst);
  void lowerShuffleSplat(ShuffleVectorInst *SI,
                         ShuffleVectorAnalyzer::SplatInfo Splat);
  bool lowerShuffleToSelect(ShuffleVectorInst *Inst);
  void lowerShuffleToMove(ShuffleVectorInst *SI);
  bool lowerShr(Instruction *Inst);
  bool lowerExtractValue(ExtractValueInst *Inst);
  bool lowerInsertValue(InsertValueInst *Inst);
  bool lowerUAddWithOverflow(CallInst *CI);
  bool lowerUAddWithSat(CallInst *CI);
  bool lowerUSubWithSat(CallInst *CI);
  bool lowerCtpop(CallInst *CI);
  bool lowerFCmpInst(FCmpInst *Inst);
  bool lowerUnorderedFCmpInst(FCmpInst *Inst);
  bool lowerMinMax(CallInst *CI, unsigned IntrinsicID);
  bool lowerSqrt(CallInst *CI);
  bool widenByteOp(Instruction *Inst);
  bool lowerLoadStore(Instruction *Inst);
  bool lowerGenXMul(CallInst *CI, unsigned IntrinsicID);
  bool lowerGenXMulSat(CallInst *CI, unsigned IntrinsicID);
  bool lowerGenXIMad(CallInst *CI, unsigned IntrinsicID);
  bool lowerMul64(Instruction *Inst);
  bool lowerLzd(Instruction *Inst);
  bool lowerTrap(CallInst *CI);
  bool lowerLLVMMaskedLoad(CallInst *CI);
  bool lowerLLVMMaskedStore(CallInst* CI);
  bool lowerLLVMMaskedGather(CallInst *CI);
  bool lowerLLVMMaskedScatter(CallInst *CI);
  bool lowerFMulAdd(CallInst *CI);
  bool lowerBitreverse(CallInst *CI);
  bool lowerFunnelShift(CallInst *CI, unsigned IntrinsicID);
  bool lowerFAbs(CallInst *CI);
  bool lowerF32MathIntrinsic(CallInst *CI, GenXIntrinsic::ID GenXID);
  bool lowerF32FastMathIntrinsic(CallInst *CI, GenXIntrinsic::ID GenXID);
  bool generatePredicatedWrrForNewLoad(CallInst *CI);
};

} // end namespace

char GenXLowering::ID = 0;
namespace llvm {
void initializeGenXLoweringPass(PassRegistry &);
}
INITIALIZE_PASS_BEGIN(GenXLowering, "GenXLowering", "GenXLowering", false,
                      false)
INITIALIZE_PASS_END(GenXLowering, "GenXLowering", "GenXLowering", false, false)

FunctionPass *llvm::createGenXLoweringPass() {
  initializeGenXLoweringPass(*PassRegistry::getPassRegistry());
  return new GenXLowering;
}

void GenXLowering::getAnalysisUsage(AnalysisUsage &AU) const {
  AU.addRequired<TargetPassConfig>();
  AU.addPreserved<DominatorTreeWrapperPass>();
  AU.addPreserved<LoopInfoWrapperPass>();
  AU.addPreserved<GenXModule>();
}

/***********************************************************************
 * GenXLowering::runOnFunction : process one function to
 *    lower instructions as required for GenX backend.
 *
 * This does a postordered depth first traversal of the CFG,
 * processing instructions within a basic block in reverse, to
 * ensure that we see a def after its uses (ignoring phi node uses).
 * This helps peephole optimizations which generally want to be
 * approached from the top down. For example, add sinking in the index
 * of an indirect region/element wants to see the trunc before the trunc
 * is lowered to a bitcast and an element access.
 */
bool GenXLowering::runOnFunction(Function &F) {
  LLVM_DEBUG(dbgs() << "GenXLowering started\n");
  auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
  DT = DTWP ? &DTWP->getDomTree() : nullptr;
  ST = &getAnalysis<TargetPassConfig>()
            .getTM<GenXTargetMachine>()
            .getGenXSubtarget();
  DL = &F.getParent()->getDataLayout();
  // First split any phi nodes with struct type.
  splitStructPhis(&F);
  // Create a list of basic blocks in the order we want to process them, before
  // we start the lowering. This is because lowering can split a basic block.
  SmallVector<BasicBlock *, 8> BBs;
  for (auto i = po_begin(&F.getEntryBlock()), e = po_end(&F.getEntryBlock());
       i != e; ++i)
    BBs.push_back(*i);
  // Process each basic block.
  for (auto i = BBs.begin(), e = BBs.end(); i != e; ++i) {
    BasicBlock *BB = *i;
    // The effect of this loop is that we process the instructions in reverse
    // order, and we re-process anything inserted before the instruction
    // being processed.
    for (Instruction *Inst = BB->getTerminator();;) {
      processInst(Inst);
      BasicBlock *Parent = Inst->getParent();
      if (Inst != &Parent->front())
        Inst = Inst->getPrevNode();
      else {
        if (Parent == BB)
          break;
        // We have reached the start of the basic block, but it is a different
        // basic block to BB, so lowering must have split a BB. Just go back to
        // the end of the previous one.
        Inst = Parent->getPrevNode()->getTerminator();
      }
    }
  }
  // Erase the instructions that we saved in ToErase.
  for (SmallVectorImpl<Instruction *>::iterator i = ToErase.begin(),
                                                e = ToErase.end();
       i != e; ++i)
    (*i)->eraseFromParent();
  ToErase.clear();
  return true;
}

// Optimize two address operands if any.
//
// An instruction with a two address opernd should be predicated. If predicate
// is a constant splat, then the old value will be over-written. In this case,
// replace the old value with undef which allows more optimizations to kick in.
//
bool GenXLowering::processTwoAddressOpnd(CallInst *CI) {
  auto OpNum = getTwoAddressOperandNum(CI);
  // Skip write regions whose OpNum is 0.
  if (OpNum && *OpNum != 0) {
    Type *Ty = CI->getArgOperand(*OpNum)->getType();
    IGC_ASSERT_MESSAGE(Ty == CI->getType(), "two address op type out of sync");

    for (unsigned i = 0; i < CI->getNumArgOperands(); ++i) {
      auto Op = dyn_cast<Constant>(CI->getArgOperand(i));
      // Check if the predicate operand is all true.
      if (Op && Op->getType()->getScalarSizeInBits() == 1) {
        if (Op->getType()->isVectorTy())
          Op = Op->getSplatValue();
        if (Op && Op->isOneValue()) {
          CI->setOperand(*OpNum, UndefValue::get(Ty));
          return true;
        }
        return false;
      }
    }
  }

  return false;
}

// Check whether given intrinsic is new load
// without predicate and old value arguments.
static bool isNewLoadInst(CallInst *Inst) {
  unsigned IID = GenXIntrinsic::getGenXIntrinsicID(Inst);
  switch (IID) {
  case GenXIntrinsic::genx_gather4_scaled2:
  case GenXIntrinsic::genx_gather_scaled2:
  case GenXIntrinsic::genx_gather4_masked_scaled2:
  case GenXIntrinsic::genx_gather_masked_scaled2:
  // Channel-wise insts:
  case GenXIntrinsic::genx_lsc_load_slm:
  case GenXIntrinsic::genx_lsc_load_stateless:
  case GenXIntrinsic::genx_lsc_load_bindless:
  case GenXIntrinsic::genx_lsc_load_bti:
  case GenXIntrinsic::genx_lsc_prefetch_bti:
  case GenXIntrinsic::genx_lsc_prefetch_stateless:
  case GenXIntrinsic::genx_lsc_prefetch_bindless:
    return true;
  default:
    return false;
  }
}

// Find single wrregion user of load instruction.
// Returns nullptr on failure.
static CallInst *getLoadWrregion(CallInst *Inst) {
  IGC_ASSERT_MESSAGE(isNewLoadInst(Inst), "Expected new load intrinsics");
  if (Inst->getNumUses() != 1)
    return nullptr;

  auto *WrR = dyn_cast<CallInst>(Inst->user_back());
  if (!WrR)
    return nullptr;
  return GenXIntrinsic::isWrRegion(WrR) ? WrR : nullptr;
}

// Find single select user of load instruction.
// Returns nullptr on failure.
// TODO: maybe just lower every select to wrregion in lowerSelect?
static SelectInst *getLoadSelect(CallInst *Inst) {
  IGC_ASSERT_MESSAGE(isNewLoadInst(Inst), "Expected new load intrinsics");
  if (Inst->getNumUses() != 1)
    return nullptr;

  auto *SI = dyn_cast<SelectInst>(Inst->user_back());
  if (!SI)
    return nullptr;
  // TODO: handle inverted selects.
  // Need to regenerate mask in this case.
  if (SI->getTrueValue() != Inst)
    return nullptr;
  return SI;
}

// Generate predicate for wrregion of splitted load.
// Returns new predicate.
static Value *generatePredicateForLoadWrregion(
    Value *OldPred, unsigned Offset, unsigned Width, unsigned NumChannels,
    Instruction *InsertBefore, const DebugLoc &DL, const Twine &Name) {

  if (Constant *C = dyn_cast<Constant>(OldPred)) {
    if (isa<ConstantInt>(OldPred))
      return OldPred;

    if (C->isAllOnesValue())
      return ConstantInt::get(Type::getInt1Ty(OldPred->getContext()), 1);
  }

  Value *Pred = OldPred;
  // If old predicate is result of rdpredregion or shufflevector then
  // we can reuse their predicate and offset to avoid double read of predicate.
  if (GenXIntrinsic::getGenXIntrinsicID(OldPred) == GenXIntrinsic::genx_rdpredregion) {
    auto *OldPredInst = cast<CallInst>(OldPred);
    Offset += cast<ConstantInt>(OldPredInst->getArgOperand(1))->getZExtValue();
    Pred = OldPredInst->getArgOperand(0);
  } else if (auto *SVI = dyn_cast<ShuffleVectorInst>(OldPred)) {
    Offset +=
        ShuffleVectorAnalyzer::getReplicatedSliceDescriptor(SVI).InitialOffset;
    Pred = SVI->getOperand(0);
  }

  // Replicate mask across channels.
  SmallVector<Constant *, 16> NewMaskVals(Width);
  IGC_ASSERT(NumChannels);
  unsigned ChannelWidth = Width / NumChannels;
  Type *Int32Ty = IntegerType::getInt32Ty(Pred->getContext());
  for (unsigned i = 0; i < NumChannels; ++i)
    std::generate_n(NewMaskVals.begin() + ChannelWidth * i, ChannelWidth,
                    [Int32Ty, Offset]() mutable {
                      return ConstantInt::get(Int32Ty, Offset++);
                    });
  Constant *NewMask = ConstantVector::get(NewMaskVals);

  Value *Undef = UndefValue::get(Pred->getType());
  auto *Res = new ShuffleVectorInst(Pred, Undef, NewMask, Name, InsertBefore);
  Res->setDebugLoc(DL);
  return Res;
}

static Value *generatePredicatedWrregion(Value *OldVal, Value *NewVal,
                                         Value *Pred, unsigned Offset,
                                         Instruction *InsertBefore,
                                         const llvm::Twine &Name = "") {
  Type *NewValType = NewVal->getType();

  Region WrR(NewValType);
  WrR.Mask = Pred;
  WrR.Offset = Offset;
  return WrR.createWrRegion(OldVal, NewVal, Name, InsertBefore,
                            InsertBefore->getDebugLoc());
}


// Generate partial write for result of splitted 1-channel load instruction.
// Initially we could have following sequence for illegal load (on gather_scaled example):
//   res = gather_scaled <32>
//   mask = rdpredregion <32> pred, offset
//   newV = wrregion <32> oldV, res, wroffset, mask
// After splitting we want to get as less extra code as possible.
// To achieve this we generate following pattern:
// bale {
//   res1 = gather_scaled <16>
//   mask1 = rdpredregion <16> pred, offset
//   partialV = wrregion <16> oldV, res1, mask1
// }
// bale {
//   res2 = gather_scaled <16>
//   mask2 = rdpredregion <16> pred, offset + 16
//   newV = wrregion <16> partialV, res2, wroffset + 16 * elemsize, mask2
// }
// Bale markers show how this will be baled later.
static Value *generate1ChannelWrrregion(Value *Target, unsigned InitialOffset,
                                        CallInst *Load, Value *OldPred,
                                        unsigned AccumulatedOffset,
                                        Instruction *InsertBefore) {
  const DebugLoc &DL = Load->getDebugLoc();
  Type *LoadType = Load->getType();
  unsigned LoadWidth =
      cast<IGCLLVM::FixedVectorType>(LoadType)->getNumElements();

  Value *Pred =
      generatePredicateForLoadWrregion(OldPred, AccumulatedOffset, LoadWidth, 1,
                                       InsertBefore, DL, "load1.pred.split");
  Region WrR(LoadType);
  WrR.Mask = Pred;
  WrR.Offset =
      InitialOffset + AccumulatedOffset * (LoadType->getScalarSizeInBits() / 8);
  return WrR.createWrRegion(Target, Load, "load1.join", InsertBefore, DL);
}

// Generate partial write for result of splitted N-channel load.
// For channelled loads we need to also shuffle result of splitted
// instructions to write back them to destination in expected order.
// Temporary splits should always be predicated in case of atomics
// because latter load and store at the same time.
// Example for gather4_scaled (with two channels enabled). Before:
//   res = gather4_scaled <32> RG
//   mask = rdpredregion <64> pred, offset ; mask is replicated across channels
//   newV = wrregion <64> oldV, res, wroffset, mask
// After:
// bale {
//   res1temp = gather4_scaled <16> RG ; create temporary (unnecessary in case of non-atomics)
//   splitmask1 = rdpredregion <32> pred, offset ; replicated
//   res1 = wrregion <32> undef, res1temp, 0, splitmask1
// }
// bale {
//   res1R = rdregion <16> res1, 0
//   mask1R = rdpredregion <16> pred, offset ; same for all channels
//   partialVR = wrregion <16> oldV, res1R, wroffset, mask1R
// }
// bale {
//   res1G = rdregion <16> res1, 16 * elemsize
//   mask1G = rdpredregion <16> pred, offset
//   partialV = wrregion <16> partialVR, res1G, wroffset + 32 * elemsize, mask1G
// }
// bale {
//   res2temp = gather4_scaled <16> RG ; second temporary
//   splitmask2 = rdpredregion <32> pred, offset + 16
//   res2 = wrregion <32> undef, res2temp, 0, splitmask2
// }
// bale {
//   res2R = rdregion <16> res2, 0
//   mask2R = rdpredregion <16> pred, offset + 16
//   newVR = wrregion <16> partialV, res2R, wroffset + 16 * elemsize, mask2R
// }
// bale {
//   res2G = rdregion <16> res2, 16 * elemsize
//   mask2G = rdpredregion <16> pred, offset + 16
//   newV = wrregion <16> newVR, res2G, wroffset + 48 * elemsize, mask2G
// }
// As it can be noticed, splitting of channeled loads is quite expensive.
// We should hope that later passes (like region collapsing) can optimize it
// by analyzing how resulting value was assembled.
static Value *generateNChannelWrregion(Value *Target, unsigned InitialOffset,
                                       CallInst *Load, Value *OldPred,
                                       unsigned SplitNum, unsigned NumSplits,
                                       unsigned NumChannels,
                                       Instruction *InsertBefore) {
  const DebugLoc &DL = Load->getDebugLoc();
  Type *LoadType = Load->getType();
  unsigned LoadWidth =
      cast<IGCLLVM::FixedVectorType>(LoadType)->getNumElements();
  IGC_ASSERT(NumChannels);
  unsigned ChannelWidth = LoadWidth / NumChannels;
  unsigned MaskOffset = ChannelWidth * SplitNum;

  // Generate temporary for load.
  Value *Pred = generatePredicateForLoadWrregion(
      OldPred, MaskOffset, LoadWidth, NumChannels, InsertBefore, DL, "loadN.pred.split");
  Region WrR(LoadType);
  WrR.Mask = Pred;
  Value *SplitRes = WrR.createWrRegion(UndefValue::get(LoadType), Load,
                                       "loadN.split", InsertBefore, DL);

  // Generate shuffle writes to the target.
  unsigned ElemByteSize = LoadType->getScalarSizeInBits() / 8;
  Type *ShuffleType =
      IGCLLVM::FixedVectorType::get(LoadType->getScalarType(), ChannelWidth);
  Region ChannelRdR(ShuffleType);
  Region ChannelWrR(ShuffleType);
  Value *ResChannel = nullptr;
  for (unsigned i = 0; i < NumChannels; ++i) {
    ChannelRdR.Offset = ChannelWidth * i * ElemByteSize;
    ResChannel = ChannelRdR.createRdRegion(SplitRes, "loadN.channel.read.join",
                                           InsertBefore, DL);
    Pred = generatePredicateForLoadWrregion(OldPred, MaskOffset, ChannelWidth,
                                            1, InsertBefore, DL,
                                            "loadN.channel.pred.join");
    ChannelWrR.Offset =
        InitialOffset +
        (ChannelWidth * SplitNum + ChannelWidth * NumSplits * i) * ElemByteSize;
    ChannelWrR.Mask = Pred;
    Target = ChannelWrR.createWrRegion(Target, ResChannel, "loadN.channel.join",
                                       InsertBefore, DL);
  }
  return Target;
}

// Try to infere predicate for load inst which will be used
// in splitting:
// - Check for mask operand first
// - Check wrr or select user
//
// If wrr or select user was found then return it's predicate
// because exactly this predicate must be used
// during splittig.
Value *getPredicateForLoadSplitting(CallInst *Load) {
  IGC_ASSERT(Load);
  Value *LoadPred = getMaskOperand(Load);
  CallInst *LoadWrr = getLoadWrregion(Load);
  SelectInst *SI = getLoadSelect(Load);

  if (!LoadWrr && !SI) {
    // No suitable user for mask inference and no predicate in load, so
    // it's not preidcated
    if (!LoadPred ||
        (isa<Constant>(LoadPred) && cast<Constant>(LoadPred)->isAllOnesValue()))
      return ConstantInt::get(IntegerType::getInt1Ty(Load->getContext()), 1);

    return LoadPred;
  }

  // Try to infer the mask from users.
  if (LoadWrr) {
    // If we found wrregion user, then use its predicate for splitted
    // instructions.
    return LoadWrr->getArgOperand(
        GenXIntrinsic::GenXRegion::PredicateOperandNum);
  }
  // Else it's select, get it's predicate
  return SI->getCondition();
}

// Get target for wrregions of splitted load.
// Returns tuple consisted of:
//  1. Target for wrregions
//  2. Predicate
//  3. Initial offset of target
//  4. Instruction to replace later
static std::tuple<Value *, Value *, unsigned, Instruction *>
getLoadTarget(CallInst *Load, const GenXSubtarget *ST) {
  Value *LoadPred = getPredicateForLoadSplitting(Load);
  CallInst *LoadWrr = getLoadWrregion(Load);
  SelectInst *SI = getLoadSelect(Load);

  if (LoadWrr) {
    // If wrregion can be represented as raw operand, we can reuse its target and offset.
    if (genx::isValueRegionOKForRaw(LoadWrr, true /* IsWrite */, ST)) {
      // TODO: mark wrregion to be erased once issue with ToErase and
      // iteration order will be resolved.
      Value *Target =
          LoadWrr->getArgOperand(GenXIntrinsic::GenXRegion::OldValueOperandNum);
      Value *Offset =
          LoadWrr->getArgOperand(GenXIntrinsic::GenXRegion::WrIndexOperandNum);
      unsigned InitialOffset = cast<ConstantInt>(Offset)->getZExtValue();
      return {Target, LoadPred, InitialOffset, LoadWrr};
    }
  } else if (SI) {
    Value *Target = SI->getFalseValue();
    return {Target, LoadPred, 0, SI};
  }

  // Create new target for load.
  Value *Target = UndefValue::get(Load->getType());
  return {Target, LoadPred, 0, Load};
}

// Get a vector of widths that are considered legal.
// InstWidth is the width of an instruction.
// InitWidth is the max legal width for that instruction, must be a power of 2.
// Currently, a width considered legal if it is less than or equal to InitWidth
// and is power of 2.
static std::vector<unsigned> calculateLegalWidths(unsigned InstWidth,
                                                  unsigned InitWidth) {
  IGC_ASSERT_MESSAGE(InstWidth, "InstWidth cannot be 0");
  IGC_ASSERT_MESSAGE(isPowerOf2_32(InitWidth), "InitWidth must be power of 2");
  std::vector<unsigned> Widths;
  while (InstWidth) {
    if (InstWidth >= InitWidth) {
      Widths.push_back(InitWidth);
      InstWidth -= InitWidth;
    } else
      // TODO: some intrinsics support very limited exec size range. Legal exec
      // sizes must be calculated more accurately.
      InitWidth /= 2;
  }

  return Widths;
}

/***********************************************************************
 * splitGatherScatter : lower gather/scatter/atomic to the width support
 * by the hardware platform.
 *
 * This performs two functions:
 *
 * 1. If the operation is wider than what hardware can support, splits it
 *    into the legal width.
 *    gather4/scatter4_* are split into instructions of the same width. Whereas
 *    others supported by this function intrinsics are split in a more flexible
 *    way with the legal width variation.
 *
 * 2. For typed gather4/scatter4, when r or both v and r are zero, replace
 *    with undef so that they are not encoded in the vISA instruction and the
 *    message skips them.
 */
bool GenXLowering::splitGatherScatter(CallInst *CI, unsigned IID) {
  enum {
    MASK_IDX = 0,
    PRED_IDX = 1,
    SURF_IDX = 2,
    U_IDX = 3,
    DATA_IDX = 6,
    NONEED = 11
  };

  unsigned MaskIdx = NONEED;
  unsigned PredIdx = NONEED;
  unsigned AddrIdx = NONEED;
  unsigned DataIdx = NONEED;
  unsigned AtomicSrcIdx = NONEED;
  bool IsTyped = false;
  int AtomicNumSrc = (-1); // -1 means not-an-atomic
  unsigned NumVectorElements = 1;

  switch (IID) {
  case GenXIntrinsic::genx_typed_atomic_add:
  case GenXIntrinsic::genx_typed_atomic_and:
  // to be open-sourced soon (need to check)
  case GenXIntrinsic::genx_typed_atomic_fadd:
  case GenXIntrinsic::genx_typed_atomic_fsub:
  case GenXIntrinsic::genx_typed_atomic_fmax:
  case GenXIntrinsic::genx_typed_atomic_fmin:
  case GenXIntrinsic::genx_typed_atomic_imax:
  case GenXIntrinsic::genx_typed_atomic_imin:
  case GenXIntrinsic::genx_typed_atomic_max:
  case GenXIntrinsic::genx_typed_atomic_min:
  case GenXIntrinsic::genx_typed_atomic_or:
  case GenXIntrinsic::genx_typed_atomic_sub:
  case GenXIntrinsic::genx_typed_atomic_xchg:
  case GenXIntrinsic::genx_typed_atomic_xor:
    AtomicSrcIdx = 2;
    PredIdx = 0;
    AddrIdx = 3;
    IsTyped = true;
    AtomicNumSrc = 1;
    break;
  case GenXIntrinsic::genx_typed_atomic_dec:
  case GenXIntrinsic::genx_typed_atomic_inc:
    PredIdx = 0;
    AddrIdx = 2;
    IsTyped = true;
    AtomicNumSrc = 0;
    break;
  case GenXIntrinsic::genx_typed_atomic_cmpxchg:
  case GenXIntrinsic::genx_typed_atomic_fcmpwr:
    AtomicSrcIdx = 2;
    PredIdx = 0;
    AddrIdx = 4;
    IsTyped = true;
    AtomicNumSrc = 2;
    break;
  case GenXIntrinsic::genx_scatter4_typed:
  case GenXIntrinsic::genx_gather4_typed:
    DataIdx = DATA_IDX;
    MaskIdx = MASK_IDX;
    PredIdx = PRED_IDX;
    AddrIdx = U_IDX;
    IsTyped = true;
    break;
  case GenXIntrinsic::genx_scatter4_scaled:
  case GenXIntrinsic::genx_gather4_scaled:
    DataIdx = 6;
    PredIdx = 0;
    MaskIdx = 1;
    AddrIdx = 5;
    break;
  case GenXIntrinsic::genx_gather4_scaled2:
    MaskIdx = 0;
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_gather4_masked_scaled2:
    PredIdx = 5;
    MaskIdx = 0;
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_svm_scatter4_scaled:
  case GenXIntrinsic::genx_svm_gather4_scaled:
    DataIdx = 5;
    PredIdx = 0;
    MaskIdx = 1;
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_scatter_scaled:
  case GenXIntrinsic::genx_gather_scaled:
    DataIdx = 6;
    PredIdx = 0;
    AddrIdx = 5;
    break;
  case GenXIntrinsic::genx_gather_scaled2:
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_gather_masked_scaled2:
    PredIdx = 5;
    AddrIdx = 4;
    break;
  case GenXIntrinsic::genx_svm_scatter:
  case GenXIntrinsic::genx_svm_gather:
    DataIdx = 3;
    PredIdx = 0;
    AddrIdx = 2;
    break;
  case GenXIntrinsic::genx_svm_atomic_dec:
  case GenXIntrinsic::genx_svm_atomic_inc:
    DataIdx = 2;
    PredIdx = 0;
    AddrIdx = 1;
    AtomicNumSrc = 0;
  case GenXIntrinsic::genx_svm_atomic_add:
  case GenXIntrinsic::genx_svm_atomic_and:
  case GenXIntrinsic::genx_svm_atomic_fmax:
  case GenXIntrinsic::genx_svm_atomic_fmin:
  case GenXIntrinsic::genx_svm_atomic_imax:
  case GenXIntrinsic::genx_svm_atomic_imin:
  case GenXIntrinsic::genx_svm_atomic_max:
  case GenXIntrinsic::genx_svm_atomic_min:
  case GenXIntrinsic::genx_svm_atomic_or:
  case GenXIntrinsic::genx_svm_atomic_sub:
  case GenXIntrinsic::genx_svm_atomic_xchg:
  case GenXIntrinsic::genx_svm_atomic_xor:
    DataIdx = 3;
    PredIdx = 0;
    AddrIdx = 1;
    AtomicSrcIdx = 2;
    AtomicNumSrc = 1;
    break;
  case GenXIntrinsic::genx_svm_atomic_cmpxchg:
  case GenXIntrinsic::genx_svm_atomic_fcmpwr:
    DataIdx = 4;
    PredIdx = 0;
    AddrIdx = 1;
    AtomicSrcIdx = 2;
    AtomicNumSrc = 2;
    break;
  case GenXIntrinsic::genx_dword_atomic_add:
  case GenXIntrinsic::genx_dword_atomic_and:
  case GenXIntrinsic::genx_dword_atomic_fadd:
  case GenXIntrinsic::genx_dword_atomic_fsub:
  case GenXIntrinsic::genx_dword_atomic_fmax:
  case GenXIntrinsic::genx_dword_atomic_fmin:
  case GenXIntrinsic::genx_dword_atomic_imax:
  case GenXIntrinsic::genx_dword_atomic_imin:
  case GenXIntrinsic::genx_dword_atomic_max:
  case GenXIntrinsic::genx_dword_atomic_min:
  case GenXIntrinsic::genx_dword_atomic_or:
  case GenXIntrinsic::genx_dword_atomic_sub:
  case GenXIntrinsic::genx_dword_atomic_xchg:
  case GenXIntrinsic::genx_dword_atomic_xor:
    DataIdx = 4;
    PredIdx = 0;
    AddrIdx = 2;
    AtomicSrcIdx = 3;
    AtomicNumSrc = 1;
    break;
  case GenXIntrinsic::genx_dword_atomic_cmpxchg:
  case GenXIntrinsic::genx_dword_atomic_fcmpwr:
    DataIdx = 5;
    PredIdx = 0;
    AddrIdx = 2;
    AtomicSrcIdx = 3;
    AtomicNumSrc = 2;
    break;
  case GenXIntrinsic::genx_dword_atomic_dec:
  case GenXIntrinsic::genx_dword_atomic_inc:
    DataIdx = 3;
    PredIdx = 0;
    AddrIdx = 2;
    AtomicNumSrc = 0;
    break;

  default:
    return false;
  }

  // nulling unused inputs for typed gather/scatter/atomic
  if (IsTyped) {
    Constant *V = dyn_cast<Constant>(CI->getArgOperand(AddrIdx + 1));
    Constant *R = dyn_cast<Constant>(CI->getArgOperand(AddrIdx + 2));
    // Only continue when R is known to be zero.
    if (R && R->isNullValue()) {
      CI->setOperand(AddrIdx + 2, UndefValue::get(R->getType()));
      if (V && V->isNullValue())
        CI->setOperand(AddrIdx + 1, UndefValue::get(V->getType()));
    }
    // check if LOD is zero for atomic
    if (AtomicNumSrc >= 0) {
      Constant *LOD = dyn_cast<Constant>(CI->getArgOperand(AddrIdx + 3));
      if (LOD && LOD->isNullValue())
        CI->setOperand(AddrIdx + 3, UndefValue::get(LOD->getType()));
    }
  }
  // Deduce intrinsic width: check predicate if exists, then check address vector.
  unsigned WidthOperand;
  if (PredIdx != NONEED)
    WidthOperand = PredIdx;
  else if (AddrIdx != NONEED)
    WidthOperand = AddrIdx;
  else
    IGC_ASSERT_EXIT_MESSAGE(0, "Cannot infer execution width of intrinsic (checked pred and addr operands)");
  auto Width =
      cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(WidthOperand)->getType())
          ->getNumElements();
  unsigned TargetWidth = IsTyped ? 8 : 16;
  if (IsTyped && ST && ST->getGRFByteSize() > 32)
    TargetWidth = 16;
  // If exec size isn't a power of 2, it must be split. For example, exec size
  // 12 -> 8 and 4.
  if (Width <= TargetWidth && isPowerOf2_64(Width))
    return false;
  IGC_ASSERT(TargetWidth);
  const std::vector<unsigned> Widths = calculateLegalWidths(Width, TargetWidth);
  IGC_ASSERT(!Widths.empty());
  unsigned NumChannels = NumVectorElements;
  if (MaskIdx != NONEED) {
    IRChecker::argOperandIsConstantInt(*CI, MaskIdx, "channel mask");
    NumChannels = (unsigned)cast<ConstantInt>(CI->getArgOperand(MaskIdx))
                      ->getZExtValue();
    NumChannels = (NumChannels & 1) + ((NumChannels & 2) >> 1) +
                  ((NumChannels & 4) >> 2) + ((NumChannels & 8) >> 3);
    IGC_ASSERT(std::all_of(Widths.begin(), Widths.end(),
                           [FirstWidth = Widths.front()](unsigned NextWidth) {
                             return FirstWidth == NextWidth;
                           }));
  }

  unsigned NumBlks = 1;
  if (IID == GenXIntrinsic::genx_svm_scatter ||
      IID == GenXIntrinsic::genx_svm_gather) {
    const unsigned NumBlocksOpIDX = 1;
    IRChecker::argOperandIsConstantInt(*CI, NumBlocksOpIDX, "log2 num blocks");
    NumBlks = (unsigned)cast<ConstantInt>(CI->getArgOperand(NumBlocksOpIDX))
                  ->getZExtValue();
    NumBlks = (1 << NumBlks);
    auto ElmSz = CI->getArgOperand(DataIdx)->getType()->getScalarSizeInBits() / 8;
    if (ElmSz == 1 && NumBlks < 4)
      NumBlks = 4;
    else if (ElmSz == 2 && NumBlks < 2)
      NumBlks = 2;
  }
  const DebugLoc &DL = CI->getDebugLoc();
  Value *NewResult = nullptr;
  if (CI->getType() && CI->getType()->isVectorTy() &&
      cast<IGCLLVM::FixedVectorType>(CI->getType())->getNumElements() >=
          Width * NumChannels * NumBlks) {
    if (DataIdx != NONEED)
      NewResult = CI->getArgOperand(DataIdx);
    else
      NewResult = UndefValue::get(CI->getType());
  }

  bool IsNewLoad = isNewLoadInst(CI);
  Value *LoadPred = nullptr;
  unsigned InitialOffset = 0;
  Instruction *InstToReplace = CI;
  if (IsNewLoad)
    std::tie(NewResult, LoadPred, InitialOffset, InstToReplace) =
        getLoadTarget(CI, ST);

  unsigned AccumulatedOffset = 0;
  for (auto CurWidth : Widths) {
    SmallVector<Value *, 8> Args;
    // initialize the args with the old values
    for (unsigned ArgI = 0; ArgI < CI->getNumArgOperands(); ++ArgI)
      Args.push_back(CI->getArgOperand(ArgI));
    // Predicate
    if (PredIdx != NONEED) {
      Value *V = CI->getArgOperand(PredIdx);
      if (auto C = dyn_cast<Constant>(V))
        Args[PredIdx] = getConstantSubvector(C, AccumulatedOffset, CurWidth);
      else
        Args[PredIdx] = Region::createRdPredRegion(
            V, AccumulatedOffset, CurWidth, "predsplit", CI, DL);
    }
    // address source
    unsigned NumAddrs = 1;
    if (IsTyped)
      NumAddrs = (AtomicNumSrc >= 0) ? 4 : 3;
    for (unsigned AddrI = 0; AddrI < NumAddrs; ++AddrI) {
      Value *V = CI->getArgOperand(AddrIdx + AddrI);
      Region R(V);
      R.Width = R.NumElements = CurWidth;
      R.Offset = AccumulatedOffset * V->getType()->getScalarSizeInBits() /
                 8; // in bytes
      Args[AddrIdx + AddrI] = R.createRdRegion(V, "addrsplit", CI, DL);
    }
    // data source
    // We need to construct a new vector with 8 elements per enabled
    // color.
    if (DataIdx != NONEED) {
      Value *V = CI->getArgOperand(DataIdx);
      auto DataTy = IGCLLVM::FixedVectorType::get(
          V->getType()->getScalarType(), CurWidth * NumChannels * NumBlks);
      auto ElmSz = V->getType()->getScalarSizeInBits() / 8;
      Value *NewVec = UndefValue::get(DataTy);
      if (!isa<UndefValue>(V)) {
        for (unsigned Channel = 0; Channel < NumChannels; ++Channel) {
          Region RdR(V);
          RdR.Width = RdR.NumElements = CurWidth * NumBlks;
          RdR.Offset =
              ElmSz * (Width * NumBlks * Channel + AccumulatedOffset * NumBlks);
          auto Rd = RdR.createRdRegion(V, "datasplit", CI, DL);
          if (NumChannels > 1) {
            Region WrR(DataTy);
            WrR.Width = WrR.NumElements = CurWidth * NumBlks;
            WrR.Offset = ElmSz * CurWidth * NumBlks * Channel;
            NewVec = WrR.createWrRegion(NewVec, Rd, "datasplit", CI, DL);
          } else
            NewVec = Rd;
        }
      }
      Args[DataIdx] = NewVec;
    }
    // atomic source operands
    if (AtomicSrcIdx != NONEED) {
      for (int SrcI = 0; SrcI < AtomicNumSrc; ++SrcI) {
        Value *V = CI->getArgOperand(AtomicSrcIdx + SrcI);
        Region R(V);
        R.Width = R.NumElements = CurWidth;
        R.Offset = AccumulatedOffset * V->getType()->getScalarSizeInBits() /
                   8; // in bytes
        Args[AtomicSrcIdx + SrcI] = R.createRdRegion(V, "addrsplit", CI, DL);
        }
    }
    // now create the new narrower instruction
    if (NewResult) {
      Type *DstTy = nullptr;
      if (DataIdx != NONEED)
        DstTy = Args[DataIdx]->getType();
      else {
        DstTy = IGCLLVM::FixedVectorType::get(CI->getType()->getScalarType(),
                                              CurWidth * NumBlks * NumChannels);
      }
      SmallVector<Type *, 4> Tys = {DstTy};

      // These two overloaded args can go in
      // arbitrary order in operand list
      unsigned IdxMin = std::min(AddrIdx, PredIdx);
      unsigned IdxMax = std::max(AddrIdx, PredIdx);
      if (IdxMin != NONEED)
        Tys.push_back(Args[IdxMin]->getType());
      if (IdxMax != NONEED)
        Tys.push_back(Args[IdxMax]->getType());

      auto Decl = GenXIntrinsic::getAnyDeclaration(CI->getModule(), IID, Tys);
      auto *Gather = CallInst::Create(Decl, Args, CI->getName() + ".split", CI);
      Gather->setDebugLoc(DL);
      Gather->copyMetadata(*CI);
      if (IsNewLoad) {
        if (NumChannels == 1)
          NewResult =
              generate1ChannelWrrregion(NewResult, InitialOffset, Gather,
                                        LoadPred, AccumulatedOffset, CI);
        else
          NewResult = generateNChannelWrregion(
              NewResult, InitialOffset, Gather, LoadPred,
              AccumulatedOffset / CurWidth, Widths.size(), NumChannels, CI);

        AccumulatedOffset += CurWidth;
        continue;
      }
      // Join the results together, starting with the old value.
      auto ElmSz = DstTy->getScalarSizeInBits() / 8;
      if (NumChannels > 1) {
        Region RdR(Gather);
        RdR.Width = RdR.NumElements = CurWidth * NumBlks;
        Region WrR(NewResult);
        WrR.Width = WrR.NumElements = CurWidth * NumBlks;
        WrR.Mask = Args[PredIdx];
        for (unsigned Channel = 0; Channel != NumChannels; ++Channel) {
          RdR.Offset = ElmSz * CurWidth * NumBlks * Channel;
          auto Rd = RdR.createRdRegion(Gather, "joint", CI, DL);
          WrR.Offset =
              ElmSz * (Width * NumBlks * Channel + NumBlks * AccumulatedOffset);
          NewResult = WrR.createWrRegion(NewResult, Rd, "join", CI, DL);
        }
      } else {
        Region WrR(NewResult);
        WrR.Width = WrR.NumElements = CurWidth * NumBlks;
        WrR.Offset = ElmSz * AccumulatedOffset * NumBlks;
        WrR.Mask = Args[PredIdx];
        NewResult = WrR.createWrRegion(NewResult, Gather, "join", CI, DL);
      }
    } else {
      IGC_ASSERT(CI->use_empty());
      IGC_ASSERT(DataIdx != NONEED);
      // Create the target-wide scatter instructions.
      Type *Tys[] = {Args[PredIdx]->getType(), Args[AddrIdx]->getType(),
                     Args[DataIdx]->getType()};
      auto Decl = GenXIntrinsic::getAnyDeclaration(CI->getModule(), IID, Tys);
      auto NewInst = CallInst::Create(Decl, Args, "", CI);
      NewInst->setDebugLoc(DL);
      NewInst->copyMetadata(*CI);
    }
    AccumulatedOffset += CurWidth;
  }

  if (NewResult)
    InstToReplace->replaceAllUsesWith(NewResult);

  if (InstToReplace != CI)
    ToErase.push_back(InstToReplace);
  ToErase.push_back(CI);
  return true;
}

static Constant* getConstVector(Type* ITy, unsigned int n, unsigned int step) {
    std::vector<Constant*> vConsts;
    unsigned v = 0;
    for (unsigned i = 0; i < n; i++) {
        vConsts.push_back(ConstantInt::get(ITy, v));
        v += step;
    }
    return ConstantVector::get(vConsts);
}


/***********************************************************************
 * translateSLMOWord : lower SLM OWord load/store to gathers/scatters
 * on legacy platform such as SKL.
 *
 * We only support the cases of 1,2,4,8 oword cases
 */
bool GenXLowering::translateSLMOWord(CallInst* CI, unsigned IID) {
  LLVMContext& CTX = CI->getContext();
  auto CIntTy = IntegerType::getInt32Ty(CTX);
  const DebugLoc& DL = CI->getDebugLoc();
  switch (IID) {
  case GenXIntrinsic::genx_oword_ld:
  case GenXIntrinsic::genx_oword_ld_unaligned: {
    constexpr unsigned BtiIdx = 1;
    constexpr unsigned AddrIdx = 2;
    Value* BtiV = CI->getArgOperand(BtiIdx);
    // only slm need this lowering
    if (!isa<ConstantInt>(BtiV))
      return false;
    if (cast<ConstantInt>(BtiV)->getZExtValue() !=
        visa::ReservedSurfaceIndex::RSI_Slm)
      return false;

    IRBuilder<> Builder(CI);
    Value* AddrV = CI->getArgOperand(AddrIdx);
    if (IID == GenXIntrinsic::genx_oword_ld) {
      AddrV = Builder.CreateShl(AddrV, llvm::ConstantInt::get(AddrV->getType(), 4));
    }
    auto *OrigVT = cast<IGCLLVM::FixedVectorType>(CI->getType());
    unsigned EltSize = OrigVT->getScalarSizeInBits();
    unsigned EltCount = OrigVT->getNumElements();
    // 1-oword is 16 bytes, using simd4 dword gather-scaled
    // 2-oword is 32 bytes, using simd8 dword gather-scaled
    // 4-oword is 64 bytes, using simd16 dword gather-scaled
    // 8-oword is 128 bytes, using 2*simd16 dword gather-scaled
    unsigned DWordCnt = (EltSize * EltCount) / 32;
    IGC_ASSERT(DWordCnt == 4 || DWordCnt == 8 || DWordCnt == 16 ||
               DWordCnt == 32);
    unsigned SimdWidth = (DWordCnt == 32) ? 16 : DWordCnt;
    auto NewVT = IGCLLVM::FixedVectorType::get(CIntTy, DWordCnt);
    auto GatherVT = IGCLLVM::FixedVectorType::get(CIntTy, SimdWidth);
    // generate gather-scaled
    auto VOffset = getConstVector(CIntTy, SimdWidth, 4);
    // create constant for predicate
    auto PredVTy = IGCLLVM::FixedVectorType::get(IntegerType::getInt1Ty(CTX), SimdWidth);
    auto OnePredV = Constant::getAllOnesValue(PredVTy);
    auto ScaleC = ConstantInt::get(Type::getInt16Ty(CTX), 0);
    std::string IntrName =
      std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) + "gather.scaled";
    auto ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
    // crease constant for num-blocks, 2 means 4-bytes
    auto NumBlksC = ConstantInt::get(CIntTy, 2);
    // create the intrinsic call
    Function* NewFDecl = GenXIntrinsic::getGenXDeclaration(
      CI->getModule(), ID, { GatherVT, PredVTy, VOffset->getType() });
    Instruction* NewInst = nullptr;
    if (DWordCnt == SimdWidth) {
      NewInst = IntrinsicInst::Create(NewFDecl,
          { OnePredV, NumBlksC, ScaleC, BtiV, AddrV,
            VOffset, UndefValue::get(GatherVT) },
          CI->getName() + ".gather", CI);
      NewInst->setDebugLoc(DL);
      LLVM_DEBUG(dbgs() << "SLM OWord Load:\n");
      LLVM_DEBUG(CI->dump());
      LLVM_DEBUG(dbgs() << "Translated to gather:\n");
      LLVM_DEBUG(NewInst->dump());
    }
    else {  // need to two gathers for 8 owords
      // 1st gather
      auto New1st = IntrinsicInst::Create(NewFDecl,
           { OnePredV, NumBlksC, ScaleC, BtiV, AddrV,
             VOffset, UndefValue::get(GatherVT) },
           CI->getName() + ".gather1", CI);
      New1st->setDebugLoc(DL);
      // 2nd gather
      AddrV = Builder.CreateAdd(AddrV, llvm::ConstantInt::get(AddrV->getType(), 64));
      auto New2nd = IntrinsicInst::Create(NewFDecl,
           { OnePredV, NumBlksC, ScaleC, BtiV, AddrV,
             VOffset, UndefValue::get(GatherVT) },
           CI->getName() + ".gather2", CI);
      New2nd->setDebugLoc(DL);
      // write region, 1st half
      Region R(NewVT);
      R.Width = SimdWidth;
      R.NumElements = SimdWidth;
      R.Stride = 1;
      R.VStride = 0;
      R.Offset = 0;
      auto PartialV = R.createWrRegion(UndefValue::get(NewVT), New1st, "", CI, CI->getDebugLoc());
      // write region, 2nd half
      R.Offset = 64;
      NewInst = R.createWrRegion(PartialV, New2nd, "", CI, CI->getDebugLoc());
      LLVM_DEBUG(dbgs() << "SLM OWord Load:\n");
      LLVM_DEBUG(CI->dump());
      LLVM_DEBUG(dbgs() << "Translated to gather:\n");
      LLVM_DEBUG(New1st->dump());
      LLVM_DEBUG(New2nd->dump());
    }
    // cast back if required
    Value* Casted = NewInst;
    if (NewVT != OrigVT)
      Casted = CastInst::CreateBitOrPointerCast(
          Casted, OrigVT, Casted->getName() + VALUE_NAME(".cast"), CI);
    CI->replaceAllUsesWith(Casted);
    ToErase.push_back(CI);
    return true;
  }
  case GenXIntrinsic::genx_oword_st: {
    constexpr unsigned DataIdx = 2;
    constexpr unsigned AddrIdx = 1;
    constexpr unsigned BtiIdx = 0;
    Value* BtiV = CI->getArgOperand(BtiIdx);
    // Only slm need this lowering
    if (!isa<ConstantInt>(BtiV))
      return false;
    if (cast<ConstantInt>(BtiV)->getZExtValue() !=
        visa::ReservedSurfaceIndex::RSI_Slm)
      return false;

    IRBuilder<> Builder(CI);
    Value* AddrV = CI->getArgOperand(AddrIdx);
    AddrV = Builder.CreateShl(AddrV, llvm::ConstantInt::get(AddrV->getType(), 4));

    Value* Datum = CI->getArgOperand(DataIdx);
    auto *OrigVT = cast<IGCLLVM::FixedVectorType>(Datum->getType());
    unsigned EltSize = OrigVT->getScalarSizeInBits();
    unsigned EltCount = OrigVT->getNumElements();
    // 1-oword is 16 bytes, using simd4 dword scatter-scaled
    // 2-oword is 32 bytes, using simd8 dword scatter-scaled
    // 4-oword is 64 bytes, using simd16 dword scatter-scaled
    // 8-oword is 128 bytes, using 2*simd16 dword scatter-scaled
    unsigned DWordCnt = (EltSize * EltCount) / 32;
    IGC_ASSERT(DWordCnt == 4 || DWordCnt == 8 || DWordCnt == 16 ||
               DWordCnt == 32);
    auto NewVT = IGCLLVM::FixedVectorType::get(CIntTy, DWordCnt);
    IGC_ASSERT_MESSAGE(CastInst::isBitCastable(NewVT, OrigVT),
        "We expect resulting vectors to be bitcastable");
    if (NewVT != OrigVT)
      Datum = CastInst::CreateBitOrPointerCast(
          Datum, NewVT, Datum->getName() + VALUE_NAME(".cast"), CI);
    unsigned SimdWidth = (DWordCnt == 32) ? 16 : DWordCnt;

    // generate scatter-scaled
    auto VOffset = getConstVector(CIntTy, SimdWidth, 4);
    // create constant for predicate
    auto PredVTy = IGCLLVM::FixedVectorType::get(IntegerType::getInt1Ty(CTX), SimdWidth);
    auto OnePredV = Constant::getAllOnesValue(PredVTy);
    auto ScaleC = ConstantInt::get(Type::getInt16Ty(CTX), 0);
    // create constant for num-blocks
    auto NumBlksC = ConstantInt::get(CIntTy, 2);
    std::string IntrName =
        std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) + "scatter.scaled";
    auto ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
    // create the intrinsic call
    auto ScatterVT = IGCLLVM::FixedVectorType::get(CIntTy, SimdWidth);
    Function* NewFDecl = GenXIntrinsic::getGenXDeclaration(
              CI->getModule(), ID, { PredVTy, VOffset->getType(), ScatterVT });
    if (DWordCnt == SimdWidth) {
      // create one scatter
      auto NewInst = Builder.CreateCall(
               NewFDecl, { OnePredV, NumBlksC, ScaleC, BtiV, AddrV, VOffset, Datum });
      NewInst->setDebugLoc(DL);
      LLVM_DEBUG(dbgs() << "SLM OWord Store:\n");
      LLVM_DEBUG(CI->dump());
      LLVM_DEBUG(dbgs() << "Translated to scatter:\n");
      LLVM_DEBUG(NewInst->dump());
    }
    else {  // 8-oword (i.e 32 dword) case
      // scatter the 1st 16 dwords
      // read region then scatter
      Region R(ScatterVT);
      R.Width = SimdWidth;
      R.NumElements = SimdWidth;
      R.Stride = 1;
      R.VStride = 0;
      R.Offset = 0;
      auto Datum1st = R.createRdRegion(Datum, "", CI, CI->getDebugLoc());
      auto New1st = Builder.CreateCall(
          NewFDecl, { OnePredV, NumBlksC, ScaleC, BtiV, AddrV, VOffset, Datum1st });
      New1st->setDebugLoc(DL);
      // scatter the 2nd 16 dwords
      // read region then scatter
      AddrV = Builder.CreateAdd(AddrV, llvm::ConstantInt::get(AddrV->getType(), 64));
      R.Offset = 64;
      auto Datum2nd = R.createRdRegion(Datum, "", CI, CI->getDebugLoc());
      auto New2nd = Builder.CreateCall(
               NewFDecl, { OnePredV, NumBlksC, ScaleC, BtiV, AddrV, VOffset, Datum2nd });
      New2nd->setDebugLoc(DL);
      LLVM_DEBUG(dbgs() << "SLM OWord Store:\n");
      LLVM_DEBUG(CI->dump());
      LLVM_DEBUG(dbgs() << "Translated to scatter:\n");
      LLVM_DEBUG(New1st->dump());
      LLVM_DEBUG(New2nd->dump());
    }
    ToErase.push_back(CI);
    return true;
  }
  default:
    break;
  }
  return false;
}

static Value *ExpandPredicate(CallInst *CI, unsigned PredIdx,
                              unsigned SIMDWidth) {
  Value *PredV = CI->getArgOperand(PredIdx);
  auto Width =
      cast<IGCLLVM::FixedVectorType>(PredV->getType())->getNumElements();
  if (Width == SIMDWidth)
    return PredV;
  IGC_ASSERT(Width < SIMDWidth);
  auto PredV32Ty = IGCLLVM::FixedVectorType::get(
      IntegerType::getInt1Ty(CI->getContext()), SIMDWidth);
  auto ZeroPred32 = Constant::getNullValue(PredV32Ty);
  if (isa<Constant>(PredV)) {
    SmallVector<Constant *, 32> PredArray;
    for (unsigned i = 0; i < Width; ++i)
      PredArray.push_back(cast<ConstantVector>(PredV)->getOperand(i));
    for (unsigned i = Width; i < SIMDWidth; ++i)
      PredArray.push_back(Constant::getNullValue(
          cast<VectorType>(PredV32Ty)->getElementType()));
    return ConstantVector::get(PredArray);
  } else {
    Region R(ZeroPred32);
    R.Width = Width;
    R.NumElements = Width;
    return R.createWrPredRegion(ZeroPred32, PredV, 0, "", CI,
                                CI->getDebugLoc());
  }
}

/***************************************************************************
 * ExpandAddrOrData : expands ArgIdx argument of CI instruction to the type
 * corresponding to SIMDWidth.
 *
 * Matrix of N x ArgWidth elements expands to matrix of N x SIMDWidth elements.
 * First elements of the rows of initial matrix become the first elements of
 * corresponding rows of a new matrix.
 *
 * TODO: After wrregion lowering we will get one mov for every row, as dst
 * register doesn't have horizontal stride. So it is possible to optimize the
 * code by copying only those rows that are used.
 *
 * Returns new value/instruction/wrregion that represents expanded argument.
 * Doesn't change the call instruction itself.
 */
static Value *ExpandAddrOrData(CallInst *CI, unsigned ArgIdx, unsigned ArgWidth,
                               unsigned NumChannels, unsigned SIMDWidth) {
  Value *InitialData = CI->getArgOperand(ArgIdx);
  if (ArgWidth == SIMDWidth)
    return InitialData;

  IGC_ASSERT(ArgWidth < SIMDWidth);
  auto InitialDataNumElem =
      cast<IGCLLVM::FixedVectorType>(InitialData->getType())->getNumElements();
  IGC_ASSERT(ArgWidth);
  IGC_ASSERT_MESSAGE(InitialDataNumElem >= ArgWidth * NumChannels,
    "there must be enough data and it should be a matrix with ArgWidth width");
  IGC_ASSERT_MESSAGE(InitialDataNumElem % ArgWidth == 0,
    "there must be enough data and it should be a matrix with ArgWidth width");
  auto InitialDataNumRows = InitialDataNumElem / ArgWidth;

  auto ExpandDataTy = IGCLLVM::FixedVectorType::get(
      cast<VectorType>(InitialData->getType())->getElementType(),
      SIMDWidth * InitialDataNumRows);
  auto ExpandDataUndef = UndefValue::get(ExpandDataTy);
  if (isa<UndefValue>(InitialData))
    return ExpandDataUndef;
  Region R(ExpandDataUndef);
  R.Width = ArgWidth;
  R.NumElements = InitialDataNumElem;
  if (NumChannels > 1)
    R.VStride = SIMDWidth;
  return R.createWrRegion(ExpandDataUndef, InitialData, "", CI,
                          CI->getDebugLoc());
}

// Widen new load intrinsics. Currently only gather4_scaled2 is supported.
// Example for gather4_scaled2 with two channels.
// Before:
//   x = <16> gather4_scaled2 <8> addrs
// After:
//   newaddrs = <16> wrregion undef, addrs
// bale {
//   tmp = <32> gather4_scaled2 newaddrs
//   wrrpred = <32> shufflevec oldpred, undef, mask
//   wrr = wrregion <32> undef, tmp, 0, xpred
//  }
// bale {
//   chR = rdregion <8> wrr, 0
//   chRpred = <8> shufflevec oldpred, undef, chmask
//   partialx = wrregion <8> oldval, chR, 0, chRpred
// }
// bale {
//   chG = rdregion <8> wrr, 16 * 4
//   chGpred = <8> shuffleve oldpred, undef, chmask
//   x = wrregion <8> partialx, chG, 8 * 4, chGpred
// }
static bool widenNewSIMD8Load(CallInst *CI, unsigned IID,
                              const GenXSubtarget *ST,
                              SmallVectorImpl<Instruction *> &ToErase) {
  switch (IID) {
  default:
    return false;
  case GenXIntrinsic::genx_gather4_scaled2:
  case GenXIntrinsic::genx_gather4_masked_scaled2:
    break;
  }

  // Probably valid only for gather.
  // Need to calculate it when other intrinsics will be normalized.
  constexpr unsigned WidenFactor = 2;

  unsigned NumChannels =
    countPopulation(cast<ConstantInt>(CI->getOperand(0))->getZExtValue());
  IGC_ASSERT(NumChannels);
  const DebugLoc &DL = CI->getDebugLoc();
  unsigned LoadWidth =
      cast<IGCLLVM::FixedVectorType>(CI->getType())->getNumElements();
  unsigned NewLoadWidth = LoadWidth * WidenFactor;
  unsigned ChannelWidth = LoadWidth / NumChannels;
  unsigned NewChannelWidth = ChannelWidth * WidenFactor;
  Type *ElemType = CI->getType()->getScalarType();

  if (ChannelWidth > 8)
    return false;

  Value *NewResult;
  Value *OldPred;
  unsigned InitialOffset;
  Instruction *InstToReplace;
  std::tie(NewResult, OldPred, InitialOffset, InstToReplace) =
      getLoadTarget(CI, ST);

  if (InstToReplace != CI)
    ToErase.push_back(InstToReplace);
  ToErase.push_back(CI);

  // Generate new addresses.
  Value *NewAddrs;
  {
    Value *Addrs = CI->getOperand(4);
    Type *AddrsType = Addrs->getType();
    Type *NewAddrsType = IGCLLVM::FixedVectorType::get(
        AddrsType->getScalarType(), NewChannelWidth);
    Region Rgn(AddrsType);
    NewAddrs = Rgn.createWrRegion(UndefValue::get(NewAddrsType), Addrs,
                                  "load.addrs.wide", CI, DL);
  }

  // Generate new wide load.
  Value *NewLoad;
  {
    Type *NewLoadType = IGCLLVM::FixedVectorType::get(ElemType, NewLoadWidth);
    SmallVector<Value *, 8> Args(CI->arg_begin(), CI->arg_end());
    SmallVector<Type *, 3> Tys;
    Tys.push_back(NewLoadType);
    switch (IID) {
    case GenXIntrinsic::genx_gather4_masked_scaled2: {
      IGC_ASSERT_MESSAGE(ChannelWidth == 8,
                         "Unexpected gather4_maked_scaled width");
      Args[4] = NewAddrs;
      // Expand predicate to the size of NewAddrs
      OldPred =
          ExpandPredicate(CI, 5 /* PredIdx */,
                          cast<IGCLLVM::FixedVectorType>(NewAddrs->getType())
                              ->getNumElements());
      Args[5] = OldPred;
      Tys.push_back(NewAddrs->getType());
      Tys.push_back(OldPred->getType());
      break;
    }
    case GenXIntrinsic::genx_gather4_scaled2: {
      IGC_ASSERT_MESSAGE(ChannelWidth == 8, "Unexpected gather4_scaled width");
      Tys.push_back(NewAddrs->getType());
      Args[4] = NewAddrs;
      break;
    }
    default:
      IGC_ASSERT_EXIT_MESSAGE(0, "Unexpected load");
    }

    Function *Decl = GenXIntrinsic::getGenXDeclaration(
        CI->getModule(), static_cast<GenXIntrinsic::ID>(IID), Tys);
    auto *NewCI = CallInst::Create(Decl, Args, CI->getName() + "wide", CI);
    NewCI->setDebugLoc(DL);
    NewLoad = NewCI;
  }

  // Generate wrregion for new load.
  {
    Type *NewLoadType = NewLoad->getType();
    Value *LoadPred = generatePredicateForLoadWrregion(
        OldPred, 0, NewLoadWidth, NumChannels, CI, DL, "load.wide.pred");
    Region Rgn(NewLoadType);
    Rgn.Mask = LoadPred;
    // If there is only one channel we can exit early.
    if (NumChannels == 1) {
      Rgn.Offset = InitialOffset;
      NewResult = Rgn.createWrRegion(NewResult, NewLoad, "load.wide.wrregion", CI, DL);
      InstToReplace->replaceAllUsesWith(NewResult);
      return true;
    }
    NewLoad = Rgn.createWrRegion(UndefValue::get(NewLoadType), NewLoad,
                                 "load.wide.wrregion", CI, DL);
  }

  // Shuffle channels.
  unsigned ElemByteSize = ElemType->getScalarSizeInBits() / 8;
  Type *ChannelType = IGCLLVM::FixedVectorType::get(ElemType, ChannelWidth);
  Region RdR(ChannelType);
  Region WrR(ChannelType);
  for (unsigned i = 0; i < NumChannels; ++i) {
    RdR.Offset = i * NewChannelWidth * ElemByteSize;
    Value *NewChannel =
        RdR.createRdRegion(NewLoad, "load.wide.channel.read", CI, DL);
    Value *NewChannelPred = generatePredicateForLoadWrregion(
        OldPred, 0, ChannelWidth, 1, CI, DL, "load.wide.channel.pred");
    WrR.Offset = InitialOffset + i * ChannelWidth * ElemByteSize;
    WrR.Mask = NewChannelPred;
    NewResult = WrR.createWrRegion(NewResult, NewChannel,
                                   "load.wide.channel.write", CI, DL);
  }

  InstToReplace->replaceAllUsesWith(NewResult);
  return true;
}

/***********************************************************************
 * widenSIMD8GatherScatter: widen simd8 gather and scatter
 *
 * This performs two functions:
 *
 * 1. If the operation is simd8 rather than simd16, expand it into
 * simd16 instructions because the simd8 gather/scatter/atomic
 * is either removed by hw (typed gather/scatter/atomic) or semantically
 * changed by hw (untyped gather4/scatter4, data payload no longer
 * contiguous in case of simd8).
 *
 * 2. For a simd16 scatter4/gather4_typed, when r or both v and r are zero,
 * replace with undef so that they are not encoded in the vISA instruction and
 * the message skips them.
 */
bool GenXLowering::widenSIMD8GatherScatter(CallInst *CI, unsigned IID) {
  if (isNewLoadInst(CI))
    return widenNewSIMD8Load(CI, IID, ST, ToErase);

  enum {
    MASK_IDX = 0,
    PRED_IDX = 1,
    SURF_IDX = 2,
    U_IDX = 3,
    DATA_IDX = 6,
    NONEED = 11
  };
  constexpr int WidenSIMD = 16;

  unsigned MaskIdx = NONEED;
  unsigned PredIdx = NONEED;
  unsigned AddrIdx = NONEED;
  unsigned DataIdx = NONEED;
  bool IsTyped = false;
  int AtomicNumSrc = (-1); // -1 means not an atomic

  switch (IID) {
  case GenXIntrinsic::genx_typed_atomic_add:
  case GenXIntrinsic::genx_typed_atomic_and:
  case GenXIntrinsic::genx_typed_atomic_fadd:
  case GenXIntrinsic::genx_typed_atomic_fsub:
  case GenXIntrinsic::genx_typed_atomic_fmax:
  case GenXIntrinsic::genx_typed_atomic_fmin:
  case GenXIntrinsic::genx_typed_atomic_imax:
  case GenXIntrinsic::genx_typed_atomic_imin:
  case GenXIntrinsic::genx_typed_atomic_max:
  case GenXIntrinsic::genx_typed_atomic_min:
  case GenXIntrinsic::genx_typed_atomic_or:
  case GenXIntrinsic::genx_typed_atomic_sub:
  case GenXIntrinsic::genx_typed_atomic_xchg:
  case GenXIntrinsic::genx_typed_atomic_xor: {
    DataIdx = 2;
    PredIdx = 0;
    AddrIdx = 3;
    IsTyped = true;
    AtomicNumSrc = 1;
  } break;
  case GenXIntrinsic::genx_typed_atomic_dec:
  case GenXIntrinsic::genx_typed_atomic_inc: {
    PredIdx = 0;
    AddrIdx = 2;
    IsTyped = true;
    AtomicNumSrc = 0;
  } break;
  case GenXIntrinsic::genx_typed_atomic_cmpxchg:
  case GenXIntrinsic::genx_typed_atomic_fcmpwr: {
    DataIdx = 2;
    PredIdx = 0;
    AddrIdx = 4;
    IsTyped = true;
    AtomicNumSrc = 2;
  } break;
  case GenXIntrinsic::genx_scatter4_typed:
  case GenXIntrinsic::genx_gather4_typed: {
    DataIdx = DATA_IDX;
    MaskIdx = MASK_IDX;
    PredIdx = PRED_IDX;
    AddrIdx = U_IDX;
    IsTyped = true;
  } break;
  case GenXIntrinsic::genx_scatter4_scaled:
  case GenXIntrinsic::genx_gather4_scaled: {
    DataIdx = 6;
    PredIdx = 0;
    MaskIdx = 1;
    AddrIdx = 5;
  } break;
  case GenXIntrinsic::genx_svm_scatter4_scaled:
  case GenXIntrinsic::genx_svm_gather4_scaled: {
    DataIdx = 5;
    PredIdx = 0;
    MaskIdx = 1;
    AddrIdx = 4;
  } break;
  default:
    return false;
  }

  if (IsTyped) {
    Constant *V = dyn_cast<Constant>(CI->getArgOperand(AddrIdx + 1));
    Constant *R = dyn_cast<Constant>(CI->getArgOperand(AddrIdx + 2));
    // Only continue when R is known to be zero.
    if (R && R->isNullValue()) {
      CI->setOperand(AddrIdx + 2, UndefValue::get(R->getType()));
      if (V && V->isNullValue())
        CI->setOperand(AddrIdx + 1, UndefValue::get(V->getType()));
    }
    // check if LOD is zero for atomic
    if (AtomicNumSrc >= 0) {
      Constant *LOD = dyn_cast<Constant>(CI->getArgOperand(AddrIdx + 3));
      if (LOD && LOD->isNullValue())
        CI->setOperand(AddrIdx + 3, UndefValue::get(LOD->getType()));
    }
  }

  auto Width =
      cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(PredIdx)->getType())
          ->getNumElements();
  if (Width == 16 || Width == 32)
    return false;

  IGC_ASSERT(Width == 1 || Width == 2 || Width == 4 || Width == 8);
  // legacy intrinsic, if width is 1/2/4/8, expand it into 16 wide
  // on PVC in legacy (no-translation) mode
  const DebugLoc &DL = CI->getDebugLoc();
  unsigned NumChannels = 1;
  if (MaskIdx != NONEED) {
    NumChannels =
        (unsigned)cast<ConstantInt>(CI->getArgOperand(MaskIdx))->getZExtValue();
    NumChannels = (NumChannels & 1) + ((NumChannels & 2) >> 1) +
                  ((NumChannels & 4) >> 2) + ((NumChannels & 8) >> 3);
  }

  SmallVector<Value *, 8> Args;
  for (unsigned i = 0; i < CI->getNumArgOperands(); ++i) {
    Args.push_back(CI->getArgOperand(i));
  }
  Args[PredIdx] = ExpandPredicate(CI, PredIdx, WidenSIMD);

  Value *AddrV = CI->getArgOperand(AddrIdx);
  auto AddrV16Ty = IGCLLVM::FixedVectorType::get(
      cast<VectorType>(AddrV->getType())->getElementType(), 16);
  auto UndefAddr16 = UndefValue::get(AddrV16Ty);
  Region R(UndefAddr16);
  R.Width = Width;
  R.NumElements = Width;
  auto WAddrR = R.createWrRegion(UndefAddr16, AddrV, "", CI, DL);
  Args[AddrIdx] = WAddrR;
  // two more address arguments for typed-gather4
  if (IsTyped) {
    AddrV = CI->getArgOperand(AddrIdx + 1);
    if (isa<UndefValue>(AddrV))
      Args[AddrIdx + 1] = UndefAddr16;
    else
      Args[AddrIdx + 1] = R.createWrRegion(UndefAddr16, AddrV, "", CI, DL);
    AddrV = CI->getArgOperand(AddrIdx + 2);
    if (isa<UndefValue>(AddrV))
      Args[AddrIdx + 2] = UndefAddr16;
    else
      Args[AddrIdx + 2] = R.createWrRegion(UndefAddr16, AddrV, "", CI, DL);
    //  typed-atomic, LOD argument
    if (AtomicNumSrc >= 0) {
      auto LODV = CI->getArgOperand(AddrIdx + 3);
      if (isa<UndefValue>(LODV))
        Args[AddrIdx + 3] = UndefAddr16;
      else
        Args[AddrIdx + 3] = R.createWrRegion(UndefAddr16, LODV, "", CI, DL);
    }
  }

  unsigned NumSrc = (AtomicNumSrc >= 0) ? AtomicNumSrc : 1;
  for (unsigned i = 0; i < NumSrc; ++i) {
    Args[DataIdx + i] =
        ExpandAddrOrData(CI, DataIdx + i, Width, NumChannels, WidenSIMD);
  }

  Instruction *NewInst = nullptr;
  switch (IID) {
  case GenXIntrinsic::genx_scatter4_typed:
  case GenXIntrinsic::genx_scatter4_scaled:
  case GenXIntrinsic::genx_svm_scatter4_scaled: {
    // Create the 16 wide scatter4 instructions.
    Type *Tys[] = {Args[PredIdx]->getType(), Args[AddrIdx]->getType(),
                   Args[DataIdx]->getType()};
    auto Decl = GenXIntrinsic::getGenXDeclaration(CI->getModule(),
                                                  (GenXIntrinsic::ID)IID, Tys);
    NewInst = CallInst::Create(Decl, Args, "", CI);
    NewInst->setDebugLoc(DL);
    ToErase.push_back(CI);
  } break;
  default: {
    // Create the 16 wide gather4 or atomic instructions
    auto DataV16Ty =
        (DataIdx == NONEED)
            ? IGCLLVM::FixedVectorType::get(
                  cast<VectorType>(CI->getType())->getElementType(), WidenSIMD)
            : Args[DataIdx]->getType();
    Type *Tys[] = {DataV16Ty, Args[PredIdx]->getType(),
                   Args[AddrIdx]->getType()};
    auto Decl = GenXIntrinsic::getAnyDeclaration(CI->getModule(), IID, Tys);
    NewInst = CallInst::Create(Decl, Args, CI->getName() + ".expand", CI);
    NewInst->setDebugLoc(DL);
    // for gather, need to generate read-region
    Region R(CI->getType());
    IGC_ASSERT(WidenSIMD);
    R.Width = Width;
    R.NumElements =
        (cast<IGCLLVM::FixedVectorType>(NewInst->getType())->getNumElements() /
         WidenSIMD) *
        Width;
    if (NumChannels > 1) {
      R.VStride = WidenSIMD;
    }
    auto NewVec = R.createRdRegion(NewInst, "", CI, DL);
    CI->replaceAllUsesWith(NewVec);
    ToErase.push_back(CI);
  } break;
  }
  return true;
}

static Constant *LSCDatumSize(Type *VTy) {
  auto ETy = cast<VectorType>(VTy)->getElementType();
  auto ElementBytes = ETy->getPrimitiveSizeInBits() / 8;
  auto DatumSize = 0;
  switch (ElementBytes) {
  case 1:
    DatumSize = LSC_DATA_SIZE_8b;
    break;
  case 2:
    DatumSize = LSC_DATA_SIZE_16b;
    break;
  case 4:
    DatumSize = LSC_DATA_SIZE_32b;
    break;
  case 8:
    DatumSize = LSC_DATA_SIZE_64b;
    break;
  default:
    IGC_ASSERT_MESSAGE(0, "datum type not support by LSC");
    break;
  }
  IntegerType *I8Ty = Type::getInt8Ty(VTy->getContext());
  return ConstantInt::get(I8Ty, DatumSize);
}

static Constant *LSCDatumSize(Value *DataArg) {
  IGC_ASSERT(DataArg);
  return LSCDatumSize(DataArg->getType());
}

static LSC_DATA_ELEMS LSCVectorSize(unsigned N) {
  switch (N) {
  case 1:
    return LSC_DATA_ELEMS_1;
  case 2:
    return LSC_DATA_ELEMS_2;
  case 3:
    return LSC_DATA_ELEMS_3;
  case 4:
    return LSC_DATA_ELEMS_4;
  case 8:
    return LSC_DATA_ELEMS_8;
  case 16:
    return LSC_DATA_ELEMS_16;
  case 32:
    return LSC_DATA_ELEMS_32;
  case 64:
    return LSC_DATA_ELEMS_64;
  default:
    break;
  }
  return LSC_DATA_ELEMS_INVALID;
}

static Constant *LSCScale(Value *ArgScale) {
  IGC_ASSERT(isa<ConstantInt>(ArgScale));
  unsigned Scale = (unsigned)cast<ConstantInt>(ArgScale)->getZExtValue();
  IntegerType *I16Ty = Type::getInt16Ty(ArgScale->getContext());
  return ConstantInt::get(I16Ty, (Scale ? Scale : 1));
}

static void TranslateDWordAtomic(LSC_OP SubOp, CallInst *CI) {
  IntegerType *I8Ty = Type::getInt8Ty(CI->getContext());
  auto CZeroInt8 = ConstantInt::get(I8Ty, 0);
  IntegerType *I16Ty = Type::getInt16Ty(CI->getContext());
  auto COneInt16 = ConstantInt::get(I16Ty, 1);
  IntegerType *I32Ty = Type::getInt32Ty(CI->getContext());
  auto CZeroInt32 = ConstantInt::get(I32Ty, 0);

  unsigned DataIdx = 3;
  unsigned PredIdx = 0;
  unsigned AddrIdx = 2;

  auto Width =
      cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(PredIdx)->getType())
          ->getNumElements();
  IGC_ASSERT(Width == 8 || Width == 16 || Width == 32);
  SmallVector<Value *, 15> Args;
  Args.push_back(ExpandPredicate(CI, PredIdx, 32));         // #0, Predicate
  Args.push_back(ConstantInt::get(I8Ty, SubOp));            // #1, subopcode
  Args.push_back(CZeroInt8);                                // #2, L1 control
  Args.push_back(CZeroInt8);                                // #3, L3 control
  Args.push_back(COneInt16);                                // #4, scale
  Args.push_back(CZeroInt32);                               // #5, immed-offset
  Args.push_back(LSCDatumSize(CI->getArgOperand(DataIdx))); // #6
  Args.push_back(ConstantInt::get(I8Ty, LSC_DATA_ORDER_NONTRANSPOSE)); // #7
  Args.push_back(
      ConstantInt::get(I8Ty, LSCVectorSize(1))); // #8, num-elem is one
  Args.push_back(CZeroInt8);                     // #9, no-channel-mask

  Value *BTIV = CI->getArgOperand(1);
  auto AtomicIID = GenXIntrinsic::genx_lsc_atomic_bti;
  if (isa<ConstantInt>(BTIV)) {
    if (cast<ConstantInt>(BTIV)->getZExtValue() ==
        visa::ReservedSurfaceIndex::RSI_Slm) {
      AtomicIID = GenXIntrinsic::genx_lsc_atomic_slm;

      // in new LSC interface, surface index visa::ReservedSurfaceIndex::RSI_Slm
      // not used we need to have 0 instead, because only type of message
      // matters
      BTIV = ConstantInt::get(BTIV->getType(), 0);
    }
  }
  Args.push_back(BTIV); // #10, SurfaceIndex

  Args.push_back(ExpandAddrOrData(CI, AddrIdx, Width, 1, 32)); // #11, address

  Value *DataV = CI->getArgOperand(DataIdx);
  auto *DataVector = cast<IGCLLVM::FixedVectorType>(DataV->getType());
  IGC_ASSERT_MESSAGE(Width == DataVector->getNumElements(),
    "Cannot translate atomic more than one element per lane");
  auto DataV32Ty = IGCLLVM::FixedVectorType::get(
      DataVector->getElementType(), 32);
  if (SubOp == LSC_ATOMIC_IINC || SubOp == LSC_ATOMIC_IDEC) {
    Args.push_back(UndefValue::get(DataV32Ty));
    Args.push_back(UndefValue::get(DataV32Ty));
  } else if (SubOp == LSC_ATOMIC_ICAS || SubOp == LSC_ATOMIC_FCAS) {
    // exchange src operand order
    Args.push_back(ExpandAddrOrData(CI, DataIdx+1, Width, 1, 32)); // #12, src1
    Args.push_back(ExpandAddrOrData(CI, DataIdx, Width, 1, 32)); // #13, src0
    DataIdx += 2;
  } else {
    Args.push_back(ExpandAddrOrData(CI, DataIdx, Width, 1, 32)); // #12, src0
    Args.push_back(UndefValue::get(DataV32Ty));
    DataIdx++;
  }

  Args.push_back(ExpandAddrOrData(CI, DataIdx, Width, 1, 32)); // #14, old-data

  // create the call to 32 wide lsc.atomic.stateless instructions.
  Type *Tys[] = {Args[0]->getType(), Args[11]->getType(), DataV32Ty};
  auto Decl =
      GenXIntrinsic::getGenXDeclaration(CI->getModule(), AtomicIID, Tys);
  auto NewInst = CallInst::Create(Decl, Args, CI->getName() + VALUE_NAME(".lsc"), CI);
  NewInst->setDebugLoc(CI->getDebugLoc());
  if (!CI->use_empty()) {
    if (Width < 32) {
      Region R(CI->getType());
      R.Width = Width;
      R.NumElements = Width;
      auto NewVec = R.createRdRegion(NewInst, "", CI, CI->getDebugLoc());
      CI->replaceAllUsesWith(NewVec);
    } else
      CI->replaceAllUsesWith(NewInst);
  }
}

static void TranslateSVMAtomic(LSC_OP SubOp, CallInst *CI) {
  IntegerType *I8Ty = Type::getInt8Ty(CI->getContext());
  auto CZeroInt8 = ConstantInt::get(I8Ty, 0);
  IntegerType *I16Ty = Type::getInt16Ty(CI->getContext());
  auto COneInt16 = ConstantInt::get(I16Ty, 1);
  IntegerType *I32Ty = Type::getInt32Ty(CI->getContext());
  auto CZeroInt32 = ConstantInt::get(I32Ty, 0);

  unsigned DataIdx = 2;
  unsigned PredIdx = 0;
  unsigned AddrIdx = 1;

  auto Width =
      cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(PredIdx)->getType())
          ->getNumElements();
  IGC_ASSERT(Width == 8 || Width == 16 || Width == 32);
  SmallVector<Value *, 15> Args;
  Args.push_back(ExpandPredicate(CI, PredIdx, 32));         // #0, Predicate
  Args.push_back(ConstantInt::get(I8Ty, SubOp));            // #1, subopcode
  Args.push_back(CZeroInt8);                                // #2, L1 control
  Args.push_back(CZeroInt8);                                // #3, L3 control
  Args.push_back(COneInt16);                                // #4, scale
  Args.push_back(CZeroInt32);                               // #5, immed-offset
  Args.push_back(LSCDatumSize(CI->getArgOperand(DataIdx))); // #6
  Args.push_back(ConstantInt::get(I8Ty, LSC_DATA_ORDER_NONTRANSPOSE)); // #7
  Args.push_back(
      ConstantInt::get(I8Ty, LSCVectorSize(1))); // #8, num-elem is one
  Args.push_back(CZeroInt8);                     // #9, no-channel-mask
  Args.push_back(CZeroInt32);                    // #10, SurfaceIndex
  Args.push_back(ExpandAddrOrData(CI, AddrIdx, Width, 1, 32)); // #11, address

  Value *DataV = CI->getArgOperand(DataIdx);
  auto *DataVector = cast<IGCLLVM::FixedVectorType>(DataV->getType());
  IGC_ASSERT_MESSAGE(Width == DataVector->getNumElements(),
    "Cannot translate svm-atomics more than one element per lane");
  auto DataV32Ty = IGCLLVM::FixedVectorType::get(
      DataVector->getElementType(), 32);
  if (SubOp == LSC_ATOMIC_IINC || SubOp == LSC_ATOMIC_IDEC) {
    Args.push_back(UndefValue::get(DataV32Ty));
    Args.push_back(UndefValue::get(DataV32Ty));
  } else if (SubOp == LSC_ATOMIC_ICAS || SubOp == LSC_ATOMIC_FCAS) {
    // exchange source ooperand order
    Args.push_back(ExpandAddrOrData(CI, DataIdx+1, Width, 1, 32)); // #12, src1
    Args.push_back(ExpandAddrOrData(CI, DataIdx, Width, 1, 32)); // #13, src0
    DataIdx += 2;
  } else {
    Args.push_back(ExpandAddrOrData(CI, DataIdx, Width, 1, 32)); // #12, src0
    Args.push_back(UndefValue::get(DataV32Ty));
    DataIdx++;
  }

  Args.push_back(ExpandAddrOrData(CI, DataIdx, Width, 1, 32)); // #14, old-data

  // create the call to 32 wide lsc.atomic.stateless instructions.
  Type *Tys[] = {Args[0]->getType(), Args[11]->getType(), DataV32Ty};
  auto Decl = GenXIntrinsic::getGenXDeclaration(
      CI->getModule(), GenXIntrinsic::genx_lsc_atomic_stateless, Tys);
  auto NewInst = CallInst::Create(Decl, Args, CI->getName() + VALUE_NAME(".lsc"), CI);
  NewInst->setDebugLoc(CI->getDebugLoc());
  if (!CI->use_empty()) {
    if (Width < 32) {
      Region R(CI->getType());
      R.Width = Width;
      R.NumElements = Width;
      auto NewVec = R.createRdRegion(NewInst, "", CI, CI->getDebugLoc());
      CI->replaceAllUsesWith(NewVec);
    } else
      CI->replaceAllUsesWith(NewInst);
  }
}

static void ComputeScatterAddress(CallInst *CI, unsigned ScaleIdx,
                                  unsigned GoffIdx, unsigned AddrIdx) {
  IRBuilder<> Builder(CI);
  Builder.SetCurrentDebugLocation(CI->getDebugLoc());
  auto ScaleVal = CI->getArgOperand(ScaleIdx);
  auto GoffVal = CI->getArgOperand(GoffIdx);
  auto AddrVal = CI->getArgOperand(AddrIdx);
  IGC_ASSERT(isa<ConstantInt>(ScaleVal));
  unsigned Scale = (unsigned)cast<ConstantInt>(ScaleVal)->getZExtValue();
  IGC_ASSERT(isa<VectorType>(AddrVal->getType()));
  auto Width =
      cast<IGCLLVM::FixedVectorType>(AddrVal->getType())->getNumElements();
  if (Scale > 1) {
    ScaleVal = ConstantInt::get(
        cast<VectorType>(AddrVal->getType())->getElementType(), Scale);
    AddrVal = Builder.CreateMul(
        AddrVal, ConstantVector::getSplat(IGCLLVM::getElementCount(Width),
                                          cast<Constant>(ScaleVal)));
  }
  // broadcast GoffIdx
  auto BC = Builder.CreateBitCast(
      GoffVal, IGCLLVM::FixedVectorType::get(GoffVal->getType(), 1));
  // Create a rdregion with a stride of 0 to represent this splat
  Region R(BC);
  R.NumElements = Width;
  R.Width = R.NumElements;
  R.Stride = 0;
  R.VStride = 0;
  R.Offset = 0;
  Instruction *Splat =
      R.createRdRegion(BC, "", CI /*InsertBefore*/, CI->getDebugLoc());
  auto AV = Builder.CreateAdd(AddrVal, Splat);
  CI->setArgOperand(AddrIdx, AV);
  CI->setArgOperand(ScaleIdx, ConstantInt::get(ScaleVal->getType(), 1));
  CI->setArgOperand(GoffIdx, ConstantInt::get(GoffVal->getType(), 0));
}

// for transposed cases only (like oword, etc)
// we want to transform:
//
//      N x i8 -> (N / 4) x i32
//      N x i16 -> (N / 2) x i32
//      N x half -> N x i16 -> (N / 2) x i32
//
// if it is possible. D8 and D16 often illegal in LSC
IGCLLVM::FixedVectorType *
GenXLowering::adjustVTForTransposedMessage(IGCLLVM::FixedVectorType *VT) {
  IGC_ASSERT(VT);
  auto *ElementType = VT->getElementType();

  if (!ElementType->isIntegerTy() && !ElementType->isFloatingPointTy())
    return VT;

  unsigned EltSize = DL->getTypeSizeInBits(ElementType);
  if ((EltSize != 16) && (EltSize != 8))
    return VT;

  unsigned EltCount = VT->getNumElements();
  if (((EltCount % 2) != 0) || ((EltSize == 8) && ((EltCount % 4) != 0)))
    return VT;

  unsigned DataSize = DL->getTypeSizeInBits(VT);
  IGC_ASSERT(EltSize == DataSize / EltCount);
  LLVM_DEBUG(dbgs() << "Vector: " << EltCount << " x i" << EltSize << "\n");

  switch (EltSize) {
  case 8:
    EltCount = EltCount / 4;
    break;
  case 16:
    EltCount = EltCount / 2;
    break;
  default:
    IGC_ASSERT_MESSAGE(0, "We shall not be there");
  }

  auto *NewVT = IGCLLVM::FixedVectorType::get(
      IntegerType::getInt32Ty(VT->getContext()), EltCount);

  return NewVT;
}

/*************************************************************************
 * translateLegacyToLSC: translate legacy genx intrinsic to LSC intrinsic
 *
 * Gather/Scatter/Atomic translated to the corresponding SIMD32 LSC
 * intrinsic with predicate-modification and temporary vector-32 in&out
 *
 * Gather4/Scatter similar to Gather/Scatter with Channel-Mask
 *
 * OWord block read/write translated to simd1-transposed LSC load/store
 *
 * media-block read/write translated to simd1 LSC load2d/store2d
 */
bool GenXLowering::translateLegacyToLSC(CallInst *CI, unsigned IID) {
  IntegerType *I8Ty = Type::getInt8Ty(CI->getContext());
  auto CZeroInt8 = ConstantInt::get(I8Ty, 0);
  IntegerType *I16Ty = Type::getInt16Ty(CI->getContext());
  auto COneInt16 = ConstantInt::get(I16Ty, 1);
  IntegerType *I32Ty = Type::getInt32Ty(CI->getContext());
  auto CZeroInt32 = ConstantInt::get(I32Ty, 0);
  // 1xi1 type for block-load and block-store
  auto PredV1Ty = IGCLLVM::FixedVectorType::get(
      IntegerType::getInt1Ty(CI->getContext()), 1);
  auto OnePred1 = Constant::getAllOnesValue(PredV1Ty);
  auto TransposeConst = ConstantInt::get(I8Ty, LSC_DATA_ORDER_TRANSPOSE);

  // transposed message may live with non-vector predicate
  auto PredTy = IntegerType::getInt1Ty(CI->getContext());
  auto OnePredScalar = Constant::getAllOnesValue(PredTy);

  auto DL = CI->getDebugLoc();
  switch (IID) {
  case GenXIntrinsic::genx_scatter4_typed:
  case GenXIntrinsic::genx_gather4_typed: {
    DiagnosticInfoLowering Err(
        CI, "Typed gather4/scatter4 typed translation not implemented",
        DS_Error);
    CI->getContext().diagnose(Err);
    return false;
  }
  case GenXIntrinsic::genx_scatter_scaled: {
    unsigned DataIdx = 6;
    unsigned PredIdx = 0;
    unsigned AddrIdx = 5;
    // lsc.load does not support non-constant global offset
    auto GlobOffset = CI->getArgOperand(4);
    if (!isa<Constant>(GlobOffset)) {
      // insert address computation
      ComputeScatterAddress(CI, 2, 4, AddrIdx);
    }
    auto Width =
        cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(PredIdx)->getType())
            ->getNumElements();
    IGC_ASSERT(Width == 8 || Width == 16 || Width == 32);
    SmallVector<Value *, 13> Args;
    Args.push_back(ExpandPredicate(CI, PredIdx, 32));  // #0, Predicate
    Args.push_back(ConstantInt::get(I8Ty, LSC_STORE)); // #1, subopcode
    Args.push_back(CZeroInt8);                         // #2, L1 control
    Args.push_back(CZeroInt8);                         // #3, L3 control
    Args.push_back(LSCScale(CI->getArgOperand(2)));    // #4, scale
    Args.push_back(CI->getArgOperand(4));              // #5, immed-offset
    Args.push_back(LSCDatumSize(CI->getArgOperand(DataIdx))); // #6
    Args.push_back(
        ConstantInt::get(I8Ty, LSCVectorSize(1))); // #7, num-elem is one
    Args.push_back(ConstantInt::get(I8Ty, LSC_DATA_ORDER_NONTRANSPOSE)); // #8
    Args.push_back(CZeroInt8); // #9, no-channel-mask
    Args.push_back(ExpandAddrOrData(CI, AddrIdx, Width, 1, 32)); // #10, offsets
    Args.push_back(ExpandAddrOrData(CI, DataIdx, Width, 1, 32)); // #11, Data
    Value *BTIV = CI->getArgOperand(3);
    auto StoreIID = GenXIntrinsic::genx_lsc_store_bti;
    if (isa<ConstantInt>(BTIV)) {
      if (cast<ConstantInt>(BTIV)->getZExtValue() ==
          visa::ReservedSurfaceIndex::RSI_Slm) {
        StoreIID = GenXIntrinsic::genx_lsc_store_slm;

        // in new LSC interface, surface index
        // visa::ReservedSurfaceIndex::RSI_Slm not used we need to have 0
        // instead, because only type of message matters
        BTIV = ConstantInt::get(BTIV->getType(), 0);
      }
    }
    Args.push_back(BTIV); // #12, SurfaceIndex

    Value *DataV = CI->getArgOperand(DataIdx);
    auto *DataVector = cast<IGCLLVM::FixedVectorType>(DataV->getType());
    IGC_ASSERT_MESSAGE(Width == DataVector->getNumElements(),
      "Cannot translate scatter more than one element per lane");
    auto DataV32Ty = IGCLLVM::FixedVectorType::get(
        DataVector->getElementType(), 32);

    // now we sure that all can be converted, so increase statistics
    NumLegacy += 1;

    // create the call to 32 wide lsc.load.bti instructions.
    Type *Tys[] = {Args[0]->getType(), Args[10]->getType(), DataV32Ty};
    auto Decl =
        GenXIntrinsic::getGenXDeclaration(CI->getModule(), StoreIID, Tys);
    auto NewInst = CallInst::Create(Decl, Args, "", CI);
    NewInst->setDebugLoc(DL);

    LLVM_DEBUG(dbgs() << "Legacy intrinsic:\n");
    LLVM_DEBUG(CI->dump());
    LLVM_DEBUG(dbgs() << "Translated to:\n");
    LLVM_DEBUG(NewInst->dump());

    CI->replaceAllUsesWith(NewInst);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_gather_scaled:
  case GenXIntrinsic::genx_gather_scaled2:
  case GenXIntrinsic::genx_gather_masked_scaled2: {
    unsigned PredIdx = 0;
    unsigned NBlocksIdx = 1;
    unsigned ScaleIdx = 2;
    unsigned SurfIdx = 3;
    unsigned OffIdx = 4;
    unsigned AddrIdx = 5;
    Value *PredValue;

    if (IID == GenXIntrinsic::genx_gather_scaled2) {
      NBlocksIdx = 0;
      ScaleIdx = 1;
      SurfIdx = 2;
      OffIdx = 3;
      AddrIdx = 4;
      PredValue = OnePredScalar;
    }

    if (IID == GenXIntrinsic::genx_gather_masked_scaled2) {
      NBlocksIdx = 0;
      ScaleIdx = 1;
      SurfIdx = 2;
      OffIdx = 3;
      AddrIdx = 4;
      PredIdx = 5;
    }

    if (IID == GenXIntrinsic::genx_gather_scaled ||
        IID == GenXIntrinsic::genx_gather_masked_scaled2) {
      PredValue = ExpandPredicate(CI, PredIdx, 32);
    }

    // data size from return type of gather
    Constant *DataSize = LSCDatumSize(CI);

    // lsc.load does not support non-constant global offset
    auto GlobOffset = CI->getArgOperand(OffIdx);
    if (!isa<Constant>(GlobOffset)) {
      // insert address computation
      ComputeScatterAddress(CI, ScaleIdx, OffIdx, AddrIdx);
    }
    auto Width =
        cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(AddrIdx)->getType())
            ->getNumElements();
    IGC_ASSERT(Width == 8 || Width == 16 || Width == 32);
    SmallVector<Value *, 12> Args;
    Args.push_back(PredValue);                        // #0, Predicate
    Args.push_back(ConstantInt::get(I8Ty, LSC_LOAD)); // #1, subopcode
    Args.push_back(CZeroInt8);                        // #2, L1 control
    Args.push_back(CZeroInt8);                        // #3, L3 control
    Args.push_back(LSCScale(CI->getArgOperand(ScaleIdx))); // #4, scale
    Args.push_back(CI->getArgOperand(OffIdx));             // #5, immed-offset
    Args.push_back(DataSize);                              // #6
    Args.push_back(
        ConstantInt::get(I8Ty, LSCVectorSize(1))); // #7, num-elem is one
    Args.push_back(ConstantInt::get(I8Ty, LSC_DATA_ORDER_NONTRANSPOSE)); // #8
    Args.push_back(CZeroInt8); // #9, no-channel-mask
    Args.push_back(ExpandAddrOrData(CI, AddrIdx, Width, 1, 32)); // #10, offsets
    Value *BTIV = CI->getArgOperand(SurfIdx);
    auto LoadIID = GenXIntrinsic::genx_lsc_load_bti;
    if (isa<ConstantInt>(BTIV)) {
      if (cast<ConstantInt>(BTIV)->getZExtValue() ==
          visa::ReservedSurfaceIndex::RSI_Slm) {
        LoadIID = GenXIntrinsic::genx_lsc_load_slm;

        // in new LSC interface, surface index
        // visa::ReservedSurfaceIndex::RSI_Slm not used we need to have 0
        // instead, because only type of message matters
        BTIV = ConstantInt::get(BTIV->getType(), 0);
      }
    }
    Args.push_back(BTIV); // #11, SurfaceIndex

    auto *DataVector = cast<IGCLLVM::FixedVectorType>(CI->getType());
    IGC_ASSERT_MESSAGE(
        Width == DataVector->getNumElements(),
        "Cannot translate gather more than one element per lane");
    auto DataV32Ty =
        IGCLLVM::FixedVectorType::get(DataVector->getElementType(), 32);

    // now we sure that all can be converted, so increase statistics
    NumLegacy += 1;

    // create the call to 32 wide lsc.load.bti instructions.
    Type *Tys[] = {DataV32Ty, Args[0]->getType(), Args[10]->getType()};
    auto Decl =
        GenXIntrinsic::getGenXDeclaration(CI->getModule(), LoadIID, Tys);
    auto NewInst = CallInst::Create(Decl, Args, CI->getName() + VALUE_NAME(".lsc"), CI);
    NewInst->setDebugLoc(DL);

    LLVM_DEBUG(dbgs() << "Legacy intrinsic:\n");
    LLVM_DEBUG(CI->dump());
    LLVM_DEBUG(dbgs() << "Translated to:\n");
    LLVM_DEBUG(NewInst->dump());

    if (Width < 32) {
      Region R(CI->getType());
      R.Width = Width;
      R.NumElements = Width;
      auto NewVec = R.createRdRegion(NewInst, "", CI, DL);
      CI->replaceAllUsesWith(NewVec);
    } else
      CI->replaceAllUsesWith(NewInst);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_scatter4_scaled: {
    unsigned DataIdx = 6;
    unsigned PredIdx = 0;
    unsigned MaskIdx = 1;
    unsigned AddrIdx = 5;
    // lsc.load does not support non-constant global offset
    auto GlobOffset = CI->getArgOperand(4);
    if (!isa<Constant>(GlobOffset)) {
      // insert address computation
      ComputeScatterAddress(CI, 2, 4, AddrIdx);
    }
    auto Width =
        cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(PredIdx)->getType())
            ->getNumElements();
    IGC_ASSERT(Width == 8 || Width == 16 || Width == 32);
    unsigned Mask =
        (unsigned)cast<ConstantInt>(CI->getArgOperand(MaskIdx))->getZExtValue();
    unsigned NumChannels =
        (Mask & 1) + ((Mask & 2) >> 1) + ((Mask & 4) >> 2) + ((Mask & 8) >> 3);
    SmallVector<Value *, 13> Args;
    Args.push_back(ExpandPredicate(CI, PredIdx, 32));       // #0, Predicate
    Args.push_back(ConstantInt::get(I8Ty, LSC_STORE_QUAD)); // #1, subopcode
    Args.push_back(CZeroInt8);                              // #2, L1 control
    Args.push_back(CZeroInt8);                              // #3, L3 control
    Args.push_back(LSCScale(CI->getArgOperand(2)));         // #4, scale
    Args.push_back(CI->getArgOperand(4));                   // #5, immed-offset
    Args.push_back(LSCDatumSize(CI->getArgOperand(DataIdx))); // #6
    Args.push_back(
        ConstantInt::get(I8Ty, LSCVectorSize(NumChannels))); // #7, num-elements
    Args.push_back(ConstantInt::get(
        I8Ty, LSC_DATA_ORDER_NONTRANSPOSE));      // #8, not-transpose
    Args.push_back(ConstantInt::get(I8Ty, Mask)); // #9, channel-mask
    Args.push_back(ExpandAddrOrData(CI, AddrIdx, Width, 1, 32)); // #10, offsets
    Args.push_back(
        ExpandAddrOrData(CI, DataIdx, Width, NumChannels, 32)); //#11 Data
    Value *BTIV = CI->getArgOperand(3);
    auto StoreIID = GenXIntrinsic::genx_lsc_store_quad_bti;
    if (isa<ConstantInt>(BTIV)) {
      if (cast<ConstantInt>(BTIV)->getZExtValue() ==
          visa::ReservedSurfaceIndex::RSI_Slm) {
        StoreIID = GenXIntrinsic::genx_lsc_store_quad_slm;

        // in new LSC interface, surface index
        // visa::ReservedSurfaceIndex::RSI_Slm not used we need to have 0
        // instead, because only type of message matters
        BTIV = ConstantInt::get(BTIV->getType(), 0);
      }
    }
    Args.push_back(BTIV); // #12, SurfaceIndex

    Value *DataV = CI->getArgOperand(DataIdx);
    auto *DataVTy = cast<IGCLLVM::FixedVectorType>(DataV->getType());
    IGC_ASSERT(Width * NumChannels == DataVTy->getNumElements());
    auto DataV32Ty = IGCLLVM::FixedVectorType::get(DataVTy->getElementType(),
                                                   32 * NumChannels);

    // now we sure that all can be converted, so increase statistics
    NumLegacy += 1;
    NumQuad += 1;

    // create the call to 32 wide lsc.load.bti instructions.
    Type *Tys[] = {Args[0]->getType(), Args[10]->getType(), DataV32Ty};
    auto Decl =
        GenXIntrinsic::getGenXDeclaration(CI->getModule(), StoreIID, Tys);
    auto NewInst = CallInst::Create(Decl, Args, "", CI);
    NewInst->setDebugLoc(DL);

    LLVM_DEBUG(dbgs() << "Legacy intrinsic:\n");
    LLVM_DEBUG(CI->dump());
    LLVM_DEBUG(dbgs() << "Translated to:\n");
    LLVM_DEBUG(NewInst->dump());

    CI->replaceAllUsesWith(NewInst);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_gather4_scaled:
  case GenXIntrinsic::genx_gather4_scaled2:
  case GenXIntrinsic::genx_gather4_masked_scaled2: {
    unsigned PredIdx  = 0;
    unsigned MaskIdx  = 1;
    unsigned ScaleIdx = 2;
    unsigned SurfIdx  = 3;
    unsigned OffIdx   = 4;
    unsigned AddrIdx  = 5;
    Value *PredValue;

    if (IID == GenXIntrinsic::genx_gather4_scaled2) {
      MaskIdx  = 0;
      ScaleIdx = 1;
      SurfIdx  = 2;
      OffIdx   = 3;
      AddrIdx  = 4;
      PredValue = OnePredScalar;
    }

    if (IID == GenXIntrinsic::genx_gather4_masked_scaled2) {
      MaskIdx  = 0;
      ScaleIdx = 1;
      SurfIdx  = 2;
      OffIdx   = 3;
      AddrIdx  = 4;
      PredIdx  = 5;
    }

    if (IID == GenXIntrinsic::genx_gather4_scaled ||
        IID == GenXIntrinsic::genx_gather4_masked_scaled2) {
      PredValue = ExpandPredicate(CI, PredIdx, 32);
    }

    // data size from return type of gather
    Constant *DataSize = LSCDatumSize(CI);

    // lsc.load does not support non-constant global offset
    auto GlobOffset = CI->getArgOperand(OffIdx);
    if (!isa<Constant>(GlobOffset)) {
      // insert address computation
      ComputeScatterAddress(CI, ScaleIdx, OffIdx, AddrIdx);
    }
    auto Width =
        cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(AddrIdx)->getType())
            ->getNumElements();
    IGC_ASSERT(Width == 8 || Width == 16 || Width == 32);
    unsigned Mask =
        (unsigned)cast<ConstantInt>(CI->getArgOperand(MaskIdx))->getZExtValue();
    unsigned NumChannels =
        (Mask & 1) + ((Mask & 2) >> 1) + ((Mask & 4) >> 2) + ((Mask & 8) >> 3);
    SmallVector<Value *, 12> Args;
    Args.push_back(PredValue);                             // #0, Predicate
    Args.push_back(ConstantInt::get(I8Ty, LSC_LOAD_QUAD)); // #1, subopcode
    Args.push_back(CZeroInt8);                             // #2, L1 control
    Args.push_back(CZeroInt8);                             // #3, L3 control
    Args.push_back(LSCScale(CI->getArgOperand(ScaleIdx))); // #4, scale
    Args.push_back(CI->getArgOperand(OffIdx));             // #5, immed-offset
    Args.push_back(DataSize);                              // #6
    Args.push_back(
        ConstantInt::get(I8Ty, LSCVectorSize(NumChannels))); // #7 num-elements
    Args.push_back(ConstantInt::get(
        I8Ty, LSC_DATA_ORDER_NONTRANSPOSE));      // #8, not-transpose
    Args.push_back(ConstantInt::get(I8Ty, Mask)); // #9, channel-mask
    Args.push_back(ExpandAddrOrData(CI, AddrIdx, Width, 1, 32)); // #10, offsets
    Value *BTIV = CI->getArgOperand(SurfIdx);
    auto LoadIID = GenXIntrinsic::genx_lsc_load_quad_bti;
    if (isa<ConstantInt>(BTIV)) {
      if (cast<ConstantInt>(BTIV)->getZExtValue() ==
          visa::ReservedSurfaceIndex::RSI_Slm) {
        LoadIID = GenXIntrinsic::genx_lsc_load_quad_slm;

        // in new LSC interface, surface index
        // visa::ReservedSurfaceIndex::RSI_Slm not used we need to have 0
        // instead, because only type of message matters
        BTIV = ConstantInt::get(BTIV->getType(), 0);
      }
    }
    Args.push_back(BTIV); // #11, SurfaceIndex

    auto *DataVector = cast<IGCLLVM::FixedVectorType>(CI->getType());
    IGC_ASSERT_MESSAGE(Width * NumChannels == DataVector->getNumElements(),
      "Cannot translate gather more than one element per lane");
    auto DataV32Ty = IGCLLVM::FixedVectorType::get(
        DataVector->getElementType(), 32 * NumChannels);

    // now we sure that all can be converted, so increase statistics
    NumLegacy += 1;
    NumQuad += 1;

    // create the call to 32 wide lsc.load.bti instructions.
    Type *Tys[] = {DataV32Ty, Args[0]->getType(), Args[10]->getType()};
    auto Decl =
        GenXIntrinsic::getGenXDeclaration(CI->getModule(), LoadIID, Tys);
    auto NewInst = CallInst::Create(Decl, Args, CI->getName() + VALUE_NAME(".lsc"), CI);
    NewInst->setDebugLoc(DL);

    LLVM_DEBUG(dbgs() << "Legacy intrinsic:\n");
    LLVM_DEBUG(CI->dump());
    LLVM_DEBUG(dbgs() << "Translated to:\n");
    LLVM_DEBUG(NewInst->dump());

    if (Width < 32) {
      Region R(CI->getType());
      R.Width = Width;
      R.NumElements = Width * NumChannels;
      R.VStride = (NumChannels > 1) ? 32 : 0;
      auto NewVec = R.createRdRegion(NewInst, "", CI, DL);
      CI->replaceAllUsesWith(NewVec);
    } else
      CI->replaceAllUsesWith(NewInst);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_scatter: {
    unsigned DataIdx = 3;
    unsigned PredIdx = 0;
    unsigned AddrIdx = 2;
    auto Width =
        cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(PredIdx)->getType())
            ->getNumElements();
    IGC_ASSERT(Width == 1 || Width == 2 || Width == 4 || Width == 8 ||
               Width == 16 || Width == 32);
    SmallVector<Value *, 13> Args;
    Args.push_back(CI->getOperand(PredIdx));           // #0, Predicate
    Args.push_back(ConstantInt::get(I8Ty, LSC_STORE)); // #1, subopcode
    Args.push_back(CZeroInt8);                         // #2, L1 control
    Args.push_back(CZeroInt8);                         // #3, L3 control
    Args.push_back(COneInt16);                         // #4, scale
    Args.push_back(CZeroInt32);                        // #5, immed-offset
    Args.push_back(LSCDatumSize(CI->getArgOperand(DataIdx))); // #6
    Args.push_back(
        ConstantInt::get(I8Ty, LSCVectorSize(1))); // #7, num-elem is one
    Args.push_back(ConstantInt::get(
        I8Ty, LSC_DATA_ORDER_NONTRANSPOSE)); // #8, not-transpose
    Args.push_back(CZeroInt8);               // #9, no-channel-mask
    Args.push_back(CI->getOperand(AddrIdx)); // #10, address
    Args.push_back(CI->getOperand(DataIdx)); // #11, data
    Args.push_back(CZeroInt32); // #12, SurfaceIndex

    Value *DataV = CI->getArgOperand(DataIdx);
    auto *DataVector = cast<IGCLLVM::FixedVectorType>(DataV->getType());
    IGC_ASSERT_MESSAGE(Width == DataVector->getNumElements(),
      "Cannot translate svm-scatter more than one element per lane");

    // now we sure that all can be converted, so increase statistics
    NumLegacy += 1;

    Type *Tys[] = {Args[0]->getType(), Args[10]->getType(), DataVector};
    auto Decl = GenXIntrinsic::getGenXDeclaration(
        CI->getModule(), GenXIntrinsic::genx_lsc_store_stateless, Tys);
    auto NewInst = CallInst::Create(Decl, Args, "", CI);
    NewInst->setDebugLoc(DL);

    LLVM_DEBUG(dbgs() << "Legacy intrinsic:\n");
    LLVM_DEBUG(CI->dump());
    LLVM_DEBUG(dbgs() << "Translated to:\n");
    LLVM_DEBUG(NewInst->dump());

    CI->replaceAllUsesWith(NewInst);
    ToErase.push_back(CI);
  } break;

  case GenXIntrinsic::genx_svm_gather: {
    unsigned DataIdx = 3;
    unsigned PredIdx = 0;
    unsigned AddrIdx = 2;
    auto Width =
        cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(PredIdx)->getType())
            ->getNumElements();
    IGC_ASSERT(Width == 1 || Width == 2 || Width == 4 || Width == 8 ||
               Width == 16 || Width == 32);
    SmallVector<Value *, 12> Args;
    Args.push_back(CI->getOperand(PredIdx));          // #0, Predicate
    Args.push_back(ConstantInt::get(I8Ty, LSC_LOAD)); // #1, subopcode
    Args.push_back(CZeroInt8);                        // #2, L1 control
    Args.push_back(CZeroInt8);                        // #3, L3 control
    Args.push_back(COneInt16);                        // #4, scale
    Args.push_back(CZeroInt32);                       // #5, immed-offset
    Args.push_back(LSCDatumSize(CI->getArgOperand(DataIdx))); // #6
    Args.push_back(
        ConstantInt::get(I8Ty, LSCVectorSize(1))); // #7, num-elem is one
    Args.push_back(ConstantInt::get(
        I8Ty, LSC_DATA_ORDER_NONTRANSPOSE)); // #8, not-transpose
    Args.push_back(CZeroInt8);               // #9, no-channel-mask
    Args.push_back(CI->getOperand(AddrIdx)); // #10, address
    Args.push_back(CZeroInt32);              // #11, SurfaceIndex

    Value *DataV = CI->getArgOperand(DataIdx);
    auto *DataVector = cast<IGCLLVM::FixedVectorType>(DataV->getType());
    IGC_ASSERT_MESSAGE(
        Width == DataVector->getNumElements(),
        "Cannot translate svm-gather more than one element per lane");

    // now we sure that all can be converted, so increase statistics
    NumLegacy += 1;

    Type *Tys[] = {DataVector, Args[0]->getType(), Args[10]->getType()};
    auto Decl = GenXIntrinsic::getGenXDeclaration(
        CI->getModule(), GenXIntrinsic::genx_lsc_load_stateless, Tys);
    auto NewInst =
        CallInst::Create(Decl, Args, CI->getName() + VALUE_NAME(".lsc"), CI);
    NewInst->setDebugLoc(DL);

    LLVM_DEBUG(dbgs() << "Legacy intrinsic:\n");
    LLVM_DEBUG(CI->dump());
    LLVM_DEBUG(dbgs() << "Translated to:\n");
    LLVM_DEBUG(NewInst->dump());
    CI->replaceAllUsesWith(NewInst);
    ToErase.push_back(CI);
  } break;

  case GenXIntrinsic::genx_svm_scatter4_scaled: {
    unsigned DataIdx = 5;
    unsigned PredIdx = 0;
    unsigned MaskIdx = 1;
    unsigned AddrIdx = 4;
    // load.lsc does not take global base address
    // add it to the address-vector
    ComputeScatterAddress(CI, 2, 3, AddrIdx);
    auto Width =
        cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(PredIdx)->getType())
            ->getNumElements();
    IGC_ASSERT(Width == 8 || Width == 16 || Width == 32);
    unsigned Mask =
        (unsigned)cast<ConstantInt>(CI->getArgOperand(MaskIdx))->getZExtValue();
    unsigned NumChannels =
        (Mask & 1) + ((Mask & 2) >> 1) + ((Mask & 4) >> 2) + ((Mask & 8) >> 3);
    SmallVector<Value *, 13> Args;
    Args.push_back(ExpandPredicate(CI, PredIdx, 32));       // #0, Predicate
    Args.push_back(ConstantInt::get(I8Ty, LSC_STORE_QUAD)); // #1, subopcode
    Args.push_back(CZeroInt8);                              // #2, L1 control
    Args.push_back(CZeroInt8);                              // #3, L3 control
    Args.push_back(LSCScale(CI->getArgOperand(2)));         // #4, scale
    Args.push_back(CZeroInt32);                             // #5, immed-offset
    Args.push_back(LSCDatumSize(CI->getArgOperand(DataIdx))); // #6
    Args.push_back(
        ConstantInt::get(I8Ty, LSCVectorSize(NumChannels))); // #7, num-elements
    Args.push_back(ConstantInt::get(
        I8Ty, LSC_DATA_ORDER_NONTRANSPOSE));      // #8, not-transpose
    Args.push_back(ConstantInt::get(I8Ty, Mask)); // #9, channel-mask
    Args.push_back(ExpandAddrOrData(CI, AddrIdx, Width, 1, 32)); // #10, offsets
    Args.push_back(
        ExpandAddrOrData(CI, DataIdx, Width, NumChannels, 32)); //#11 Data
    auto StoreIID = GenXIntrinsic::genx_lsc_store_quad_stateless;
    Args.push_back(CZeroInt32);   // #12, SurfaceIndex

    Value *DataV = CI->getArgOperand(DataIdx);
    auto *DataVTy = cast<IGCLLVM::FixedVectorType>(DataV->getType());
    IGC_ASSERT(Width * NumChannels == DataVTy->getNumElements());
    auto DataV32Ty = IGCLLVM::FixedVectorType::get(DataVTy->getElementType(),
                                                   32 * NumChannels);

    // now we sure that all can be converted, so increase statistics
    NumLegacy += 1;
    NumQuad += 1;

    // create the call to 32 wide lsc.load.bti instructions.
    Type *Tys[] = {Args[0]->getType(), Args[10]->getType(), DataV32Ty};
    auto Decl =
        GenXIntrinsic::getGenXDeclaration(CI->getModule(), StoreIID, Tys);
    auto NewInst = CallInst::Create(Decl, Args, "", CI);
    NewInst->setDebugLoc(DL);

    LLVM_DEBUG(dbgs() << "Legacy intrinsic:\n");
    LLVM_DEBUG(CI->dump());
    LLVM_DEBUG(dbgs() << "Translated to:\n");
    LLVM_DEBUG(NewInst->dump());

    CI->replaceAllUsesWith(NewInst);
    ToErase.push_back(CI);
  }
  break;

  case GenXIntrinsic::genx_svm_gather4_scaled: {
    unsigned DataIdx = 5;
    unsigned PredIdx = 0;
    unsigned MaskIdx = 1;
    unsigned AddrIdx = 4;
    // load.lsc does not take global base address
    // add it to the address-vector
    ComputeScatterAddress(CI, 2, 3, AddrIdx);
    auto Width =
        cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(PredIdx)->getType())
            ->getNumElements();
    IGC_ASSERT(Width == 8 || Width == 16 || Width == 32);
    unsigned Mask =
        (unsigned)cast<ConstantInt>(CI->getArgOperand(MaskIdx))->getZExtValue();
    unsigned NumChannels =
        (Mask & 1) + ((Mask & 2) >> 1) + ((Mask & 4) >> 2) + ((Mask & 8) >> 3);
    SmallVector<Value *, 12> Args;
    Args.push_back(ExpandPredicate(CI, PredIdx, 32));      // #0, Predicate
    Args.push_back(ConstantInt::get(I8Ty, LSC_LOAD_QUAD)); // #1, subopcode
    Args.push_back(CZeroInt8);                             // #2, L1 control
    Args.push_back(CZeroInt8);                             // #3, L3 control
    Args.push_back(LSCScale(CI->getArgOperand(2)));        // #4, scale
    Args.push_back(CZeroInt32);                            // #5, immed-offset
    Args.push_back(LSCDatumSize(CI->getArgOperand(DataIdx))); // #6
    Args.push_back(
        ConstantInt::get(I8Ty, LSCVectorSize(NumChannels))); // #7 num-elements
    Args.push_back(ConstantInt::get(
        I8Ty, LSC_DATA_ORDER_NONTRANSPOSE));      // #8, not-transpose
    Args.push_back(ConstantInt::get(I8Ty, Mask)); // #9, channel-mask
    Args.push_back(ExpandAddrOrData(CI, AddrIdx, Width, 1, 32)); // #10, address
    Args.push_back(CZeroInt32);            // #11, SurfaceIndex

    Value *DataV = CI->getArgOperand(DataIdx);
    auto *DataVector = cast<IGCLLVM::FixedVectorType>(DataV->getType());
    IGC_ASSERT_MESSAGE(Width * NumChannels == DataVector->getNumElements(),
      "Cannot translate scatter more than one element per lane");
    auto DataV32Ty = IGCLLVM::FixedVectorType::get(
        DataVector->getElementType(), 32 * NumChannels);

    // now we sure that all can be converted, so increase statistics
    NumLegacy += 1;
    NumQuad += 1;

    // create the call to 32 wide lsc.load.bti instructions.
    Type *Tys[] = {DataV32Ty, Args[0]->getType(), Args[10]->getType()};
    auto Decl = GenXIntrinsic::getGenXDeclaration(
        CI->getModule(), GenXIntrinsic::genx_lsc_load_quad_stateless, Tys);
    auto NewInst = CallInst::Create(Decl, Args, CI->getName() + VALUE_NAME(".lsc"), CI);
    NewInst->setDebugLoc(DL);

    LLVM_DEBUG(dbgs() << "Legacy intrinsic:\n");
    LLVM_DEBUG(CI->dump());
    LLVM_DEBUG(dbgs() << "Translated to:\n");
    LLVM_DEBUG(NewInst->dump());

    if (Width < 32) {
      Region R(CI->getType());
      R.Width = Width;
      R.NumElements = Width * NumChannels;
      R.VStride = (NumChannels > 1) ? 32 : 0;
      auto NewVec = R.createRdRegion(NewInst, "", CI, DL);
      CI->replaceAllUsesWith(NewVec);
    } else
      CI->replaceAllUsesWith(NewInst);
    ToErase.push_back(CI);
  }
  break;
  case GenXIntrinsic::genx_svm_atomic_add: {
    TranslateSVMAtomic(LSC_ATOMIC_IADD, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_and: {
    TranslateSVMAtomic(LSC_ATOMIC_AND, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_cmpxchg: {
    TranslateSVMAtomic(LSC_ATOMIC_ICAS, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_dec: {
    TranslateSVMAtomic(LSC_ATOMIC_IDEC, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_fcmpwr: {
    TranslateSVMAtomic(LSC_ATOMIC_FCAS, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_fmax: {
    TranslateSVMAtomic(LSC_ATOMIC_FMAX, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_fmin: {
    TranslateSVMAtomic(LSC_ATOMIC_FMIN, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_imax: {
    TranslateSVMAtomic(LSC_ATOMIC_SMAX, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_imin: {
    TranslateSVMAtomic(LSC_ATOMIC_SMIN, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_inc: {
    TranslateSVMAtomic(LSC_ATOMIC_IINC, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_max: {
    TranslateSVMAtomic(LSC_ATOMIC_UMAX, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_min: {
    TranslateSVMAtomic(LSC_ATOMIC_UMIN, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_or: {
    TranslateSVMAtomic(LSC_ATOMIC_OR, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_sub: {
    TranslateSVMAtomic(LSC_ATOMIC_ISUB, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_xchg: {
    TranslateSVMAtomic(LSC_ATOMIC_STORE, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_atomic_xor: {
    TranslateSVMAtomic(LSC_ATOMIC_XOR, CI);
    ToErase.push_back(CI);
  } break;

  case GenXIntrinsic::genx_dword_atomic_add: {
    TranslateDWordAtomic(LSC_ATOMIC_IADD, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_and: {
    TranslateDWordAtomic(LSC_ATOMIC_AND, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_cmpxchg: {
    TranslateDWordAtomic(LSC_ATOMIC_ICAS, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_dec: {
    TranslateDWordAtomic(LSC_ATOMIC_IDEC, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_fadd: {
    TranslateDWordAtomic(LSC_ATOMIC_FADD, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_fcmpwr: {
    TranslateDWordAtomic(LSC_ATOMIC_FCAS, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_fmax: {
    TranslateDWordAtomic(LSC_ATOMIC_FMAX, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_fmin: {
    TranslateDWordAtomic(LSC_ATOMIC_FMIN, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_fsub: {
    TranslateDWordAtomic(LSC_ATOMIC_FSUB, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_imax: {
    TranslateDWordAtomic(LSC_ATOMIC_SMAX, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_imin: {
    TranslateDWordAtomic(LSC_ATOMIC_SMIN, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_inc: {
    TranslateDWordAtomic(LSC_ATOMIC_IINC, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_max: {
    TranslateDWordAtomic(LSC_ATOMIC_UMAX, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_min: {
    TranslateDWordAtomic(LSC_ATOMIC_UMIN, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_or: {
    TranslateDWordAtomic(LSC_ATOMIC_OR, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_sub: {
    TranslateDWordAtomic(LSC_ATOMIC_ISUB, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_xchg: {
    TranslateDWordAtomic(LSC_ATOMIC_STORE, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_dword_atomic_xor: {
    TranslateDWordAtomic(LSC_ATOMIC_XOR, CI);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_oword_ld:
  case GenXIntrinsic::genx_oword_ld_unaligned: {
    constexpr unsigned BtiIdx = 1;
    constexpr unsigned AddrIdx = 2;

    // for now FE don't generate and VISA don't support scaled
    // so lets artificially scale address
    IRBuilder<> Builder(CI);
    Value *AddrV = CI->getArgOperand(AddrIdx);
    if (IID == GenXIntrinsic::genx_oword_ld) {
      AddrV =
          Builder.CreateShl(AddrV, llvm::ConstantInt::get(AddrV->getType(), 4));
    }

    auto *OldVT = cast<IGCLLVM::FixedVectorType>(CI->getType());
    auto *VT = adjustVTForTransposedMessage(OldVT);
    auto Length = VT->getNumElements();
    IGC_ASSERT(Length <= 128);
    SmallVector<Value *, 12> Args;
    Args.push_back(OnePredScalar);                    // #0, i1 Predicate
    Args.push_back(ConstantInt::get(I8Ty, LSC_LOAD)); // #1, subopcode
    Args.push_back(CZeroInt8);                        // #2, L1 control
    Args.push_back(CZeroInt8);                        // #3, L3 control
    Args.push_back(ConstantInt::get(I16Ty, 1));       // #4, scale
    Args.push_back(CZeroInt32);                       // #5, immed-offset
    Args.push_back(LSCDatumSize(VT));                 // #6, datum size
    Args.push_back(ConstantInt::get(
        I8Ty, LSCVectorSize(Length))); // #7, num-elem is the Length
    Args.push_back(TransposeConst);    // #8, transpose
    Args.push_back(CZeroInt8);         // #9, no-channel-mask
    Args.push_back(AddrV);             // #10, address
    Value *BTIV = CI->getArgOperand(BtiIdx);
    auto LoadIID = GenXIntrinsic::genx_lsc_load_bti;
    if (isa<ConstantInt>(BTIV)) {
      if (cast<ConstantInt>(BTIV)->getZExtValue() ==
          visa::ReservedSurfaceIndex::RSI_Slm) {
        LoadIID = GenXIntrinsic::genx_lsc_load_slm;

        // in new LSC interface, surface index
        // visa::ReservedSurfaceIndex::RSI_Slm not used we need to have 0
        // instead, because only type of message matters
        BTIV = ConstantInt::get(BTIV->getType(), 0);
      }
    }
    Args.push_back(BTIV); // #11, SurfaceIndex

    // now we sure that all can be converted, so increase statistics
    NumLegacy += 1;
    NumOword += 1;

    // create the call to lsc.load.bti instruction
    Type *Tys[] = {VT, Args[0]->getType(), Args[10]->getType()};
    auto Decl =
        GenXIntrinsic::getGenXDeclaration(CI->getModule(), LoadIID, Tys);
    auto NewInst = CallInst::Create(Decl, Args, CI->getName() + VALUE_NAME(".lsc"), CI);
    NewInst->setDebugLoc(DL);
    LLVM_DEBUG(dbgs() << "Legacy intrinsic:\n");
    LLVM_DEBUG(CI->dump());
    LLVM_DEBUG(dbgs() << "Translated to:\n");
    LLVM_DEBUG(NewInst->dump());

    // cast back if required
    Value *Casted = NewInst;
    if (VT != OldVT)
      Casted = CastInst::CreateBitOrPointerCast(
          Casted, OldVT, Casted->getName() + VALUE_NAME(".cast.lsc"), CI);
    CI->replaceAllUsesWith(Casted);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_oword_st: {
    constexpr unsigned DataIdx = 2;
    constexpr unsigned AddrIdx = 1;

    Value *Datum = CI->getArgOperand(DataIdx);

    // same as for oword_ld
    IRBuilder<> Builder(CI);
    Value *AddrV = CI->getArgOperand(AddrIdx);
    AddrV =
        Builder.CreateShl(AddrV, llvm::ConstantInt::get(AddrV->getType(), 4));

    auto *OldVT = cast<IGCLLVM::FixedVectorType>(Datum->getType());
    auto *VT = adjustVTForTransposedMessage(OldVT);

    IGC_ASSERT_MESSAGE(CastInst::isBitCastable(VT, Datum->getType()),
                       "We expect resulting vectors to be bitcastable");

    if (VT != OldVT)
      Datum = CastInst::CreateBitOrPointerCast(
          Datum, VT, Datum->getName() + VALUE_NAME(".cast.lsc"), CI);

    auto Length = VT->getNumElements();
    IGC_ASSERT(Length <= 128);
    auto VS = ConstantInt::get(I8Ty, LSCVectorSize(Length));

    SmallVector<Value *, 13> Args;
    Args.push_back(OnePredScalar);                     // #0, i1 Predicate
    Args.push_back(ConstantInt::get(I8Ty, LSC_STORE)); // #1, subopcode
    Args.push_back(CZeroInt8);                         // #2, L1 control
    Args.push_back(CZeroInt8);                         // #3, L3 control
    Args.push_back(ConstantInt::get(I16Ty, 1));        // #4, scale
    Args.push_back(CZeroInt32);                        // #5, immed-offset
    Args.push_back(LSCDatumSize(VT));                  // #6, DS
    Args.push_back(VS);                                // #7, VS
    Args.push_back(TransposeConst);                    // #8, transpose
    Args.push_back(CZeroInt8);                         // #9, no-channel-mask
    Args.push_back(AddrV);                             // #10, address
    Args.push_back(Datum);                             // #11, Data
    Value *BTIV = CI->getArgOperand(0);

    // access to surface visa::ReservedSurfaceIndex::RSI_Slm is BTI access to
    // SLM
    auto StoreIID = GenXIntrinsic::genx_lsc_store_bti;
    if (isa<ConstantInt>(BTIV)) {
      if (cast<ConstantInt>(BTIV)->getZExtValue() ==
          visa::ReservedSurfaceIndex::RSI_Slm) {
        StoreIID = GenXIntrinsic::genx_lsc_store_slm;

        // in new LSC interface, surface index
        // visa::ReservedSurfaceIndex::RSI_Slm not used we need to have 0
        // instead, because only type of message matters
        BTIV = ConstantInt::get(BTIV->getType(), 0);
      }
    }

    Args.push_back(BTIV); //#12, SurfaceIndex

    // now we sure that all can be converted, so increase statistics
    NumLegacy += 1;
    NumOword += 1;

    // create the call to lsc.store.bti instruction
    Type *Tys[] = {Args[0]->getType(), Args[10]->getType(), VT};
    auto Decl =
        GenXIntrinsic::getGenXDeclaration(CI->getModule(), StoreIID, Tys);
    auto NewInst = CallInst::Create(Decl, Args, "", CI);

    LLVM_DEBUG(dbgs() << "Legacy intrinsic:\n");
    LLVM_DEBUG(CI->dump());
    LLVM_DEBUG(dbgs() << "Translated to:\n");
    LLVM_DEBUG(NewInst->dump());

    NewInst->setDebugLoc(DL);
    CI->replaceAllUsesWith(NewInst);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_block_ld:
  case GenXIntrinsic::genx_svm_block_ld_unaligned: {
    unsigned AddrIdx = 0;
    auto Length =
        cast<IGCLLVM::FixedVectorType>(CI->getType())->getNumElements();
    IGC_ASSERT(Length <= 128);
    SmallVector<Value *, 12> Args;
    Args.push_back(OnePred1);                         // #0, 1xi1 Predicate
    Args.push_back(ConstantInt::get(I8Ty, LSC_LOAD)); // #1, subopcode
    Args.push_back(CZeroInt8);                        // #2, L1 control
    Args.push_back(CZeroInt8);                        // #3, L3 control
    if (IID == GenXIntrinsic::genx_svm_block_ld_unaligned)
      Args.push_back(ConstantInt::get(I16Ty, 1)); // #4, scale
    else
      Args.push_back(ConstantInt::get(I16Ty, 16)); // #4, scale
    Args.push_back(CZeroInt32);                    // #5, immed-offset
    Args.push_back(LSCDatumSize(CI));              // #6
    Args.push_back(ConstantInt::get(
        I8Ty, LSCVectorSize(Length))); // #7, num-elem is the Length
    Args.push_back(
        ConstantInt::get(I8Ty, LSC_DATA_ORDER_TRANSPOSE)); // #8, transpose
    Args.push_back(CZeroInt8); // #9, no-channel-mask
                               // create 1x vector for address
    Value *AddrV = CI->getArgOperand(AddrIdx);
    auto AddrV1Ty = IGCLLVM::FixedVectorType::get(AddrV->getType(), 1);
    auto AddrV1 = CastInst::CreateBitOrPointerCast(
        AddrV, AddrV1Ty, AddrV->getName() + VALUE_NAME(".lsc"), CI);
    Args.push_back(AddrV1);     // #10, address
    Args.push_back(CZeroInt32); // #11, SurfaceIndex

    // now we sure that all can be converted, so increase statistics
    NumLegacy += 1;

    // create the call to 32 wide lsc.load.stateless instructions.
    Type *Tys[] = {CI->getType(), Args[0]->getType(), Args[10]->getType()};
    auto Decl = GenXIntrinsic::getGenXDeclaration(
        CI->getModule(), GenXIntrinsic::genx_lsc_load_stateless, Tys);
    auto NewInst = CallInst::Create(Decl, Args, CI->getName() + VALUE_NAME(".lsc"), CI);

    LLVM_DEBUG(dbgs() << "Legacy intrinsic:\n");
    LLVM_DEBUG(CI->dump());
    LLVM_DEBUG(dbgs() << "Translated to:\n");
    LLVM_DEBUG(NewInst->dump());

    NewInst->setDebugLoc(DL);
    CI->replaceAllUsesWith(NewInst);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_svm_block_st: {
    unsigned DataIdx = 1;
    unsigned AddrIdx = 0;
    auto Length =
        cast<IGCLLVM::FixedVectorType>(CI->getArgOperand(DataIdx)->getType())
            ->getNumElements();
    IGC_ASSERT(Length <= 128);
    SmallVector<Value *, 13> Args;
    Args.push_back(OnePred1);                          // #0, 1xi1 Predicate
    Args.push_back(ConstantInt::get(I8Ty, LSC_STORE)); // #1, subopcode
    Args.push_back(CZeroInt8);                         // #2, L1 control
    Args.push_back(CZeroInt8);                         // #3, L3 control
    Args.push_back(ConstantInt::get(I16Ty, 16));       // #4, scale
    Args.push_back(CZeroInt32);                        // #5, immed-offset
    Args.push_back(LSCDatumSize(CI->getArgOperand(DataIdx))); // #6
    Args.push_back(ConstantInt::get(
        I8Ty, LSCVectorSize(Length))); // #7, num-elem is the Length
    Args.push_back(
        ConstantInt::get(I8Ty, LSC_DATA_ORDER_TRANSPOSE)); // #8, transpose
    Args.push_back(CZeroInt8); // #9, no-channel-mask
                               // create 1x vector for address
    Value *AddrV = CI->getArgOperand(AddrIdx);
    auto AddrV1Ty = IGCLLVM::FixedVectorType::get(AddrV->getType(), 1);
    auto AddrV1 = CastInst::CreateBitOrPointerCast(
        AddrV, AddrV1Ty, AddrV->getName() + VALUE_NAME(".lsc"), CI);
    Args.push_back(AddrV1);                     // #10, address
    Args.push_back(CI->getArgOperand(DataIdx)); // #11, Data
    Args.push_back(CZeroInt32);                 // #12, SurfaceIndex

    // now we sure that all can be converted, so increase statistics
    NumLegacy += 1;

    // create the call to 32 wide lsc.store.stateless instructions.
    Type *Tys[] = {Args[0]->getType(), Args[10]->getType(),
                   Args[11]->getType()};
    auto Decl = GenXIntrinsic::getGenXDeclaration(
        CI->getModule(), GenXIntrinsic::genx_lsc_store_stateless, Tys);
    auto NewInst = CallInst::Create(Decl, Args, "", CI);

    LLVM_DEBUG(dbgs() << "Legacy intrinsic:\n");
    LLVM_DEBUG(CI->dump());
    LLVM_DEBUG(dbgs() << "Translated to:\n");
    LLVM_DEBUG(NewInst->dump());

    NewInst->setDebugLoc(DL);
    CI->replaceAllUsesWith(NewInst);
    ToErase.push_back(CI);
  } break;
  case GenXIntrinsic::genx_typed_atomic_add:
  case GenXIntrinsic::genx_typed_atomic_and:
  case GenXIntrinsic::genx_typed_atomic_cmpxchg:
  case GenXIntrinsic::genx_typed_atomic_dec:
  case GenXIntrinsic::genx_typed_atomic_fadd:
  case GenXIntrinsic::genx_typed_atomic_fsub:
  case GenXIntrinsic::genx_typed_atomic_fcmpwr:
  case GenXIntrinsic::genx_typed_atomic_fmax:
  case GenXIntrinsic::genx_typed_atomic_fmin:
  case GenXIntrinsic::genx_typed_atomic_imax:
  case GenXIntrinsic::genx_typed_atomic_imin:
  case GenXIntrinsic::genx_typed_atomic_inc:
  case GenXIntrinsic::genx_typed_atomic_max:
  case GenXIntrinsic::genx_typed_atomic_min:
  case GenXIntrinsic::genx_typed_atomic_or:
  case GenXIntrinsic::genx_typed_atomic_sub:
  case GenXIntrinsic::genx_typed_atomic_xchg:
  case GenXIntrinsic::genx_typed_atomic_xor:
    IGC_ASSERT_MESSAGE(0, "typed atomic translation not implemented");
  default:
    return false;
  }
  return true;
}

/***********************************************************************
 * lowerMediaIntrinsic : lower media walker intrinsic calls
 */
bool GenXLowering::lowerMediaWalkerAPIs(CallInst *CI, unsigned IID) {
  // translate genx_thread_x -> genx_group_id_x
  //           genx_thread_y -> genx_group_id_y
  if (ST->translateMediaWalker()) {
    auto NewIID = GenXIntrinsic::not_any_intrinsic;
    switch (IID) {
    case GenXIntrinsic::genx_thread_x:
      NewIID = GenXIntrinsic::genx_group_id_x;
      break;
    case GenXIntrinsic::genx_thread_y:
      NewIID = GenXIntrinsic::genx_group_id_y;
      break;
    case GenXIntrinsic::genx_get_color:
      CI->getContext().emitError(CI,
                                 "get_color not supported on " + ST->getCPU());
    default:
      break;
    }

    if (GenXIntrinsic::isAnyNonTrivialIntrinsic(NewIID)) {
      IRBuilder<> Builder(CI);
      auto Fn = GenXIntrinsic::getGenXDeclaration(CI->getModule(), NewIID);
      Value *Val = Builder.CreateCall(Fn);
      Val = Builder.CreateTruncOrBitCast(Val, CI->getType());
      CI->replaceAllUsesWith(Val);
      ToErase.push_back(CI);
      return true;
    }
  }
  return false;
}

/***********************************************************************
 * generatePrecicatedWrrForNewLoad : Generate predicated wrr if result
 *                                   of a load that needs no splits
 * Return: true if predicated wrr was generated
 */

bool GenXLowering::generatePredicatedWrrForNewLoad(CallInst *CI) {
  IGC_ASSERT_MESSAGE(isNewLoadInst(CI), "New load expected");
  // Generate predicated wrr if result of a load is predicated with a select
  if (auto *SI = getLoadSelect(CI)) {
    Value *NewResult = SI->getFalseValue();
    Value *LoadPred = SI->getCondition();
    NewResult =
        generatePredicatedWrregion(NewResult, CI, LoadPred, 0 /* Offset */,
                                   SI->getNextNode(), "lowerpred");
    SI->replaceAllUsesWith(NewResult);
    ToErase.push_back(SI);
    return true;
  }
  return false;
}

/***********************************************************************
 * processInst : process one instruction in GenXLowering
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::processInst(Instruction *Inst) {
  if (isa<InsertElementInst>(Inst))
    return lowerInsertElement(Inst);
  if (isa<ExtractElementInst>(Inst))
    return lowerExtractElement(Inst);
  if (isa<TruncInst>(Inst))
    return lowerTrunc(Inst);
  if (isa<CastInst>(Inst))
    return lowerCast(Inst);
  if (auto SI = dyn_cast<SelectInst>(Inst)) {
    if (SI->getType()->getScalarType()->isIntegerTy(1)) {
      if (SI->getType() == SI->getCondition()->getType())
        return lowerBoolVectorSelect(SI);
      return lowerBoolScalarSelect(SI);
    }
    // Widen byte op if it necessary.
    return widenByteOp(SI);
  }
  if (auto SI = dyn_cast<ShuffleVectorInst>(Inst)) {
    if (SI->getType()->getScalarType()->isIntegerTy(1))
      return lowerBoolShuffle(SI);
    return lowerShuffle(SI);
  }
  if (isa<BinaryOperator>(Inst)) {
    if (widenByteOp(Inst))
      return true;
    if (Inst->getOpcode() == Instruction::AShr ||
        Inst->getOpcode() == Instruction::LShr)
      return lowerShr(Inst);
    if (Inst->getOpcode() == Instruction::Mul)
      return lowerMul64(Inst);
    return false;
  }
  if (Inst->getOpcode() == Instruction::ICmp)
    return widenByteOp(Inst);
  else if (auto CI = dyn_cast<FCmpInst>(Inst))
    return lowerFCmpInst(CI);
  if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
    if (CI->isInlineAsm())
      return false;
    processTwoAddressOpnd(CI);
    unsigned IntrinsicID = GenXIntrinsic::not_any_intrinsic;
    if (Function *Callee = CI->getCalledFunction()) {
      IntrinsicID = vc::getAnyIntrinsicID(Callee);
      IGC_ASSERT(CI->getNumArgOperands() < GenXIntrinsicInfo::OPNDMASK);
    }
    if (ST) {
      // use gather/scatter to implement SLM oword load/store on
      // legacy platforms
      if (!ST->isICLLPplus()) {
        if (translateSLMOWord(CI, IntrinsicID))
          return true;
      }
    }
    if (ST) {
      if (ST->translateLegacyMessages()) {
        if (translateLegacyToLSC(CI, IntrinsicID))
          return true;
        // Error if Message was not translated but it should
        // supported platform works only for GenxIntrinsics
      } else {
        if (GenXIntrinsic::isGenXIntrinsic(IntrinsicID) &&
            !GenXIntrinsic::isSupportedPlatform(ST->getCPU().str(),
                                                IntrinsicID)) {
          Inst->getContext().emitError(
              Inst, "<" + CI->getCalledFunction()->getName() + "> intrinsic " +
                        "is unsupported by target platform <" + ST->getCPU() +
                        ">. Please, enable legacy intrinsics translation");
        }
        if (ST->getGRFByteSize() > 32) {
          if (widenSIMD8GatherScatter(CI, IntrinsicID))
            return true;
        }
      }
    }
    // split gather/scatter/atomic into the width legal to the target
    if (splitGatherScatter(CI, IntrinsicID))
      return true;
    else if (isNewLoadInst(CI))
      return generatePredicatedWrrForNewLoad(CI);

    switch (IntrinsicID) {
    case GenXIntrinsic::genx_rdregioni:
    case GenXIntrinsic::genx_rdregionf:
      return lowerRdRegion(Inst);
    case GenXIntrinsic::genx_wrregioni:
    case GenXIntrinsic::genx_wrregionf:
      return lowerWrRegion(Inst);
    case GenXIntrinsic::genx_rdpredregion:
      return lowerRdPredRegion(Inst);
    case GenXIntrinsic::genx_wrpredregion:
      return lowerWrPredRegion(Inst);
    case GenXIntrinsic::not_any_intrinsic:
      break;
    case GenXIntrinsic::genx_absf:
    case GenXIntrinsic::genx_absi:
      break; // ignore
    case GenXIntrinsic::genx_thread_x:
    case GenXIntrinsic::genx_thread_y:
    case GenXIntrinsic::genx_get_color:
      return lowerMediaWalkerAPIs(CI, IntrinsicID);
    default:
    case GenXIntrinsic::genx_constantpred:
    case GenXIntrinsic::genx_constanti:
    case GenXIntrinsic::genx_constantf:
      break; // ignore
    case GenXIntrinsic::genx_vload: {
      if (!Inst->use_empty()) {
        Value *Ptr = Inst->getOperand(0);
        LoadInst *LI = new LoadInst(Ptr->getType()->getPointerElementType(),
                                    Ptr, "", /*volatile*/ true, Inst);
        LI->takeName(Inst);
        LI->setDebugLoc(Inst->getDebugLoc());
        Inst->replaceAllUsesWith(LI);
      }
      ToErase.push_back(Inst);
      return true;
    }
    case GenXIntrinsic::genx_vstore: {
      Value *Val = Inst->getOperand(0);
      Value *Ptr = Inst->getOperand(1);
      auto ST = new StoreInst(Val, Ptr, /*volatile*/ true, Inst);
      ST->setDebugLoc(Inst->getDebugLoc());
      ToErase.push_back(Inst);
      return true;
    }
    case GenXIntrinsic::genx_ssmul:
    case GenXIntrinsic::genx_sumul:
    case GenXIntrinsic::genx_usmul:
    case GenXIntrinsic::genx_uumul:
      return lowerGenXMul(CI, IntrinsicID);
    case GenXIntrinsic::genx_ssmul_sat:
    case GenXIntrinsic::genx_sumul_sat:
    case GenXIntrinsic::genx_usmul_sat:
    case GenXIntrinsic::genx_uumul_sat:
      return lowerGenXMulSat(CI, IntrinsicID);
    case GenXIntrinsic::genx_simad:
    case GenXIntrinsic::genx_uimad:
      return lowerGenXIMad(CI, IntrinsicID);
    case GenXIntrinsic::genx_lzd:
      return lowerLzd(Inst);
    case Intrinsic::masked_load:
      return lowerLLVMMaskedLoad(CI);
    case Intrinsic::masked_store:
      return lowerLLVMMaskedStore(CI);
    case Intrinsic::masked_gather:
      return lowerLLVMMaskedGather(CI);
    case Intrinsic::masked_scatter:
      return lowerLLVMMaskedScatter(CI);
    case Intrinsic::trap:
      return lowerTrap(CI);
    case Intrinsic::ctpop:
      return lowerCtpop(CI);
    case Intrinsic::minnum:
    case Intrinsic::maxnum:
      return lowerMinMax(CI, IntrinsicID);
    case Intrinsic::sqrt:
      return lowerSqrt(CI);
    case Intrinsic::sadd_sat:
    case Intrinsic::ssub_sat:
      vc::diagnose(Inst->getContext(), "GenXLowering", Inst,
                   "Sorry not implemented: GenX backend cannot handle this "
                   "intrinsic yet");
      break;
    case Intrinsic::uadd_sat:
      return lowerUAddWithSat(CI);
    case Intrinsic::usub_sat:
      return lowerUSubWithSat(CI);
    case Intrinsic::uadd_with_overflow:
      return lowerUAddWithOverflow(CI);
    case Intrinsic::sadd_with_overflow:
    case Intrinsic::ssub_with_overflow:
    case Intrinsic::usub_with_overflow:
    case Intrinsic::smul_with_overflow:
    case Intrinsic::umul_with_overflow:
      Inst->getContext().emitError(
          Inst, "GenX backend cannot handle overflowing intrinsics yet");
      break;
    case Intrinsic::assume:
      ToErase.push_back(Inst);
      return true;
    case Intrinsic::expect:
      llvm_unreachable("Expect intrinsic should be lowered before");
    case Intrinsic::fmuladd:
      return lowerFMulAdd(CI);
    case Intrinsic::bitreverse:
      return lowerBitreverse(CI);
    case Intrinsic::fshl:
    case Intrinsic::fshr:
      return lowerFunnelShift(CI, IntrinsicID);
    case Intrinsic::fabs:
      return lowerFAbs(CI);
    case Intrinsic::ceil:
      return lowerF32MathIntrinsic(CI, GenXIntrinsic::genx_rndu);
    case Intrinsic::floor:
      return lowerF32MathIntrinsic(CI, GenXIntrinsic::genx_rndd);
    case Intrinsic::trunc:
      return lowerF32MathIntrinsic(CI, GenXIntrinsic::genx_rndz);
    case Intrinsic::exp2:
      return lowerF32FastMathIntrinsic(CI, GenXIntrinsic::genx_exp);
    case Intrinsic::log2:
      return lowerF32FastMathIntrinsic(CI, GenXIntrinsic::genx_log);
    case Intrinsic::pow:
      return lowerF32FastMathIntrinsic(CI, GenXIntrinsic::genx_pow);
    case Intrinsic::sin:
      return lowerF32FastMathIntrinsic(CI, GenXIntrinsic::genx_sin);
    case Intrinsic::cos:
      return lowerF32FastMathIntrinsic(CI, GenXIntrinsic::genx_cos);
    }
    return false;
  }
  if (ExtractValueInst *EV = dyn_cast<ExtractValueInst>(Inst))
    return lowerExtractValue(EV);
  if (InsertValueInst *IV = dyn_cast<InsertValueInst>(Inst))
    return lowerInsertValue(IV);
  if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst))
    return lowerLoadStore(Inst);
  if (isa<AllocaInst>(Inst) && !ST->isOCLRuntime())
    Inst->getContext().emitError(Inst,
        "GenX backend cannot handle allocas with CMRT yet");
  return false;
}

/***********************************************************************
 * lowerRdRegion : handle read region instruction
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * 1. If index is variable do add sinking on it. (This in itself does not
 *    cause this function to return true, because it does not cause the
 *    original instruction to be replaced.)
 */
bool GenXLowering::lowerRdRegion(Instruction *Inst) {
  // Sink add in address calculation.
  Use *U = &Inst->getOperandUse(GenXIntrinsic::GenXRegion::RdIndexOperandNum);
  *U = sinkAdd(*U);
  return false;
}

/***********************************************************************
 * lowerWrRegion : handle write region instruction
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * 1. If index is variable do add sinking on it. (This in itself does not
 *    cause this function to return true, because it does not cause the
 *    original instruction to be replaced.)
 *
 * 2. If it is a predicated byte wrregion, see if it can be widened.
 */
bool GenXLowering::lowerWrRegion(Instruction *Inst) {
  // Sink add in address calculation.
  Use *U = &Inst->getOperandUse(GenXIntrinsic::GenXRegion::WrIndexOperandNum);
  *U = sinkAdd(*U);
  // See if a predicated byte wrregion can be widened.
  return widenByteOp(Inst);
}

/***********************************************************************
 * lowerRdPredRegion : handle read predicate region instruction
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * rdpredregion is a GenX backend internal intrinsic, and was thus created
 * within this GenXLowering pass. However it is considered legal only if its
 * uses are all in select or wrregion or wrpredpredregion; if not we lower
 * it further here. If a use is in rdpredregion, we need to combine the two
 * rdpredregions into one.
 */
bool GenXLowering::lowerRdPredRegion(Instruction *Inst) {
  SmallVector<CallInst *, 4> RdPredRegionUsers;
  bool Ok = true;
  for (auto ui = Inst->use_begin(), ue = Inst->use_end(); ui != ue; ++ui) {
    auto User = cast<Instruction>(ui->getUser());
    if (isa<SelectInst>(User))
      continue;
    unsigned IID = vc::getAnyIntrinsicID(User);
    if (GenXIntrinsic::isWrRegion(IID))
      continue;
    if (IID == GenXIntrinsic::genx_wrpredpredregion)
      continue;
    if (IID == GenXIntrinsic::genx_rdpredregion) {
      RdPredRegionUsers.push_back(cast<CallInst>(User));
      continue;
    }
    if (IID == GenXIntrinsic::not_any_intrinsic) {
      Ok = false;
      break;
    }
    if (cast<CallInst>(User)->doesNotAccessMemory()) {
      Ok = false;
      break;
    }
  }
  unsigned Start = cast<ConstantInt>(Inst->getOperand(1))->getZExtValue();
  unsigned Size =
      cast<IGCLLVM::FixedVectorType>(Inst->getType())->getNumElements();
  if (Ok) {
    // All uses in select/wrregion/rdpredregion/non-ALU intrinsic, so we can
    // keep the rdpredregion.  Check for uses in another rdpredregion; we need
    // to combine those.
    for (auto ui = RdPredRegionUsers.begin(), ue = RdPredRegionUsers.end();
         ui != ue; ++ui) {
      auto User = *ui;
      unsigned UserStart =
          cast<ConstantInt>(User->getOperand(1))->getZExtValue();
      unsigned UserSize =
          cast<IGCLLVM::FixedVectorType>(User->getType())->getNumElements();
      auto Combined =
          Region::createRdPredRegion(Inst->getOperand(0), Start + UserStart,
                                     UserSize, "", User, User->getDebugLoc());
      Combined->takeName(User);
      User->replaceAllUsesWith(Combined);
      ToErase.push_back(User);
    }
    return false;
  }
  // Need to lower it further.
  const DebugLoc &DL = Inst->getDebugLoc();
  // Convert input to vector of short.
  auto In = Inst->getOperand(0);
  Type *I16Ty = Type::getInt16Ty(Inst->getContext());
  Type *InI16Ty = IGCLLVM::FixedVectorType::get(
      I16Ty, cast<IGCLLVM::FixedVectorType>(In->getType())->getNumElements());
  auto InI16 = CastInst::Create(Instruction::ZExt, In, InI16Ty,
                                Inst->getName() + ".lower1", Inst);
  InI16->setDebugLoc(DL);
  // Use rdregion to extract the region.
  Region R(InI16);
  R.getSubregion(Start, Size);
  auto Rd = R.createRdRegion(InI16, Inst->getName() + ".lower3", Inst, DL);
  // Convert back to predicate.
  auto Res = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, Rd,
                             Constant::getNullValue(Rd->getType()),
                             Inst->getName() + ".lower4", Inst);
  Res->setDebugLoc(DL);
  // Replace uses and erase.
  Inst->replaceAllUsesWith(Res);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerWrPredRegion : handle write predicate region instruction
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * wrpredregion is a GenX backend internal intrinsic, and was thus created
 * within this GenXLowering pass. However it is considered legal only if its
 * "new value" input is a compare; if not we lower it further here.
 */
bool GenXLowering::lowerWrPredRegion(Instruction *Inst) {
  auto NewVal = Inst->getOperand(1);
  if (isa<CmpInst>(NewVal))
    return false;
  // Need to lower it further.
  const DebugLoc &DL = Inst->getDebugLoc();
  // Convert "old value" input to vector of short.
  auto OldVal = Inst->getOperand(0);
  Type *I16Ty = Type::getInt16Ty(Inst->getContext());
  Type *OldValI16Ty = IGCLLVM::FixedVectorType::get(
      I16Ty,
      cast<IGCLLVM::FixedVectorType>(OldVal->getType())->getNumElements());
  auto OldValI16 = CastInst::Create(Instruction::ZExt, OldVal, OldValI16Ty,
                                    Inst->getName() + ".lower1", Inst);
  OldValI16->setDebugLoc(DL);
  // Convert "new value" input to vector of short.
  Type *NewValI16Ty = IGCLLVM::FixedVectorType::get(
      I16Ty,
      cast<IGCLLVM::FixedVectorType>(NewVal->getType())->getNumElements());
  auto NewValI16 = CastInst::Create(Instruction::ZExt, NewVal, NewValI16Ty,
                                    Inst->getName() + ".lower2", Inst);
  NewValI16->setDebugLoc(DL);
  // Use wrregion to write the new value into the old value.
  Region R(OldValI16);
  R.getSubregion(cast<ConstantInt>(Inst->getOperand(2))->getZExtValue(),
                 cast<IGCLLVM::FixedVectorType>(NewValI16Ty)->getNumElements());
  auto Wr = R.createWrRegion(OldValI16, NewValI16, Inst->getName() + ".lower3",
                             Inst, DL);
  // Convert back to predicate.
  auto Res = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, Wr,
                             Constant::getNullValue(Wr->getType()),
                             Inst->getName() + ".lower4", Inst);
  Res->setDebugLoc(DL);
  // Replace uses and erase.
  Inst->replaceAllUsesWith(Res);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerInsertElement : lower InsertElement to wrregion, multiplying the
 *      index by the element size
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::lowerInsertElement(Instruction *Inst) {
  Instruction *NewInst = NULL;
  // Special case - if the result has 1 element (usually turning scalar into 1
  // element vector) then simply transform the insert element into a bitcast We
  // don't need to worry about the index since if it is not zero the result is
  // undef anyway (and can be set to anything we like) We also don't need to
  // worry about what the original vector is (usually undef) since it will be
  // overwritten or undef
  auto *VT = dyn_cast<IGCLLVM::FixedVectorType>(Inst->getType());
  IGC_ASSERT(VT);
  unsigned NumElements = VT->getNumElements();
  const DebugLoc &DL = Inst->getDebugLoc();
  if (NumElements == 1) {
    Value *ToInsert = Inst->getOperand(1);
    NewInst = CastInst::Create(Instruction::BitCast, ToInsert, VT,
                               Inst->getName(), Inst);
    NewInst->setDebugLoc(DL);
  } else if (!Inst->getType()->getScalarType()->isIntegerTy(1)) {
    // Cast and scale the index.
    Value *IdxVal = scaleInsertExtractElementIndex(
        Inst->getOperand(2), Inst->getOperand(1)->getType(), Inst);
    // Sink adds in the address calculation.
    IdxVal = sinkAdd(IdxVal);
    // Create the new wrregion
    Value *Src = Inst->getOperand(1);
    Region R(Src);
    R.Indirect = IdxVal;
    NewInst = R.createWrRegion(Inst->getOperand(0), Src, Inst->getName(),
                               Inst /*InsertBefore*/, DL);
  } else {
    // Boolean insertelement. We have to cast everything to i16, do the
    // insertelement, and cast it back again. All this gets further lowered
    // subsequently.
    auto I16Ty = Type::getIntNTy(Inst->getContext(), 16);
    auto VecTy = IGCLLVM::FixedVectorType::get(
        I16Ty,
        cast<IGCLLVM::FixedVectorType>(Inst->getType())->getNumElements());
    auto CastVec =
        CastInst::Create(Instruction::ZExt, Inst->getOperand(0), VecTy,
                         Inst->getOperand(0)->getName() + ".casti16", Inst);
    CastVec->setDebugLoc(DL);
    auto CastEl =
        CastInst::Create(Instruction::ZExt, Inst->getOperand(1), I16Ty,
                         Inst->getOperand(1)->getName() + ".casti16", Inst);
    CastEl->setDebugLoc(DL);
    auto NewInsert = InsertElementInst::Create(CastVec, CastEl,
                                               Inst->getOperand(2), "", Inst);
    NewInsert->takeName(Inst);
    NewInsert->setDebugLoc(DL);
    NewInst = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, NewInsert,
                              Constant::getNullValue(VecTy),
                              NewInsert->getName() + ".casti1", Inst);
    NewInst->setDebugLoc(DL);
  }
  // Change uses and mark the old inst for erasing.
  Inst->replaceAllUsesWith(NewInst);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerExtractElement : lower ExtractElement to rdregion, multiplying the
 *      index by the element size
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::lowerExtractElement(Instruction *Inst) {
  Instruction *NewInst = nullptr;
  if (!Inst->getType()->isIntegerTy(1)) {
    // Cast and scale the index.
    Type *ElTy = Inst->getType();
    Value *IdxVal =
        scaleInsertExtractElementIndex(Inst->getOperand(1), ElTy, Inst);
    // Sink adds in the address calculation.
    IdxVal = sinkAdd(IdxVal);
    // Create the new rdregion.
    Region R(Inst);
    R.Indirect = IdxVal;
    NewInst = R.createRdRegion(Inst->getOperand(0), Inst->getName(),
                               Inst /*InsertBefore*/, Inst->getDebugLoc(),
                               true /*AllowScalar*/);
  } else {
    // Boolean extractelement. We have to cast everything to i16, do the
    // extractelement, and cast it back again. All this gets further lowered
    // subsequently.
    auto I16Ty = Type::getIntNTy(Inst->getContext(), 16);
    auto VecTy = IGCLLVM::FixedVectorType::get(
        I16Ty, cast<IGCLLVM::FixedVectorType>(Inst->getOperand(0)->getType())
                   ->getNumElements());
    auto CastVec =
        CastInst::Create(Instruction::ZExt, Inst->getOperand(0), VecTy,
                         Inst->getOperand(0)->getName() + ".casti16", Inst);
    const DebugLoc &DL = Inst->getDebugLoc();
    CastVec->setDebugLoc(DL);
    auto NewExtract =
        ExtractElementInst::Create(CastVec, Inst->getOperand(1), "", Inst);
    NewExtract->takeName(Inst);
    NewExtract->setDebugLoc(DL);
    NewInst = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_NE, NewExtract,
                              Constant::getNullValue(I16Ty),
                              NewExtract->getName() + ".casti1", Inst);
    NewInst->setDebugLoc(DL);
  }
  // Change uses and mark the old inst for erasing.
  Inst->replaceAllUsesWith(NewInst);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * scaleInsertExtractElementIndex : scale index by element byte size,
 *      and ensure it is an i16
 */
Value *GenXLowering::scaleInsertExtractElementIndex(Value *IdxVal, Type *ElTy,
                                                    Instruction *InsertBefore) {
  // Do the cast and multiply.
  static_assert(genx::ByteBits);
  unsigned ElementBytes = DL->getTypeSizeInBits(ElTy) / genx::ByteBits;
  IntegerType *I16Ty = Type::getInt16Ty(IdxVal->getContext());
  if (ConstantInt *CI = dyn_cast<ConstantInt>(IdxVal))
    return ConstantInt::get(I16Ty, CI->getSExtValue() * ElementBytes);
  // Ensure the variable offset is i16.
  Instruction *IdxInst = CastInst::CreateIntegerCast(
      IdxVal, I16Ty, false /*isSigned*/, "cast", InsertBefore);
  IdxInst->setDebugLoc(InsertBefore->getDebugLoc());
  // Multiply it by the element size in bytes.
  if (ElementBytes != 1) {
    IdxInst = BinaryOperator::Create(
        Instruction::Shl, IdxInst,
        ConstantInt::get(I16Ty, genx::log2(ElementBytes)), "scale",
        InsertBefore);
    IdxInst->setDebugLoc(InsertBefore->getDebugLoc());
  }
  return IdxInst;
}

/***********************************************************************
 * lowerTrunc : lower a TruncInst
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * A Trunc is lowered to a bitcast then a region/element read with a stride.
 * GenXCoalescing will coalesce the bitcast, so this will hopefully save
 * an instruction.
 */
bool GenXLowering::lowerTrunc(Instruction *Inst) {
  Value *InValue = Inst->getOperand(0);
  // Check for the trunc's input being a sext/zext where the original element
  // size is the same as the result of the trunc. We can just remove the
  // whole thing then. (This can arise from GenXReduceIntSize.)
  if (auto CI = dyn_cast<CastInst>(InValue)) {
    if ((isa<SExtInst>(CI) || isa<ZExtInst>(CI)) &&
        CI->getOperand(0)->getType() == Inst->getType()) {
      // Just replace uses with the original unextended value.
      Inst->replaceAllUsesWith(CI->getOperand(0));
      ToErase.push_back(Inst);
      return true;
    }
  }

  // Lower "trunc i8 %v to i1" into "cmp.ne (%v & 1), 0"
  if (Inst->getType()->isIntOrIntVectorTy(1)) {
    IRBuilder<> Builder(Inst);
    auto V =
        Builder.CreateAnd(InValue, ConstantInt::get(InValue->getType(), 1));
    V = Builder.CreateICmpNE(V, ConstantInt::get(V->getType(), 0));
    if (auto I = dyn_cast<Instruction>(V))
      I->setDebugLoc(Inst->getDebugLoc());
    Inst->replaceAllUsesWith(V);
    ToErase.push_back(Inst);
    return true;
  }

  Type *InElementTy = InValue->getType();
  Type *OutElementTy = Inst->getType();
  unsigned NumElements = 1;
  if (auto *VT = dyn_cast<IGCLLVM::FixedVectorType>(InElementTy)) {
    InElementTy = VT->getElementType();
    OutElementTy = cast<VectorType>(OutElementTy)->getElementType();
    NumElements = VT->getNumElements();
  }

  // Lower "trunc <32 x i16> %v to <32 x i1>" into "cmp.ne (%v & 1), 0"
  if (NumElements > 1 && OutElementTy->isIntegerTy(1)) {
    IRBuilder<> Builder(Inst);
    unsigned N = NumElements;
    Value *Os = ConstantVector::getSplat(IGCLLVM::getElementCount(N),
                                         ConstantInt::get(InElementTy, 1));
    Value *Zs = ConstantVector::getSplat(IGCLLVM::getElementCount(N),
                                         ConstantInt::get(InElementTy, 0));
    auto V = Builder.CreateAnd(InValue, Os);
    if (auto I = dyn_cast<Instruction>(V))
      I->setDebugLoc(Inst->getDebugLoc());
    V = Builder.CreateICmpNE(V, Zs);
    if (auto I = dyn_cast<Instruction>(V))
      I->setDebugLoc(Inst->getDebugLoc());
    Inst->replaceAllUsesWith(V);
    ToErase.push_back(Inst);
    return true;
  }

  IGC_ASSERT(OutElementTy->getPrimitiveSizeInBits());
  unsigned Stride = InElementTy->getPrimitiveSizeInBits() /
                    OutElementTy->getPrimitiveSizeInBits();
  // Create the new bitcast.
  Instruction *BC = CastInst::Create(
      Instruction::BitCast, InValue,
      IGCLLVM::FixedVectorType::get(OutElementTy, Stride * NumElements),
      Inst->getName(), Inst /*InsertBefore*/);
  BC->setDebugLoc(Inst->getDebugLoc());
  // Create the new rdregion.
  Region R(BC);
  R.NumElements = NumElements;
  R.Stride = Stride;
  R.Width = NumElements;
  R.VStride = R.Stride * R.Width;
  Instruction *NewInst = R.createRdRegion(
      BC, Inst->getName(), Inst /*InsertBefore*/, Inst->getDebugLoc(),
      !isa<VectorType>(Inst->getType()) /*AllowScalar*/);
  // Change uses and mark the old inst for erasing.
  Inst->replaceAllUsesWith(NewInst);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerCast : lower a CastInst
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::lowerCast(Instruction *Inst) {
  // if it is a bitcast for vector of i1 predicate to vector,
  // create bitcast to scalar and them bitcast to vector due to visa
  // requirements
  if (Inst->getOpcode() == Instruction::BitCast &&
      Inst->getOperand(0)->getType()->getScalarType()->isIntegerTy(1) &&
      Inst->getType()->isVectorTy() &&
      !Inst->getType()->getScalarType()->isIntegerTy(1)) {
    IRBuilder<> Builder(Inst);
    Value *operand = Inst->getOperand(0);
    Type *originTy = Inst->getType();
    Type *scalarIntTy =
        Type::getIntNTy(Inst->getContext(), DL->getTypeSizeInBits(originTy));
    const Twine Name = "pred";
    Value *Scalar = Builder.CreateBitCast(operand, scalarIntTy, Name);
    Value *Res = Builder.CreateBitCast(Scalar, originTy, Name);
    Inst->replaceAllUsesWith(Res);
    ToErase.push_back(Inst);
    return true;
  }

  // If it is zext/sext/UIToFP from (vector of) i1, turn into a select.
  if (Inst->getOperand(0)->getType()->getScalarType()->isIntegerTy(1) &&
      Inst->getOpcode() != Instruction::BitCast) {
    int OneVal = 0;
    switch (Inst->getOpcode()) {
    case Instruction::ZExt:
      OneVal = 1;
      break;
    case Instruction::SExt:
      OneVal = -1;
      break;
    case Instruction::UIToFP:
      OneVal = 1;
      break;
    default:
      IGC_ASSERT_MESSAGE(0, "unknown opcode in lowerCast");
    }

    Instruction *NewInst;
    if (Inst->getType()->isFPOrFPVectorTy())
      NewInst = SelectInst::Create(
          Inst->getOperand(0), ConstantFP::get(Inst->getType(), OneVal),
          ConstantFP::get(Inst->getType(), 0), Inst->getName(), Inst);
    else
      NewInst = SelectInst::Create(
          Inst->getOperand(0), ConstantInt::get(Inst->getType(), OneVal),
          ConstantInt::get(Inst->getType(), 0), Inst->getName(), Inst);
    NewInst->setDebugLoc(Inst->getDebugLoc());
    Inst->replaceAllUsesWith(NewInst);
    ToErase.push_back(Inst);
    return true;
  }
  return false;
}

/***********************************************************************
 * lowerBoolScalarSelect : lower a SelectInst on vector of i1
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * This is a select on vector of i1 where the condition is scalar. This only
 * happens in simd control flow where an LLVM pass has optimized away the
 * conditional branch. We restore the conditional branch and create an
 * if..else..endif.
 */
bool GenXLowering::lowerBoolScalarSelect(SelectInst *SI) {
  //         BB1
  //        /  |
  // false /   | true
  //      /    |
  //   BB2     |
  //      \    |
  //       \   |
  //        \  |
  //         BB4
  //
  auto BB1 = SI->getParent();
  auto BB2 = SplitBlock(BB1, SI, DT);
  auto BB4 = SplitEdge(BB1, BB2, DT);
  BB2->setName("select.false");
  BB4->setName("select.true");

  auto OldTerm = BB1->getTerminator();
  auto NewInst = BranchInst::Create(BB4, BB2, SI->getCondition(), OldTerm);
  NewInst->setDebugLoc(SI->getDebugLoc());
  OldTerm->eraseFromParent();
  // Since additional edge is added between BB1 and BB4 instead of through BB2
  // only. BB4 is not immediately dominated by BB2 anymore. Instead, BB4 is
  // dominated by BB1 immediately.
  if (DT)
    DT->changeImmediateDominator(BB4, BB1);
  // Replace 'select' with 'phi'
  auto Phi = PHINode::Create(SI->getType(), /*NumReservedValues=*/2, "",
                             &BB4->front());
  Phi->takeName(SI);
  Phi->addIncoming(SI->getTrueValue(), BB1);
  Phi->addIncoming(SI->getFalseValue(), BB2);
  Phi->setDebugLoc(SI->getDebugLoc());
  SI->replaceAllUsesWith(Phi);
  ToErase.push_back(SI);
  // Split the (critical) edge from BB1 to BB4 to avoid having critical edge.
  auto BB3 = SplitEdge(BB1, BB4, DT);
  BB3->setName("select.crit");
  return true;
}

/***********************************************************************
 * lowerBoolVectorSelect : lower a SelectInst on (vector of) i1
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * A select on (vector of) i1 is lowered to the equivalent and/or/xor
 * instructions. No simplification is done even if an input is a constant.
 *
 * However, if the selector looks like an EM value, and the "true" operand is
 * a cmp, it is instead lowered to an llvm.genx.wrpredpredregion. Baling will
 * bale the cmp into it, resulting in a masked cmp instruction that sets bits
 * of the flag only if the corresponding EM bit is set.
 *
 * FIXME: I have seen a case where the two inputs are all false and all true.
 * Rather than try and simplify that here in the GenX backend, we should
 * try and work out how to stop LLVM generating it in the first place.
 */
bool GenXLowering::lowerBoolVectorSelect(SelectInst *Inst) {
  if (isa<CmpInst>(Inst->getTrueValue())) {
    // Check for the condition being an EM value. It might be a shufflevector
    // that slices the EM value at index 0.
    bool IsEM = GotoJoin::isEMValue(Inst->getCondition());
    if (!IsEM) {
      if (auto SV = dyn_cast<ShuffleVectorInst>(Inst->getCondition())) {
        ShuffleVectorAnalyzer SVA(SV);
        if (!SVA.getAsSlice()) {
          // Slice at index 0.
          IsEM = GotoJoin::isEMValue(SV->getOperand(0));
        }
      }
    }
    if (IsEM) {
      // Can be lowered to llvm.genx.wrpredpredregion. It always has an index of
      // 0 and the "new value" operand the same vector width as the whole vector
      // here. That might get changed if it is split up in legalization.
      auto NewInst = Region::createWrPredPredRegion(
          Inst->getFalseValue(), Inst->getTrueValue(), 0, Inst->getCondition(),
          "", Inst, Inst->getDebugLoc());
      NewInst->takeName(Inst);
      Inst->replaceAllUsesWith(NewInst);
      ToErase.push_back(Inst);
      return true;
    }
  }
  // Normal lowering to some bit twiddling.
  Instruction *NewInst1 =
      BinaryOperator::Create(BinaryOperator::And, Inst->getOperand(0),
                             Inst->getOperand(1), Inst->getName(), Inst);
  NewInst1->setDebugLoc(Inst->getDebugLoc());
  Instruction *NewInst2 = BinaryOperator::Create(
      BinaryOperator::Xor, Inst->getOperand(0),
      Constant::getAllOnesValue(Inst->getType()), Inst->getName(), Inst);
  NewInst2->setDebugLoc(Inst->getDebugLoc());
  Instruction *NewInst3 =
      BinaryOperator::Create(BinaryOperator::And, Inst->getOperand(2), NewInst2,
                             Inst->getName(), Inst);
  NewInst3->setDebugLoc(Inst->getDebugLoc());
  Instruction *NewInst4 = BinaryOperator::Create(
      BinaryOperator::Or, NewInst1, NewInst3, Inst->getName(), Inst);
  NewInst4->setDebugLoc(Inst->getDebugLoc());
  Inst->replaceAllUsesWith(NewInst4);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerBoolShuffle : lower a shufflevector (element type i1)
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * We handle four cases:
 *
 * 1. A slice of the vector, which can be turned into rdpredregion.
 *
 * 2. A splat. By default we need to lower that to a select to
 *    0 or -1 then a bitcast to the vector of i1. But if the input is the
 *    result of a cmp then we can splat the cmp as an optimization.
 *
 * 3. An unslice of the vector, which can be turned into wrpredregion.
 *
 * 4. General case. Like in the splat case we convert input via select and
 *    result is then bitcasted back to vector of i1. Converted vectors are
 *    then handled by lowerShuffleToMove
 */
bool GenXLowering::lowerBoolShuffle(ShuffleVectorInst *SI) {
  ShuffleVectorAnalyzer SVA(SI);
  // 1. Check for a slice.
  int SliceStart = SVA.getAsSlice();
  if (SliceStart >= 0) {
    unsigned Width =
        cast<IGCLLVM::FixedVectorType>(SI->getType())->getNumElements();
    auto RPR = Region::createRdPredRegion(SI->getOperand(0), SliceStart, Width,
                                          "", SI, SI->getDebugLoc());
    RPR->takeName(SI);
    SI->replaceAllUsesWith(RPR);
    ToErase.push_back(SI);
    return true;
  }
  // 2. Check for a splat.
  auto Splat = SVA.getAsSplat();
  if (Splat.Input)
    return lowerBoolSplat(SI, Splat.Input, Splat.Index);
  // 3. Check for an unslice. The "old value" input is operand 0 of the
  // shufflevector; the "new value" input is operand 0 of the shufflevector
  // that is operand 1 of SI. We create a wrpredregion, but GenXLowering might
  // subsequently decide that it is illegal because its "new value" input is not
  // a compare, in which case it is further lowered.
  int UnsliceStart = SVA.getAsUnslice();
  if (UnsliceStart >= 0) {
    auto InnerSI = cast<ShuffleVectorInst>(SI->getOperand(1));
    auto WPR =
        Region::createWrPredRegion(SI->getOperand(0), InnerSI->getOperand(0),
                                   UnsliceStart, "", SI, SI->getDebugLoc());
    WPR->takeName(SI);
    SI->replaceAllUsesWith(WPR);
    // Undef out the operand for InnerSI in SI, so we can directly erase InnerSI
    // if SI was its only use.
    SI->setOperand(1, UndefValue::get(InnerSI->getType()));
    ToErase.push_back(SI);
    if (InnerSI->use_empty())
      InnerSI->eraseFromParent();
    return true;
  }

  // Do not lower replicated slices.
  if (SVA.isReplicatedSlice())
    return false;

  // 4. General case.

  // The idea is to convert input i1 vector to i16 vector via select,
  // then do a shufflevector lowering for non-bool case
  // and convert back to i1 vector via icmp instruction.

  IRBuilder<> B(SI);
  unsigned WidthInput =
      cast<IGCLLVM::FixedVectorType>(SI->getOperand(0)->getType())
          ->getNumElements();
  unsigned WidthResult =
      cast<IGCLLVM::FixedVectorType>(SI->getType())->getNumElements();
  Constant *C1 = ConstantVector::getSplat(IGCLLVM::getElementCount(WidthInput),
                                          B.getInt16(1));
  Constant *C0 = ConstantVector::getSplat(IGCLLVM::getElementCount(WidthInput),
                                          B.getInt16(0));
  Value *V1 = B.CreateSelect(SI->getOperand(0), C1, C0);
  Value *V2 = B.CreateSelect(SI->getOperand(1), C1, C0);
  Value *SI1 = B.CreateShuffleVector(V1, V2, IGCLLVM::getShuffleMaskForBitcode(SI), SI->getName());
  Constant *C2 = ConstantVector::getSplat(IGCLLVM::getElementCount(WidthResult),
                                          B.getInt16(0));
  Value *Result = B.CreateICmpNE(SI1, C2);
  SI->replaceAllUsesWith(Result);
  ToErase.push_back(SI);

  return true;
}

/***********************************************************************
 * lowerBoolSplat : lower a shufflevector (element type i1) that is a splat
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 */
bool GenXLowering::lowerBoolSplat(ShuffleVectorInst *SI, Value *In,
                                  unsigned Idx) {
  auto IsFixedVectorOfWidth = [](const Type *Ty, unsigned Width) {
    IGC_ASSERT(Ty);
    const auto *VTy = dyn_cast<IGCLLVM::FixedVectorType>(Ty);
    if (!VTy)
      return false;
    return VTy->getNumElements() == Width;
  };

  unsigned Width = cast<IGCLLVM::FixedVectorType>(SI->getType())->getNumElements();
  if (IsFixedVectorOfWidth(In->getType(), Width)) {
    IRBuilder<> B(SI);
    Constant *C1 = ConstantVector::getSplat(IGCLLVM::getElementCount(Width),
                                            B.getInt16(1));
    Constant *C0 = ConstantVector::getSplat(IGCLLVM::getElementCount(Width),
                                            B.getInt16(0));
    Value *V = B.CreateSelect(In, C1, C0);
    Region R(V);
    R.NumElements = Width;
    R.Stride = 0;
    R.VStride = 0;
    R.Offset = (int)Idx;
    V = R.createRdRegion(V, "splat", SI, SI->getDebugLoc());
    V = B.CreateICmpNE(V, C0);
    SI->replaceAllUsesWith(V);
    ToErase.push_back(SI);
    return true;
  }
  // This is a splat. See if the input is a cmp, possibly via a bitcast.
  if (auto BC = dyn_cast<BitCastInst>(In))
    In = BC->getOperand(0);
  if (auto Cmp = dyn_cast<CmpInst>(In)) {
    // Create a splatted version of the cmp.
    Value *CmpOpnds[2];
    Region R(Cmp->getOperand(0));
    R.NumElements = Width;
    R.Width = R.NumElements;
    R.Stride = 0;
    R.VStride = 0;
    for (unsigned i = 0; i != 2; ++i) {
      auto Opnd = Cmp->getOperand(i);
      if (auto C = dyn_cast<Constant>(Opnd)) {
        Constant *SplatC = C->getType()->isVectorTy() ?
          C->getAggregateElement(Idx) : C;
        CmpOpnds[i] = ConstantVector::getSplat(
            IGCLLVM::getElementCount(R.NumElements), SplatC);
        continue;
      }
      if (!isa<VectorType>(Opnd->getType())) {
        auto NewBC =
            CastInst::Create(Instruction::BitCast, Opnd,
                             IGCLLVM::FixedVectorType::get(Opnd->getType(), 1),
                             Opnd->getName() + ".bc", Cmp);
        NewBC->setDebugLoc(Cmp->getDebugLoc());
        Opnd = NewBC;
      }
      CmpOpnds[i] =
          R.createRdRegion(Opnd, Cmp->getOperand(i)->getName() + ".splat",
                           Cmp /*InsertBefore*/, Cmp->getDebugLoc());
    }
    auto NewCmp = CmpInst::Create(
        Cmp->getOpcode(), Cmp->getPredicate(), CmpOpnds[0], CmpOpnds[1],
        Cmp->getName() + ".splat", Cmp /*InsertBefore*/);
    NewCmp->setDebugLoc(Cmp->getDebugLoc());
    SI->replaceAllUsesWith(NewCmp);
    ToErase.push_back(SI);
    return true;
  }
  // Default code. Select int and bitcast to vector of i1.
  if (isa<VectorType>(In->getType())) {
    // First convert v1i1 to i1.
    auto NewBC = CastInst::Create(Instruction::BitCast, In,
                                  In->getType()->getScalarType(),
                                  In->getName() + ".scalar", SI);
    NewBC->setDebugLoc(SI->getDebugLoc());
    In = NewBC;
  }
  if (Width == 8 || Width == 16 || Width == 32) {
    auto IntTy = Type::getIntNTy(SI->getContext(), Width);
    auto Sel = SelectInst::Create(In, Constant::getAllOnesValue(IntTy),
                                  Constant::getNullValue(IntTy),
                                  SI->getName() + ".sel", SI);
    Sel->setDebugLoc(SI->getDebugLoc());
    auto NewBC =
        CastInst::Create(Instruction::BitCast, Sel, SI->getType(), "", SI);
    NewBC->takeName(SI);
    NewBC->setDebugLoc(SI->getDebugLoc());
    SI->replaceAllUsesWith(NewBC);
    ToErase.push_back(SI);
    return true;
  }

  IRBuilder<> Builder(SI);
  auto Val = Builder.CreateSelect(In, Builder.getInt16(1), Builder.getInt16(0),
                                  SI->getName() + ".sel");
  if (auto Inst = dyn_cast<Instruction>(Val))
    Inst->setDebugLoc(SI->getDebugLoc());
  Val = Builder.CreateBitCast(
      Val, IGCLLVM::FixedVectorType::get(Builder.getInt16Ty(), 1));
  if (auto Inst = dyn_cast<Instruction>(Val))
    Inst->setDebugLoc(SI->getDebugLoc());

  Region R(Val);
  R.Offset = 0;
  R.Width = 1;
  R.Stride = R.VStride = 0;
  R.NumElements = Width;
  Val = R.createRdRegion(Val, "", SI, SI->getDebugLoc());
  Val = Builder.CreateICmpNE(Val, ConstantVector::getNullValue(Val->getType()));
  Val->takeName(SI);
  if (auto Inst = dyn_cast<Instruction>(Val))
    Inst->setDebugLoc(SI->getDebugLoc());
  SI->replaceAllUsesWith(Val);
  ToErase.push_back(SI);
  return true;
}

/***********************************************************************
 * lowerShuffleSplat : lower a ShuffleInst (element type not i1) when it is
 *                     a splat (repetition of the same element)
 */
void GenXLowering::lowerShuffleSplat(ShuffleVectorInst *SI,
                                     ShuffleVectorAnalyzer::SplatInfo Splat) {
  // This is a splat. Turn it into a splatting rdregion.
  if (!isa<VectorType>(Splat.Input->getType())) {
    // The input is a scalar rather than a 1-vector. Bitcast it to a 1-vector.
    auto *BC = CastInst::Create(
        Instruction::BitCast, Splat.Input,
        IGCLLVM::FixedVectorType::get(Splat.Input->getType(), 1), SI->getName(),
        SI);
    BC->setDebugLoc(SI->getDebugLoc());
    Splat.Input = BC;
  }
  // Create a rdregion with a stride of 0 to represent this splat
  Region R(Splat.Input);
  R.NumElements =
      cast<IGCLLVM::FixedVectorType>(SI->getType())->getNumElements();
  R.Width = R.NumElements;
  R.Stride = 0;
  R.VStride = 0;
  R.Offset = Splat.Index * R.ElementBytes;
  Instruction *NewInst =
      R.createRdRegion(Splat.Input, "", SI /*InsertBefore*/, SI->getDebugLoc());
  NewInst->takeName(SI);
  NewInst->setDebugLoc(SI->getDebugLoc());
  SI->replaceAllUsesWith(NewInst);
  ToErase.push_back(SI);
}

/***********************************************************************
 * lowerShuffle : lower a ShuffleInst (element type not i1)
 *
 * Mostly these are splats. These are lowered to a rdregion
 * Any other shuffle is currently unsupported
 */
bool GenXLowering::lowerShuffle(ShuffleVectorInst *SI) {
  auto Splat = ShuffleVectorAnalyzer(SI).getAsSplat();
  if (Splat.Input) {
    lowerShuffleSplat(SI, Splat);
    return true;
  }
  if (lowerShuffleToSelect(SI))
    return true;
  lowerShuffleToMove(SI);
  return true;
}

// Lower those shufflevector that can be implemented efficiently as select.
bool GenXLowering::lowerShuffleToSelect(ShuffleVectorInst *SI) {
  int NumElements =
      cast<IGCLLVM::FixedVectorType>(SI->getType())->getNumElements();
  int NumOpnd = SI->getNumOperands();
  for (int i = 0; i < NumOpnd; ++i) {
    if (cast<IGCLLVM::FixedVectorType>(SI->getOperand(i)->getType())
            ->getNumElements() != NumElements)
      return false;
  }
  for (int i = 0; i < NumElements; ++i) {
    int idx = SI->getMaskValue(i);
    // undef index returns -1.
    if (idx < 0)
      continue;
    if (idx != i && idx != i + NumElements)
      return false;
  }
  IRBuilder<> Builder(SI);
  Type *Int1Ty = Builder.getInt1Ty();
  SmallVector<Constant *, 16> MaskVec;
  MaskVec.reserve(NumElements);
  for (int i = 0; i < NumElements; ++i) {
    int idx = SI->getMaskValue(i);
    // undef index returns -1.
    if (idx == i || idx < 0)
      MaskVec.push_back(ConstantInt::get(Int1Ty, 1));
    else
      MaskVec.push_back(ConstantInt::get(Int1Ty, 0));
  }
  Value *Mask = ConstantVector::get(MaskVec);
  auto NewSel =
      SelectInst::Create(Mask, SI->getOperand(0), SI->getOperand(1), "", SI);
  NewSel->takeName(SI);
  NewSel->setDebugLoc(SI->getDebugLoc());
  SI->replaceAllUsesWith(NewSel);
  ToErase.push_back(SI);
  return true;
}

template <typename Iter> Iter skipUndefs(Iter First, Iter Last) {
  return std::find_if(First, Last, [](int MaskVal) { return MaskVal != -1; });
}

/***********************************************************************
 * lowerShuffleToMove : lower a ShuffleInst (element type is not i1) to a
 *                      sequence of rd/wrregion intrinsics
 */
void GenXLowering::lowerShuffleToMove(ShuffleVectorInst *SI) {
  ShuffleVectorAnalyzer Analyzer(SI);
  std::vector<ShuffleVectorAnalyzer::OperandRegionInfo> RdRegions;
  std::vector<Region> WrRegions;
  auto MaskVals = SI->getShuffleMask();

  // Filling read and write regions based on shuffle mask.
  for (auto It = skipUndefs(MaskVals.begin(), MaskVals.end());
       It != MaskVals.end();
       It = skipUndefs(std::next(It, RdRegions.back().R.NumElements),
                       MaskVals.end())) {
    int Idx = It - MaskVals.begin();
    auto OpRegion = Analyzer.getMaskRegionPrefix(Idx);
    IGC_ASSERT_MESSAGE(OpRegion.R.NumElements > 0,
      "should've match at least 1 element region");
    Region WrRegion(SI);
    WrRegion.Offset = Idx * WrRegion.ElementBytes;
    WrRegion.NumElements = WrRegion.Width = OpRegion.R.NumElements;
    RdRegions.push_back(std::move(OpRegion));
    WrRegions.push_back(std::move(WrRegion));
  }

  // Building rdregion intrinsics or promoting the operand if possible.
  std::vector<Value *> RdRegionInsts;
  std::transform(
      RdRegions.begin(), RdRegions.end(), std::back_inserter(RdRegionInsts),
      [SI](ShuffleVectorAnalyzer::OperandRegionInfo &OpRegion) -> Value * {
        if (OpRegion.R.isWhole(OpRegion.Op->getType()))
          return OpRegion.Op;
        return OpRegion.R.createRdRegion(
            OpRegion.Op, SI->getName() + ".shuffle.rd", SI, SI->getDebugLoc());
      });

  // Obtaining SI replacement (sequence of wrregion intrinsics in the
  // most common case).
  Value *Result;
  if (WrRegions.size() == 0)
    Result = UndefValue::get(SI->getType());
  else if (WrRegions.size() == 1 &&
           WrRegions.front().NumElements ==
               cast<IGCLLVM::FixedVectorType>(SI->getType())->getNumElements())
    Result = RdRegionInsts.back();
  else {
    auto WrRegionArgs = zip(WrRegions, RdRegionInsts);
    Result = std::accumulate(
        WrRegionArgs.begin(), WrRegionArgs.end(),
        static_cast<Value *>(UndefValue::get(SI->getType())),
        [SI](Value *PrevWrRegionInst,
             const std::tuple<Region &, Value *> &Args) {
          return std::get<0>(Args).createWrRegion(
              PrevWrRegionInst, std::get<1>(Args),
              SI->getName() + ".shuffle.wr", SI, SI->getDebugLoc());
        });
  }

  SI->replaceAllUsesWith(Result);
  ToErase.push_back(SI);
}

/***********************************************************************
 * lowerShr : lower Shl followed by AShr/LShr by the same amount
 *    into trunc+sext/zext
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * See convertShlShr below.
 */
bool GenXLowering::lowerShr(Instruction *Inst) {
  Instruction *NewInst = convertShlShr(Inst);
  if (!NewInst)
    return false; // no conversion done
  ToErase.push_back(Inst);
  auto Shl = cast<Instruction>(Inst->getOperand(0));
  if (Shl->hasOneUse())
    ToErase.push_back(Shl);
  return true;
}

/***********************************************************************
 * convertShlShr : convert Shl followed by AShr/LShr by the same amount
 *    into trunc+sext/zext
 *
 * Enter:   Inst = the AShr or LShr instruction
 *
 * Return:  0 if no conversion done, else the new SExt/ZExt instruction.
 *          The original AShr/LShr is now unused, but neither original
 *          instruction is erased.
 *
 * This is the opposite to what instruction combining does! We want to change
 * it back to trunc then extend because the trunc can then be lowered into
 * a region, and the extend can sometimes be baled into whatever uses it.
 *
 * This is a separately callable global function so it can also be used
 * from GenXReduceIntSize, which for other reasons of convenience runs before
 * GenXLowering.
 */
Instruction *llvm::genx::convertShlShr(Instruction *Inst) {
  unsigned NumBits = Inst->getType()->getScalarType()->getPrimitiveSizeInBits();
  auto C = dyn_cast<Constant>(Inst->getOperand(1));
  if (!C)
    return nullptr;
  auto Shl = dyn_cast<Instruction>(Inst->getOperand(0));
  if (!Shl)
    return nullptr;
  if (Shl->getOpcode() != Instruction::Shl)
    return nullptr;
  if (Shl->getOperand(1) != C)
    return nullptr;
  if (isa<VectorType>(C->getType())) {
    C = C->getSplatValue();
    if (!C)
      return nullptr;
  }
  unsigned ShiftBits = cast<ConstantInt>(C)->getSExtValue();
  unsigned RemainingBits = NumBits - ShiftBits;
  if (RemainingBits != 8 && RemainingBits != 16)
    return nullptr;
  // We have Shl+AShr or Shl+LShr that can be turned into trunc+sext/zext.
  Type *ConvTy = Type::getIntNTy(Inst->getContext(), RemainingBits);
  if (auto *VT = dyn_cast<IGCLLVM::FixedVectorType>(Inst->getType()))
    ConvTy = IGCLLVM::FixedVectorType::get(ConvTy, VT->getNumElements());
  auto Trunc = CastInst::Create(Instruction::Trunc, Shl->getOperand(0), ConvTy,
                                "", Inst);
  Trunc->takeName(Shl);
  Trunc->setDebugLoc(Inst->getDebugLoc());
  auto Ext = CastInst::Create(Inst->getOpcode() == Instruction::AShr
                                  ? Instruction::SExt
                                  : Instruction::ZExt,
                              Trunc, Inst->getType(), "", Inst);
  Ext->takeName(Inst);
  Ext->setDebugLoc(Inst->getDebugLoc());
  Inst->replaceAllUsesWith(Ext);
  return Ext;
}

/***********************************************************************
 * lowerExtractValue : remove extractvalue if possible
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * If we can trace the input of the extractvalue to the point where the
 * value was inserted, use that value instead.
 *
 * Because we have already split struct phi nodes, we should just be left
 * with insertvalue/extractvalue pairs that we can remove here. The
 * exception is when a struct is passed in to or returned from a call.
 * Then we leave the extractvalue for later handling in the register
 * allocator.
 */
bool GenXLowering::lowerExtractValue(ExtractValueInst *Inst) {
  ArrayRef<unsigned> EVIndices = Inst->getIndices();
  ArrayRef<unsigned> Indices = EVIndices;
  Value *V = Inst->getAggregateOperand();
  for (;;) {
    InsertValueInst *IV = dyn_cast<InsertValueInst>(V);
    if (!IV) {
      // If we used up any indices, create a new extractvalue for the
      // remaining ones.
      if (Indices.size() != EVIndices.size()) {
        Instruction *NewIV = ExtractValueInst::Create(
            Inst->getAggregateOperand(), Indices, Inst->getName(), Inst);
        NewIV->setDebugLoc(Inst->getDebugLoc());
        Inst->replaceAllUsesWith(NewIV);
        ToErase.push_back(Inst);
        return true;
      }
      return false;
    }
    // We have an insertvalue. See how many of the indices agree.
    ArrayRef<unsigned> IVIndices = IV->getIndices();
    unsigned Match = 0;
    while (Match < Indices.size() && Match < IVIndices.size() &&
           Indices[Match] == IVIndices[Match])
      ++Match;
    if (!Match) {
      // No match at all. Go back to the previous insertvalue.
      V = IV->getAggregateOperand();
      continue;
    }
    // Use the inserted value here.
    V = IV->getInsertedValueOperand();
    // Chop off the indices we have used up. If none left, we have finished.
    Indices = Indices.slice(Match);
    if (!Indices.size())
      break;
  }
  // We have found the struct element value V.
  Inst->replaceAllUsesWith(V);
  ToErase.push_back(Inst);
  return true;
}

/***********************************************************************
 * lowerInsertValue : remove insertvalue if possible
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * In most cases, by the time we get to an insertvalue, it will be unused
 * because of extractvalue removal.
 *
 * In a case where it is still used (probably because this function has an
 * arg or return value that is a struct, or we call a function like that),
 * the struct value is dealt with in register allocation.
 */
bool GenXLowering::lowerInsertValue(InsertValueInst *Inst) {
  if (Inst->use_empty()) {
    ToErase.push_back(Inst);
    return true;
  }
  return false;
}

/***********************************************************************
 * lowerUAddWithOverflow : lower llvm.uadd.with.overflow
 *
 * This could potentially be implemented with the vISA addc instruction.
 * However an intrinsic for that would need extra GenX backend support for
 * returning a struct containing two vectors, and that support does not exist
 * now.
 *
 * So for now we use the old DEC Alpha trick of comparing the result with
 * one of the operands.
 */
bool GenXLowering::lowerUAddWithOverflow(CallInst *CI) {
  const DebugLoc &DL = CI->getDebugLoc();
  // Do the add.
  auto Add =
      BinaryOperator::Create(Instruction::Add, CI->getArgOperand(0),
                             CI->getArgOperand(1), CI->getName() + ".add", CI);
  Add->setDebugLoc(DL);
  // Do the comparison. (An unsigned add has overflowed if the result is
  // smaller than one of the operands, and, if it has overflowed, the result
  // is smaller than both of the operands. So it doesn't matter which operand
  // we use for the comparison.)
  auto Cmp = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_ULT, Add,
                             CI->getArgOperand(1), CI->getName() + ".cmp", CI);
  Cmp->setDebugLoc(DL);
  // For any extractvalue use of the result of the original add with overflow,
  // replace it directly.
  SmallVector<ExtractValueInst *, 4> Extracts;
  for (auto ui = CI->use_begin(), ue = CI->use_end(); ui != ue; ++ui)
    if (auto EVI = dyn_cast<ExtractValueInst>(ui->getUser()))
      Extracts.push_back(EVI);
  for (auto ei = Extracts.begin(), ee = Extracts.end(); ei != ee; ++ei) {
    auto EVI = *ei;
    EVI->replaceAllUsesWith(EVI->getIndices()[0] ? (Value *)Cmp : (Value *)Add);
    EVI->setOperand(0, UndefValue::get(CI->getType()));
    ToErase.push_back(EVI);
  }
  // If any uses of the original intrinsic remain, recreate the struct value.
  if (!CI->use_empty()) {
    auto Insert = InsertValueInst::Create(UndefValue::get(CI->getType()), Add,
                                          0, CI->getName() + ".insertadd", CI);
    Insert->setDebugLoc(DL);
    Insert = InsertValueInst::Create(Insert, Cmp, 1,
                                     CI->getName() + ".insertcmp", CI);
    Insert->setDebugLoc(DL);
    // ... and use it to replace the original intrinsic.
    CI->replaceAllUsesWith(Insert);
  }
  ToErase.push_back(CI);
  return true;
}

// common subroutine for sub+sat and add+sat: builds uadd with sat
static CallInst *buildUAddWithSat(CallInst *CI, Value *Arg0, Value *Arg1,
                                  Instruction *InsertPoint, bool IsSignedSrc) {
  IGC_ASSERT(CI);
  const DebugLoc &DL = CI->getDebugLoc();
  Module *M = CI->getModule();
  IRBuilder<> Builder(InsertPoint);
  Type *ArgTypes[] = {CI->getType(), Arg0->getType()};
  Value *Args[] = {Arg0, Arg1};
  auto Fn = GenXIntrinsic::getGenXDeclaration(
      M,
      IsSignedSrc ? GenXIntrinsic::genx_usadd_sat
                  : GenXIntrinsic::genx_uuadd_sat,
      ArgTypes);
  auto *UUAddInst = Builder.CreateCall(Fn, Args, CI->getName());
  UUAddInst->setDebugLoc(DL);
  return UUAddInst;
}

// llvm.uadd.sat
// here we are lucky to reuse genx intrinsic
bool GenXLowering::lowerUAddWithSat(CallInst *CI) {
  IGC_ASSERT(CI);
  Value *Arg0 = CI->getArgOperand(0);
  Value *Arg1 = CI->getArgOperand(1);
  auto *UUAddInst = buildUAddWithSat(CI, Arg0, Arg1, CI, /*IsSignedSrc*/ false);
  CI->replaceAllUsesWith(UUAddInst);
  ToErase.push_back(CI);
  return true;
}

// llvm.usub.sat i.e. sat(a - b) we can treat as sat(a + (-b))
// i.e. we are building -b and then uadd with saturation
bool GenXLowering::lowerUSubWithSat(CallInst *CI) {
  IGC_ASSERT(CI);
  Value *Arg0 = CI->getArgOperand(0);
  Value *Arg1 = IRBuilder<>(CI).CreateNeg(CI->getArgOperand(1), CI->getName());
  auto *UUAddInst = buildUAddWithSat(CI, Arg0, Arg1, CI, /*IsSignedSrc*/ true);
  CI->replaceAllUsesWith(UUAddInst);
  ToErase.push_back(CI);
  return true;
}

bool GenXLowering::lowerTrap(CallInst *CI) {
  Module *M = CI->getModule();
  IRBuilder<> Builder(CI);
  auto &Ctx = CI->getContext();
  unsigned EMWidth = 32;
  Type *ArgTypes[] = {
      IGCLLVM::FixedVectorType::get(Type::getInt1Ty(Ctx), EMWidth),
      IGCLLVM::FixedVectorType::get(Type::getInt16Ty(Ctx), EMWidth)};
  auto Fn = GenXIntrinsic::getGenXDeclaration(M,
    GenXIntrinsic::genx_raw_send_noresult, ArgTypes);
  SmallVector<Value *, 8> Args;
  // send
  Args.push_back(ConstantInt::get(Type::getInt32Ty(Ctx), 0));
  // predicate all lanes
  Args.push_back(ConstantVector::getSplat(IGCLLVM::getElementCount(EMWidth),
                                          ConstantInt::getTrue(Ctx)));
  // EOT
  Args.push_back(ConstantInt::get(Type::getInt32Ty(Ctx), 0x27));
  Args.push_back(ConstantInt::get(Type::getInt32Ty(Ctx), 0x02000010));
  Args.push_back(
      ConstantVector::getSplat(IGCLLVM::getElementCount(EMWidth),
                               Constant::getNullValue(Type::getInt16Ty(Ctx))));
  Builder.CreateCall(Fn, Args);
  ToErase.push_back(CI);

  return true;
}

bool GenXLowering::lowerCtpop(CallInst *CI) {
  Module *M = CI->getModule();
  IRBuilder<> Builder(CI);
  Builder.SetCurrentDebugLocation(CI->getDebugLoc());

  Type *Int32Ty = IntegerType::getInt32Ty(CI->getContext());
  Type *RetTy = nullptr;
  if (auto *VT = dyn_cast<IGCLLVM::FixedVectorType>(CI->getType()))
    RetTy = IGCLLVM::FixedVectorType::get(Int32Ty, VT->getNumElements());
  else
    RetTy = Int32Ty;

  auto *CBitDecl = GenXIntrinsic::getGenXDeclaration(
      M, GenXIntrinsic::genx_cbit, {RetTy, CI->getType()});
  Value *CBitInst =
      Builder.CreateCall(CBitDecl, CI->getOperand(0), CI->getName());

  CBitInst = Builder.CreateZExtOrTrunc(CBitInst, CI->getType());
  CI->replaceAllUsesWith(CBitInst);
  ToErase.push_back(CI);

  return true;
}

// Lower cmp instructions that GenX cannot deal with.
bool GenXLowering::lowerFCmpInst(FCmpInst *Inst) {
  IRBuilder<> Builder(Inst);
  Builder.SetCurrentDebugLocation(Inst->getDebugLoc());
  Value *Ops[] = {Inst->getOperand(0), Inst->getOperand(1)};

  switch (Inst->getPredicate()) {
  default:
    return lowerUnorderedFCmpInst(Inst);
  case CmpInst::FCMP_ORD: // True if ordered (no nans)
  {
    // %c = fcmp ord %a %b
    // =>
    // %1 = fcmp oeq %a %a
    // %2 = fcmp oeq %b %b
    // %c = and %1 %2
    Value *LHS = Builder.CreateFCmpOEQ(Ops[0], Ops[0]);
    Value *RHS = Builder.CreateFCmpOEQ(Ops[1], Ops[1]);
    Value *New = Builder.CreateAnd(LHS, RHS);
    Inst->replaceAllUsesWith(New);
    ToErase.push_back(Inst);
    return true;
  }
  case CmpInst::FCMP_UNO: // True if unordered: isnan(X) | isnan(Y)
  {
    // %c = fcmp uno %a %b
    // =>
    // %1 = fcmp une %a %a
    // %2 = fcmp une %b %b
    // %c = or %1 %2
    Value *LHS = Builder.CreateFCmpUNE(Ops[0], Ops[0]);
    Value *RHS = Builder.CreateFCmpUNE(Ops[1], Ops[1]);
    Value *New = Builder.CreateOr(LHS, RHS);
    Inst->replaceAllUsesWith(New);
    ToErase.push_back(Inst);
    return true;
  }
  case CmpInst::FCMP_UEQ: // UEQ cannot be replaced with NOT ONE because we do
                          // not have ONE
  {
    // %c = fcmp ueq %a %b
    // =>
    // %1 = fcmp olt %a %b
    // %2 = fcmp ogt %a %b
    // %3 = or %1 %2
    // %c = not %3
    Value *LHS = Builder.CreateFCmpOLT(Ops[0], Ops[1]);
    Value *RHS = Builder.CreateFCmpOGT(Ops[0], Ops[1]);
    Value *Or = Builder.CreateOr(LHS, RHS);
    Value *New = Builder.CreateNot(Or);
    Inst->replaceAllUsesWith(New);
    ToErase.push_back(Inst);
    return true;
  }
  case CmpInst::FCMP_ONE: // NE is unordered
  {
    // %c = fcmp one %a %b
    // =>
    // %1 = fcmp olt %a %b
    // %2 = fcmp ogt %a %b
    // %c = or %1 %2
    Value *LHS = Builder.CreateFCmpOLT(Ops[0], Ops[1]);
    Value *RHS = Builder.CreateFCmpOGT(Ops[0], Ops[1]);
    Value *New = Builder.CreateOr(LHS, RHS);
    Inst->replaceAllUsesWith(New);
    ToErase.push_back(Inst);
    return true;
  }
  }

  return false;
}

// FCmp with NE is the only one supported unordered cmp inst. All the rest must
// be lowered.
bool GenXLowering::lowerUnorderedFCmpInst(FCmpInst *Inst) {
  CmpInst::Predicate Pred = Inst->getPredicate();
  if (CmpInst::isOrdered(Pred))
    return false;

  // We support UNE.
  if (Pred == CmpInst::FCMP_UNE)
    return false;

  // For UNO and UEQ we have replacement in lowerFCmpInst.
  IGC_ASSERT(Pred != CmpInst::FCMP_UNO);
  IGC_ASSERT(Pred != CmpInst::FCMP_UEQ);

  CmpInst *InverseFCmp = CmpInst::Create(
      Inst->getOpcode(), CmpInst::getInversePredicate(Pred),
      Inst->getOperand(0), Inst->getOperand(1),
      Inst->getName() + ".ordered.inversed", Inst->getNextNode());
  Instruction *Result = BinaryOperator::CreateNot(
      InverseFCmp, InverseFCmp->getName() + ".not", InverseFCmp->getNextNode());
  InverseFCmp->setDebugLoc(Inst->getDebugLoc());
  Result->setDebugLoc(Inst->getDebugLoc());
  Inst->replaceAllUsesWith(Result);
  ToErase.push_back(Inst);

  return true;
}

static GenXIntrinsic::ID convertMinMaxIntrinsic(unsigned ID) {
  switch (ID) {
  case Intrinsic::minnum:
    return GenXIntrinsic::genx_fmin;
  case Intrinsic::maxnum:
    return GenXIntrinsic::genx_fmax;
  };
  IGC_ASSERT_MESSAGE(0, "unknown min/max intrinsic");
  return GenXIntrinsic::not_any_intrinsic;
}

// Lower llvm minnum/maxnum to genx fmin/fmax.
bool GenXLowering::lowerMinMax(CallInst *CI, unsigned IntrinsicID) {
  auto *ResTy = CI->getType();
  auto *MinMaxDecl = GenXIntrinsic::getGenXDeclaration(
      CI->getModule(), convertMinMaxIntrinsic(IntrinsicID), {ResTy, ResTy});
  Value *Result = IRBuilder<>(CI).CreateCall(
      MinMaxDecl, {CI->getArgOperand(0), CI->getArgOperand(1)}, CI->getName());
  CI->replaceAllUsesWith(Result);
  ToErase.push_back(CI);
  return true;
}

// Lower llvm.sqrt to genx.ieee.sqrt equivalent.
bool GenXLowering::lowerSqrt(CallInst *CI) {
  IGC_ASSERT_MESSAGE(vc::getAnyIntrinsicID(CI) == Intrinsic::sqrt,
    "llvm.sqrt expected");
  auto *ResTy = CI->getType();

  GenXIntrinsic::ID SqrtID = (CI->getType()->getScalarType()->isDoubleTy() ||
                              !CI->getFastMathFlags().isFast())
                                 ? GenXIntrinsic::genx_ieee_sqrt
                                 : GenXIntrinsic::genx_sqrt;

  auto *SqrtDecl =
      GenXIntrinsic::getGenXDeclaration(CI->getModule(), SqrtID, {ResTy});
  Value *Result = IRBuilder<>(CI).CreateCall(SqrtDecl, {CI->getArgOperand(0)},
                                             CI->getName());
  CI->replaceAllUsesWith(Result);
  ToErase.push_back(CI);
  return true;
}

// Some GenX-specific mul intrinsics require special lowering
bool GenXLowering::lowerGenXMul(CallInst *CI, unsigned IID) {
  // DxD -> Q mul should be handled specially if MulDDQ is NOT desired
  if (ST->useMulDDQ())
    return false;
  if (IID == GenXIntrinsic::genx_sumul || IID == GenXIntrinsic::genx_usmul)
    return false;
  IGC_ASSERT(IID == GenXIntrinsic::genx_uumul ||
             IID == GenXIntrinsic::genx_ssmul);

  auto ScalarType = [](Value *V) { return V->getType()->getScalarType(); };
  if (!ScalarType(CI)->isIntegerTy(64))
    return false;
  Value *LH = CI->getOperand(0);
  Value *RH = CI->getOperand(1);
  if (!ScalarType(LH)->isIntegerTy(32) || !ScalarType(RH)->isIntegerTy(32))
    return false;

  auto *M = CI->getModule();
  Type *tys[2] = {LH->getType(), RH->getType()};

  Function *MulHFunc =
      (IID == GenXIntrinsic::genx_uumul)
          ? GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_umulh, tys)
          : GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_smulh,
                                              tys);

  IRBuilder<> B(CI);
  Value *MulLo = B.CreateMul(LH, RH, CI->getName() + ".lo.");
  Value *MulHi = B.CreateCall(MulHFunc, {LH, RH}, CI->getName() + ".hi.");

  IGC_ASSERT(MulLo->getType() == MulHi->getType());
  if (!MulLo->getType()->isVectorTy()) {
    auto *VTy = IGCLLVM::FixedVectorType::get(B.getInt32Ty(), 1);
    MulLo = B.CreateBitCast(MulLo, VTy, MulLo->getName() + "v.");
    MulHi = B.CreateBitCast(MulHi, VTy, MulHi->getName() + "v.");
  }

  IVSplitter SplitBuilder(*CI);
  Value *Result = SplitBuilder.combineLoHiSplit(
      {MulLo, MulHi}, CI->getName() + ".", CI->getType()->isIntegerTy());
  CI->replaceAllUsesWith(Result);
  ToErase.push_back(CI);
  return true;
}
// Lower integer mul with saturation since VISA support mul.sat only for float.
bool GenXLowering::lowerGenXMulSat(CallInst *CI, unsigned IntrinsicID) {
  auto IsSignedMulSat = [](unsigned ID) -> std::pair<bool, bool> {
    switch (ID) {
    case GenXIntrinsic::genx_uumul_sat:
      return {false, false};
    case GenXIntrinsic::genx_usmul_sat:
      return {false, true};
    case GenXIntrinsic::genx_sumul_sat:
      return {true, false};
    case GenXIntrinsic::genx_ssmul_sat:
      return {true, true};
    default:
      llvm_unreachable("Inst should be *mul.sat intrinsic");
    }
  };

  auto GetTruncSatIntrinsicId = [](bool ResSigned, bool OpSigned) {
    return ResSigned ? (OpSigned ? GenXIntrinsic::genx_sstrunc_sat
                                 : GenXIntrinsic::genx_sutrunc_sat)
                     : (OpSigned ? GenXIntrinsic::genx_ustrunc_sat
                                 : GenXIntrinsic::genx_uutrunc_sat);
  };

  auto [IsSignedRes, IsSignedOps] = IsSignedMulSat(IntrinsicID);

  Type *ResType = CI->getType();
  IGC_ASSERT(ResType->isIntOrIntVectorTy());

  IGC_ASSERT(CI->getOperand(0)->getType() == CI->getOperand(1)->getType());
  Type *OpType = CI->getOperand(0)->getType();
  IGC_ASSERT(OpType->isIntOrIntVectorTy());

  // Create type that doesn't overflow in multiplication.
  unsigned OpTypeWidth =
      cast<IntegerType>(OpType->getScalarType())->getBitWidth();
  IGC_ASSERT_MESSAGE(OpTypeWidth != 64, "i64 types are not supported");
  Type *MulType = IntegerType::get(OpType->getContext(), 2 * OpTypeWidth);
  if (auto *OpVTy = dyn_cast<IGCLLVM::FixedVectorType>(OpType))
    MulType = IGCLLVM::FixedVectorType::get(MulType, OpVTy->getNumElements());

  IRBuilder<> B(CI);
  auto ExtendMulOperand = [&, IsSignedOps=IsSignedOps](Value *Val) {
    return IsSignedOps ? B.CreateSExt(Val, MulType, Val->getName() + ".sext")
                       : B.CreateZExt(Val, MulType, Val->getName() + ".zext");
  };
  Value *FstMulOpnd = ExtendMulOperand(CI->getOperand(0)),
        *SndMulOpnd = ExtendMulOperand(CI->getOperand(1));
  Value *Mul = B.CreateMul(FstMulOpnd, SndMulOpnd, CI->getName());
  Function *TruncSatFunc = GenXIntrinsic::getGenXDeclaration(
      CI->getModule(), GetTruncSatIntrinsicId(IsSignedRes, IsSignedOps),
      {ResType, MulType});
  Value *Result = B.CreateCall(TruncSatFunc, {Mul}, CI->getName() + ".sat");
  CI->replaceAllUsesWith(Result);
  ToErase.push_back(CI);
  return true;
}

// Generic lowering for mul64
// GenX backend does not support 64-bit multiplication, so we try
// To lower it to a sequence of 32-bit ops
bool GenXLowering::lowerMul64(Instruction *Inst) {

  IVSplitter SplitBuilder(*Inst);
  if (!SplitBuilder.IsI64Operation())
    return false;

  IRBuilder<> Builder(Inst);

  auto Src0 = SplitBuilder.splitOperandLoHi(0);
  auto Src1 = SplitBuilder.splitOperandLoHi(1);

  // create muls and adds
  auto *ResL = Builder.CreateMul(Src0.Lo, Src1.Lo);
  // create the mulh intrinsic to the get the carry-part
  Type *tys[2] = {ResL->getType(), Src0.Lo->getType()};
  // build argument list
  SmallVector<llvm::Value *, 2> args{Src0.Lo, Src1.Lo};
  auto *M = Inst->getModule();
  Function *IntrinFunc =
      GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_umulh, tys);

  auto *Cari = Builder.CreateCall(IntrinFunc, args, ".cari");
  auto *Temp0 = Builder.CreateMul(Src0.Lo, Src1.Hi);
  auto *Temp1 = Builder.CreateAdd(Cari, Temp0);
  auto *Temp2 = Builder.CreateMul(Src0.Hi, Src1.Lo);
  auto *ResH = Builder.CreateAdd(Temp2, Temp1);

  // create the bitcast to the destination-type
  auto *Replace = SplitBuilder.combineLoHiSplit({ ResL, ResH }, "mul64",
                                                Inst->getType()->isIntegerTy());
  Inst->replaceAllUsesWith(Replace);
  ToErase.push_back(Inst);
  return true;
}

struct LoHiRes {
  Value *Lo;
  Value *Hi;
};

// Result of madw is a single vector with width == 2GRF/sizeof(i32),
// first GRF contains hi part of result, second -- lo part.
// This function extract hi and lo parts and return those as struct {Lo, Hi}.
static LoHiRes getLoHiFromMadw(CallInst *Madw, IRBuilder<> &IRB) {
  IGC_ASSERT(Madw);
  IGC_ASSERT_MESSAGE(Madw->getType()->isVectorTy(),
                     "result of madw must be i32 vector type");

  unsigned OpWidth =
      cast<IGCLLVM::FixedVectorType>(Madw->getArgOperand(0)->getType())
          ->getNumElements();
  unsigned ResWidth =
      cast<IGCLLVM::FixedVectorType>(Madw->getType())->getNumElements();

  Region AddrHi{Madw};
  AddrHi.getSubregion(ResWidth / 2, OpWidth);
  auto *RdrHi = AddrHi.createRdRegion(Madw, Madw->getName() + ".hi",
                                      &*IRB.GetInsertPoint(),
                                      IRB.getCurrentDebugLocation());

  Region AddrLo{Madw};
  AddrLo.getSubregion(0, OpWidth);
  auto *RdrLo = AddrLo.createRdRegion(Madw, Madw->getName() + ".lo",
                                      &*IRB.GetInsertPoint(),
                                      IRB.getCurrentDebugLocation());

  return {RdrLo, RdrHi};
}

CallInst *buildLegalMadw(ArrayRef<Value *> Args, bool Signed, const Twine &Name,
                         const GenXSubtarget &ST, IRBuilder<> &IRB) {
  IGC_ASSERT_MESSAGE(Args.size() == 3, "madw must have 3 arguments");
  IGC_ASSERT_MESSAGE(Args[0]->getType()->isVectorTy() &&
                         Args[0]->getType()->isIntOrIntVectorTy(),
                     "madw support only i32 vector types");

  unsigned TargetWidth = ST.getGRFByteSize() / genx::DWordBytes;
  auto *VecTy = cast<IGCLLVM::FixedVectorType>(Args[0]->getType());
  IGC_ASSERT(std::all_of(Args.begin(), Args.end(), [VecTy](auto Arg) {
    return Arg->getType() == VecTy;
  }));
  unsigned OpWidth = VecTy->getNumElements();
  IGC_ASSERT_MESSAGE(OpWidth <= TargetWidth && isPowerOf2_32(OpWidth),
                     "attempt to build madw with incorrect argument width");

  unsigned ResWidth = 2 * TargetWidth;
  Module *M = IRB.GetInsertPoint()->getModule();

  auto *RetTy = IGCLLVM::FixedVectorType::get(VecTy->getScalarType(), ResWidth);
  Type *Tys[2] = {RetTy, VecTy};
  Function *IntrinFunc = GenXIntrinsic::getGenXDeclaration(
      M, Signed ? GenXIntrinsic::genx_smadw : GenXIntrinsic::genx_umadw, Tys);
  return IRB.CreateCall(IntrinFunc, Args, Name);
}

// Join one result of madw to another.
// Insert the JoinedRes.(Lo|Hi) vector into the Res.(Lo|Hi) vector starting
// from the StartIdx.
static LoHiRes joinLoHiResults(const LoHiRes &Res, const LoHiRes &JoinedRes,
                               unsigned StartIdx, IRBuilder<> &IRB) {
  IGC_ASSERT_MESSAGE(JoinedRes.Hi->getType() == JoinedRes.Lo->getType(),
                     "Lo and Hi parts of madw result must have the same type");
  IGC_ASSERT_MESSAGE(Res.Hi->getType() == Res.Lo->getType(),
                     "Lo and Hi parts of madw result must have the same type");
  IGC_ASSERT_MESSAGE(JoinedRes.Hi->getType()->isVectorTy(),
                     "result of madw must be i32 vector");
  IGC_ASSERT_MESSAGE(Res.Hi->getType()->isVectorTy(),
                     "result of madw must be i32 vector");

  auto *VTy = cast<IGCLLVM::FixedVectorType>(JoinedRes.Hi->getType());
  unsigned SplitWidth = VTy->getNumElements();

  Region JoinSplitRes{Res.Hi};
  JoinSplitRes.getSubregion(StartIdx, SplitWidth);

  Value *Hi = JoinSplitRes.createWrRegion(
      Res.Hi, JoinedRes.Hi,
      JoinedRes.Hi->getName() + ".join.hi" + Twine(StartIdx),
      &*IRB.GetInsertPoint(), IRB.getCurrentDebugLocation());

  Value *Lo = JoinSplitRes.createWrRegion(
      Res.Lo, JoinedRes.Lo,
      JoinedRes.Lo->getName() + ".join.lo" + Twine(StartIdx),
      &*IRB.GetInsertPoint(), IRB.getCurrentDebugLocation());

  return {Lo, Hi};
}

Value *getSplitOperand(Value *Opnd, unsigned StartIdx, unsigned Size,
                       IRBuilder<> &IRB) {
  if (auto *C = dyn_cast<Constant>(Opnd))
    return getConstantSubvector(C, StartIdx, Size);
  auto *OpTy = Opnd->getType();
  Region R{OpTy};
  R.getSubregion(StartIdx, Size);
  return R.createRdRegion(Opnd, Opnd->getName() + ".split" + Twine(StartIdx),
                          &*IRB.GetInsertPoint(),
                          IRB.getCurrentDebugLocation());
}

static LoHiRes buildIMadWithMadw(ArrayRef<Value *> Args, bool Signed,
                                 const Twine &Name, const GenXSubtarget &ST,
                                 IRBuilder<> &IRB) {
  IGC_ASSERT_MESSAGE(Args.size() == 3, "madw must have 3 arguments");
  IGC_ASSERT_MESSAGE(Args[0]->getType()->isVectorTy() &&
                         Args[0]->getType()->isIntOrIntVectorTy(),
                     "madw support only i32 vector types");

  auto *OpTy = cast<IGCLLVM::FixedVectorType>(Args[0]->getType());
  IGC_ASSERT(std::all_of(Args.begin(), Args.end(),
                         [OpTy](auto Arg) { return Arg->getType() == OpTy; }));

  unsigned OpWidth = OpTy->getNumElements();
  unsigned TargetWidth = ST.getGRFByteSize() / genx::DWordBytes;

  if (OpWidth <= TargetWidth && isPowerOf2_32(OpWidth)) {
    CallInst *Madw = buildLegalMadw(Args, Signed, Name, ST, IRB);
    return getLoHiFromMadw(Madw, IRB);
  }

  LoHiRes Res = {UndefValue::get(OpTy), UndefValue::get(OpTy)};
  unsigned StartIdx = 0;
  while (StartIdx < OpWidth) {
    unsigned SplitWidth = std::min(
        TargetWidth, static_cast<unsigned>(PowerOf2Floor(OpWidth - StartIdx)));

    std::array<Value *, 3> SplitArgs;
    for (unsigned i = 0; i < Args.size(); ++i)
      SplitArgs[i] = getSplitOperand(Args[i], StartIdx, SplitWidth, IRB);

    CallInst *Madw = buildLegalMadw(SplitArgs, Signed,
                                    Name + ".split" + Twine(StartIdx), ST, IRB);

    auto SplitRes = getLoHiFromMadw(Madw, IRB);
    Res = joinLoHiResults(Res, SplitRes, StartIdx, IRB);

    StartIdx += SplitWidth;
  }

  return Res;
}

static LoHiRes buildIMadWithMulDDQ(ArrayRef<Value *> Args, bool Signed,
                                   const Twine &Name, IRBuilder<> &IRB) {
  IGC_ASSERT_MESSAGE(Args.size() == 3, "imad must have 3 arguments");
  IGC_ASSERT_MESSAGE(Args[0]->getType()->isVectorTy() &&
                         Args[0]->getType()->isIntOrIntVectorTy(32),
                     "i32 vector type expected");

  auto *OpTy = Args[0]->getType();
  IGC_ASSERT(std::all_of(Args.begin(), Args.end(),
                         [OpTy](auto Arg) { return Arg->getType() == OpTy; }));
  unsigned Width = cast<IGCLLVM::FixedVectorType>(OpTy)->getNumElements();
  auto *Int64Ty = Type::getInt64Ty(OpTy->getContext());
  auto *VInt64Ty = IGCLLVM::FixedVectorType::get(Int64Ty, Width);

  Type *Tys[] = {VInt64Ty, OpTy};
  Module *M = IRB.GetInsertPoint()->getModule();
  Function *I64MulFunc =
      Signed
          ? GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_ssmul, Tys)
          : GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_uumul,
                                              Tys);

  Value *Res = IRB.CreateCall(I64MulFunc, {Args[0], Args[1]}, Name + ".mul64");

  // If last argument of imad isn't zeroinitiliazer we should produce add
  auto *C = dyn_cast<Constant>(Args[2]);
  if (!(C && C->isZeroValue())) {
    auto *Ext = CastInst::Create(
        Signed ? Instruction::SExt : Instruction::ZExt, Args[2], VInt64Ty,
        Args[2]->getName() + ".cast", &*IRB.GetInsertPoint());
    Ext->setDebugLoc(IRB.getCurrentDebugLocation());
    Res = IRB.CreateAdd(Res, Ext, Name + ".add64");
  }

  unsigned BaseOpIdx = 0;
  IVSplitter SplitBuilder(*IRB.GetInsertPoint(), &BaseOpIdx);
  auto Split = SplitBuilder.splitValueLoHi(*Res);

  return {Split.Lo, Split.Hi};
}

void replaceUsesOfIMad(CallInst *IMad, const LoHiRes &Res, IRBuilder<> &IRB) {
  bool NeedsInsertValues = false;
  SmallVector<Instruction *, 2> ExtValsToErase;
  for (const auto &U : IMad->uses()) {
    IGC_ASSERT_MESSAGE(isa<Instruction>(U.getUser()),
                       "expected that user is instruction");
    if (auto *ExtVal = dyn_cast<ExtractValueInst>(U.getUser())) {
      unsigned Idx = *ExtVal->idx_begin();
      IGC_ASSERT_MESSAGE(
          ExtVal->getNumIndices() == 1 && (Idx == 0 || Idx == 1),
          "index of extractvalue from imad result must be 0 or 1");
      if (Idx == 0) // Hi part
        ExtVal->replaceAllUsesWith(Res.Hi);
      else // Idx == 1, Lo part
        ExtVal->replaceAllUsesWith(Res.Lo);
      ExtValsToErase.push_back(ExtVal);
    } else {
      NeedsInsertValues = true;
    }
  }

  // Erase extractvalues.
  // This is necessary cos we don't want to replace useless imad users.
  for (auto *ExtVal : ExtValsToErase)
    ExtVal->eraseFromParent();

  if (!NeedsInsertValues)
    return;

  auto *InsertHi =
      IRB.CreateInsertValue(UndefValue::get(IMad->getType()), Res.Hi, 0);
  auto *InsertLo = IRB.CreateInsertValue(InsertHi, Res.Lo, 1);
  IMad->replaceAllUsesWith(InsertLo);
}

// Lower imad to sequence of madw
// After that imad isn't supported
bool GenXLowering::lowerGenXIMad(CallInst *CI, unsigned IntrinsicID) {
  IGC_ASSERT_MESSAGE(IntrinsicID == GenXIntrinsic::genx_simad ||
                         IntrinsicID == GenXIntrinsic::genx_uimad,
                     "expected genx_*imad");

  std::array<Value *, 3> Args = {CI->getArgOperand(0), CI->getArgOperand(1),
                                 CI->getArgOperand(2)};
  auto *OpTy = CI->getArgOperand(0)->getType();
  IGC_ASSERT(std::all_of(Args.begin(), Args.end(),
                         [OpTy](auto Arg) { return Arg->getType() == OpTy; }));

  IRBuilder<> Builder{CI};
  if (!OpTy->isVectorTy()) {
    auto *VecTy = IGCLLVM::FixedVectorType::get(OpTy, 1);
    std::for_each(Args.begin(), Args.end(), [VecTy, &Builder](auto &Arg) {
      Arg = Builder.CreateBitCast(Arg, VecTy, Arg->getName() + ".bitcast");
    });
  }

  bool Signed = (IntrinsicID == GenXIntrinsic::genx_simad);
  LoHiRes Res =
      ST->useMulDDQ()
          ? buildIMadWithMulDDQ(Args, Signed, CI->getName(), Builder)
          : buildIMadWithMadw(Args, Signed, CI->getName(), *ST, Builder);

  if (!OpTy->isVectorTy()) {
    Res.Hi =
        Builder.CreateBitCast(Res.Hi, OpTy, Res.Hi->getName() + ".bitcast");
    Res.Lo =
        Builder.CreateBitCast(Res.Lo, OpTy, Res.Lo->getName() + ".bitcast");
  }

  replaceUsesOfIMad(CI, Res, Builder);

  ToErase.push_back(CI);
  return true;
}

bool GenXLowering::lowerLzd(Instruction *Inst) {
  const unsigned OpIndex = 0;
  IVSplitter SplitBuilder(*Inst, &OpIndex);
  if (!SplitBuilder.IsI64Operation())
    return false;

  IRBuilder<> Builder(Inst);

  auto Src = SplitBuilder.splitOperandLoHi(0);

  auto *VTy32 = cast<VectorType>(Src.Lo->getType());
  IGC_ASSERT(VTy32->getScalarType() == Builder.getInt32Ty());
  auto *Zero = ConstantInt::getNullValue(VTy32);
  auto *K32 = ConstantInt::get(VTy32, 32);

  auto *LzdF = GenXIntrinsic::getAnyDeclaration(
      Inst->getModule(), GenXIntrinsic::genx_lzd, {Src.Lo->getType()});

  // Lzd64 is lowered as:
  // LoLzd  = lzd(Src.Lo)
  // HiLzd  = lzd(Src.Hi)
  // Result = (Src.Hi == 0) ? (LoLzd + 32) : HiLzd
  auto *VlzdLo = Builder.CreateCall(LzdF, Src.Lo, "lower.lzd64.lo.");
  auto *VlzdHi = Builder.CreateCall(LzdF, Src.Hi, "lower.lzd64.hi.");
  auto *FlagHiZero = Builder.CreateICmpEQ(Src.Hi, Zero, "lower.lzd64.hicmp.");
  auto *LoPathResult = Builder.CreateAdd(VlzdLo, K32, "lower.lzd64.lores.");
  auto *Result =
      Builder.CreateSelect(FlagHiZero, LoPathResult, VlzdHi, "lower.lzd32.");
  // TODO: allow lzd to have type of destination != operand type
  if (Inst->getType()->getScalarType()->isIntegerTy(64)) {
    Result = Builder.CreateZExt(Result,
                                VectorType::getExtendedElementVectorType(VTy32),
                                "lower.lzd64.");
  }
  auto *CastedResult =
      scalarizeOrVectorizeIfNeeded(cast<Instruction>(Result), Inst);
  if (CastedResult)
    Result = CastedResult;
  Inst->replaceAllUsesWith(Result);
  Result->takeName(Inst);
  ToErase.push_back(Inst);
  return true;
}

#define SYCL_SLM_AS  3
bool GenXLowering::lowerLLVMMaskedLoad(CallInst* CallOp) {
  auto PtrV = CallOp->getArgOperand(0);
  IGC_ASSERT(PtrV->getType()->isPointerTy());
  auto AS = cast<PointerType>(PtrV->getType())->getAddressSpace();
  IGC_ASSERT_MESSAGE(AS != SYCL_SLM_AS, "do not expect masked load from SLM");
  auto DTy = CallOp->getType();
  // convert to unaligned-block-load then select
  std::string IntrName =
      std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) +
      "svm.block.ld.unaligned";
  auto ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
  Function* NewFDecl = GenXIntrinsic::getGenXDeclaration(
      CallOp->getModule(), ID, { DTy, PtrV->getType() });
  auto VecDT = CallInst::Create(NewFDecl, { PtrV },
      "svm.block.ld.unaligned", CallOp);
  VecDT->setDebugLoc(CallOp->getDebugLoc());
  IRBuilder<> Builder(CallOp);
  auto RepI = Builder.CreateSelect(CallOp->getArgOperand(2), VecDT,
      CallOp->getArgOperand(3), CallOp->getName());
  CallOp->replaceAllUsesWith(RepI);
  ToErase.push_back(CallOp);
  return true;
}

bool GenXLowering::lowerLLVMMaskedStore(CallInst* CallOp) {
  auto PtrV = CallOp->getArgOperand(1);
  IGC_ASSERT(PtrV->getType()->isPointerTy());
  auto AS = cast<PointerType>(PtrV->getType())->getAddressSpace();
  IGC_ASSERT_MESSAGE(AS != SYCL_SLM_AS, "do not expected masked store to SLM");
  auto DTV = CallOp->getArgOperand(0);
  auto DL = CallOp->getDebugLoc();
  // convert to unaligned-block-load then select
  // then block-store
  std::string IntrName =
      std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) +
      "svm.block.ld.unaligned";
  auto ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
  Function* NewFDecl = GenXIntrinsic::getGenXDeclaration(
      CallOp->getModule(), ID, { DTV->getType(), PtrV->getType() });
  auto VecDT = CallInst::Create(NewFDecl, { PtrV },
      "svm.block.ld.unaligned", CallOp);
  VecDT->setDebugLoc(DL);
  IRBuilder<> Builder(CallOp);
  auto SelI = Builder.CreateSelect(CallOp->getArgOperand(3), DTV, VecDT);

  IntrName = std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) +
       "svm.block.st";
  ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
  NewFDecl = GenXIntrinsic::getGenXDeclaration(
      CallOp->getModule(), ID, { PtrV->getType(), DTV->getType() });
  auto NewInst = CallInst::Create(
      NewFDecl, { PtrV, SelI },
      NewFDecl->getReturnType()->isVoidTy() ? "" : "svm.block.st",
      CallOp);
  NewInst->setDebugLoc(DL);
  ToErase.push_back(CallOp);
  return true;
}

bool GenXLowering::lowerLLVMMaskedGather(CallInst* CallOp) {
  auto PtrV = CallOp->getArgOperand(0);
  auto MaskV = CallOp->getArgOperand(2);
  auto OldV = CallOp->getArgOperand(3);
  auto DTy = CallOp->getType();
  IGC_ASSERT(PtrV->getType()->isVectorTy());
  IGC_ASSERT(DTy->isVectorTy());
  auto PtrETy = cast<VectorType>(PtrV->getType())->getElementType();
  IGC_ASSERT(PtrETy->isPointerTy());
  auto AS = cast<PointerType>(PtrETy)->getAddressSpace();
  IGC_ASSERT_MESSAGE(AS != SYCL_SLM_AS, "do not expect masked gather from SLM");
  auto EltTy = cast<VectorType>(DTy)->getElementType();
  auto NumElts = cast<IGCLLVM::FixedVectorType>(DTy)->getNumElements();
  auto EltBytes = EltTy->getPrimitiveSizeInBits() / 8;
  int EltsPerLane = 1;
  // for short or byte type, load-data is 4 bytes per lane
  if (EltBytes < 4) {
    EltTy = Type::getInt8Ty(EltTy->getContext());
    EltsPerLane = 4;
  }
  auto VDTy = IGCLLVM::FixedVectorType::get(EltTy, NumElts * EltsPerLane);
  auto CIntTy = Type::getInt32Ty(EltTy->getContext());
  int nb = (EltBytes == 2) ? 1 : 0; // 0/1/2/3 means 1/2/4/8 blks
  auto NumBlksC = ConstantInt::get(CIntTy, nb);
  Value* RepI = nullptr;
  IRBuilder<> Builder(CallOp);
  // need write-region for old-data in byte or short type
  if (EltBytes < 4) {
    if (isa<UndefValue>(OldV))
      OldV = UndefValue::get(VDTy);
    else {
      // zext to i32 type
      auto VInt32 = IGCLLVM::FixedVectorType::get(CIntTy, NumElts);
      auto ExtI = Builder.CreateZExt(OldV, VInt32, OldV->getName());
      // bitcast to 4xi8
      OldV = Builder.CreateBitCast(ExtI, VDTy, ExtI->getName());
    }
  }
  std::string IntrName =
      std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) + "svm.gather";
  auto ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
  Function* NewFDecl = GenXIntrinsic::getGenXDeclaration(
      CallOp->getModule(), ID, { VDTy, MaskV->getType(), PtrV->getType() });
  auto NewInst = CallInst::Create(NewFDecl, { MaskV, NumBlksC, PtrV, OldV },
      CallOp->getName(), CallOp);
  NewInst->setDebugLoc(CallOp->getDebugLoc());
  RepI = NewInst;
  if (EltBytes < 4) {
    // bitcast to i32 type
    auto VInt32 = IGCLLVM::FixedVectorType::get(CIntTy, NumElts);
    auto CastI = Builder.CreateBitCast(RepI, VInt32, RepI->getName());
    // trunc to the dst type
    RepI = Builder.CreateTrunc(CastI, DTy, CastI->getName());
  }
  CallOp->replaceAllUsesWith(RepI);
  ToErase.push_back(CallOp);
  return true;
}

bool GenXLowering::lowerLLVMMaskedScatter(CallInst *CallOp) {
  auto DataV = CallOp->getArgOperand(0);
  auto PtrV = CallOp->getArgOperand(1);
  auto MaskV = CallOp->getArgOperand(3);
  auto DTy = DataV->getType();
  IGC_ASSERT(PtrV->getType()->isVectorTy());
  IGC_ASSERT(DTy->isVectorTy());
  auto PtrETy = cast<VectorType>(PtrV->getType())->getElementType();
  IGC_ASSERT(PtrETy->isPointerTy());
  auto AS = cast<PointerType>(PtrETy)->getAddressSpace();
  IGC_ASSERT_MESSAGE(AS != SYCL_SLM_AS, "do not expect masked scatter to SLM");
  auto EltTy = cast<VectorType>(DTy)->getElementType();
  auto NumElts = cast<IGCLLVM::FixedVectorType>(DTy)->getNumElements();
  auto EltBytes = EltTy->getPrimitiveSizeInBits() / 8;
  int EltsPerLane = 1;
  // for short or byte type, store-data is 4 bytes per lane
  if (EltBytes < 4) {
    EltTy = Type::getInt8Ty(EltTy->getContext());
    EltsPerLane = 4;
  }
  auto VDTy = IGCLLVM::FixedVectorType::get(EltTy, NumElts * EltsPerLane);
  auto CIntTy = Type::getInt32Ty(EltTy->getContext());
  int nb = (EltBytes == 2) ? 1 : 0; // 0/1/2/3 means 1/2/4/8 blks
  auto NumBlksC = ConstantInt::get(CIntTy, nb);
  // need write-region for old-data in byte or short type
  IRBuilder<> Builder(CallOp);
  if (EltBytes < 4) {
    // zext to i32 type
    auto VInt32 = IGCLLVM::FixedVectorType::get(CIntTy, NumElts);
    auto ExtI = Builder.CreateZExt(DataV, VInt32, DataV->getName());
    // bitcast to 4xi8
    DataV = Builder.CreateBitCast(ExtI, VDTy, ExtI->getName());
  }
  std::string IntrName =
      std::string(GenXIntrinsic::getGenXIntrinsicPrefix()) + "svm.scatter";
  auto ID = GenXIntrinsic::lookupGenXIntrinsicID(IntrName);
  Function* NewFDecl = GenXIntrinsic::getGenXDeclaration(
      CallOp->getModule(), ID, { MaskV->getType(), PtrV->getType(), VDTy });
  auto NewInst = CallInst::Create(NewFDecl, { MaskV, NumBlksC, PtrV, DataV },
      CallOp->getName(), CallOp);
  NewInst->setDebugLoc(CallOp->getDebugLoc());
  ToErase.push_back(CallOp);
  return true;
}

// %1 = call i8 @llvm.bitreverse.i8(i8 %in)
// to
// %1.zext = zext i8 %in to i32
// %halfres.bigger = call i32 @genx.bfrev.i32(i32 %1.zext)
// // calculated val = 32 - 8
// %res.bigger = lshr i32 %halfres.bigger, val
// %res = trunc i32 %res.bigger to i8
bool GenXLowering::lowerBitreverse(CallInst *CI) {
  IGC_ASSERT(CI);
  LLVMContext &Ctx = CI->getContext();

  Value *ValueToBitReverse = CI->getOperand(0);
  auto IDBfrev = GenXIntrinsic::genx_bfrev;
  Type *OriginalType = ValueToBitReverse->getType();
  Type *BfrevType =
      isa<IGCLLVM::FixedVectorType>(OriginalType)
          ? cast<Type>(IGCLLVM::FixedVectorType::get(
                Type::getInt32Ty(Ctx),
                cast<IGCLLVM::FixedVectorType>(OriginalType)->getNumElements()))
          : Type::getInt32Ty(Ctx);

  auto *DeclGenXBfReverse =
      GenXIntrinsic::getGenXDeclaration(CI->getModule(), IDBfrev, {BfrevType});
  auto OriginalElementBitSize = OriginalType->getScalarSizeInBits();
  int ShiftSize = 32 - OriginalElementBitSize;
  if (ShiftSize < 0) {
    DiagnosticInfoLowering Err(
        CI,
        "currently llvm.bitreverse with bitsize bigger than 32 is not supported",
        DS_Error);
    CI->getContext().diagnose(Err);
  }
  Value *ShiftSizeVal = ConstantInt::get(BfrevType, ShiftSize);
  IRBuilder<> Builder(CI);
  Value *Zext = Builder.CreateZExt(ValueToBitReverse, BfrevType);
  Value *WiderRes = Builder.CreateCall(DeclGenXBfReverse->getFunctionType(),
                                       DeclGenXBfReverse, {Zext}, "bfRev");
  if (ShiftSize > 0)
    WiderRes = Builder.CreateLShr(WiderRes, ShiftSizeVal, "lshl");
  Value *Res = Builder.CreateTrunc(WiderRes, OriginalType);

  CI->replaceAllUsesWith(Res);
  ToErase.push_back(CI);

  return true;
}

bool GenXLowering::lowerFunnelShift(CallInst *CI, unsigned IntrinsicID) {
  IGC_ASSERT(CI);
  unsigned BitWidth = CI->getType()->getScalarSizeInBits();
  IGC_ASSERT(isPowerOf2_32(BitWidth) && BitWidth <= 64);
  Value *Fst = CI->getOperand(0), *Snd = CI->getOperand(1);
  Value *ShiftAmnt = CI->getOperand(2);
  IRBuilder<> Builder(CI);
  // If Fst == Snd, funnel shift is equivalent to rotate. Lower to appropriate
  // intrinsic on supported platforms (>= ICLLP)
  // 64-bit rotates are supported only on PVC+. On unsupported platforms lower
  // it as generic funnel shift, so it will be emulated later.
  if (((ST->hasBitRotate() && BitWidth != 64) || ST->has64BitRotate()) &&
      Fst == Snd) {
    auto RotateIntrinsicID = IntrinsicID == Intrinsic::fshl
                                 ? GenXIntrinsic::genx_rol
                                 : GenXIntrinsic::genx_ror;
    auto *Decl = GenXIntrinsic::getGenXDeclaration(
        CI->getModule(), RotateIntrinsicID, {CI->getType(), CI->getType()});
    Value *Rotate = Builder.CreateCall(Decl, {Fst, ShiftAmnt}, "rotate");
    CI->replaceAllUsesWith(Rotate);
    ToErase.push_back(CI);
    return true;
  }
  // Otherwise, emulate it with next sequence:
  // ShiftAmnt &= (BitWidth - 1)
  // For fshl: Res = (Fst << ShiftAmnt) | (Snd >> (BitWidth - ShiftAmnt))
  // For fshr: Res = (Fst << (BitWidth - ShiftAmnt)) | (Snd >> ShiftAmnt)
  ShiftAmnt = Builder.CreateAnd(ShiftAmnt, BitWidth - 1, "shiftamnt");
  Constant *BitWidthConstant =
      Constant::getIntegerValue(CI->getType(), APInt(BitWidth, BitWidth));
  Value *ComplementShiftAmnt =
      Builder.CreateSub(BitWidthConstant, ShiftAmnt, "complementshiftamnt");
  Fst = Builder.CreateShl(
      Fst, IntrinsicID == Intrinsic::fshl ? ShiftAmnt : ComplementShiftAmnt,
      "fstpart");
  Snd = Builder.CreateLShr(
      Snd, IntrinsicID == Intrinsic::fshl ? ComplementShiftAmnt : ShiftAmnt,
      "sndpart");
  Value *Res = Builder.CreateOr(Fst, Snd, "funnelshift");
  CI->replaceAllUsesWith(Res);
  ToErase.push_back(CI);
  return true;
}

bool GenXLowering::lowerFMulAdd(CallInst *CI) {
  IGC_ASSERT(CI);
  auto *Decl = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
                                         {CI->getType()});
  SmallVector<Value *, 3> Args{CI->args()};
  auto *FMA = CallInst::Create(Decl, Args, CI->getName(), CI);
  FMA->setDebugLoc(CI->getDebugLoc());
  CI->replaceAllUsesWith(FMA);

  ToErase.push_back(CI);
  return true;
}

bool GenXLowering::lowerFAbs(CallInst *CI) {
  IGC_ASSERT(CI);
  auto *Decl = GenXIntrinsic::getGenXDeclaration(
      CI->getModule(), GenXIntrinsic::genx_absf, {CI->getType()});
  SmallVector<Value *, 2> Args{CI->args()};
  IRBuilder<> Builder{CI};
  auto *Res = Builder.CreateCall(Decl, Args, CI->getName());
  CI->replaceAllUsesWith(Res);
  ToErase.push_back(CI);
  return true;
}

bool GenXLowering::lowerF32MathIntrinsic(CallInst *CI,
                                         GenXIntrinsic::ID GenXID) {
  IGC_ASSERT(CI);
  auto *ResTy = CI->getType();
  if (!ResTy->getScalarType()->isFloatTy())
    vc::diagnose(CI->getContext(), "GenXLowering", CI,
                 "Sorry there is only f32 native instruction");
  auto *Decl = GenXIntrinsic::getGenXDeclaration(CI->getModule(), GenXID,
                                                 {CI->getType()});
  SmallVector<Value *, 2> Args{CI->args()};
  IRBuilder<> Builder{CI};
  auto *Res = Builder.CreateCall(Decl, Args, CI->getName());
  CI->replaceAllUsesWith(Res);
  ToErase.push_back(CI);
  return true;
}

bool GenXLowering::lowerF32FastMathIntrinsic(CallInst *CI,
                                             GenXIntrinsic::ID GenXID) {
  if (!CI->getFastMathFlags().isFast())
    vc::diagnose(CI->getContext(), "GenXLowering", CI,
                 "Sorry there is only low precision native instruction");
  return lowerF32MathIntrinsic(CI, GenXID);
}

/***********************************************************************
 * widenByteOp : widen a vector byte operation to short if that might
 *               improve code
 *
 * Return:  whether any change was made, and thus the current instruction
 *          is now marked for erasing
 *
 * Gen has restrictions on byte operands. The jitter copes with that, but
 * sometimes it needs to do even-odd splitting, which can lead to suboptimal
 * code if cmps and predicates are involved.
 * Here we attempt to pick up the common cases by converting a byte operation
 * to short.
 *
 * Note that we might end up with the extends being baled into the instruction
 * anyway, resulting in a byte operation in vISA.
 */
bool GenXLowering::widenByteOp(Instruction *Inst) {
  if (!EnableGenXByteWidening)
    return false;
  Type *Ty = Inst->getType();
  if (isa<CmpInst>(Inst))
    Ty = Inst->getOperand(0)->getType();
  if (!isa<VectorType>(Ty) || !Ty->getScalarType()->isIntegerTy(8))
    return false; // not byte operation
  if (Inst->use_empty())
    return false; // result unused
  // check use, if use is a phi, stop widenning
  if (!isa<CmpInst>(Inst)) {
    for (auto ui = Inst->use_begin(), ue = Inst->use_end(); ui != ue; ++ui) {
      auto User = cast<Instruction>(ui->getUser());
      if (isa<PHINode>(User))
        return false;
    }
  }
  // For a predicated wrregion, widen by separating the predication into a
  // rdregion and select, which can then be widened.
  if (GenXIntrinsic::isWrRegion(Inst)) {
    Region R = makeRegionFromBaleInfo(Inst, BaleInfo());
    if (R.NumElements == 1 || !R.Mask)
      return false;
    // Can only do this if the predicate is the right size. (We could handle
    // the wrong size case by adding an rdpredregion, but then we would need
    // to ensure that GenXLegalization can cope with an arbitrary size
    // rdpredregion.)
    if (cast<IGCLLVM::FixedVectorType>(R.Mask->getType())->getNumElements() !=
        R.NumElements)
      return false;
    // Create the rdregion and select.
    auto NewRd =
        R.createRdRegion(Inst->getOperand(0), Inst->getName() + ".byteselrdr",
                         Inst, Inst->getDebugLoc());
    auto NewSel =
        SelectInst::Create(R.Mask, Inst->getOperand(1), NewRd, "", Inst);
    NewSel->takeName(Inst);
    NewSel->setDebugLoc(Inst->getDebugLoc());
    // Modify the existing wrregion.
    Inst->setName(NewSel->getName() + ".byteselwrr");
    Inst->setOperand(1, NewSel);
    Inst->setOperand(GenXIntrinsic::GenXRegion::PredicateOperandNum,
                     Constant::getAllOnesValue(R.Mask->getType()));
    // Fall through for the select to get widened.
    Inst = NewSel;
  }
  // Do the widening for:
  // 1. a compare or select
  // 2. used in a zext that indicates that the user has probably already been
  //    widened by this code.
  bool Widen = false;
  if (isa<CmpInst>(Inst) || isa<SelectInst>(Inst))
    Widen = true;
  else {
    auto user = cast<Instruction>(Inst->use_begin()->getUser());
    if (isa<ZExtInst>(user))
      Widen = true;
  }
  if (!Widen)
    return false;
  // Widen to short.
  // Decide whether to zero or sign extend. Also decide whether the result is
  // guaranteed to have all 0 bits in the extended part.
  Instruction::CastOps ExtOpcode = Instruction::ZExt;
  bool ExtendedIsZero = false;
  switch (Inst->getOpcode()) {
  case Instruction::SDiv:
  case Instruction::AShr:
    ExtOpcode = Instruction::SExt;
    break;
  case Instruction::And:
  case Instruction::Or:
  case Instruction::Xor:
  case Instruction::LShr:
    ExtendedIsZero = true;
    break;
  case Instruction::ICmp:
    if (cast<CmpInst>(Inst)->isSigned())
      ExtOpcode = Instruction::SExt;
    break;
  default:
    break;
  }
  // Get the range of operands to process.
  unsigned StartIdx = 0, EndIdx = Inst->getNumOperands();
  if (auto CI = dyn_cast<CallInst>(Inst))
    EndIdx = CI->getNumArgOperands();
  else if (isa<SelectInst>(Inst))
    StartIdx = 1;
  // Extend the operands.
  auto ExtTy = IGCLLVM::FixedVectorType::get(
      Type::getInt16Ty(Inst->getContext()),
      cast<IGCLLVM::FixedVectorType>(Inst->getOperand(StartIdx)->getType())
          ->getNumElements());
  SmallVector<Value *, 4> Opnds;
  for (unsigned Idx = 0; Idx != EndIdx; ++Idx) {
    Value *Opnd = Inst->getOperand(Idx);
    if (Idx >= StartIdx) {
      if (auto C = dyn_cast<Constant>(Opnd))
        Opnd = ConstantExpr::getCast(ExtOpcode, C, ExtTy);
      else {
        auto NewExt = CastInst::Create(ExtOpcode, Opnd, ExtTy,
                                       Inst->getName() + ".byteext", Inst);
        NewExt->setDebugLoc(Inst->getDebugLoc());
        Opnd = NewExt;
      }
    }
    Opnds.push_back(Opnd);
  }
  // Create the replacement instruction.
  Instruction *NewInst = nullptr;
  if (isa<BinaryOperator>(Inst))
    NewInst = BinaryOperator::Create((Instruction::BinaryOps)Inst->getOpcode(),
                                     Opnds[0], Opnds[1], "", Inst);
  else if (auto CI = dyn_cast<CmpInst>(Inst))
    NewInst = CmpInst::Create(CI->getOpcode(), CI->getPredicate(), Opnds[0],
                              Opnds[1], "", CI);
  else if (isa<SelectInst>(Inst))
    NewInst = SelectInst::Create(Opnds[0], Opnds[1], Opnds[2], "", Inst);
  else
    IGC_ASSERT_EXIT_MESSAGE(0, "unhandled instruction in widenByteOp");
  NewInst->takeName(Inst);
  NewInst->setDebugLoc(Inst->getDebugLoc());
  if (ExtendedIsZero) {
    // We know that the extended part of the result contains 0 bits. If we
    // find that any use is a zext (probably from also being byte widened
    // in this code), we can replace the use directly and save the
    // trunc/zext pair. First put the uses in a vector as the use list will
    // change under our feet.
    SmallVector<Use *, 4> Uses;
    for (auto ui = Inst->use_begin(), ue = Inst->use_end(); ui != ue; ++ui)
      Uses.push_back(&*ui);
    for (auto ui = Uses.begin(), ue = Uses.end(); ui != ue; ++ui) {
      if (auto user = dyn_cast<ZExtInst>((*ui)->getUser())) {
        if (user->getType() == NewInst->getType()) {
          user->replaceAllUsesWith(NewInst);
          ToErase.push_back(user);
          // Remove the use of Inst from the trunc so we can tell whether there
          // are any uses left below.
          *(*ui) = UndefValue::get(Inst->getType());
        }
      }
    }
  }
  if (!Inst->use_empty()) {
    // Truncate the result.
    if (!isa<CmpInst>(Inst)) {
      NewInst = CastInst::Create(Instruction::Trunc, NewInst, Inst->getType(),
                                 Inst->getName() + ".bytetrunc", Inst);
      NewInst->setDebugLoc(Inst->getDebugLoc());
    }
    // Replace uses.
    Inst->replaceAllUsesWith(NewInst);
  }
  ToErase.push_back(Inst);
  return true;
}

namespace {

// Helper class to translate load/store into proper GenX intrinsic calls.
class LoadStoreResolver {
  Instruction *Inst;
  const GenXSubtarget *ST;
  IRBuilder<> Builder;

public:
  LoadStoreResolver(Instruction *Inst, const GenXSubtarget *ST)
      : Inst(Inst), ST(ST), Builder(Inst) {}

  // Resolve this instruction and return true on success.
  bool resolve();

private:
  bool isLoad() const { return isa<LoadInst>(Inst); }
  bool isStore() const { return isa<StoreInst>(Inst); }

  const DataLayout &getDL() const {
    Function *F = Inst->getParent()->getParent();
    return F->getParent()->getDataLayout();
  }

  // Find a proper GenX intrinsic ID for this load/store instruction.
  GenXIntrinsic::ID getGenXIntrinsicID() const;

  unsigned getPointerSizeInBits() const {
    unsigned AddrSp = 0;
    if (auto LI = dyn_cast<LoadInst>(Inst))
      AddrSp = LI->getPointerAddressSpace();
    else if (auto SI = dyn_cast<StoreInst>(Inst))
      AddrSp = SI->getPointerAddressSpace();
    return getDL().getPointerSizeInBits(AddrSp);
  }

  unsigned getValueSizeInBits(Type *T) const {
    if (auto PT = dyn_cast<PointerType>(T)) {
      unsigned AddrSp = PT->getAddressSpace();
      return getDL().getPointerSizeInBits(AddrSp);
    }
    return T->getPrimitiveSizeInBits();
  }

  bool isVectorInst() const {
    if (auto *LI = dyn_cast<LoadInst>(Inst))
      return LI->getType()->isVectorTy();

    auto *SI = cast<StoreInst>(Inst);
    return SI->getValueOperand()->getType()->isVectorTy();
  }

  // Return true if this load/store can be translated.
  bool isSupported() const;

  // Emit actual intrinsic calls.
  bool emitGather();
  bool emitScatter();
  bool emitSVMGather();
  bool emitSVMScatter();
  bool emitSVMBlockLD();
  bool emitSVMBlockST();
  bool emitOwordLD();
  bool emitOwordST();

  // Helpers
  std::tuple<IGCLLVM::FixedVectorType *, Type *, Value *>
  castValueIfNeeded(IGCLLVM::FixedVectorType *ValTy, Value *Val = nullptr);
};

} // namespace

// Translate store instructions into genx builtins.
bool GenXLowering::lowerLoadStore(Instruction *Inst) {
  LoadStoreResolver Resolver(Inst, ST);
  if (Resolver.resolve()) {
    ToErase.push_back(Inst);
    return true;
  }
  return false;
}

bool LoadStoreResolver::resolve() {
  if (!isSupported())
    return false;

  GenXIntrinsic::ID ID = getGenXIntrinsicID();
  switch (ID) {
  case GenXIntrinsic::genx_gather_scaled:
    return emitGather();
  case GenXIntrinsic::genx_scatter_scaled:
    return emitScatter();
  case GenXIntrinsic::genx_svm_gather:
    return emitSVMGather();
  case GenXIntrinsic::genx_svm_scatter:
    return emitSVMScatter();
  case GenXIntrinsic::genx_svm_block_ld:
    return emitSVMBlockLD();
  case GenXIntrinsic::genx_svm_block_st:
    return emitSVMBlockST();
  case GenXIntrinsic::genx_oword_ld:
    return emitOwordLD();
  case GenXIntrinsic::genx_oword_st:
    return emitOwordST();
  default:
    break;
  }

  return false;
}

// Return true if this load/store can be translated.
bool LoadStoreResolver::isSupported() const {
  auto IsGlobalLoadStore = [=]() {
    Value *Ptr = nullptr;
    if (auto LI = dyn_cast<LoadInst>(Inst))
      Ptr = LI->getPointerOperand();
    if (auto SI = dyn_cast<StoreInst>(Inst))
      Ptr = SI->getPointerOperand();
    return vc::getUnderlyingGlobalVariable(Ptr) != nullptr;
  };

  if (IsGlobalLoadStore())
    return false;

  Type *ValTy = Inst->getType();
  if (auto SI = dyn_cast<StoreInst>(Inst))
    ValTy = SI->getValueOperand()->getType();

  // Use scalar type for vector: it will be resolved separately
  unsigned VecWidth = 0;
  if (auto *VecTy = dyn_cast<IGCLLVM::FixedVectorType>(ValTy)) {
    ValTy = VecTy->getScalarType();
    VecWidth = VecTy->getNumElements();
  }

  // Only scalar data types.
  if (!ValTy->isFloatingPointTy() && !ValTy->isIntegerTy() &&
      !ValTy->isPointerTy()) {
    Inst->getContext().emitError(Inst, "unsupported type for load/store");
    return false;
  }

  // Only legal types: float, double, half, i8, i16, 132, i64, pointer types.
  unsigned NumBits = getValueSizeInBits(ValTy);
  if (NumBits < 8 || NumBits > 64 || !isPowerOf2_32(NumBits)) {
    Inst->getContext().emitError("unsupported integer type for load/store");
    return false;
  }

  // Only legal vec size: 16/32/64/128 bytes
  // TODO: it is possible to handle any vector by dividing it into
  // separate instructions. This is uncommon case so it is not
  // implemented now.
  if (VecWidth) {
    unsigned NumBytes = VecWidth * (NumBits / ByteBits);
    if (NumBytes < 16 || NumBytes > 128 || !isPowerOf2_32(NumBytes)) {
      Inst->getContext().emitError("unsupported vector type for load/store");
    }
  }

  // Translate this instruction.
  return true;
}

// Find a proper GenX intrinsic ID for this load/store instruction.
GenXIntrinsic::ID LoadStoreResolver::getGenXIntrinsicID() const {
  unsigned NBits = getPointerSizeInBits();
  if (isVectorInst()) {
    if (NBits == 32)
      return isLoad() ? GenXIntrinsic::genx_oword_ld
                      : GenXIntrinsic::genx_oword_st;
    return isLoad() ? GenXIntrinsic::genx_svm_block_ld
                    : GenXIntrinsic::genx_svm_block_st;
  }

  // A32 byte scattered stateless messages only work on CNL+.
  if (NBits == 32 && ST && !ST->WaNoA32ByteScatteredStatelessMessages())
    return isLoad() ? GenXIntrinsic::genx_gather_scaled
                    : GenXIntrinsic::genx_scatter_scaled;

  return isLoad() ? GenXIntrinsic::genx_svm_gather
                  : GenXIntrinsic::genx_svm_scatter;
}

bool LoadStoreResolver::emitGather() {
  unsigned NBits = getPointerSizeInBits();
  Type *IntTy = IntegerType::get(Inst->getContext(), NBits);
  auto LI = cast<LoadInst>(Inst);

  // Global offset.
  Value *Addr = LI->getPointerOperand();
  Addr = Builder.CreatePtrToInt(Addr, IntTy);

  unsigned NBlocks = getValueSizeInBits(LI->getType()) / 8;
  unsigned NBlocksLog2 = llvm::Log2_32(NBlocks);

  // If this is more than 4 bytes, use a larger SIMD size.
  unsigned SIMD = 1;
  if (NBlocks > 4) {
    SIMD = NBlocks / 4;
    NBlocksLog2 = 2;
  }

  // The old value is undef.
  Type *ValTy = LI->getType();
  if (ValTy->isPointerTy())
    ValTy = Builder.getIntNTy(getValueSizeInBits(ValTy));
  Type *DataTy = IGCLLVM::FixedVectorType::get(ValTy, 1);
  if (SIMD > 1)
    DataTy = IGCLLVM::FixedVectorType::get(Builder.getInt32Ty(), SIMD);
  Value *OldVal = UndefValue::get(DataTy);

  // Offset.
  Type *EltOffsetTy = IGCLLVM::FixedVectorType::get(Builder.getInt32Ty(), SIMD);
  Value *EltOffset = Constant::getNullValue(EltOffsetTy);
  if (SIMD > 1) {
    SmallVector<uint32_t, 2> Offsets(SIMD);
    for (unsigned i = 0; i < SIMD; ++i)
      // Increase offset by 4 bytes for each lane.
      Offsets[i] = i * 4;
    EltOffset = ConstantDataVector::get(Inst->getContext(), Offsets);
  }

  // Arguments.
  Value *Args[] = {
      Constant::getAllOnesValue(
          IGCLLVM::FixedVectorType::get(Builder.getInt1Ty(), SIMD)),
      Builder.getInt32(NBlocksLog2), // log[2](NBlocks)
      Builder.getInt16(0),           // scale
      Builder.getInt32(visa::getReservedSurfaceIndex(
          PreDefined_Surface::PREDEFINED_SURFACE_T255)), // surface
      Addr,                                              // global offset
      EltOffset,                                         // element offset
      OldVal                                             // old value
  };

  // Overload with return type, predicate type and element offset type
  Type *Tys[] = {OldVal->getType(), Args[0]->getType(), EltOffsetTy};
  Module *M = Inst->getModule();
  auto Fn = GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_gather_scaled, Tys);

  Value *NewVal = Builder.CreateCall(Fn, Args);
  NewVal = Builder.CreateBitCast(NewVal, ValTy);
  LI->replaceAllUsesWith(NewVal);
  return true;
}

bool LoadStoreResolver::emitScatter() {
  unsigned NBits = getPointerSizeInBits();
  Type *IntTy = IntegerType::get(Inst->getContext(), NBits);
  auto SI = cast<StoreInst>(Inst);

  // Global offset.
  Value *Addr = SI->getPointerOperand();
  Addr = Builder.CreatePtrToInt(Addr, IntTy);

  Value *Val = SI->getValueOperand();
  unsigned NBlocks = getValueSizeInBits(Val->getType()) / 8;
  unsigned NBlocksLog2 = llvm::Log2_32(NBlocks);

  // If this is more than 4 bytes, use a larger SIMD size.
  unsigned SIMD = 1;
  if (NBlocks > 4) {
    SIMD = NBlocks / 4;
    NBlocksLog2 = 2;
  }

  // Value to write.
  Type *ValTy = (SIMD > 1) ? Builder.getInt32Ty() : Val->getType();
  if (ValTy->isPointerTy())
    ValTy = Builder.getIntNTy(getValueSizeInBits(ValTy));
  Val = Builder.CreateBitCast(Val, IGCLLVM::FixedVectorType::get(ValTy, SIMD));

  // Offset.
  Type *EltOffsetTy = IGCLLVM::FixedVectorType::get(Builder.getInt32Ty(), SIMD);
  Value *EltOffset = Constant::getNullValue(EltOffsetTy);
  if (SIMD > 1) {
    SmallVector<uint32_t, 2> Offsets(SIMD);
    // Increase offset by 4 bytes for each lane.
    for (unsigned i = 0; i < SIMD; ++i)
      Offsets[i] = i * 4;
    EltOffset = ConstantDataVector::get(Inst->getContext(), Offsets);
  }

  // Arguments.
  Value *Args[] = {
      Constant::getAllOnesValue(
          IGCLLVM::FixedVectorType::get(Builder.getInt1Ty(), SIMD)),
      Builder.getInt32(NBlocksLog2), // log[2](NBlocks)
      Builder.getInt16(0),           // scale
      Builder.getInt32(visa::getReservedSurfaceIndex(
          PreDefined_Surface::PREDEFINED_SURFACE_T255)), // surface
      Addr,                                              // global offset
      EltOffset,                                         // element offset
      Val                                                // value to write
  };

  // Overload with predicate type, element offset type, value to write type.
  Type *Tys[] = {Args[0]->getType(), EltOffsetTy, Val->getType()};
  Module *M = Inst->getModule();
  auto Fn = GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_scatter_scaled, Tys);
  Builder.CreateCall(Fn, Args);
  return true;
}

// Compute the block size and the number of blocks for svm gather/scatter.
//
// Block_Size, 1, 4, 8
// Num_Blocks, 1, 2, 4,
//             8 only valid for 4 byte blocks and execution size 8.
//
static unsigned getBlockCount(Type *Ty) {
  unsigned NumBytes = Ty->getPrimitiveSizeInBits() / 8;
  IGC_ASSERT_MESSAGE(NumBytes <= 8, "out of sync");

  // If this is N = 2 byte data, use 2 blocks;
  // otherwise, use 1 block of N bytes.
  return (NumBytes == 2) ? NumBytes : 1U;
}

// Translate store to svm scatter.
bool LoadStoreResolver::emitSVMGather() {
  unsigned NBits = getPointerSizeInBits();
  Type *IntTy = IntegerType::get(Inst->getContext(), NBits);
  auto LI = cast<LoadInst>(Inst);

  // Address.
  Value *Addr = LI->getPointerOperand();
  Addr = Builder.CreatePtrToInt(Addr, IntTy);
  if (NBits == 32)
    Addr = Builder.CreateZExt(Addr, Builder.getInt64Ty());
  Addr = Builder.CreateBitCast(
      Addr, IGCLLVM::FixedVectorType::get(Addr->getType(), 1));

  // The old value is undef.
  Type *ValTy = LI->getType();
  if (ValTy->isPointerTy())
    ValTy = Builder.getIntNTy(getValueSizeInBits(ValTy));
  Type *DataTy = IGCLLVM::FixedVectorType::get(ValTy, 1);
  Value *OldVal = UndefValue::get(DataTy);

  // Num of blocks.
  unsigned NBlocks = getBlockCount(OldVal->getType());
  unsigned NBlocksLog2 = llvm::Log2_32(NBlocks);

  Value *Args[] = {
      Constant::getAllOnesValue(
          IGCLLVM::FixedVectorType::get(Builder.getInt1Ty(), 1)),
      Builder.getInt32(NBlocksLog2), // log2(num_of_blocks)
      Addr,                          // addresses
      OldVal                         // old value
  };

  // Overload with return type, predicate type and address vector type
  Type *Tys[] = {OldVal->getType(), Args[0]->getType(), Addr->getType()};
  Module *M = Inst->getModule();
  auto Fn = GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_svm_gather, Tys);

  Value *NewVal = Builder.CreateCall(Fn, Args);
  NewVal = Builder.CreateBitCast(NewVal, ValTy);
  if (LI->getType()->isPointerTy())
    NewVal = Builder.CreateIntToPtr(NewVal, LI->getType());
  LI->replaceAllUsesWith(NewVal);
  return true;
}

bool LoadStoreResolver::emitSVMScatter() {
  unsigned NBits = getPointerSizeInBits();
  Type *IntTy = IntegerType::get(Inst->getContext(), NBits);
  auto SI = cast<StoreInst>(Inst);

  // Address
  Value *Addr = SI->getPointerOperand();
  Addr = Builder.CreatePtrToInt(Addr, IntTy);
  if (NBits == 32)
    Addr = Builder.CreateZExt(Addr, Builder.getInt64Ty());
  Addr = Builder.CreateBitCast(
      Addr, IGCLLVM::FixedVectorType::get(Addr->getType(), 1));

  // data to write.
  Value *Val = SI->getValueOperand();
  Type *ValTy = Val->getType();
  if (ValTy->isPointerTy()) {
    ValTy = Builder.getIntNTy(getValueSizeInBits(ValTy));
    Val = Builder.CreatePtrToInt(Val, ValTy);
  }
  Val = Builder.CreateBitCast(Val, IGCLLVM::FixedVectorType::get(ValTy, 1));

  // Num of blocks.
  unsigned NBlocks = getBlockCount(Val->getType());
  unsigned NBlocksLog2 = llvm::Log2_32(NBlocks);

  Value *Args[] = {
      Constant::getAllOnesValue(
          IGCLLVM::FixedVectorType::get(Builder.getInt1Ty(), 1)),
      Builder.getInt32(NBlocksLog2), // log2(num_of_blocks)
      Addr,                          // addresses
      Val                            // value to write
  };

  // Overload with predicate type, address vector type, and data type
  Type *Tys[] = {Args[0]->getType(), Addr->getType(), Val->getType()};
  Module *M = Inst->getModule();
  auto Fn = GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_svm_scatter, Tys);

  Builder.CreateCall(Fn, Args);
  return true;
}

// Helper for pointer value cast.
// Returns [Casted ValTy, Old ScalarTy, Casted Value]
std::tuple<IGCLLVM::FixedVectorType *, Type *, Value *>
LoadStoreResolver::castValueIfNeeded(IGCLLVM::FixedVectorType *ValTy,
                                     Value *Val) {
  Type *ScalarTy = ValTy->getScalarType();
  if (ScalarTy->isPointerTy()) {
    unsigned VecWidth = ValTy->getNumElements();
    ValTy = IGCLLVM::FixedVectorType::get(
        Builder.getIntNTy(getValueSizeInBits(ScalarTy)), VecWidth);
    if (Val)
      Val = Builder.CreatePtrToInt(Val, ValTy);
  }

  return {ValTy, ScalarTy, Val};
}

// Translate load to svm block ld.
bool LoadStoreResolver::emitSVMBlockLD() {
  IGC_ASSERT_MESSAGE(getPointerSizeInBits() == 64,
                     "Incorrect ptr size for svm block ld");
  auto *LI = cast<LoadInst>(Inst);

  // Address argument should be int
  Value *Addr = LI->getPointerOperand();
  Type *IntPtrTy = getDL().getIntPtrType(Addr->getType());
  Addr = Builder.CreatePtrToInt(Addr, IntPtrTy);

  // Analyze types
  auto *ValTy = cast<IGCLLVM::FixedVectorType>(LI->getType());
  Type *BasicScalarTy = nullptr;
  std::tie(ValTy, BasicScalarTy, std::ignore) = castValueIfNeeded(ValTy);

  // Overload with return type and address type
  Type *Tys[] = {ValTy, Addr->getType()};
  Module *M = Inst->getModule();
  auto *Fn = GenXIntrinsic::getGenXDeclaration(
      M, GenXIntrinsic::genx_svm_block_ld, Tys);

  auto NewInst = Builder.CreateCall(Fn, Addr);
  NewInst->setDebugLoc(LI->getDebugLoc());
  Value *NewVal = NewInst;
  if (BasicScalarTy->isPointerTy())
    NewVal = Builder.CreateIntToPtr(NewVal, LI->getType());
  LI->replaceAllUsesWith(NewVal);
  return true;
}

// Translate store to svm block st.
bool LoadStoreResolver::emitSVMBlockST() {
  IGC_ASSERT_MESSAGE(getPointerSizeInBits() == 64,
                     "Incorrect ptr size for svm block st");
  auto *SI = cast<StoreInst>(Inst);

  // Address argument should be int
  Value *Addr = SI->getPointerOperand();
  Type *IntPtrTy = getDL().getIntPtrType(Addr->getType());
  Addr = Builder.CreatePtrToInt(Addr, IntPtrTy);

  // Data to write
  Value *Val = SI->getValueOperand();
  auto *ValTy = cast<IGCLLVM::FixedVectorType>(Val->getType());
  std::tie(ValTy, std::ignore, Val) = castValueIfNeeded(ValTy, Val);

  // Overload with predicate type, address vector type, and data type
  Type *Tys[] = {Addr->getType(), ValTy};
  Module *M = Inst->getModule();
  auto *Fn = GenXIntrinsic::getGenXDeclaration(
      M, GenXIntrinsic::genx_svm_block_st, Tys);

  Builder.CreateCall(Fn, {Addr, Val});
  return true;
}

// Translate load to oword ld.
bool LoadStoreResolver::emitOwordLD() {
  IGC_ASSERT_MESSAGE(getPointerSizeInBits() == 32,
                     "Incorrect ptr size for oword ld");
  auto *LI = cast<LoadInst>(Inst);

  // Offset argument should be int
  Value *Addr = LI->getPointerOperand();
  Type *IntPtrTy = getDL().getIntPtrType(Addr->getType());
  Addr = Builder.CreatePtrToInt(Addr, IntPtrTy);

  // Analyze types
  auto *ValTy = cast<IGCLLVM::FixedVectorType>(LI->getType());
  Type *BasicScalarTy = nullptr;
  std::tie(ValTy, BasicScalarTy, std::ignore) = castValueIfNeeded(ValTy);

  // Arguments
  Value *Args[] = {
      Builder.getInt32(1), // is modified (ignored according to vISA spec)
      Builder.getInt32(visa::getReservedSurfaceIndex(
          PreDefined_Surface::PREDEFINED_SURFACE_T255)), // surface
      Addr                                               // offset
  };

  // Overload with return type
  Module *M = Inst->getModule();
  auto *Fn =
      GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_oword_ld, ValTy);

  Value *NewVal = Builder.CreateCall(Fn, Args);
  if (BasicScalarTy->isPointerTy())
    NewVal = Builder.CreateIntToPtr(NewVal, LI->getType());
  LI->replaceAllUsesWith(NewVal);
  return true;
}

// Translate store to oword st.
bool LoadStoreResolver::emitOwordST() {
  IGC_ASSERT_MESSAGE(getPointerSizeInBits() == 32,
                     "Incorrect ptr size for oword st");
  auto *SI = cast<StoreInst>(Inst);

  // Offset argument should be int
  Value *Addr = SI->getPointerOperand();
  Type *IntPtrTy = getDL().getIntPtrType(Addr->getType());
  Addr = Builder.CreatePtrToInt(Addr, IntPtrTy);

  // Value to write
  Value *Val = SI->getValueOperand();
  auto *ValTy = cast<IGCLLVM::FixedVectorType>(Val->getType());
  std::tie(ValTy, std::ignore, Val) = castValueIfNeeded(ValTy, Val);

  // Arguments.
  Value *Args[] = {
      Builder.getInt32(visa::getReservedSurfaceIndex(
          PreDefined_Surface::PREDEFINED_SURFACE_T255)), // surface
      Addr,                                              // offset
      Val                                                // value to write
  };

  // Overload with value to write type.
  Module *M = Inst->getModule();
  auto *Fn =
      GenXIntrinsic::getGenXDeclaration(M, GenXIntrinsic::genx_oword_st, ValTy);
  Builder.CreateCall(Fn, Args);
  return true;
}
