xref: /art/compiler/optimizing/code_generator_riscv64.cc

/*
 * Copyright (C) 2023 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "code_generator_riscv64.h"

#include "android-base/logging.h"
#include "android-base/macros.h"
#include "arch/riscv64/jni_frame_riscv64.h"
#include "arch/riscv64/registers_riscv64.h"
#include "base/arena_containers.h"
#include "base/macros.h"
#include "class_root-inl.h"
#include "code_generator_utils.h"
#include "dwarf/register.h"
#include "gc/heap.h"
#include "gc/space/image_space.h"
#include "heap_poisoning.h"
#include "intrinsics_list.h"
#include "intrinsics_riscv64.h"
#include "jit/profiling_info.h"
#include "linker/linker_patch.h"
#include "mirror/class-inl.h"
#include "optimizing/nodes.h"
#include "optimizing/profiling_info_builder.h"
#include "runtime.h"
#include "scoped_thread_state_change-inl.h"
#include "stack_map_stream.h"
#include "trace.h"
#include "utils/label.h"
#include "utils/riscv64/assembler_riscv64.h"
#include "utils/stack_checks.h"

namespace art HIDDEN {
namespace riscv64 {

// Placeholder values embedded in instructions, patched at link time.
constexpr uint32_t kLinkTimeOffsetPlaceholderHigh = 0x12345;
constexpr uint32_t kLinkTimeOffsetPlaceholderLow = 0x678;

// Compare-and-jump packed switch generates approx. 3 + 1.5 * N 32-bit
// instructions for N cases.
// Table-based packed switch generates approx. 10 32-bit instructions
// and N 32-bit data words for N cases.
// We switch to the table-based method starting with 6 entries.
static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 6;

static constexpr XRegister kCoreCalleeSaves[] = {
    // S1(TR) is excluded as the ART thread register.
    S0, S2, S3, S4, S5, S6, S7, S8, S9, S10, S11, RA
};

static constexpr FRegister kFpuCalleeSaves[] = {
    FS0, FS1, FS2, FS3, FS4, FS5, FS6, FS7, FS8, FS9, FS10, FS11
};

#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kRiscv64PointerSize, x).Int32Value()

Location RegisterOrZeroBitPatternLocation(HInstruction* instruction) {
  DCHECK(!DataType::IsFloatingPointType(instruction->GetType()));
  return IsZeroBitPattern(instruction)
      ? Location::ConstantLocation(instruction)
      : Location::RequiresRegister();
}

Location FpuRegisterOrZeroBitPatternLocation(HInstruction* instruction) {
  DCHECK(DataType::IsFloatingPointType(instruction->GetType()));
  return IsZeroBitPattern(instruction)
      ? Location::ConstantLocation(instruction)
      : Location::RequiresFpuRegister();
}

XRegister InputXRegisterOrZero(Location location) {
  if (location.IsConstant()) {
    DCHECK(location.GetConstant()->IsZeroBitPattern());
    return Zero;
  } else {
    return location.AsRegister<XRegister>();
  }
}

Location ValueLocationForStore(HInstruction* value) {
  if (IsZeroBitPattern(value)) {
    return Location::ConstantLocation(value);
  } else if (DataType::IsFloatingPointType(value->GetType())) {
    return Location::RequiresFpuRegister();
  } else {
    return Location::RequiresRegister();
  }
}

Location Riscv64ReturnLocation(DataType::Type return_type) {
  switch (return_type) {
    case DataType::Type::kBool:
    case DataType::Type::kUint8:
    case DataType::Type::kInt8:
    case DataType::Type::kUint16:
    case DataType::Type::kInt16:
    case DataType::Type::kUint32:
    case DataType::Type::kInt32:
    case DataType::Type::kReference:
    case DataType::Type::kUint64:
    case DataType::Type::kInt64:
      return Location::RegisterLocation(A0);

    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64:
      return Location::FpuRegisterLocation(FA0);

    case DataType::Type::kVoid:
      return Location::NoLocation();
  }
}

static RegisterSet OneRegInReferenceOutSaveEverythingCallerSaves() {
  InvokeRuntimeCallingConvention calling_convention;
  RegisterSet caller_saves = RegisterSet::Empty();
  caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
  DCHECK_EQ(
      calling_convention.GetRegisterAt(0),
      calling_convention.GetReturnLocation(DataType::Type::kReference).AsRegister<XRegister>());
  return caller_saves;
}

template <ClassStatus kStatus>
static constexpr int64_t ShiftedSignExtendedClassStatusValue() {
  // This is used only for status values that have the highest bit set.
  static_assert(CLZ(enum_cast<uint32_t>(kStatus)) == kClassStatusLsbPosition);
  constexpr uint32_t kShiftedStatusValue = enum_cast<uint32_t>(kStatus) << kClassStatusLsbPosition;
  static_assert(kShiftedStatusValue >= 0x80000000u);
  return static_cast<int64_t>(kShiftedStatusValue) - (INT64_C(1) << 32);
}

// Split a 64-bit address used by JIT to the nearest 4KiB-aligned base address and a 12-bit
// signed offset. It is usually cheaper to materialize the aligned address than the full address.
std::pair<uint64_t, int32_t> SplitJitAddress(uint64_t address) {
  uint64_t bits0_11 = address & UINT64_C(0xfff);
  uint64_t bit11 = address & UINT64_C(0x800);
  // Round the address to nearest 4KiB address because the `imm12` has range [-0x800, 0x800).
  uint64_t base_address = (address & ~UINT64_C(0xfff)) + (bit11 << 1);
  int32_t imm12 = dchecked_integral_cast<int32_t>(bits0_11) -
                  dchecked_integral_cast<int32_t>(bit11 << 1);
  return {base_address, imm12};
}

int32_t ReadBarrierMarkEntrypointOffset(Location ref) {
  DCHECK(ref.IsRegister());
  int reg = ref.reg();
  DCHECK(T0 <= reg && reg <= T6 && reg != TR) << reg;
  // Note: Entrypoints for registers X30 (T5) and X31 (T6) are stored in entries
  // for X0 (Zero) and X1 (RA) because these are not valid registers for marking
  // and we currently have slots only up to register 29.
  int entry_point_number = (reg >= 30) ? reg - 30 : reg;
  return Thread::ReadBarrierMarkEntryPointsOffset<kRiscv64PointerSize>(entry_point_number);
}

Location InvokeRuntimeCallingConvention::GetReturnLocation(DataType::Type return_type) {
  return Riscv64ReturnLocation(return_type);
}

Location InvokeDexCallingConventionVisitorRISCV64::GetReturnLocation(DataType::Type type) const {
  return Riscv64ReturnLocation(type);
}

Location InvokeDexCallingConventionVisitorRISCV64::GetMethodLocation() const {
  return Location::RegisterLocation(kArtMethodRegister);
}

Location InvokeDexCallingConventionVisitorRISCV64::GetNextLocation(DataType::Type type) {
  Location next_location;
  if (type == DataType::Type::kVoid) {
    LOG(FATAL) << "Unexpected parameter type " << type;
  }

  // Note: Unlike the RISC-V C/C++ calling convention, managed ABI does not use
  // GPRs to pass FP args when we run out of FPRs.
  if (DataType::IsFloatingPointType(type) &&
      float_index_ < calling_convention.GetNumberOfFpuRegisters()) {
    next_location =
        Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++));
  } else if (!DataType::IsFloatingPointType(type) &&
             (gp_index_ < calling_convention.GetNumberOfRegisters())) {
    next_location = Location::RegisterLocation(calling_convention.GetRegisterAt(gp_index_++));
  } else {
    size_t stack_offset = calling_convention.GetStackOffsetOf(stack_index_);
    next_location = DataType::Is64BitType(type) ? Location::DoubleStackSlot(stack_offset) :
                                                  Location::StackSlot(stack_offset);
  }

  // Space on the stack is reserved for all arguments.
  stack_index_ += DataType::Is64BitType(type) ? 2 : 1;

  return next_location;
}

Location CriticalNativeCallingConventionVisitorRiscv64::GetNextLocation(DataType::Type type) {
  DCHECK_NE(type, DataType::Type::kReference);

  Location location = Location::NoLocation();
  if (DataType::IsFloatingPointType(type)) {
    if (fpr_index_ < kParameterFpuRegistersLength) {
      location = Location::FpuRegisterLocation(kParameterFpuRegisters[fpr_index_]);
      ++fpr_index_;
    } else {
      // Native ABI allows passing excessive FP args in GPRs. This is facilitated by
      // inserting fake conversion intrinsic calls (`Double.doubleToRawLongBits()`
      // or `Float.floatToRawIntBits()`) by `CriticalNativeAbiFixupRiscv64`.
      // Remaining FP args shall be passed on the stack.
      CHECK_EQ(gpr_index_, kRuntimeParameterCoreRegistersLength);
    }
  } else {
    // Native ABI uses the same core registers as a runtime call.
    if (gpr_index_ < kRuntimeParameterCoreRegistersLength) {
      location = Location::RegisterLocation(kRuntimeParameterCoreRegisters[gpr_index_]);
      ++gpr_index_;
    }
  }
  if (location.IsInvalid()) {
    // Only a `float` gets a single slot. Integral args need to be sign-extended to 64 bits.
    if (type == DataType::Type::kFloat32) {
      location = Location::StackSlot(stack_offset_);
    } else {
      location = Location::DoubleStackSlot(stack_offset_);
    }
    stack_offset_ += kFramePointerSize;

    if (for_register_allocation_) {
      location = Location::Any();
    }
  }
  return location;
}

Location CriticalNativeCallingConventionVisitorRiscv64::GetReturnLocation(
    DataType::Type type) const {
  // The result is returned the same way in native ABI and managed ABI. No result conversion is
  // needed, see comments in `Riscv64JniCallingConvention::RequiresSmallResultTypeExtension()`.
  InvokeDexCallingConventionVisitorRISCV64 dex_calling_convention;
  return dex_calling_convention.GetReturnLocation(type);
}

Location CriticalNativeCallingConventionVisitorRiscv64::GetMethodLocation() const {
  // Pass the method in the hidden argument T0.
  return Location::RegisterLocation(T0);
}

#define __ down_cast<CodeGeneratorRISCV64*>(codegen)->GetAssembler()->  // NOLINT

void LocationsBuilderRISCV64::HandleInvoke(HInvoke* instruction) {
  InvokeDexCallingConventionVisitorRISCV64 calling_convention_visitor;
  CodeGenerator::CreateCommonInvokeLocationSummary(instruction, &calling_convention_visitor);
}

class CompileOptimizedSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  CompileOptimizedSlowPathRISCV64(HSuspendCheck* suspend_check, XRegister base, int32_t imm12)
      : SlowPathCodeRISCV64(suspend_check),
        base_(base),
        imm12_(imm12) {}

  void EmitNativeCode(CodeGenerator* codegen) override {
    uint32_t entrypoint_offset =
        GetThreadOffset<kRiscv64PointerSize>(kQuickCompileOptimized).Int32Value();
    __ Bind(GetEntryLabel());
    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);
    riscv64::ScratchRegisterScope srs(riscv64_codegen->GetAssembler());
    XRegister counter = srs.AllocateXRegister();
    __ LoadConst32(counter, ProfilingInfo::GetOptimizeThreshold());
    __ Sh(counter, base_, imm12_);
    if (instruction_ != nullptr) {
      // Only saves live vector regs for SIMD.
      SaveLiveRegisters(codegen, instruction_->GetLocations());
    }
    __ Loadd(RA, TR, entrypoint_offset);
    // Note: we don't record the call here (and therefore don't generate a stack
    // map), as the entrypoint should never be suspended.
    __ Jalr(RA);
    if (instruction_ != nullptr) {
      // Only restores live vector regs for SIMD.
      RestoreLiveRegisters(codegen, instruction_->GetLocations());
    }
    __ J(GetExitLabel());
  }

  const char* GetDescription() const override { return "CompileOptimizedSlowPath"; }

 private:
  XRegister base_;
  const int32_t imm12_;

  DISALLOW_COPY_AND_ASSIGN(CompileOptimizedSlowPathRISCV64);
};

class SuspendCheckSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  SuspendCheckSlowPathRISCV64(HSuspendCheck* instruction, HBasicBlock* successor)
      : SlowPathCodeRISCV64(instruction), successor_(successor) {}

  void EmitNativeCode(CodeGenerator* codegen) override {
    LocationSummary* locations = instruction_->GetLocations();
    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);
    __ Bind(GetEntryLabel());
    SaveLiveRegisters(codegen, locations);  // Only saves live vector registers for SIMD.
    riscv64_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this);
    CheckEntrypointTypes<kQuickTestSuspend, void, void>();
    RestoreLiveRegisters(codegen, locations);  // Only restores live vector registers for SIMD.
    if (successor_ == nullptr) {
      __ J(GetReturnLabel());
    } else {
      __ J(riscv64_codegen->GetLabelOf(successor_));
    }
  }

  Riscv64Label* GetReturnLabel() {
    DCHECK(successor_ == nullptr);
    return &return_label_;
  }

  const char* GetDescription() const override { return "SuspendCheckSlowPathRISCV64"; }

  HBasicBlock* GetSuccessor() const { return successor_; }

 private:
  // If not null, the block to branch to after the suspend check.
  HBasicBlock* const successor_;

  // If `successor_` is null, the label to branch to after the suspend check.
  Riscv64Label return_label_;

  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathRISCV64);
};

class NullCheckSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  explicit NullCheckSlowPathRISCV64(HNullCheck* instr) : SlowPathCodeRISCV64(instr) {}

  void EmitNativeCode(CodeGenerator* codegen) override {
    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);
    __ Bind(GetEntryLabel());
    if (instruction_->CanThrowIntoCatchBlock()) {
      // Live registers will be restored in the catch block if caught.
      SaveLiveRegisters(codegen, instruction_->GetLocations());
    }
    riscv64_codegen->InvokeRuntime(
        kQuickThrowNullPointer, instruction_, instruction_->GetDexPc(), this);
    CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
  }

  bool IsFatal() const override { return true; }

  const char* GetDescription() const override { return "NullCheckSlowPathRISCV64"; }

 private:
  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathRISCV64);
};

class BoundsCheckSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  explicit BoundsCheckSlowPathRISCV64(HBoundsCheck* instruction)
      : SlowPathCodeRISCV64(instruction) {}

  void EmitNativeCode(CodeGenerator* codegen) override {
    LocationSummary* locations = instruction_->GetLocations();
    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);
    __ Bind(GetEntryLabel());
    if (instruction_->CanThrowIntoCatchBlock()) {
      // Live registers will be restored in the catch block if caught.
      SaveLiveRegisters(codegen, instruction_->GetLocations());
    }
    // We're moving two locations to locations that could overlap, so we need a parallel
    // move resolver.
    InvokeRuntimeCallingConvention calling_convention;
    codegen->EmitParallelMoves(locations->InAt(0),
                               Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
                               DataType::Type::kInt32,
                               locations->InAt(1),
                               Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
                               DataType::Type::kInt32);
    QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() ?
                                         kQuickThrowStringBounds :
                                         kQuickThrowArrayBounds;
    riscv64_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this);
    CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>();
    CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
  }

  bool IsFatal() const override { return true; }

  const char* GetDescription() const override { return "BoundsCheckSlowPathRISCV64"; }

 private:
  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathRISCV64);
};

class LoadClassSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  LoadClassSlowPathRISCV64(HLoadClass* cls, HInstruction* at) : SlowPathCodeRISCV64(at), cls_(cls) {
    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
    DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_);
  }

  void EmitNativeCode(CodeGenerator* codegen) override {
    LocationSummary* locations = instruction_->GetLocations();
    Location out = locations->Out();
    const uint32_t dex_pc = instruction_->GetDexPc();
    bool must_resolve_type = instruction_->IsLoadClass() && cls_->MustResolveTypeOnSlowPath();
    bool must_do_clinit = instruction_->IsClinitCheck() || cls_->MustGenerateClinitCheck();

    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);
    __ Bind(GetEntryLabel());
    SaveLiveRegisters(codegen, locations);

    InvokeRuntimeCallingConvention calling_convention;
    if (must_resolve_type) {
      DCHECK(IsSameDexFile(cls_->GetDexFile(), riscv64_codegen->GetGraph()->GetDexFile()) ||
             riscv64_codegen->GetCompilerOptions().WithinOatFile(&cls_->GetDexFile()) ||
             ContainsElement(Runtime::Current()->GetClassLinker()->GetBootClassPath(),
                             &cls_->GetDexFile()));
      dex::TypeIndex type_index = cls_->GetTypeIndex();
      __ LoadConst32(calling_convention.GetRegisterAt(0), type_index.index_);
      if (cls_->NeedsAccessCheck()) {
        CheckEntrypointTypes<kQuickResolveTypeAndVerifyAccess, void*, uint32_t>();
        riscv64_codegen->InvokeRuntime(
            kQuickResolveTypeAndVerifyAccess, instruction_, dex_pc, this);
      } else {
        CheckEntrypointTypes<kQuickResolveType, void*, uint32_t>();
        riscv64_codegen->InvokeRuntime(kQuickResolveType, instruction_, dex_pc, this);
      }
      // If we also must_do_clinit, the resolved type is now in the correct register.
    } else {
      DCHECK(must_do_clinit);
      Location source = instruction_->IsLoadClass() ? out : locations->InAt(0);
      riscv64_codegen->MoveLocation(
          Location::RegisterLocation(calling_convention.GetRegisterAt(0)), source, cls_->GetType());
    }
    if (must_do_clinit) {
      riscv64_codegen->InvokeRuntime(kQuickInitializeStaticStorage, instruction_, dex_pc, this);
      CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, mirror::Class*>();
    }

    // Move the class to the desired location.
    if (out.IsValid()) {
      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
      DataType::Type type = DataType::Type::kReference;
      DCHECK_EQ(type, instruction_->GetType());
      riscv64_codegen->MoveLocation(out, calling_convention.GetReturnLocation(type), type);
    }
    RestoreLiveRegisters(codegen, locations);

    __ J(GetExitLabel());
  }

  const char* GetDescription() const override { return "LoadClassSlowPathRISCV64"; }

 private:
  // The class this slow path will load.
  HLoadClass* const cls_;

  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathRISCV64);
};

class DeoptimizationSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  explicit DeoptimizationSlowPathRISCV64(HDeoptimize* instruction)
      : SlowPathCodeRISCV64(instruction) {}

  void EmitNativeCode(CodeGenerator* codegen) override {
    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);
    __ Bind(GetEntryLabel());
    LocationSummary* locations = instruction_->GetLocations();
    SaveLiveRegisters(codegen, locations);
    InvokeRuntimeCallingConvention calling_convention;
    __ LoadConst32(calling_convention.GetRegisterAt(0),
                   static_cast<uint32_t>(instruction_->AsDeoptimize()->GetDeoptimizationKind()));
    riscv64_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this);
    CheckEntrypointTypes<kQuickDeoptimize, void, DeoptimizationKind>();
  }

  const char* GetDescription() const override { return "DeoptimizationSlowPathRISCV64"; }

 private:
  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathRISCV64);
};

// Slow path generating a read barrier for a GC root.
class ReadBarrierForRootSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  ReadBarrierForRootSlowPathRISCV64(HInstruction* instruction, Location out, Location root)
      : SlowPathCodeRISCV64(instruction), out_(out), root_(root) {
  }

  void EmitNativeCode(CodeGenerator* codegen) override {
    DCHECK(codegen->EmitReadBarrier());
    LocationSummary* locations = instruction_->GetLocations();
    DataType::Type type = DataType::Type::kReference;
    XRegister reg_out = out_.AsRegister<XRegister>();
    DCHECK(locations->CanCall());
    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
    DCHECK(instruction_->IsLoadClass() ||
           instruction_->IsLoadString() ||
           (instruction_->IsInvoke() && instruction_->GetLocations()->Intrinsified()))
        << "Unexpected instruction in read barrier for GC root slow path: "
        << instruction_->DebugName();

    __ Bind(GetEntryLabel());
    SaveLiveRegisters(codegen, locations);

    InvokeRuntimeCallingConvention calling_convention;
    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);
    riscv64_codegen->MoveLocation(Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
                                  root_,
                                  DataType::Type::kReference);
    riscv64_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow,
                                   instruction_,
                                   instruction_->GetDexPc(),
                                   this);
    CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
    riscv64_codegen->MoveLocation(out_, calling_convention.GetReturnLocation(type), type);

    RestoreLiveRegisters(codegen, locations);
    __ J(GetExitLabel());
  }

  const char* GetDescription() const override { return "ReadBarrierForRootSlowPathRISCV64"; }

 private:
  const Location out_;
  const Location root_;

  DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathRISCV64);
};

class MethodEntryExitHooksSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  explicit MethodEntryExitHooksSlowPathRISCV64(HInstruction* instruction)
      : SlowPathCodeRISCV64(instruction) {}

  void EmitNativeCode(CodeGenerator* codegen) override {
    LocationSummary* locations = instruction_->GetLocations();
    QuickEntrypointEnum entry_point =
        (instruction_->IsMethodEntryHook()) ? kQuickMethodEntryHook : kQuickMethodExitHook;
    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);
    __ Bind(GetEntryLabel());
    SaveLiveRegisters(codegen, locations);
    if (instruction_->IsMethodExitHook()) {
      __ Li(A4, riscv64_codegen->GetFrameSize());
    }
    riscv64_codegen->InvokeRuntime(entry_point, instruction_, instruction_->GetDexPc(), this);
    RestoreLiveRegisters(codegen, locations);
    __ J(GetExitLabel());
  }

  const char* GetDescription() const override {
    return "MethodEntryExitHooksSlowPathRISCV";
  }

 private:
  DISALLOW_COPY_AND_ASSIGN(MethodEntryExitHooksSlowPathRISCV64);
};

class ArraySetSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  explicit ArraySetSlowPathRISCV64(HInstruction* instruction) : SlowPathCodeRISCV64(instruction) {}

  void EmitNativeCode(CodeGenerator* codegen) override {
    LocationSummary* locations = instruction_->GetLocations();
    __ Bind(GetEntryLabel());
    SaveLiveRegisters(codegen, locations);

    InvokeRuntimeCallingConvention calling_convention;
    HParallelMove parallel_move(codegen->GetGraph()->GetAllocator());
    parallel_move.AddMove(
        locations->InAt(0),
        Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
        DataType::Type::kReference,
        nullptr);
    parallel_move.AddMove(
        locations->InAt(1),
        Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
        DataType::Type::kInt32,
        nullptr);
    parallel_move.AddMove(
        locations->InAt(2),
        Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
        DataType::Type::kReference,
        nullptr);
    codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);

    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);
    riscv64_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this);
    CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
    RestoreLiveRegisters(codegen, locations);
    __ J(GetExitLabel());
  }

  const char* GetDescription() const override { return "ArraySetSlowPathRISCV64"; }

 private:
  DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathRISCV64);
};

class TypeCheckSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  explicit TypeCheckSlowPathRISCV64(HInstruction* instruction, bool is_fatal)
      : SlowPathCodeRISCV64(instruction), is_fatal_(is_fatal) {}

  void EmitNativeCode(CodeGenerator* codegen) override {
    LocationSummary* locations = instruction_->GetLocations();

    uint32_t dex_pc = instruction_->GetDexPc();
    DCHECK(instruction_->IsCheckCast()
           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);

    __ Bind(GetEntryLabel());
    if (!is_fatal_ || instruction_->CanThrowIntoCatchBlock()) {
      SaveLiveRegisters(codegen, locations);
    }

    // We're moving two locations to locations that could overlap, so we need a parallel
    // move resolver.
    InvokeRuntimeCallingConvention calling_convention;
    codegen->EmitParallelMoves(locations->InAt(0),
                               Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
                               DataType::Type::kReference,
                               locations->InAt(1),
                               Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
                               DataType::Type::kReference);
    if (instruction_->IsInstanceOf()) {
      riscv64_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, instruction_, dex_pc, this);
      CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>();
      DataType::Type ret_type = instruction_->GetType();
      Location ret_loc = calling_convention.GetReturnLocation(ret_type);
      riscv64_codegen->MoveLocation(locations->Out(), ret_loc, ret_type);
    } else {
      DCHECK(instruction_->IsCheckCast());
      riscv64_codegen->InvokeRuntime(kQuickCheckInstanceOf, instruction_, dex_pc, this);
      CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>();
    }

    if (!is_fatal_) {
      RestoreLiveRegisters(codegen, locations);
      __ J(GetExitLabel());
    }
  }

  const char* GetDescription() const override { return "TypeCheckSlowPathRISCV64"; }

  bool IsFatal() const override { return is_fatal_; }

 private:
  const bool is_fatal_;

  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathRISCV64);
};

class DivZeroCheckSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  explicit DivZeroCheckSlowPathRISCV64(HDivZeroCheck* instruction)
      : SlowPathCodeRISCV64(instruction) {}

  void EmitNativeCode(CodeGenerator* codegen) override {
    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);
    __ Bind(GetEntryLabel());
    riscv64_codegen->InvokeRuntime(
        kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this);
    CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
  }

  bool IsFatal() const override { return true; }

  const char* GetDescription() const override { return "DivZeroCheckSlowPathRISCV64"; }

 private:
  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathRISCV64);
};

class ReadBarrierMarkSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  ReadBarrierMarkSlowPathRISCV64(HInstruction* instruction, Location ref, Location entrypoint)
      : SlowPathCodeRISCV64(instruction), ref_(ref), entrypoint_(entrypoint) {
    DCHECK(entrypoint.IsRegister());
  }

  const char* GetDescription() const override { return "ReadBarrierMarkSlowPathRISCV64"; }

  void EmitNativeCode(CodeGenerator* codegen) override {
    DCHECK(codegen->EmitReadBarrier());
    LocationSummary* locations = instruction_->GetLocations();
    XRegister ref_reg = ref_.AsRegister<XRegister>();
    DCHECK(locations->CanCall());
    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg;
    DCHECK(instruction_->IsInstanceFieldGet() ||
           instruction_->IsStaticFieldGet() ||
           instruction_->IsArrayGet() ||
           instruction_->IsArraySet() ||
           instruction_->IsLoadClass() ||
           instruction_->IsLoadString() ||
           instruction_->IsInstanceOf() ||
           instruction_->IsCheckCast() ||
           (instruction_->IsInvoke() && instruction_->GetLocations()->Intrinsified()))
        << "Unexpected instruction in read barrier marking slow path: "
        << instruction_->DebugName();

    __ Bind(GetEntryLabel());
    // No need to save live registers; it's taken care of by the
    // entrypoint. Also, there is no need to update the stack mask,
    // as this runtime call will not trigger a garbage collection.
    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);
    DCHECK(ref_reg >= T0 && ref_reg != TR);

    // "Compact" slow path, saving two moves.
    //
    // Instead of using the standard runtime calling convention (input
    // and output in A0 and V0 respectively):
    //
    //   A0 <- ref
    //   V0 <- ReadBarrierMark(A0)
    //   ref <- V0
    //
    // we just use rX (the register containing `ref`) as input and output
    // of a dedicated entrypoint:
    //
    //   rX <- ReadBarrierMarkRegX(rX)
    //
    riscv64_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this);
    DCHECK_NE(entrypoint_.AsRegister<XRegister>(), TMP);  // A taken branch can clobber `TMP`.
    __ Jalr(entrypoint_.AsRegister<XRegister>());  // Clobbers `RA` (used as the `entrypoint_`).
    __ J(GetExitLabel());
  }

 private:
  // The location (register) of the marked object reference.
  const Location ref_;

  // The location of the already loaded entrypoint.
  const Location entrypoint_;

  DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathRISCV64);
};

class LoadStringSlowPathRISCV64 : public SlowPathCodeRISCV64 {
 public:
  explicit LoadStringSlowPathRISCV64(HLoadString* instruction)
      : SlowPathCodeRISCV64(instruction) {}

  void EmitNativeCode(CodeGenerator* codegen) override {
    DCHECK(instruction_->IsLoadString());
    DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry);
    LocationSummary* locations = instruction_->GetLocations();
    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
    const dex::StringIndex string_index = instruction_->AsLoadString()->GetStringIndex();
    CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen);
    InvokeRuntimeCallingConvention calling_convention;
    __ Bind(GetEntryLabel());
    SaveLiveRegisters(codegen, locations);

    __ LoadConst32(calling_convention.GetRegisterAt(0), string_index.index_);
    riscv64_codegen->InvokeRuntime(
        kQuickResolveString, instruction_, instruction_->GetDexPc(), this);
    CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();

    DataType::Type type = DataType::Type::kReference;
    DCHECK_EQ(type, instruction_->GetType());
    riscv64_codegen->MoveLocation(
        locations->Out(), calling_convention.GetReturnLocation(type), type);
    RestoreLiveRegisters(codegen, locations);

    __ J(GetExitLabel());
  }

  const char* GetDescription() const override { return "LoadStringSlowPathRISCV64"; }

 private:
  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathRISCV64);
};

#undef __
#define __ down_cast<Riscv64Assembler*>(GetAssembler())->  // NOLINT

template <typename Reg,
          void (Riscv64Assembler::*opS)(Reg, FRegister, FRegister),
          void (Riscv64Assembler::*opD)(Reg, FRegister, FRegister)>
inline void InstructionCodeGeneratorRISCV64::FpBinOp(
    Reg rd, FRegister rs1, FRegister rs2, DataType::Type type) {
  Riscv64Assembler* assembler = down_cast<CodeGeneratorRISCV64*>(codegen_)->GetAssembler();
  if (type == DataType::Type::kFloat32) {
    (assembler->*opS)(rd, rs1, rs2);
  } else {
    DCHECK_EQ(type, DataType::Type::kFloat64);
    (assembler->*opD)(rd, rs1, rs2);
  }
}

void InstructionCodeGeneratorRISCV64::FAdd(
    FRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) {
  FpBinOp<FRegister, &Riscv64Assembler::FAddS, &Riscv64Assembler::FAddD>(rd, rs1, rs2, type);
}

inline void InstructionCodeGeneratorRISCV64::FSub(
    FRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) {
  FpBinOp<FRegister, &Riscv64Assembler::FSubS, &Riscv64Assembler::FSubD>(rd, rs1, rs2, type);
}

inline void InstructionCodeGeneratorRISCV64::FDiv(
    FRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) {
  FpBinOp<FRegister, &Riscv64Assembler::FDivS, &Riscv64Assembler::FDivD>(rd, rs1, rs2, type);
}

inline void InstructionCodeGeneratorRISCV64::FMul(
    FRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) {
  FpBinOp<FRegister, &Riscv64Assembler::FMulS, &Riscv64Assembler::FMulD>(rd, rs1, rs2, type);
}

inline void InstructionCodeGeneratorRISCV64::FMin(
    FRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) {
  FpBinOp<FRegister, &Riscv64Assembler::FMinS, &Riscv64Assembler::FMinD>(rd, rs1, rs2, type);
}

inline void InstructionCodeGeneratorRISCV64::FMax(
    FRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) {
  FpBinOp<FRegister, &Riscv64Assembler::FMaxS, &Riscv64Assembler::FMaxD>(rd, rs1, rs2, type);
}

inline void InstructionCodeGeneratorRISCV64::FEq(
    XRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) {
  FpBinOp<XRegister, &Riscv64Assembler::FEqS, &Riscv64Assembler::FEqD>(rd, rs1, rs2, type);
}

inline void InstructionCodeGeneratorRISCV64::FLt(
    XRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) {
  FpBinOp<XRegister, &Riscv64Assembler::FLtS, &Riscv64Assembler::FLtD>(rd, rs1, rs2, type);
}

inline void InstructionCodeGeneratorRISCV64::FLe(
    XRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) {
  FpBinOp<XRegister, &Riscv64Assembler::FLeS, &Riscv64Assembler::FLeD>(rd, rs1, rs2, type);
}

template <typename Reg,
          void (Riscv64Assembler::*opS)(Reg, FRegister),
          void (Riscv64Assembler::*opD)(Reg, FRegister)>
inline void InstructionCodeGeneratorRISCV64::FpUnOp(
    Reg rd, FRegister rs1, DataType::Type type) {
  Riscv64Assembler* assembler = down_cast<CodeGeneratorRISCV64*>(codegen_)->GetAssembler();
  if (type == DataType::Type::kFloat32) {
    (assembler->*opS)(rd, rs1);
  } else {
    DCHECK_EQ(type, DataType::Type::kFloat64);
    (assembler->*opD)(rd, rs1);
  }
}

inline void InstructionCodeGeneratorRISCV64::FAbs(
    FRegister rd, FRegister rs1, DataType::Type type) {
  FpUnOp<FRegister, &Riscv64Assembler::FAbsS, &Riscv64Assembler::FAbsD>(rd, rs1, type);
}

inline void InstructionCodeGeneratorRISCV64::FNeg(
    FRegister rd, FRegister rs1, DataType::Type type) {
  FpUnOp<FRegister, &Riscv64Assembler::FNegS, &Riscv64Assembler::FNegD>(rd, rs1, type);
}

inline void InstructionCodeGeneratorRISCV64::FMv(
    FRegister rd, FRegister rs1, DataType::Type type) {
  FpUnOp<FRegister, &Riscv64Assembler::FMvS, &Riscv64Assembler::FMvD>(rd, rs1, type);
}

inline void InstructionCodeGeneratorRISCV64::FMvX(
    XRegister rd, FRegister rs1, DataType::Type type) {
  FpUnOp<XRegister, &Riscv64Assembler::FMvXW, &Riscv64Assembler::FMvXD>(rd, rs1, type);
}

void InstructionCodeGeneratorRISCV64::FClass(
    XRegister rd, FRegister rs1, DataType::Type type) {
  FpUnOp<XRegister, &Riscv64Assembler::FClassS, &Riscv64Assembler::FClassD>(rd, rs1, type);
}

void InstructionCodeGeneratorRISCV64::Load(
    Location out, XRegister rs1, int32_t offset, DataType::Type type) {
  switch (type) {
    case DataType::Type::kBool:
    case DataType::Type::kUint8:
      __ Loadbu(out.AsRegister<XRegister>(), rs1, offset);
      break;
    case DataType::Type::kInt8:
      __ Loadb(out.AsRegister<XRegister>(), rs1, offset);
      break;
    case DataType::Type::kUint16:
      __ Loadhu(out.AsRegister<XRegister>(), rs1, offset);
      break;
    case DataType::Type::kInt16:
      __ Loadh(out.AsRegister<XRegister>(), rs1, offset);
      break;
    case DataType::Type::kInt32:
      __ Loadw(out.AsRegister<XRegister>(), rs1, offset);
      break;
    case DataType::Type::kInt64:
      __ Loadd(out.AsRegister<XRegister>(), rs1, offset);
      break;
    case DataType::Type::kReference:
      __ Loadwu(out.AsRegister<XRegister>(), rs1, offset);
      break;
    case DataType::Type::kFloat32:
      __ FLoadw(out.AsFpuRegister<FRegister>(), rs1, offset);
      break;
    case DataType::Type::kFloat64:
      __ FLoadd(out.AsFpuRegister<FRegister>(), rs1, offset);
      break;
    case DataType::Type::kUint32:
    case DataType::Type::kUint64:
    case DataType::Type::kVoid:
      LOG(FATAL) << "Unreachable type " << type;
      UNREACHABLE();
  }
}

void InstructionCodeGeneratorRISCV64::Store(
    Location value, XRegister rs1, int32_t offset, DataType::Type type) {
  DCHECK_IMPLIES(value.IsConstant(), IsZeroBitPattern(value.GetConstant()));
  if (kPoisonHeapReferences && type == DataType::Type::kReference && !value.IsConstant()) {
    riscv64::ScratchRegisterScope srs(GetAssembler());
    XRegister tmp = srs.AllocateXRegister();
    __ Mv(tmp, value.AsRegister<XRegister>());
    codegen_->PoisonHeapReference(tmp);
    __ Storew(tmp, rs1, offset);
    return;
  }
  switch (type) {
    case DataType::Type::kBool:
    case DataType::Type::kUint8:
    case DataType::Type::kInt8:
      __ Storeb(InputXRegisterOrZero(value), rs1, offset);
      break;
    case DataType::Type::kUint16:
    case DataType::Type::kInt16:
      __ Storeh(InputXRegisterOrZero(value), rs1, offset);
      break;
    case DataType::Type::kFloat32:
      if (!value.IsConstant()) {
        __ FStorew(value.AsFpuRegister<FRegister>(), rs1, offset);
        break;
      }
      FALLTHROUGH_INTENDED;
    case DataType::Type::kInt32:
    case DataType::Type::kReference:
      __ Storew(InputXRegisterOrZero(value), rs1, offset);
      break;
    case DataType::Type::kFloat64:
      if (!value.IsConstant()) {
        __ FStored(value.AsFpuRegister<FRegister>(), rs1, offset);
        break;
      }
      FALLTHROUGH_INTENDED;
    case DataType::Type::kInt64:
      __ Stored(InputXRegisterOrZero(value), rs1, offset);
      break;
    case DataType::Type::kUint32:
    case DataType::Type::kUint64:
    case DataType::Type::kVoid:
      LOG(FATAL) << "Unreachable type " << type;
      UNREACHABLE();
  }
}

void InstructionCodeGeneratorRISCV64::StoreSeqCst(Location value,
                                                  XRegister rs1,
                                                  int32_t offset,
                                                  DataType::Type type,
                                                  HInstruction* instruction) {
  if (DataType::Size(type) >= 4u) {
    // Use AMOSWAP for 32-bit and 64-bit data types.
    ScratchRegisterScope srs(GetAssembler());
    XRegister swap_src = kNoXRegister;
    if (kPoisonHeapReferences && type == DataType::Type::kReference && !value.IsConstant()) {
      swap_src = srs.AllocateXRegister();
      __ Mv(swap_src, value.AsRegister<XRegister>());
      codegen_->PoisonHeapReference(swap_src);
    } else if (DataType::IsFloatingPointType(type) && !value.IsConstant()) {
      swap_src = srs.AllocateXRegister();
      FMvX(swap_src, value.AsFpuRegister<FRegister>(), type);
    } else {
      swap_src = InputXRegisterOrZero(value);
    }
    XRegister addr = rs1;
    if (offset != 0) {
      addr = srs.AllocateXRegister();
      __ AddConst64(addr, rs1, offset);
    }
    if (DataType::Is64BitType(type)) {
      __ AmoSwapD(Zero, swap_src, addr, AqRl::kRelease);
    } else {
      __ AmoSwapW(Zero, swap_src, addr, AqRl::kRelease);
    }
    if (instruction != nullptr) {
      codegen_->MaybeRecordImplicitNullCheck(instruction);
    }
  } else {
    // Use fences for smaller data types.
    codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
    Store(value, rs1, offset, type);
    if (instruction != nullptr) {
      codegen_->MaybeRecordImplicitNullCheck(instruction);
    }
    codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
  }
}

void InstructionCodeGeneratorRISCV64::ShNAdd(
    XRegister rd, XRegister rs1, XRegister rs2, DataType::Type type) {
  switch (type) {
    case DataType::Type::kBool:
    case DataType::Type::kUint8:
    case DataType::Type::kInt8:
      DCHECK_EQ(DataType::SizeShift(type), 0u);
      __ Add(rd, rs1, rs2);
      break;
    case DataType::Type::kUint16:
    case DataType::Type::kInt16:
      DCHECK_EQ(DataType::SizeShift(type), 1u);
      __ Sh1Add(rd, rs1, rs2);
      break;
    case DataType::Type::kInt32:
    case DataType::Type::kReference:
    case DataType::Type::kFloat32:
      DCHECK_EQ(DataType::SizeShift(type), 2u);
      __ Sh2Add(rd, rs1, rs2);
      break;
    case DataType::Type::kInt64:
    case DataType::Type::kFloat64:
      DCHECK_EQ(DataType::SizeShift(type), 3u);
      __ Sh3Add(rd, rs1, rs2);
      break;
    case DataType::Type::kUint32:
    case DataType::Type::kUint64:
    case DataType::Type::kVoid:
      LOG(FATAL) << "Unreachable type " << type;
      UNREACHABLE();
  }
}

Riscv64Assembler* ParallelMoveResolverRISCV64::GetAssembler() const {
  return codegen_->GetAssembler();
}

void ParallelMoveResolverRISCV64::EmitMove(size_t index) {
  MoveOperands* move = moves_[index];
  codegen_->MoveLocation(move->GetDestination(), move->GetSource(), move->GetType());
}

void ParallelMoveResolverRISCV64::EmitSwap(size_t index) {
  MoveOperands* move = moves_[index];
  codegen_->SwapLocations(move->GetDestination(), move->GetSource(), move->GetType());
}

void ParallelMoveResolverRISCV64::SpillScratch([[maybe_unused]] int reg) {
  LOG(FATAL) << "Unimplemented";
  UNREACHABLE();
}

void ParallelMoveResolverRISCV64::RestoreScratch([[maybe_unused]] int reg) {
  LOG(FATAL) << "Unimplemented";
  UNREACHABLE();
}

void ParallelMoveResolverRISCV64::Exchange(int index1, int index2, bool double_slot) {
  // We have 2 scratch X registers and 1 scratch F register that we can use. We prefer
  // to use X registers for the swap but if both offsets are too big, we need to reserve
  // one of the X registers for address adjustment and use an F register.
  bool use_fp_tmp2 = false;
  if (!IsInt<12>(index2)) {
    if (!IsInt<12>(index1)) {
      use_fp_tmp2 = true;
    } else {
      std::swap(index1, index2);
    }
  }
  DCHECK_IMPLIES(!IsInt<12>(index2), use_fp_tmp2);

  Location loc1(double_slot ? Location::DoubleStackSlot(index1) : Location::StackSlot(index1));
  Location loc2(double_slot ? Location::DoubleStackSlot(index2) : Location::StackSlot(index2));
  riscv64::ScratchRegisterScope srs(GetAssembler());
  Location tmp = Location::RegisterLocation(srs.AllocateXRegister());
  DataType::Type tmp_type = double_slot ? DataType::Type::kInt64 : DataType::Type::kInt32;
  Location tmp2 = use_fp_tmp2
      ? Location::FpuRegisterLocation(srs.AllocateFRegister())
      : Location::RegisterLocation(srs.AllocateXRegister());
  DataType::Type tmp2_type = use_fp_tmp2
      ? (double_slot ? DataType::Type::kFloat64 : DataType::Type::kFloat32)
      : tmp_type;

  codegen_->MoveLocation(tmp, loc1, tmp_type);
  codegen_->MoveLocation(tmp2, loc2, tmp2_type);
  if (use_fp_tmp2) {
    codegen_->MoveLocation(loc2, tmp, tmp_type);
  } else {
    // We cannot use `Stored()` or `Storew()` via `MoveLocation()` because we have
    // no more scratch registers available. Use `Sd()` or `Sw()` explicitly.
    DCHECK(IsInt<12>(index2));
    if (double_slot) {
      __ Sd(tmp.AsRegister<XRegister>(), SP, index2);
    } else {
      __ Sw(tmp.AsRegister<XRegister>(), SP, index2);
    }
    srs.FreeXRegister(tmp.AsRegister<XRegister>());  // Free a temporary for `MoveLocation()`.
  }
  codegen_->MoveLocation(loc1, tmp2, tmp2_type);
}

InstructionCodeGeneratorRISCV64::InstructionCodeGeneratorRISCV64(HGraph* graph,
                                                                 CodeGeneratorRISCV64* codegen)
    : InstructionCodeGenerator(graph, codegen),
      assembler_(codegen->GetAssembler()),
      codegen_(codegen) {}

void InstructionCodeGeneratorRISCV64::GenerateClassInitializationCheck(
    SlowPathCodeRISCV64* slow_path, XRegister class_reg) {
  ScratchRegisterScope srs(GetAssembler());
  XRegister tmp = srs.AllocateXRegister();
  XRegister tmp2 = srs.AllocateXRegister();

  // We shall load the full 32-bit status word with sign-extension and compare as unsigned
  // to a sign-extended shifted status value. This yields the same comparison as loading and
  // materializing unsigned but the constant is materialized with a single LUI instruction.
  __ Loadw(tmp, class_reg, mirror::Class::StatusOffset().SizeValue());  // Sign-extended.
  __ Li(tmp2, ShiftedSignExtendedClassStatusValue<ClassStatus::kVisiblyInitialized>());
  __ Bltu(tmp, tmp2, slow_path->GetEntryLabel());
  __ Bind(slow_path->GetExitLabel());
}

void InstructionCodeGeneratorRISCV64::GenerateBitstringTypeCheckCompare(
    HTypeCheckInstruction* instruction, XRegister temp) {
  UNUSED(instruction);
  UNUSED(temp);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::GenerateSuspendCheck(HSuspendCheck* instruction,
                                                           HBasicBlock* successor) {
  if (instruction->IsNoOp()) {
    if (successor != nullptr) {
      __ J(codegen_->GetLabelOf(successor));
    }
    return;
  }

  if (codegen_->CanUseImplicitSuspendCheck()) {
    LOG(FATAL) << "Unimplemented ImplicitSuspendCheck";
    return;
  }

  SuspendCheckSlowPathRISCV64* slow_path =
      down_cast<SuspendCheckSlowPathRISCV64*>(instruction->GetSlowPath());

  if (slow_path == nullptr) {
    slow_path =
        new (codegen_->GetScopedAllocator()) SuspendCheckSlowPathRISCV64(instruction, successor);
    instruction->SetSlowPath(slow_path);
    codegen_->AddSlowPath(slow_path);
    if (successor != nullptr) {
      DCHECK(successor->IsLoopHeader());
    }
  } else {
    DCHECK_EQ(slow_path->GetSuccessor(), successor);
  }

  ScratchRegisterScope srs(GetAssembler());
  XRegister tmp = srs.AllocateXRegister();
  __ Loadw(tmp, TR, Thread::ThreadFlagsOffset<kRiscv64PointerSize>().Int32Value());
  static_assert(Thread::SuspendOrCheckpointRequestFlags() != std::numeric_limits<uint32_t>::max());
  static_assert(IsPowerOfTwo(Thread::SuspendOrCheckpointRequestFlags() + 1u));
  // Shift out other bits. Use an instruction that can be 16-bit with the "C" Standard Extension.
  __ Slli(tmp, tmp, CLZ(static_cast<uint64_t>(Thread::SuspendOrCheckpointRequestFlags())));
  if (successor == nullptr) {
    __ Bnez(tmp, slow_path->GetEntryLabel());
    __ Bind(slow_path->GetReturnLabel());
  } else {
    __ Beqz(tmp, codegen_->GetLabelOf(successor));
    __ J(slow_path->GetEntryLabel());
    // slow_path will return to GetLabelOf(successor).
  }
}

void InstructionCodeGeneratorRISCV64::GenerateReferenceLoadOneRegister(
    HInstruction* instruction,
    Location out,
    uint32_t offset,
    Location maybe_temp,
    ReadBarrierOption read_barrier_option) {
  XRegister out_reg = out.AsRegister<XRegister>();
  if (read_barrier_option == kWithReadBarrier) {
    DCHECK(codegen_->EmitReadBarrier());
    if (kUseBakerReadBarrier) {
      // Load with fast path based Baker's read barrier.
      // /* HeapReference<Object> */ out = *(out + offset)
      codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction,
                                                      out,
                                                      out_reg,
                                                      offset,
                                                      maybe_temp,
                                                      /* needs_null_check= */ false);
    } else {
      // Load with slow path based read barrier.
      // Save the value of `out` into `maybe_temp` before overwriting it
      // in the following move operation, as we will need it for the
      // read barrier below.
      __ Mv(maybe_temp.AsRegister<XRegister>(), out_reg);
      // /* HeapReference<Object> */ out = *(out + offset)
      __ Loadwu(out_reg, out_reg, offset);
      codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset);
    }
  } else {
    // Plain load with no read barrier.
    // /* HeapReference<Object> */ out = *(out + offset)
    __ Loadwu(out_reg, out_reg, offset);
    codegen_->MaybeUnpoisonHeapReference(out_reg);
  }
}

void InstructionCodeGeneratorRISCV64::GenerateReferenceLoadTwoRegisters(
    HInstruction* instruction,
    Location out,
    Location obj,
    uint32_t offset,
    Location maybe_temp,
    ReadBarrierOption read_barrier_option) {
  XRegister out_reg = out.AsRegister<XRegister>();
  XRegister obj_reg = obj.AsRegister<XRegister>();
  if (read_barrier_option == kWithReadBarrier) {
    DCHECK(codegen_->EmitReadBarrier());
    if (kUseBakerReadBarrier) {
      // Load with fast path based Baker's read barrier.
      // /* HeapReference<Object> */ out = *(obj + offset)
      codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction,
                                                      out,
                                                      obj_reg,
                                                      offset,
                                                      maybe_temp,
                                                      /* needs_null_check= */ false);
    } else {
      // Load with slow path based read barrier.
      // /* HeapReference<Object> */ out = *(obj + offset)
      __ Loadwu(out_reg, obj_reg, offset);
      codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset);
    }
  } else {
    // Plain load with no read barrier.
    // /* HeapReference<Object> */ out = *(obj + offset)
    __ Loadwu(out_reg, obj_reg, offset);
    codegen_->MaybeUnpoisonHeapReference(out_reg);
  }
}

SlowPathCodeRISCV64* CodeGeneratorRISCV64::AddGcRootBakerBarrierBarrierSlowPath(
    HInstruction* instruction, Location root, Location temp) {
  SlowPathCodeRISCV64* slow_path =
      new (GetScopedAllocator()) ReadBarrierMarkSlowPathRISCV64(instruction, root, temp);
  AddSlowPath(slow_path);
  return slow_path;
}

void CodeGeneratorRISCV64::EmitBakerReadBarierMarkingCheck(
    SlowPathCodeRISCV64* slow_path, Location root, Location temp) {
  const int32_t entry_point_offset = ReadBarrierMarkEntrypointOffset(root);
  // Loading the entrypoint does not require a load acquire since it is only changed when
  // threads are suspended or running a checkpoint.
  __ Loadd(temp.AsRegister<XRegister>(), TR, entry_point_offset);
  __ Bnez(temp.AsRegister<XRegister>(), slow_path->GetEntryLabel());
  __ Bind(slow_path->GetExitLabel());
}

void CodeGeneratorRISCV64::GenerateGcRootFieldLoad(HInstruction* instruction,
                                                   Location root,
                                                   XRegister obj,
                                                   uint32_t offset,
                                                   ReadBarrierOption read_barrier_option,
                                                   Riscv64Label* label_low) {
  DCHECK_IMPLIES(label_low != nullptr, offset == kLinkTimeOffsetPlaceholderLow) << offset;
  XRegister root_reg = root.AsRegister<XRegister>();
  if (read_barrier_option == kWithReadBarrier) {
    DCHECK(EmitReadBarrier());
    if (kUseBakerReadBarrier) {
      // Note that we do not actually check the value of `GetIsGcMarking()`
      // to decide whether to mark the loaded GC root or not.  Instead, we
      // load into `temp` (T6) the read barrier mark entry point corresponding
      // to register `root`. If `temp` is null, it means that `GetIsGcMarking()`
      // is false, and vice versa.
      //
      //     GcRoot<mirror::Object> root = *(obj+offset);  // Original reference load.
      //     temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
      //     if (temp != null) {
      //       root = temp(root)
      //     }
      //
      // TODO(riscv64): Introduce a "marking register" that holds the pointer to one of the
      // register marking entrypoints if marking (null if not marking) and make sure that
      // marking entrypoints for other registers are at known offsets, so that we can call
      // them using the "marking register" plus the offset embedded in the JALR instruction.

      if (label_low != nullptr) {
        __ Bind(label_low);
      }
      // /* GcRoot<mirror::Object> */ root = *(obj + offset)
      __ Loadwu(root_reg, obj, offset);
      static_assert(
          sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
          "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> "
          "have different sizes.");
      static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t),
                    "art::mirror::CompressedReference<mirror::Object> and int32_t "
                    "have different sizes.");

      // Use RA as temp. It is clobbered in the slow path anyway.
      Location temp = Location::RegisterLocation(RA);
      SlowPathCodeRISCV64* slow_path =
          AddGcRootBakerBarrierBarrierSlowPath(instruction, root, temp);
      EmitBakerReadBarierMarkingCheck(slow_path, root, temp);
    } else {
      // GC root loaded through a slow path for read barriers other
      // than Baker's.
      // /* GcRoot<mirror::Object>* */ root = obj + offset
      if (label_low != nullptr) {
        __ Bind(label_low);
      }
      __ AddConst32(root_reg, obj, offset);
      // /* mirror::Object* */ root = root->Read()
      GenerateReadBarrierForRootSlow(instruction, root, root);
    }
  } else {
    // Plain GC root load with no read barrier.
    // /* GcRoot<mirror::Object> */ root = *(obj + offset)
    if (label_low != nullptr) {
      __ Bind(label_low);
    }
    __ Loadwu(root_reg, obj, offset);
    // Note that GC roots are not affected by heap poisoning, thus we
    // do not have to unpoison `root_reg` here.
  }
}

void InstructionCodeGeneratorRISCV64::GenerateTestAndBranch(HInstruction* instruction,
                                                            size_t condition_input_index,
                                                            Riscv64Label* true_target,
                                                            Riscv64Label* false_target) {
  HInstruction* cond = instruction->InputAt(condition_input_index);

  if (true_target == nullptr && false_target == nullptr) {
    // Nothing to do. The code always falls through.
    return;
  } else if (cond->IsIntConstant()) {
    // Constant condition, statically compared against "true" (integer value 1).
    if (cond->AsIntConstant()->IsTrue()) {
      if (true_target != nullptr) {
        __ J(true_target);
      }
    } else {
      DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue();
      if (false_target != nullptr) {
        __ J(false_target);
      }
    }
    return;
  }

  // The following code generates these patterns:
  //  (1) true_target == nullptr && false_target != nullptr
  //        - opposite condition true => branch to false_target
  //  (2) true_target != nullptr && false_target == nullptr
  //        - condition true => branch to true_target
  //  (3) true_target != nullptr && false_target != nullptr
  //        - condition true => branch to true_target
  //        - branch to false_target
  if (IsBooleanValueOrMaterializedCondition(cond)) {
    // The condition instruction has been materialized, compare the output to 0.
    Location cond_val = instruction->GetLocations()->InAt(condition_input_index);
    DCHECK(cond_val.IsRegister());
    if (true_target == nullptr) {
      __ Beqz(cond_val.AsRegister<XRegister>(), false_target);
    } else {
      __ Bnez(cond_val.AsRegister<XRegister>(), true_target);
    }
  } else {
    // The condition instruction has not been materialized, use its inputs as
    // the comparison and its condition as the branch condition.
    HCondition* condition = cond->AsCondition();
    DataType::Type type = condition->InputAt(0)->GetType();
    LocationSummary* locations = condition->GetLocations();
    IfCondition if_cond = condition->GetCondition();
    Riscv64Label* branch_target = true_target;

    if (true_target == nullptr) {
      if_cond = condition->GetOppositeCondition();
      branch_target = false_target;
    }

    switch (type) {
      case DataType::Type::kFloat32:
      case DataType::Type::kFloat64:
        GenerateFpCondition(if_cond, condition->IsGtBias(), type, locations, branch_target);
        break;
      default:
        // Integral types and reference equality.
        GenerateIntLongCompareAndBranch(if_cond, locations, branch_target);
        break;
    }
  }

  // If neither branch falls through (case 3), the conditional branch to `true_target`
  // was already emitted (case 2) and we need to emit a jump to `false_target`.
  if (true_target != nullptr && false_target != nullptr) {
    __ J(false_target);
  }
}

void InstructionCodeGeneratorRISCV64::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
  DCHECK(instruction->IsDiv() || instruction->IsRem());
  DataType::Type type = instruction->GetResultType();

  LocationSummary* locations = instruction->GetLocations();
  Location second = locations->InAt(1);
  DCHECK(second.IsConstant());

  XRegister out = locations->Out().AsRegister<XRegister>();
  XRegister dividend = locations->InAt(0).AsRegister<XRegister>();
  int64_t imm = Int64FromConstant(second.GetConstant());
  DCHECK(imm == 1 || imm == -1);

  if (instruction->IsRem()) {
    __ Mv(out, Zero);
  } else {
    if (imm == -1) {
      if (type == DataType::Type::kInt32) {
        __ Subw(out, Zero, dividend);
      } else {
        DCHECK_EQ(type, DataType::Type::kInt64);
        __ Sub(out, Zero, dividend);
      }
    } else if (out != dividend) {
      __ Mv(out, dividend);
    }
  }
}

void InstructionCodeGeneratorRISCV64::DivRemByPowerOfTwo(HBinaryOperation* instruction) {
  DCHECK(instruction->IsDiv() || instruction->IsRem());
  DataType::Type type = instruction->GetResultType();
  DCHECK(type == DataType::Type::kInt32 || type == DataType::Type::kInt64) << type;

  LocationSummary* locations = instruction->GetLocations();
  Location second = locations->InAt(1);
  DCHECK(second.IsConstant());

  XRegister out = locations->Out().AsRegister<XRegister>();
  XRegister dividend = locations->InAt(0).AsRegister<XRegister>();
  int64_t imm = Int64FromConstant(second.GetConstant());
  int64_t abs_imm = static_cast<uint64_t>(AbsOrMin(imm));
  int ctz_imm = CTZ(abs_imm);
  DCHECK_GE(ctz_imm, 1);  // Division by +/-1 is handled by `DivRemOneOrMinusOne()`.

  ScratchRegisterScope srs(GetAssembler());
  XRegister tmp = srs.AllocateXRegister();
  // Calculate the negative dividend adjustment `tmp = dividend < 0 ? abs_imm - 1 : 0`.
  // This adjustment is needed for rounding the division result towards zero.
  if (type == DataType::Type::kInt32 || ctz_imm == 1) {
    // A 32-bit dividend is sign-extended to 64-bit, so we can use the upper bits.
    // And for a 64-bit division by +/-2, we need just the sign bit.
    DCHECK_IMPLIES(type == DataType::Type::kInt32, ctz_imm < 32);
    __ Srli(tmp, dividend, 64 - ctz_imm);
  } else {
    // For other 64-bit divisions, we need to replicate the sign bit.
    __ Srai(tmp, dividend, 63);
    __ Srli(tmp, tmp, 64 - ctz_imm);
  }
  // The rest of the calculation can use 64-bit operations even for 32-bit div/rem.
  __ Add(tmp, tmp, dividend);
  if (instruction->IsDiv()) {
    __ Srai(out, tmp, ctz_imm);
    if (imm < 0) {
      __ Neg(out, out);
    }
  } else {
    if (ctz_imm <= 11) {
      __ Andi(tmp, tmp, -abs_imm);
    } else {
      ScratchRegisterScope srs2(GetAssembler());
      XRegister tmp2 = srs2.AllocateXRegister();
      __ Li(tmp2, -abs_imm);
      __ And(tmp, tmp, tmp2);
    }
    __ Sub(out, dividend, tmp);
  }
}

void InstructionCodeGeneratorRISCV64::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
  DCHECK(instruction->IsDiv() || instruction->IsRem());
  LocationSummary* locations = instruction->GetLocations();
  XRegister dividend = locations->InAt(0).AsRegister<XRegister>();
  XRegister out = locations->Out().AsRegister<XRegister>();
  Location second = locations->InAt(1);
  int64_t imm = Int64FromConstant(second.GetConstant());
  DataType::Type type = instruction->GetResultType();
  ScratchRegisterScope srs(GetAssembler());
  XRegister tmp = srs.AllocateXRegister();

  // TODO: optimize with constant.
  __ LoadConst64(tmp, imm);
  if (instruction->IsDiv()) {
    if (type == DataType::Type::kInt32) {
      __ Divw(out, dividend, tmp);
    } else {
      __ Div(out, dividend, tmp);
    }
  } else {
    if (type == DataType::Type::kInt32)  {
      __ Remw(out, dividend, tmp);
    } else {
      __ Rem(out, dividend, tmp);
    }
  }
}

void InstructionCodeGeneratorRISCV64::GenerateDivRemIntegral(HBinaryOperation* instruction) {
  DCHECK(instruction->IsDiv() || instruction->IsRem());
  DataType::Type type = instruction->GetResultType();
  DCHECK(type == DataType::Type::kInt32 || type == DataType::Type::kInt64) << type;

  LocationSummary* locations = instruction->GetLocations();
  XRegister out = locations->Out().AsRegister<XRegister>();
  Location second = locations->InAt(1);

  if (second.IsConstant()) {
    int64_t imm = Int64FromConstant(second.GetConstant());
    if (imm == 0) {
      // Do not generate anything. DivZeroCheck would prevent any code to be executed.
    } else if (imm == 1 || imm == -1) {
      DivRemOneOrMinusOne(instruction);
    } else if (IsPowerOfTwo(AbsOrMin(imm))) {
      DivRemByPowerOfTwo(instruction);
    } else {
      DCHECK(imm <= -2 || imm >= 2);
      GenerateDivRemWithAnyConstant(instruction);
    }
  } else {
    XRegister dividend = locations->InAt(0).AsRegister<XRegister>();
    XRegister divisor = second.AsRegister<XRegister>();
    if (instruction->IsDiv()) {
      if (type == DataType::Type::kInt32) {
        __ Divw(out, dividend, divisor);
      } else {
        __ Div(out, dividend, divisor);
      }
    } else {
      if (type == DataType::Type::kInt32) {
        __ Remw(out, dividend, divisor);
      } else {
        __ Rem(out, dividend, divisor);
      }
    }
  }
}

void InstructionCodeGeneratorRISCV64::GenerateIntLongCondition(IfCondition cond,
                                                               LocationSummary* locations) {
  XRegister rd = locations->Out().AsRegister<XRegister>();
  GenerateIntLongCondition(cond, locations, rd, /*to_all_bits=*/ false);
}

void InstructionCodeGeneratorRISCV64::GenerateIntLongCondition(IfCondition cond,
                                                               LocationSummary* locations,
                                                               XRegister rd,
                                                               bool to_all_bits) {
  XRegister rs1 = locations->InAt(0).AsRegister<XRegister>();
  Location rs2_location = locations->InAt(1);
  bool use_imm = rs2_location.IsConstant();
  int64_t imm = use_imm ? CodeGenerator::GetInt64ValueOf(rs2_location.GetConstant()) : 0;
  XRegister rs2 = use_imm ? kNoXRegister : rs2_location.AsRegister<XRegister>();
  bool reverse_condition = false;
  switch (cond) {
    case kCondEQ:
    case kCondNE:
      if (!use_imm) {
        __ Sub(rd, rs1, rs2);  // SUB is OK here even for 32-bit comparison.
      } else if (imm != 0) {
        DCHECK(IsInt<12>(-imm));
        __ Addi(rd, rs1, -imm);  // ADDI is OK here even for 32-bit comparison.
      }  // else test `rs1` directly without subtraction for `use_imm && imm == 0`.
      if (cond == kCondEQ) {
        __ Seqz(rd, (use_imm && imm == 0) ? rs1 : rd);
      } else {
        __ Snez(rd, (use_imm && imm == 0) ? rs1 : rd);
      }
      break;

    case kCondLT:
    case kCondGE:
      if (use_imm) {
        DCHECK(IsInt<12>(imm));
        __ Slti(rd, rs1, imm);
      } else {
        __ Slt(rd, rs1, rs2);
      }
      // Calculate `rs1 >= rhs` as `!(rs1 < rhs)` since there's only the SLT but no SGE.
      reverse_condition = (cond == kCondGE);
      break;

    case kCondLE:
    case kCondGT:
      if (use_imm) {
        // Calculate `rs1 <= imm` as `rs1 < imm + 1`.
        DCHECK(IsInt<12>(imm + 1));  // The value that overflows would fail this check.
        __ Slti(rd, rs1, imm + 1);
      } else {
        __ Slt(rd, rs2, rs1);
      }
      // Calculate `rs1 > imm` as `!(rs1 < imm + 1)` and calculate
      // `rs1 <= rs2` as `!(rs2 < rs1)` since there's only the SLT but no SGE.
      reverse_condition = ((cond == kCondGT) == use_imm);
      break;

    case kCondB:
    case kCondAE:
      if (use_imm) {
        // Sltiu sign-extends its 12-bit immediate operand before the comparison
        // and thus lets us compare directly with unsigned values in the ranges
        // [0, 0x7ff] and [0x[ffffffff]fffff800, 0x[ffffffff]ffffffff].
        DCHECK(IsInt<12>(imm));
        __ Sltiu(rd, rs1, imm);
      } else {
        __ Sltu(rd, rs1, rs2);
      }
      // Calculate `rs1 AE rhs` as `!(rs1 B rhs)` since there's only the SLTU but no SGEU.
      reverse_condition = (cond == kCondAE);
      break;

    case kCondBE:
    case kCondA:
      if (use_imm) {
        // Calculate `rs1 BE imm` as `rs1 B imm + 1`.
        // Sltiu sign-extends its 12-bit immediate operand before the comparison
        // and thus lets us compare directly with unsigned values in the ranges
        // [0, 0x7ff] and [0x[ffffffff]fffff800, 0x[ffffffff]ffffffff].
        DCHECK(IsInt<12>(imm + 1));  // The value that overflows would fail this check.
        __ Sltiu(rd, rs1, imm + 1);
      } else {
        __ Sltu(rd, rs2, rs1);
      }
      // Calculate `rs1 A imm` as `!(rs1 B imm + 1)` and calculate
      // `rs1 BE rs2` as `!(rs2 B rs1)` since there's only the SLTU but no SGEU.
      reverse_condition = ((cond == kCondA) == use_imm);
      break;
  }
  if (to_all_bits) {
    // Store the result to all bits; in other words, "true" is represented by -1.
    if (reverse_condition) {
      __ Addi(rd, rd, -1);  // 0 -> -1, 1 -> 0
    } else {
      __ Neg(rd, rd);  // 0 -> 0, 1 -> -1
    }
  } else {
    if (reverse_condition) {
      __ Xori(rd, rd, 1);
    }
  }
}

void InstructionCodeGeneratorRISCV64::GenerateIntLongCompareAndBranch(IfCondition cond,
                                                                      LocationSummary* locations,
                                                                      Riscv64Label* label) {
  XRegister left = locations->InAt(0).AsRegister<XRegister>();
  Location right_location = locations->InAt(1);
  if (right_location.IsConstant()) {
    DCHECK_EQ(CodeGenerator::GetInt64ValueOf(right_location.GetConstant()), 0);
    switch (cond) {
      case kCondEQ:
      case kCondBE:  // <= 0 if zero
        __ Beqz(left, label);
        break;
      case kCondNE:
      case kCondA:  // > 0 if non-zero
        __ Bnez(left, label);
        break;
      case kCondLT:
        __ Bltz(left, label);
        break;
      case kCondGE:
        __ Bgez(left, label);
        break;
      case kCondLE:
        __ Blez(left, label);
        break;
      case kCondGT:
        __ Bgtz(left, label);
        break;
      case kCondB:  // always false
        break;
      case kCondAE:  // always true
        __ J(label);
        break;
    }
  } else {
    XRegister right_reg = right_location.AsRegister<XRegister>();
    switch (cond) {
      case kCondEQ:
        __ Beq(left, right_reg, label);
        break;
      case kCondNE:
        __ Bne(left, right_reg, label);
        break;
      case kCondLT:
        __ Blt(left, right_reg, label);
        break;
      case kCondGE:
        __ Bge(left, right_reg, label);
        break;
      case kCondLE:
        __ Ble(left, right_reg, label);
        break;
      case kCondGT:
        __ Bgt(left, right_reg, label);
        break;
      case kCondB:
        __ Bltu(left, right_reg, label);
        break;
      case kCondAE:
        __ Bgeu(left, right_reg, label);
        break;
      case kCondBE:
        __ Bleu(left, right_reg, label);
        break;
      case kCondA:
        __ Bgtu(left, right_reg, label);
        break;
    }
  }
}

void InstructionCodeGeneratorRISCV64::GenerateFpCondition(IfCondition cond,
                                                          bool gt_bias,
                                                          DataType::Type type,
                                                          LocationSummary* locations,
                                                          Riscv64Label* label) {
  DCHECK_EQ(label != nullptr, locations->Out().IsInvalid());
  ScratchRegisterScope srs(GetAssembler());
  XRegister rd =
      (label != nullptr) ? srs.AllocateXRegister() : locations->Out().AsRegister<XRegister>();
  GenerateFpCondition(cond, gt_bias, type, locations, label, rd, /*to_all_bits=*/ false);
}

void InstructionCodeGeneratorRISCV64::GenerateFpCondition(IfCondition cond,
                                                          bool gt_bias,
                                                          DataType::Type type,
                                                          LocationSummary* locations,
                                                          Riscv64Label* label,
                                                          XRegister rd,
                                                          bool to_all_bits) {
  // RISCV-V FP compare instructions yield the following values:
  //                      l<r  l=r  l>r Unordered
  //             FEQ l,r   0    1    0    0
  //             FLT l,r   1    0    0    0
  //             FLT r,l   0    0    1    0
  //             FLE l,r   1    1    0    0
  //             FLE r,l   0    1    1    0
  //
  // We can calculate the `Compare` results using the following formulas:
  //                      l<r  l=r  l>r Unordered
  //     Compare/gt_bias  -1    0    1    1       = ((FLE l,r) ^ 1) - (FLT l,r)
  //     Compare/lt_bias  -1    0    1   -1       = ((FLE r,l) - 1) + (FLT r,l)
  // These are emitted in `VisitCompare()`.
  //
  // This function emits a fused `Condition(Compare(., .), 0)`. If we compare the
  // `Compare` results above with 0, we get the following values and formulas:
  //                      l<r  l=r  l>r Unordered
  //     CondEQ/-          0    1    0    0       = (FEQ l, r)
  //     CondNE/-          1    0    1    1       = (FEQ l, r) ^ 1
  //     CondLT/gt_bias    1    0    0    0       = (FLT l,r)
  //     CondLT/lt_bias    1    0    0    1       = (FLE r,l) ^ 1
  //     CondLE/gt_bias    1    1    0    0       = (FLE l,r)
  //     CondLE/lt_bias    1    1    0    1       = (FLT r,l) ^ 1
  //     CondGT/gt_bias    0    0    1    1       = (FLE l,r) ^ 1
  //     CondGT/lt_bias    0    0    1    0       = (FLT r,l)
  //     CondGE/gt_bias    0    1    1    1       = (FLT l,r) ^ 1
  //     CondGE/lt_bias    0    1    1    0       = (FLE r,l)
  // (CondEQ/CondNE comparison with zero yields the same result with gt_bias and lt_bias.)
  //
  // If the condition is not materialized, the `^ 1` is not emitted,
  // instead the condition is reversed by emitting BEQZ instead of BNEZ.

  FRegister rs1 = locations->InAt(0).AsFpuRegister<FRegister>();
  FRegister rs2 = locations->InAt(1).AsFpuRegister<FRegister>();

  bool reverse_condition = false;
  switch (cond) {
    case kCondEQ:
      FEq(rd, rs1, rs2, type);
      break;
    case kCondNE:
      FEq(rd, rs1, rs2, type);
      reverse_condition = true;
      break;
    case kCondLT:
      if (gt_bias) {
        FLt(rd, rs1, rs2, type);
      } else {
        FLe(rd, rs2, rs1, type);
        reverse_condition = true;
      }
      break;
    case kCondLE:
      if (gt_bias) {
        FLe(rd, rs1, rs2, type);
      } else {
        FLt(rd, rs2, rs1, type);
        reverse_condition = true;
      }
      break;
    case kCondGT:
      if (gt_bias) {
        FLe(rd, rs1, rs2, type);
        reverse_condition = true;
      } else {
        FLt(rd, rs2, rs1, type);
      }
      break;
    case kCondGE:
      if (gt_bias) {
        FLt(rd, rs1, rs2, type);
        reverse_condition = true;
      } else {
        FLe(rd, rs2, rs1, type);
      }
      break;
    default:
      LOG(FATAL) << "Unexpected floating-point condition " << cond;
      UNREACHABLE();
  }

  if (label != nullptr) {
    if (reverse_condition) {
      __ Beqz(rd, label);
    } else {
      __ Bnez(rd, label);
    }
  } else if (to_all_bits) {
    // Store the result to all bits; in other words, "true" is represented by -1.
    if (reverse_condition) {
      __ Addi(rd, rd, -1);  // 0 -> -1, 1 -> 0
    } else {
      __ Neg(rd, rd);  // 0 -> 0, 1 -> -1
    }
  } else {
    if (reverse_condition) {
      __ Xori(rd, rd, 1);
    }
  }
}

void CodeGeneratorRISCV64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
                                                                 Location ref,
                                                                 XRegister obj,
                                                                 uint32_t offset,
                                                                 Location temp,
                                                                 bool needs_null_check) {
  GenerateReferenceLoadWithBakerReadBarrier(
      instruction, ref, obj, offset, /*index=*/ Location::NoLocation(), temp, needs_null_check);
}

void CodeGeneratorRISCV64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction,
                                                                 Location ref,
                                                                 XRegister obj,
                                                                 uint32_t data_offset,
                                                                 Location index,
                                                                 Location temp,
                                                                 bool needs_null_check) {
  GenerateReferenceLoadWithBakerReadBarrier(
      instruction, ref, obj, data_offset, index, temp, needs_null_check);
}

void CodeGeneratorRISCV64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction,
                                                                     Location ref,
                                                                     XRegister obj,
                                                                     uint32_t offset,
                                                                     Location index,
                                                                     Location temp,
                                                                     bool needs_null_check) {
  // For now, use the same approach as for GC roots plus unpoison the reference if needed.
  // TODO(riscv64): Implement checking if the holder is black.
  UNUSED(temp);

  DCHECK(EmitBakerReadBarrier());
  XRegister reg = ref.AsRegister<XRegister>();
  if (index.IsValid()) {
    DCHECK(!needs_null_check);
    DCHECK(index.IsRegister());
    DataType::Type type = DataType::Type::kReference;
    DCHECK_EQ(type, instruction->GetType());
    if (instruction->IsArrayGet()) {
      // /* HeapReference<Object> */ ref = *(obj + index * element_size + offset)
      instruction_visitor_.ShNAdd(reg, index.AsRegister<XRegister>(), obj, type);
    } else {
      // /* HeapReference<Object> */ ref = *(obj + index + offset)
      DCHECK(instruction->IsInvoke());
      DCHECK(instruction->GetLocations()->Intrinsified());
      __ Add(reg, index.AsRegister<XRegister>(), obj);
    }
    __ Loadwu(reg, reg, offset);
  } else {
    // /* HeapReference<Object> */ ref = *(obj + offset)
    __ Loadwu(reg, obj, offset);
    if (needs_null_check) {
      MaybeRecordImplicitNullCheck(instruction);
    }
  }
  MaybeUnpoisonHeapReference(reg);

  // Slow path marking the reference.
  XRegister tmp = RA;  // Use RA as temp. It is clobbered in the slow path anyway.
  SlowPathCodeRISCV64* slow_path = new (GetScopedAllocator()) ReadBarrierMarkSlowPathRISCV64(
      instruction, ref, Location::RegisterLocation(tmp));
  AddSlowPath(slow_path);

  const int32_t entry_point_offset = ReadBarrierMarkEntrypointOffset(ref);
  // Loading the entrypoint does not require a load acquire since it is only changed when
  // threads are suspended or running a checkpoint.
  __ Loadd(tmp, TR, entry_point_offset);
  __ Bnez(tmp, slow_path->GetEntryLabel());
  __ Bind(slow_path->GetExitLabel());
}

SlowPathCodeRISCV64* CodeGeneratorRISCV64::AddReadBarrierSlowPath(HInstruction* instruction,
                                                                  Location out,
                                                                  Location ref,
                                                                  Location obj,
                                                                  uint32_t offset,
                                                                  Location index) {
  UNUSED(instruction);
  UNUSED(out);
  UNUSED(ref);
  UNUSED(obj);
  UNUSED(offset);
  UNUSED(index);
  LOG(FATAL) << "Unimplemented";
  UNREACHABLE();
}

void CodeGeneratorRISCV64::GenerateReadBarrierSlow(HInstruction* instruction,
                                                   Location out,
                                                   Location ref,
                                                   Location obj,
                                                   uint32_t offset,
                                                   Location index) {
  UNUSED(instruction);
  UNUSED(out);
  UNUSED(ref);
  UNUSED(obj);
  UNUSED(offset);
  UNUSED(index);
  LOG(FATAL) << "Unimplemented";
}

void CodeGeneratorRISCV64::MaybeGenerateReadBarrierSlow(HInstruction* instruction,
                                                        Location out,
                                                        Location ref,
                                                        Location obj,
                                                        uint32_t offset,
                                                        Location index) {
  if (EmitReadBarrier()) {
    // Baker's read barriers shall be handled by the fast path
    // (CodeGeneratorRISCV64::GenerateReferenceLoadWithBakerReadBarrier).
    DCHECK(!kUseBakerReadBarrier);
    // If heap poisoning is enabled, unpoisoning will be taken care of
    // by the runtime within the slow path.
    GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index);
  } else if (kPoisonHeapReferences) {
    UnpoisonHeapReference(out.AsRegister<XRegister>());
  }
}

void CodeGeneratorRISCV64::GenerateReadBarrierForRootSlow(HInstruction* instruction,
                                                          Location out,
                                                          Location root) {
  DCHECK(EmitReadBarrier());

  // Insert a slow path based read barrier *after* the GC root load.
  //
  // Note that GC roots are not affected by heap poisoning, so we do
  // not need to do anything special for this here.
  SlowPathCodeRISCV64* slow_path =
      new (GetScopedAllocator()) ReadBarrierForRootSlowPathRISCV64(instruction, out, root);
  AddSlowPath(slow_path);

  __ J(slow_path->GetEntryLabel());
  __ Bind(slow_path->GetExitLabel());
}

void InstructionCodeGeneratorRISCV64::HandleGoto(HInstruction* instruction,
                                                 HBasicBlock* successor) {
  if (successor->IsExitBlock()) {
    DCHECK(instruction->GetPrevious()->AlwaysThrows());
    return;  // no code needed
  }

  HBasicBlock* block = instruction->GetBlock();
  HInstruction* previous = instruction->GetPrevious();
  HLoopInformation* info = block->GetLoopInformation();

  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
    codegen_->MaybeIncrementHotness(info->GetSuspendCheck(), /*is_frame_entry=*/ false);
    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
    return;  // `GenerateSuspendCheck()` emitted the jump.
  }
  if (block->IsEntryBlock() && previous != nullptr && previous->IsSuspendCheck()) {
    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
  }
  if (!codegen_->GoesToNextBlock(block, successor)) {
    __ J(codegen_->GetLabelOf(successor));
  }
}

void InstructionCodeGeneratorRISCV64::GenPackedSwitchWithCompares(XRegister adjusted,
                                                                  XRegister temp,
                                                                  uint32_t num_entries,
                                                                  HBasicBlock* switch_block) {
  // Note: The `adjusted` register holds `value - lower_bound`. If the `lower_bound` is 0,
  // `adjusted` is the original `value` register and we must not clobber it. Otherwise,
  // `adjusted` is the `temp`. The caller already emitted the `adjusted < num_entries` check.

  // Create a set of compare/jumps.
  ArrayRef<HBasicBlock* const> successors(switch_block->GetSuccessors());
  uint32_t index = 0;
  for (; num_entries - index >= 2u; index += 2u) {
    // Jump to `successors[index]` if `value == lower_bound + index`.
    // Note that `adjusted` holds `value - lower_bound - index`.
    __ Beqz(adjusted, codegen_->GetLabelOf(successors[index]));
    if (num_entries - index == 2u) {
      break;  // The last entry shall match, so the branch shall be unconditional.
    }
    // Jump to `successors[index + 1]` if `value == lower_bound + index + 1`.
    // Modify `adjusted` to hold `value - lower_bound - index - 2` for this comparison.
    __ Addi(temp, adjusted, -2);
    adjusted = temp;
    __ Bltz(adjusted, codegen_->GetLabelOf(successors[index + 1]));
  }
  // For the last entry, unconditionally jump to `successors[num_entries - 1]`.
  __ J(codegen_->GetLabelOf(successors[num_entries - 1u]));
}

void InstructionCodeGeneratorRISCV64::GenTableBasedPackedSwitch(XRegister adjusted,
                                                                XRegister temp,
                                                                uint32_t num_entries,
                                                                HBasicBlock* switch_block) {
  // Note: The `adjusted` register holds `value - lower_bound`. If the `lower_bound` is 0,
  // `adjusted` is the original `value` register and we must not clobber it. Otherwise,
  // `adjusted` is the `temp`. The caller already emitted the `adjusted < num_entries` check.

  // Create a jump table.
  ArenaVector<Riscv64Label*> labels(num_entries,
                                    __ GetAllocator()->Adapter(kArenaAllocSwitchTable));
  const ArenaVector<HBasicBlock*>& successors = switch_block->GetSuccessors();
  for (uint32_t i = 0; i < num_entries; i++) {
    labels[i] = codegen_->GetLabelOf(successors[i]);
  }
  JumpTable* table = __ CreateJumpTable(std::move(labels));

  // Load the address of the jump table.
  // Note: The `LoadLabelAddress()` emits AUIPC+ADD. It is possible to avoid the ADD and
  // instead embed that offset in the LW below as well as all jump table entries but
  // that would need some invasive changes in the jump table handling in the assembler.
  ScratchRegisterScope srs(GetAssembler());
  XRegister table_base = srs.AllocateXRegister();
  __ LoadLabelAddress(table_base, table->GetLabel());

  // Load the PC difference from the jump table.
  // TODO(riscv64): Use SH2ADD from the Zba extension.
  __ Slli(temp, adjusted, 2);
  __ Add(temp, temp, table_base);
  __ Lw(temp, temp, 0);

  // Compute the absolute target address by adding the table start address
  // (the table contains offsets to targets relative to its start).
  __ Add(temp, temp, table_base);
  // And jump.
  __ Jr(temp);
}

int32_t InstructionCodeGeneratorRISCV64::VecAddress(LocationSummary* locations,
                                                    size_t size,
                                                    /*out*/ XRegister* adjusted_base) {
  UNUSED(locations);
  UNUSED(size);
  UNUSED(adjusted_base);
  LOG(FATAL) << "Unimplemented";
  UNREACHABLE();
}

void LocationsBuilderRISCV64::HandleBinaryOp(HBinaryOperation* instruction) {
  DCHECK_EQ(instruction->InputCount(), 2u);
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  DataType::Type type = instruction->GetResultType();
  switch (type) {
    case DataType::Type::kInt32:
    case DataType::Type::kInt64: {
      locations->SetInAt(0, Location::RequiresRegister());
      HInstruction* right = instruction->InputAt(1);
      bool can_use_imm = false;
      if (instruction->IsMin() || instruction->IsMax()) {
        can_use_imm = IsZeroBitPattern(instruction);
      } else if (right->IsConstant()) {
        int64_t imm = CodeGenerator::GetInt64ValueOf(right->AsConstant());
        can_use_imm = IsInt<12>(instruction->IsSub() ? -imm : imm);
      }
      if (can_use_imm) {
        locations->SetInAt(1, Location::ConstantLocation(right));
      } else {
        locations->SetInAt(1, Location::RequiresRegister());
      }
      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
      break;
    }

    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64:
      locations->SetInAt(0, Location::RequiresFpuRegister());
      locations->SetInAt(1, Location::RequiresFpuRegister());
      if (instruction->IsMin() || instruction->IsMax()) {
        locations->SetOut(Location::RequiresFpuRegister(), Location::kOutputOverlap);
      } else {
        locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
      }
      break;

    default:
      LOG(FATAL) << "Unexpected " << instruction->DebugName() << " type " << type;
      UNREACHABLE();
  }
}

void InstructionCodeGeneratorRISCV64::HandleBinaryOp(HBinaryOperation* instruction) {
  DataType::Type type = instruction->GetType();
  LocationSummary* locations = instruction->GetLocations();

  switch (type) {
    case DataType::Type::kInt32:
    case DataType::Type::kInt64: {
      XRegister rd = locations->Out().AsRegister<XRegister>();
      XRegister rs1 = locations->InAt(0).AsRegister<XRegister>();
      Location rs2_location = locations->InAt(1);

      bool use_imm = rs2_location.IsConstant();
      XRegister rs2 = use_imm ? kNoXRegister : rs2_location.AsRegister<XRegister>();
      int64_t imm = use_imm ? CodeGenerator::GetInt64ValueOf(rs2_location.GetConstant()) : 0;

      if (instruction->IsAnd()) {
        if (use_imm) {
          __ Andi(rd, rs1, imm);
        } else {
          __ And(rd, rs1, rs2);
        }
      } else if (instruction->IsOr()) {
        if (use_imm) {
          __ Ori(rd, rs1, imm);
        } else {
          __ Or(rd, rs1, rs2);
        }
      } else if (instruction->IsXor()) {
        if (use_imm) {
          __ Xori(rd, rs1, imm);
        } else {
          __ Xor(rd, rs1, rs2);
        }
      } else if (instruction->IsAdd() || instruction->IsSub()) {
        if (type == DataType::Type::kInt32) {
          if (use_imm) {
            __ Addiw(rd, rs1, instruction->IsSub() ? -imm : imm);
          } else if (instruction->IsAdd()) {
            __ Addw(rd, rs1, rs2);
          } else {
            DCHECK(instruction->IsSub());
            __ Subw(rd, rs1, rs2);
          }
        } else {
          if (use_imm) {
            __ Addi(rd, rs1, instruction->IsSub() ? -imm : imm);
          } else if (instruction->IsAdd()) {
            __ Add(rd, rs1, rs2);
          } else {
            DCHECK(instruction->IsSub());
            __ Sub(rd, rs1, rs2);
          }
        }
      } else if (instruction->IsMin()) {
        DCHECK_IMPLIES(use_imm, imm == 0);
        __ Min(rd, rs1, use_imm ? Zero : rs2);
      } else {
        DCHECK(instruction->IsMax());
        DCHECK_IMPLIES(use_imm, imm == 0);
        __ Max(rd, rs1, use_imm ? Zero : rs2);
      }
      break;
    }
    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64: {
      FRegister rd = locations->Out().AsFpuRegister<FRegister>();
      FRegister rs1 = locations->InAt(0).AsFpuRegister<FRegister>();
      FRegister rs2 = locations->InAt(1).AsFpuRegister<FRegister>();
      if (instruction->IsAdd()) {
        FAdd(rd, rs1, rs2, type);
      } else if (instruction->IsSub()) {
        FSub(rd, rs1, rs2, type);
      } else {
        DCHECK(instruction->IsMin() || instruction->IsMax());
        // If one of the operands is NaN and the other is not, riscv64 instructions FMIN/FMAX
        // return the other operand while we want to return the NaN operand.
        DCHECK_NE(rd, rs1);  // Requested `Location::kOutputOverlap`.
        DCHECK_NE(rd, rs2);  // Requested `Location::kOutputOverlap`.
        ScratchRegisterScope srs(GetAssembler());
        XRegister tmp = srs.AllocateXRegister();
        XRegister tmp2 = srs.AllocateXRegister();
        Riscv64Label done;
        // Return `rs1` if it's NaN.
        FClass(tmp, rs1, type);
        __ Li(tmp2, kFClassNaNMinValue);
        FMv(rd, rs1, type);
        __ Bgeu(tmp, tmp2, &done);
        // Return `rs2` if it's NaN.
        FClass(tmp, rs2, type);
        FMv(rd, rs2, type);
        __ Bgeu(tmp, tmp2, &done);
        // Calculate Min/Max for non-NaN arguments.
        if (instruction->IsMin()) {
          FMin(rd, rs1, rs2, type);
        } else {
          FMax(rd, rs1, rs2, type);
        }
        __ Bind(&done);
      }
      break;
    }
    default:
      LOG(FATAL) << "Unexpected binary operation type " << type;
      UNREACHABLE();
  }
}

void LocationsBuilderRISCV64::HandleCondition(HCondition* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  switch (instruction->InputAt(0)->GetType()) {
    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64:
      locations->SetInAt(0, Location::RequiresFpuRegister());
      locations->SetInAt(1, Location::RequiresFpuRegister());
      break;

    default: {
      locations->SetInAt(0, Location::RequiresRegister());
      HInstruction* rhs = instruction->InputAt(1);
      bool use_imm = false;
      if (rhs->IsConstant()) {
        int64_t imm = CodeGenerator::GetInt64ValueOf(rhs->AsConstant());
        if (instruction->IsEmittedAtUseSite()) {
          // For `HIf`, materialize all non-zero constants with an `HParallelMove`.
          // Note: For certain constants and conditions, the code could be improved.
          // For example, 2048 takes two instructions to materialize but the negative
          // -2048 could be embedded in ADDI for EQ/NE comparison.
          use_imm = (imm == 0);
        } else {
          // Constants that cannot be embedded in an instruction's 12-bit immediate shall be
          // materialized with an `HParallelMove`. This simplifies the code and avoids cases
          // with arithmetic overflow. Adjust the `imm` if needed for a particular instruction.
          switch (instruction->GetCondition()) {
            case kCondEQ:
            case kCondNE:
              imm = -imm;  // ADDI with negative immediate (there is no SUBI).
              break;
            case kCondLE:
            case kCondGT:
            case kCondBE:
            case kCondA:
              imm += 1;    // SLTI/SLTIU with adjusted immediate (there is no SLEI/SLEIU).
              break;
            default:
              break;
          }
          use_imm = IsInt<12>(imm);
        }
      }
      if (use_imm) {
        locations->SetInAt(1, Location::ConstantLocation(rhs));
      } else {
        locations->SetInAt(1, Location::RequiresRegister());
      }
      break;
    }
  }
  if (!instruction->IsEmittedAtUseSite()) {
    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
  }
}

void InstructionCodeGeneratorRISCV64::HandleCondition(HCondition* instruction) {
  if (instruction->IsEmittedAtUseSite()) {
    return;
  }

  DataType::Type type = instruction->InputAt(0)->GetType();
  LocationSummary* locations = instruction->GetLocations();
  switch (type) {
    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64:
      GenerateFpCondition(instruction->GetCondition(), instruction->IsGtBias(), type, locations);
      return;
    default:
      // Integral types and reference equality.
      GenerateIntLongCondition(instruction->GetCondition(), locations);
      return;
  }
}

void LocationsBuilderRISCV64::HandleShift(HBinaryOperation* instruction) {
  DCHECK(instruction->IsShl() ||
         instruction->IsShr() ||
         instruction->IsUShr() ||
         instruction->IsRor());

  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  DataType::Type type = instruction->GetResultType();
  switch (type) {
    case DataType::Type::kInt32:
    case DataType::Type::kInt64: {
      locations->SetInAt(0, Location::RequiresRegister());
      locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
      break;
    }
    default:
      LOG(FATAL) << "Unexpected shift type " << type;
      UNREACHABLE();
  }
}

void InstructionCodeGeneratorRISCV64::HandleShift(HBinaryOperation* instruction) {
  DCHECK(instruction->IsShl() ||
         instruction->IsShr() ||
         instruction->IsUShr() ||
         instruction->IsRor());
  LocationSummary* locations = instruction->GetLocations();
  DataType::Type type = instruction->GetType();

  switch (type) {
    case DataType::Type::kInt32:
    case DataType::Type::kInt64: {
      XRegister rd = locations->Out().AsRegister<XRegister>();
      XRegister rs1 = locations->InAt(0).AsRegister<XRegister>();
      Location rs2_location = locations->InAt(1);

      if (rs2_location.IsConstant()) {
        int64_t imm = CodeGenerator::GetInt64ValueOf(rs2_location.GetConstant());
        uint32_t shamt =
            imm & (type == DataType::Type::kInt32 ? kMaxIntShiftDistance : kMaxLongShiftDistance);

        if (shamt == 0) {
          if (rd != rs1) {
            __ Mv(rd, rs1);
          }
        } else if (type == DataType::Type::kInt32) {
          if (instruction->IsShl()) {
            __ Slliw(rd, rs1, shamt);
          } else if (instruction->IsShr()) {
            __ Sraiw(rd, rs1, shamt);
          } else if (instruction->IsUShr()) {
            __ Srliw(rd, rs1, shamt);
          } else {
            DCHECK(instruction->IsRor());
            __ Roriw(rd, rs1, shamt);
          }
        } else {
          if (instruction->IsShl()) {
            __ Slli(rd, rs1, shamt);
          } else if (instruction->IsShr()) {
            __ Srai(rd, rs1, shamt);
          } else if (instruction->IsUShr()) {
            __ Srli(rd, rs1, shamt);
          } else {
            DCHECK(instruction->IsRor());
            __ Rori(rd, rs1, shamt);
          }
        }
      } else {
        XRegister rs2 = rs2_location.AsRegister<XRegister>();
        if (type == DataType::Type::kInt32) {
          if (instruction->IsShl()) {
            __ Sllw(rd, rs1, rs2);
          } else if (instruction->IsShr()) {
            __ Sraw(rd, rs1, rs2);
          } else if (instruction->IsUShr()) {
            __ Srlw(rd, rs1, rs2);
          } else {
            DCHECK(instruction->IsRor());
            __ Rorw(rd, rs1, rs2);
          }
        } else {
          if (instruction->IsShl()) {
            __ Sll(rd, rs1, rs2);
          } else if (instruction->IsShr()) {
            __ Sra(rd, rs1, rs2);
          } else if (instruction->IsUShr()) {
            __ Srl(rd, rs1, rs2);
          } else {
            DCHECK(instruction->IsRor());
            __ Ror(rd, rs1, rs2);
          }
        }
      }
      break;
    }
    default:
      LOG(FATAL) << "Unexpected shift operation type " << type;
  }
}

void CodeGeneratorRISCV64::MaybeMarkGCCard(XRegister object,
                                           XRegister value,
                                           bool value_can_be_null) {
  Riscv64Label done;
  if (value_can_be_null) {
    __ Beqz(value, &done);
  }
  MarkGCCard(object);
  __ Bind(&done);
}

void CodeGeneratorRISCV64::MarkGCCard(XRegister object) {
  ScratchRegisterScope srs(GetAssembler());
  XRegister card = srs.AllocateXRegister();
  XRegister temp = srs.AllocateXRegister();
  // Load the address of the card table into `card`.
  __ Loadd(card, TR, Thread::CardTableOffset<kRiscv64PointerSize>().Int32Value());

  // Calculate the address of the card corresponding to `object`.
  __ Srli(temp, object, gc::accounting::CardTable::kCardShift);
  __ Add(temp, card, temp);
  // Write the `art::gc::accounting::CardTable::kCardDirty` value into the
  // `object`'s card.
  //
  // Register `card` contains the address of the card table. Note that the card
  // table's base is biased during its creation so that it always starts at an
  // address whose least-significant byte is equal to `kCardDirty` (see
  // art::gc::accounting::CardTable::Create). Therefore the SB instruction
  // below writes the `kCardDirty` (byte) value into the `object`'s card
  // (located at `card + object >> kCardShift`).
  //
  // This dual use of the value in register `card` (1. to calculate the location
  // of the card to mark; and 2. to load the `kCardDirty` value) saves a load
  // (no need to explicitly load `kCardDirty` as an immediate value).
  __ Sb(card, temp, 0);  // No scratch register left for `Storeb()`.
}

void CodeGeneratorRISCV64::CheckGCCardIsValid(XRegister object) {
  Riscv64Label done;
  ScratchRegisterScope srs(GetAssembler());
  XRegister card = srs.AllocateXRegister();
  XRegister temp = srs.AllocateXRegister();
  // Load the address of the card table into `card`.
  __ Loadd(card, TR, Thread::CardTableOffset<kRiscv64PointerSize>().Int32Value());

  // Calculate the address of the card corresponding to `object`.
  __ Srli(temp, object, gc::accounting::CardTable::kCardShift);
  __ Add(temp, card, temp);
  // assert (!clean || !self->is_gc_marking)
  __ Lb(temp, temp, 0);
  static_assert(gc::accounting::CardTable::kCardClean == 0);
  __ Bnez(temp, &done);
  __ Loadw(temp, TR, Thread::IsGcMarkingOffset<kRiscv64PointerSize>().Int32Value());
  __ Beqz(temp, &done);
  __ Unimp();
  __ Bind(&done);
}

void LocationsBuilderRISCV64::HandleFieldSet(HInstruction* instruction) {
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kNoCall);
  locations->SetInAt(0, Location::RequiresRegister());
  locations->SetInAt(1, ValueLocationForStore(instruction->InputAt(1)));
}

void InstructionCodeGeneratorRISCV64::HandleFieldSet(HInstruction* instruction,
                                                     const FieldInfo& field_info,
                                                     bool value_can_be_null,
                                                     WriteBarrierKind write_barrier_kind) {
  DataType::Type type = field_info.GetFieldType();
  LocationSummary* locations = instruction->GetLocations();
  XRegister obj = locations->InAt(0).AsRegister<XRegister>();
  Location value = locations->InAt(1);
  DCHECK_IMPLIES(value.IsConstant(), IsZeroBitPattern(value.GetConstant()));
  bool is_volatile = field_info.IsVolatile();
  uint32_t offset = field_info.GetFieldOffset().Uint32Value();

  if (is_volatile) {
    StoreSeqCst(value, obj, offset, type, instruction);
  } else {
    Store(value, obj, offset, type);
    codegen_->MaybeRecordImplicitNullCheck(instruction);
  }

  bool needs_write_barrier =
      codegen_->StoreNeedsWriteBarrier(type, instruction->InputAt(1), write_barrier_kind);
  if (needs_write_barrier) {
    if (value.IsConstant()) {
      DCHECK_EQ(write_barrier_kind, WriteBarrierKind::kEmitBeingReliedOn);
      codegen_->MarkGCCard(obj);
    } else {
      codegen_->MaybeMarkGCCard(
          obj,
          value.AsRegister<XRegister>(),
          value_can_be_null && write_barrier_kind == WriteBarrierKind::kEmitNotBeingReliedOn);
    }
  } else if (codegen_->ShouldCheckGCCard(type, instruction->InputAt(1), write_barrier_kind)) {
    codegen_->CheckGCCardIsValid(obj);
  }
}

void LocationsBuilderRISCV64::HandleFieldGet(HInstruction* instruction) {
  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());

  bool object_field_get_with_read_barrier =
      (instruction->GetType() == DataType::Type::kReference) && codegen_->EmitReadBarrier();
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(
      instruction,
      object_field_get_with_read_barrier
          ? LocationSummary::kCallOnSlowPath
          : LocationSummary::kNoCall);

  // Input for object receiver.
  locations->SetInAt(0, Location::RequiresRegister());

  if (DataType::IsFloatingPointType(instruction->GetType())) {
    locations->SetOut(Location::RequiresFpuRegister());
  } else {
    // The output overlaps for an object field get when read barriers
    // are enabled: we do not want the load to overwrite the object's
    // location, as we need it to emit the read barrier.
    locations->SetOut(
        Location::RequiresRegister(),
        object_field_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap);
  }

  if (object_field_get_with_read_barrier && kUseBakerReadBarrier) {
    locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty());  // No caller-save registers.
    // We need a temporary register for the read barrier marking slow
    // path in CodeGeneratorRISCV64::GenerateFieldLoadWithBakerReadBarrier.
    locations->AddTemp(Location::RequiresRegister());
  }
}

void InstructionCodeGeneratorRISCV64::HandleFieldGet(HInstruction* instruction,
                                                     const FieldInfo& field_info) {
  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
  DCHECK_EQ(DataType::Size(field_info.GetFieldType()), DataType::Size(instruction->GetType()));
  DataType::Type type = instruction->GetType();
  LocationSummary* locations = instruction->GetLocations();
  Location obj_loc = locations->InAt(0);
  XRegister obj = obj_loc.AsRegister<XRegister>();
  Location dst_loc = locations->Out();
  bool is_volatile = field_info.IsVolatile();
  uint32_t offset = field_info.GetFieldOffset().Uint32Value();

  if (is_volatile) {
    codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
  }

  if (type == DataType::Type::kReference && codegen_->EmitBakerReadBarrier()) {
    // /* HeapReference<Object> */ dst = *(obj + offset)
    Location temp_loc = locations->GetTemp(0);
    // Note that a potential implicit null check is handled in this
    // CodeGeneratorRISCV64::GenerateFieldLoadWithBakerReadBarrier call.
    codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction,
                                                    dst_loc,
                                                    obj,
                                                    offset,
                                                    temp_loc,
                                                    /* needs_null_check= */ true);
  } else {
    Load(dst_loc, obj, offset, type);
    codegen_->MaybeRecordImplicitNullCheck(instruction);
  }

  if (is_volatile) {
    codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
  }

  if (type == DataType::Type::kReference && !codegen_->EmitBakerReadBarrier()) {
    // If read barriers are enabled, emit read barriers other than
    // Baker's using a slow path (and also unpoison the loaded
    // reference, if heap poisoning is enabled).
    codegen_->MaybeGenerateReadBarrierSlow(instruction, dst_loc, dst_loc, obj_loc, offset);
  }
}

void InstructionCodeGeneratorRISCV64::GenerateMethodEntryExitHook(HInstruction* instruction) {
  SlowPathCodeRISCV64* slow_path =
      new (codegen_->GetScopedAllocator()) MethodEntryExitHooksSlowPathRISCV64(instruction);
  codegen_->AddSlowPath(slow_path);

  ScratchRegisterScope temps(GetAssembler());
  XRegister tmp = temps.AllocateXRegister();

  if (instruction->IsMethodExitHook()) {
    // Check if we are required to check if the caller needs a deoptimization. Strictly speaking it
    // would be sufficient to check if CheckCallerForDeopt bit is set. Though it is faster to check
    // if it is just non-zero. kCHA bit isn't used in debuggable runtimes as cha optimization is
    // disabled in debuggable runtime. The other bit is used when this method itself requires a
    // deoptimization due to redefinition. So it is safe to just check for non-zero value here.
    __ Loadwu(tmp, SP, codegen_->GetStackOffsetOfShouldDeoptimizeFlag());
    __ Bnez(tmp, slow_path->GetEntryLabel());
  }

  uint64_t hook_offset = instruction->IsMethodExitHook() ?
      instrumentation::Instrumentation::HaveMethodExitListenersOffset().SizeValue() :
      instrumentation::Instrumentation::HaveMethodEntryListenersOffset().SizeValue();
  auto [base_hook_address, hook_imm12] = SplitJitAddress(
      reinterpret_cast64<uint64_t>(Runtime::Current()->GetInstrumentation()) + hook_offset);
  __ LoadConst64(tmp, base_hook_address);
  __ Lbu(tmp, tmp, hook_imm12);
  // Check if there are any method entry / exit listeners. If no, continue.
  __ Beqz(tmp, slow_path->GetExitLabel());
  // Check if there are any slow (jvmti / trace with thread cpu time) method entry / exit listeners.
  // If yes, just take the slow path.
  static_assert(instrumentation::Instrumentation::kFastTraceListeners == 1u);
  __ Addi(tmp, tmp, -1);
  __ Bnez(tmp, slow_path->GetEntryLabel());

  // Check if there is place in the buffer to store a new entry, if no, take the slow path.
  int32_t trace_buffer_index_offset =
      Thread::TraceBufferIndexOffset<kRiscv64PointerSize>().Int32Value();
  __ Loadd(tmp, TR, trace_buffer_index_offset);
  __ Addi(tmp, tmp, -dchecked_integral_cast<int32_t>(kNumEntriesForWallClock));
  __ Bltz(tmp, slow_path->GetEntryLabel());

  // Update the index in the `Thread`.
  __ Stored(tmp, TR, trace_buffer_index_offset);

  // Allocate second core scratch register. We can no longer use `Stored()`
  // and similar macro instructions because there is no core scratch register left.
  XRegister tmp2 = temps.AllocateXRegister();

  // Calculate the entry address in the buffer.
  // /*addr*/ tmp = TR->GetMethodTraceBuffer() + sizeof(void*) * /*index*/ tmp;
  __ Loadd(tmp2, TR, Thread::TraceBufferPtrOffset<kRiscv64PointerSize>().SizeValue());
  __ Sh3Add(tmp, tmp, tmp2);

  // Record method pointer and trace action.
  __ Ld(tmp2, SP, 0);
  // Use last two bits to encode trace method action. For MethodEntry it is 0
  // so no need to set the bits since they are 0 already.
  DCHECK_GE(ArtMethod::Alignment(kRuntimePointerSize), static_cast<size_t>(4));
  static_assert(enum_cast<int32_t>(TraceAction::kTraceMethodEnter) == 0);
  static_assert(enum_cast<int32_t>(TraceAction::kTraceMethodExit) == 1);
  if (instruction->IsMethodExitHook()) {
    __ Ori(tmp2, tmp2, enum_cast<int32_t>(TraceAction::kTraceMethodExit));
  }
  static_assert(IsInt<12>(kMethodOffsetInBytes));  // No free scratch register for `Stored()`.
  __ Sd(tmp2, tmp, kMethodOffsetInBytes);

  // Record the timestamp.
  __ RdTime(tmp2);
  static_assert(IsInt<12>(kTimestampOffsetInBytes));  // No free scratch register for `Stored()`.
  __ Sd(tmp2, tmp, kTimestampOffsetInBytes);

  __ Bind(slow_path->GetExitLabel());
}

void LocationsBuilderRISCV64::VisitAbove(HAbove* instruction) {
  HandleCondition(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitAbove(HAbove* instruction) {
  HandleCondition(instruction);
}

void LocationsBuilderRISCV64::VisitAboveOrEqual(HAboveOrEqual* instruction) {
  HandleCondition(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitAboveOrEqual(HAboveOrEqual* instruction) {
  HandleCondition(instruction);
}

void LocationsBuilderRISCV64::VisitAbs(HAbs* abs) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(abs);
  switch (abs->GetResultType()) {
    case DataType::Type::kInt32:
    case DataType::Type::kInt64:
      locations->SetInAt(0, Location::RequiresRegister());
      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
      break;
    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64:
      locations->SetInAt(0, Location::RequiresFpuRegister());
      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
      break;
    default:
      LOG(FATAL) << "Unexpected abs type " << abs->GetResultType();
  }
}

void InstructionCodeGeneratorRISCV64::VisitAbs(HAbs* abs) {
  LocationSummary* locations = abs->GetLocations();
  switch (abs->GetResultType()) {
    case DataType::Type::kInt32: {
      XRegister in = locations->InAt(0).AsRegister<XRegister>();
      XRegister out = locations->Out().AsRegister<XRegister>();
      ScratchRegisterScope srs(GetAssembler());
      XRegister tmp = srs.AllocateXRegister();
      __ Sraiw(tmp, in, 31);
      __ Xor(out, in, tmp);
      __ Subw(out, out, tmp);
      break;
    }
    case DataType::Type::kInt64: {
      XRegister in = locations->InAt(0).AsRegister<XRegister>();
      XRegister out = locations->Out().AsRegister<XRegister>();
      ScratchRegisterScope srs(GetAssembler());
      XRegister tmp = srs.AllocateXRegister();
      __ Srai(tmp, in, 63);
      __ Xor(out, in, tmp);
      __ Sub(out, out, tmp);
      break;
    }
    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64:
      FAbs(locations->Out().AsFpuRegister<FRegister>(),
           locations->InAt(0).AsFpuRegister<FRegister>(),
           abs->GetResultType());
      break;
    default:
      LOG(FATAL) << "Unexpected abs type " << abs->GetResultType();
  }
}

void LocationsBuilderRISCV64::VisitAdd(HAdd* instruction) {
  HandleBinaryOp(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitAdd(HAdd* instruction) {
  HandleBinaryOp(instruction);
}

void LocationsBuilderRISCV64::VisitAnd(HAnd* instruction) {
  HandleBinaryOp(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitAnd(HAnd* instruction) {
  HandleBinaryOp(instruction);
}

void LocationsBuilderRISCV64::VisitArrayGet(HArrayGet* instruction) {
  DataType::Type type = instruction->GetType();
  bool object_array_get_with_read_barrier =
      (type == DataType::Type::kReference) && codegen_->EmitReadBarrier();
  LocationSummary* locations = new (GetGraph()->GetAllocator())
      LocationSummary(instruction,
                      object_array_get_with_read_barrier ? LocationSummary::kCallOnSlowPath :
                                                           LocationSummary::kNoCall);
  locations->SetInAt(0, Location::RequiresRegister());
  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
  if (DataType::IsFloatingPointType(type)) {
    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
  } else {
    // The output overlaps in the case of an object array get with
    // read barriers enabled: we do not want the move to overwrite the
    // array's location, as we need it to emit the read barrier.
    locations->SetOut(
        Location::RequiresRegister(),
        object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap);
  }
  if (object_array_get_with_read_barrier && kUseBakerReadBarrier) {
    locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty());  // No caller-save registers.
    // We need a temporary register for the read barrier marking slow
    // path in CodeGeneratorRISCV64::GenerateArrayLoadWithBakerReadBarrier.
    locations->AddTemp(Location::RequiresRegister());
  }
}

void InstructionCodeGeneratorRISCV64::VisitArrayGet(HArrayGet* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  Location obj_loc = locations->InAt(0);
  XRegister obj = obj_loc.AsRegister<XRegister>();
  Location out_loc = locations->Out();
  Location index = locations->InAt(1);
  uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction);
  DataType::Type type = instruction->GetType();
  const bool maybe_compressed_char_at =
      mirror::kUseStringCompression && instruction->IsStringCharAt();

  Riscv64Label string_char_at_done;
  if (maybe_compressed_char_at) {
    DCHECK_EQ(type, DataType::Type::kUint16);
    uint32_t count_offset = mirror::String::CountOffset().Uint32Value();
    Riscv64Label uncompressed_load;
    {
      ScratchRegisterScope srs(GetAssembler());
      XRegister tmp = srs.AllocateXRegister();
      __ Loadw(tmp, obj, count_offset);
      codegen_->MaybeRecordImplicitNullCheck(instruction);
      __ Andi(tmp, tmp, 0x1);
      static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u,
                    "Expecting 0=compressed, 1=uncompressed");
      __ Bnez(tmp, &uncompressed_load);
    }
    XRegister out = out_loc.AsRegister<XRegister>();
    if (index.IsConstant()) {
        int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue();
      __ Loadbu(out, obj, data_offset + const_index);
    } else {
      __ Add(out, obj, index.AsRegister<XRegister>());
      __ Loadbu(out, out, data_offset);
    }
    __ J(&string_char_at_done);
    __ Bind(&uncompressed_load);
  }

  if (type == DataType::Type::kReference && codegen_->EmitBakerReadBarrier()) {
    static_assert(
        sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
        "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
    // /* HeapReference<Object> */ out =
    //     *(obj + data_offset + index * sizeof(HeapReference<Object>))
    // Note that a potential implicit null check could be handled in these
    // `CodeGeneratorRISCV64::Generate{Array,Field}LoadWithBakerReadBarrier()` calls
    // but we currently do not support implicit null checks on `HArrayGet`.
    DCHECK(!instruction->CanDoImplicitNullCheckOn(instruction->InputAt(0)));
    Location temp = locations->GetTemp(0);
    if (index.IsConstant()) {
      // Array load with a constant index can be treated as a field load.
      static constexpr size_t shift = DataType::SizeShift(DataType::Type::kReference);
      size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << shift) + data_offset;
      codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction,
                                                      out_loc,
                                                      obj,
                                                      offset,
                                                      temp,
                                                      /* needs_null_check= */ false);
    } else {
      codegen_->GenerateArrayLoadWithBakerReadBarrier(instruction,
                                                      out_loc,
                                                      obj,
                                                      data_offset,
                                                      index,
                                                      temp,
                                                      /* needs_null_check= */ false);
    }
  } else if (index.IsConstant()) {
    int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue();
    int32_t offset = data_offset + (const_index << DataType::SizeShift(type));
    Load(out_loc, obj, offset, type);
    if (!maybe_compressed_char_at) {
      codegen_->MaybeRecordImplicitNullCheck(instruction);
    }
    if (type == DataType::Type::kReference) {
      DCHECK(!codegen_->EmitBakerReadBarrier());
      // If read barriers are enabled, emit read barriers other than Baker's using
      // a slow path (and also unpoison the loaded reference, if heap poisoning is enabled).
      codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset);
    }
  } else {
    ScratchRegisterScope srs(GetAssembler());
    XRegister tmp = srs.AllocateXRegister();
    ShNAdd(tmp, index.AsRegister<XRegister>(), obj, type);
    Load(out_loc, tmp, data_offset, type);
    if (!maybe_compressed_char_at) {
      codegen_->MaybeRecordImplicitNullCheck(instruction);
    }
    if (type == DataType::Type::kReference) {
      DCHECK(!codegen_->EmitBakerReadBarrier());
      // If read barriers are enabled, emit read barriers other than Baker's using
      // a slow path (and also unpoison the loaded reference, if heap poisoning is enabled).
      codegen_->MaybeGenerateReadBarrierSlow(
          instruction, out_loc, out_loc, obj_loc, data_offset, index);
    }
  }

  if (maybe_compressed_char_at) {
    __ Bind(&string_char_at_done);
  }
}

void LocationsBuilderRISCV64::VisitArrayLength(HArrayLength* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  locations->SetInAt(0, Location::RequiresRegister());
  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}

void InstructionCodeGeneratorRISCV64::VisitArrayLength(HArrayLength* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction);
  XRegister obj = locations->InAt(0).AsRegister<XRegister>();
  XRegister out = locations->Out().AsRegister<XRegister>();
  __ Loadwu(out, obj, offset);  // Unsigned for string length; does not matter for other arrays.
  codegen_->MaybeRecordImplicitNullCheck(instruction);
  // Mask out compression flag from String's array length.
  if (mirror::kUseStringCompression && instruction->IsStringLength()) {
    __ Srli(out, out, 1u);
  }
}

void LocationsBuilderRISCV64::VisitArraySet(HArraySet* instruction) {
  bool needs_type_check = instruction->NeedsTypeCheck();
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(
      instruction,
      needs_type_check ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall);
  locations->SetInAt(0, Location::RequiresRegister());
  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
  locations->SetInAt(2, ValueLocationForStore(instruction->GetValue()));
  if (kPoisonHeapReferences &&
      instruction->GetComponentType() == DataType::Type::kReference &&
      !locations->InAt(1).IsConstant() &&
      !locations->InAt(2).IsConstant()) {
    locations->AddTemp(Location::RequiresRegister());
  }
}

void InstructionCodeGeneratorRISCV64::VisitArraySet(HArraySet* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  XRegister array = locations->InAt(0).AsRegister<XRegister>();
  Location index = locations->InAt(1);
  Location value = locations->InAt(2);
  DataType::Type value_type = instruction->GetComponentType();
  bool needs_type_check = instruction->NeedsTypeCheck();
  const WriteBarrierKind write_barrier_kind = instruction->GetWriteBarrierKind();
  bool needs_write_barrier =
      codegen_->StoreNeedsWriteBarrier(value_type, instruction->GetValue(), write_barrier_kind);
  size_t data_offset = mirror::Array::DataOffset(DataType::Size(value_type)).Uint32Value();
  SlowPathCodeRISCV64* slow_path = nullptr;

  if (needs_write_barrier) {
    DCHECK_EQ(value_type, DataType::Type::kReference);
    DCHECK_IMPLIES(value.IsConstant(), value.GetConstant()->IsArithmeticZero());
    const bool storing_constant_zero = value.IsConstant();
    // The WriteBarrierKind::kEmitNotBeingReliedOn case is able to skip the write barrier when its
    // value is null (without an extra CompareAndBranchIfZero since we already checked if the
    // value is null for the type check).
    bool skip_marking_gc_card = false;
    Riscv64Label skip_writing_card;
    if (!storing_constant_zero) {
      Riscv64Label do_store;

      bool can_value_be_null = instruction->GetValueCanBeNull();
      skip_marking_gc_card =
          can_value_be_null && write_barrier_kind == WriteBarrierKind::kEmitNotBeingReliedOn;
      if (can_value_be_null) {
        if (skip_marking_gc_card) {
          __ Beqz(value.AsRegister<XRegister>(), &skip_writing_card);
        } else {
          __ Beqz(value.AsRegister<XRegister>(), &do_store);
        }
      }

      if (needs_type_check) {
        slow_path = new (codegen_->GetScopedAllocator()) ArraySetSlowPathRISCV64(instruction);
        codegen_->AddSlowPath(slow_path);

        uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
        uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
        uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();

        ScratchRegisterScope srs(GetAssembler());
        XRegister temp1 = srs.AllocateXRegister();
        XRegister temp2 = srs.AllocateXRegister();

        // Note that when read barriers are enabled, the type checks are performed
        // without read barriers.  This is fine, even in the case where a class object
        // is in the from-space after the flip, as a comparison involving such a type
        // would not produce a false positive; it may of course produce a false
        // negative, in which case we would take the ArraySet slow path.

        // /* HeapReference<Class> */ temp1 = array->klass_
        __ Loadwu(temp1, array, class_offset);
        codegen_->MaybeRecordImplicitNullCheck(instruction);
        codegen_->MaybeUnpoisonHeapReference(temp1);

        // /* HeapReference<Class> */ temp2 = temp1->component_type_
        __ Loadwu(temp2, temp1, component_offset);
        // /* HeapReference<Class> */ temp1 = value->klass_
        __ Loadwu(temp1, value.AsRegister<XRegister>(), class_offset);
        // If heap poisoning is enabled, no need to unpoison `temp1`
        // nor `temp2`, as we are comparing two poisoned references.
        if (instruction->StaticTypeOfArrayIsObjectArray()) {
          Riscv64Label do_put;
          __ Beq(temp1, temp2, &do_put);
          // If heap poisoning is enabled, the `temp2` reference has
          // not been unpoisoned yet; unpoison it now.
          codegen_->MaybeUnpoisonHeapReference(temp2);

          // /* HeapReference<Class> */ temp1 = temp2->super_class_
          __ Loadwu(temp1, temp2, super_offset);
          // If heap poisoning is enabled, no need to unpoison
          // `temp1`, as we are comparing against null below.
          __ Bnez(temp1, slow_path->GetEntryLabel());
          __ Bind(&do_put);
        } else {
          __ Bne(temp1, temp2, slow_path->GetEntryLabel());
        }
      }

      if (can_value_be_null && !skip_marking_gc_card) {
        DCHECK(do_store.IsLinked());
        __ Bind(&do_store);
      }
    }

    DCHECK_NE(write_barrier_kind, WriteBarrierKind::kDontEmit);
    DCHECK_IMPLIES(storing_constant_zero,
                   write_barrier_kind == WriteBarrierKind::kEmitBeingReliedOn);
    codegen_->MarkGCCard(array);

    if (skip_marking_gc_card) {
      // Note that we don't check that the GC card is valid as it can be correctly clean.
      DCHECK(skip_writing_card.IsLinked());
      __ Bind(&skip_writing_card);
    }
  } else if (codegen_->ShouldCheckGCCard(value_type, instruction->GetValue(), write_barrier_kind)) {
    codegen_->CheckGCCardIsValid(array);
  }

  if (index.IsConstant()) {
    int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue();
    int32_t offset = data_offset + (const_index << DataType::SizeShift(value_type));
    Store(value, array, offset, value_type);
  } else {
    ScratchRegisterScope srs(GetAssembler());
    // Heap poisoning needs two scratch registers in `Store()`, except for null constants.
    XRegister tmp =
        (kPoisonHeapReferences && value_type == DataType::Type::kReference && !value.IsConstant())
            ? locations->GetTemp(0).AsRegister<XRegister>()
            : srs.AllocateXRegister();
    ShNAdd(tmp, index.AsRegister<XRegister>(), array, value_type);
    Store(value, tmp, data_offset, value_type);
  }
  // There must be no instructions between the `Store()` and the `MaybeRecordImplicitNullCheck()`.
  // We can avoid this if the type check makes the null check unconditionally.
  DCHECK_IMPLIES(needs_type_check, needs_write_barrier);
  if (!(needs_type_check && !instruction->GetValueCanBeNull())) {
    codegen_->MaybeRecordImplicitNullCheck(instruction);
  }

  if (slow_path != nullptr) {
    __ Bind(slow_path->GetExitLabel());
  }
}

void LocationsBuilderRISCV64::VisitBelow(HBelow* instruction) {
  HandleCondition(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitBelow(HBelow* instruction) {
  HandleCondition(instruction);
}

void LocationsBuilderRISCV64::VisitBelowOrEqual(HBelowOrEqual* instruction) {
  HandleCondition(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitBelowOrEqual(HBelowOrEqual* instruction) {
  HandleCondition(instruction);
}

void LocationsBuilderRISCV64::VisitBooleanNot(HBooleanNot* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  locations->SetInAt(0, Location::RequiresRegister());
  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}

void InstructionCodeGeneratorRISCV64::VisitBooleanNot(HBooleanNot* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  __ Xori(locations->Out().AsRegister<XRegister>(), locations->InAt(0).AsRegister<XRegister>(), 1);
}

void LocationsBuilderRISCV64::VisitBoundsCheck(HBoundsCheck* instruction) {
  RegisterSet caller_saves = RegisterSet::Empty();
  InvokeRuntimeCallingConvention calling_convention;
  caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
  caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
  LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves);

  HInstruction* index = instruction->InputAt(0);
  HInstruction* length = instruction->InputAt(1);

  bool const_index = false;
  bool const_length = false;

  if (length->IsConstant()) {
    if (index->IsConstant()) {
      const_index = true;
      const_length = true;
    } else {
      int32_t length_value = length->AsIntConstant()->GetValue();
      if (length_value == 0 || length_value == 1) {
        const_length = true;
      }
    }
  } else if (index->IsConstant()) {
    int32_t index_value = index->AsIntConstant()->GetValue();
    if (index_value <= 0) {
      const_index = true;
    }
  }

  locations->SetInAt(
      0,
      const_index ? Location::ConstantLocation(index) : Location::RequiresRegister());
  locations->SetInAt(
      1,
      const_length ? Location::ConstantLocation(length) : Location::RequiresRegister());
}

void InstructionCodeGeneratorRISCV64::VisitBoundsCheck(HBoundsCheck* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  Location index_loc = locations->InAt(0);
  Location length_loc = locations->InAt(1);

  if (length_loc.IsConstant()) {
    int32_t length = length_loc.GetConstant()->AsIntConstant()->GetValue();
    if (index_loc.IsConstant()) {
      int32_t index = index_loc.GetConstant()->AsIntConstant()->GetValue();
      if (index < 0 || index >= length) {
        BoundsCheckSlowPathRISCV64* slow_path =
            new (codegen_->GetScopedAllocator()) BoundsCheckSlowPathRISCV64(instruction);
        codegen_->AddSlowPath(slow_path);
        __ J(slow_path->GetEntryLabel());
      } else {
        // Nothing to be done.
      }
      return;
    }

    BoundsCheckSlowPathRISCV64* slow_path =
        new (codegen_->GetScopedAllocator()) BoundsCheckSlowPathRISCV64(instruction);
    codegen_->AddSlowPath(slow_path);
    XRegister index = index_loc.AsRegister<XRegister>();
    if (length == 0) {
      __ J(slow_path->GetEntryLabel());
    } else {
      DCHECK_EQ(length, 1);
      __ Bnez(index, slow_path->GetEntryLabel());
    }
  } else {
    XRegister length = length_loc.AsRegister<XRegister>();
    BoundsCheckSlowPathRISCV64* slow_path =
        new (codegen_->GetScopedAllocator()) BoundsCheckSlowPathRISCV64(instruction);
    codegen_->AddSlowPath(slow_path);
    if (index_loc.IsConstant()) {
      int32_t index = index_loc.GetConstant()->AsIntConstant()->GetValue();
      if (index < 0) {
        __ J(slow_path->GetEntryLabel());
      } else {
        DCHECK_EQ(index, 0);
        __ Blez(length, slow_path->GetEntryLabel());
      }
    } else {
      XRegister index = index_loc.AsRegister<XRegister>();
      __ Bgeu(index, length, slow_path->GetEntryLabel());
    }
  }
}

void LocationsBuilderRISCV64::VisitBoundType([[maybe_unused]] HBoundType* instruction) {
  // Nothing to do, this should be removed during prepare for register allocator.
  LOG(FATAL) << "Unreachable";
}

void InstructionCodeGeneratorRISCV64::VisitBoundType([[maybe_unused]] HBoundType* instruction) {
  // Nothing to do, this should be removed during prepare for register allocator.
  LOG(FATAL) << "Unreachable";
}

// Temp is used for read barrier.
static size_t NumberOfInstanceOfTemps(bool emit_read_barrier, TypeCheckKind type_check_kind) {
  if (emit_read_barrier &&
      (kUseBakerReadBarrier ||
       type_check_kind == TypeCheckKind::kAbstractClassCheck ||
       type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
       type_check_kind == TypeCheckKind::kArrayObjectCheck)) {
    return 1;
  }
  return 0;
}

// Interface case has 3 temps, one for holding the number of interfaces, one for the current
// interface pointer, one for loading the current interface.
// The other checks have one temp for loading the object's class and maybe a temp for read barrier.
static size_t NumberOfCheckCastTemps(bool emit_read_barrier, TypeCheckKind type_check_kind) {
  if (type_check_kind == TypeCheckKind::kInterfaceCheck) {
    return 3;
  }
  return 1 + NumberOfInstanceOfTemps(emit_read_barrier, type_check_kind);
}

void LocationsBuilderRISCV64::VisitCheckCast(HCheckCast* instruction) {
  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
  LocationSummary::CallKind call_kind = codegen_->GetCheckCastCallKind(instruction);
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, call_kind);
  locations->SetInAt(0, Location::RequiresRegister());
  if (type_check_kind == TypeCheckKind::kBitstringCheck) {
    locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)));
    locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)));
    locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)));
  } else {
    locations->SetInAt(1, Location::RequiresRegister());
  }
  locations->AddRegisterTemps(NumberOfCheckCastTemps(codegen_->EmitReadBarrier(), type_check_kind));
}

void InstructionCodeGeneratorRISCV64::VisitCheckCast(HCheckCast* instruction) {
TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
  LocationSummary* locations = instruction->GetLocations();
  Location obj_loc = locations->InAt(0);
  XRegister obj = obj_loc.AsRegister<XRegister>();
  Location cls = (type_check_kind == TypeCheckKind::kBitstringCheck)
      ? Location::NoLocation()
      : locations->InAt(1);
  Location temp_loc = locations->GetTemp(0);
  XRegister temp = temp_loc.AsRegister<XRegister>();
  const size_t num_temps = NumberOfCheckCastTemps(codegen_->EmitReadBarrier(), type_check_kind);
  DCHECK_GE(num_temps, 1u);
  DCHECK_LE(num_temps, 3u);
  Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation();
  Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation();
  const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
  const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
  const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
  const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
  const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value();
  const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value();
  const uint32_t object_array_data_offset =
      mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value();
  Riscv64Label done;

  bool is_type_check_slow_path_fatal = codegen_->IsTypeCheckSlowPathFatal(instruction);
  SlowPathCodeRISCV64* slow_path =
      new (codegen_->GetScopedAllocator()) TypeCheckSlowPathRISCV64(
          instruction, is_type_check_slow_path_fatal);
  codegen_->AddSlowPath(slow_path);

  // Avoid this check if we know `obj` is not null.
  if (instruction->MustDoNullCheck()) {
    __ Beqz(obj, &done);
  }

  switch (type_check_kind) {
    case TypeCheckKind::kExactCheck:
    case TypeCheckKind::kArrayCheck: {
      // /* HeapReference<Class> */ temp = obj->klass_
      GenerateReferenceLoadTwoRegisters(instruction,
                                        temp_loc,
                                        obj_loc,
                                        class_offset,
                                        maybe_temp2_loc,
                                        kWithoutReadBarrier);
      // Jump to slow path for throwing the exception or doing a
      // more involved array check.
      __ Bne(temp, cls.AsRegister<XRegister>(), slow_path->GetEntryLabel());
      break;
    }

    case TypeCheckKind::kAbstractClassCheck: {
      // /* HeapReference<Class> */ temp = obj->klass_
      GenerateReferenceLoadTwoRegisters(instruction,
                                        temp_loc,
                                        obj_loc,
                                        class_offset,
                                        maybe_temp2_loc,
                                        kWithoutReadBarrier);
      // If the class is abstract, we eagerly fetch the super class of the
      // object to avoid doing a comparison we know will fail.
      Riscv64Label loop;
      __ Bind(&loop);
      // /* HeapReference<Class> */ temp = temp->super_class_
      GenerateReferenceLoadOneRegister(instruction,
                                       temp_loc,
                                       super_offset,
                                       maybe_temp2_loc,
                                       kWithoutReadBarrier);
      // If the class reference currently in `temp` is null, jump to the slow path to throw the
      // exception.
      __ Beqz(temp, slow_path->GetEntryLabel());
      // Otherwise, compare the classes.
      __ Bne(temp, cls.AsRegister<XRegister>(), &loop);
      break;
    }

    case TypeCheckKind::kClassHierarchyCheck: {
      // /* HeapReference<Class> */ temp = obj->klass_
      GenerateReferenceLoadTwoRegisters(instruction,
                                        temp_loc,
                                        obj_loc,
                                        class_offset,
                                        maybe_temp2_loc,
                                        kWithoutReadBarrier);
      // Walk over the class hierarchy to find a match.
      Riscv64Label loop;
      __ Bind(&loop);
      __ Beq(temp, cls.AsRegister<XRegister>(), &done);
      // /* HeapReference<Class> */ temp = temp->super_class_
      GenerateReferenceLoadOneRegister(instruction,
                                       temp_loc,
                                       super_offset,
                                       maybe_temp2_loc,
                                       kWithoutReadBarrier);
      // If the class reference currently in `temp` is null, jump to the slow path to throw the
      // exception. Otherwise, jump to the beginning of the loop.
      __ Bnez(temp, &loop);
      __ J(slow_path->GetEntryLabel());
      break;
    }

    case TypeCheckKind::kArrayObjectCheck: {
      // /* HeapReference<Class> */ temp = obj->klass_
      GenerateReferenceLoadTwoRegisters(instruction,
                                        temp_loc,
                                        obj_loc,
                                        class_offset,
                                        maybe_temp2_loc,
                                        kWithoutReadBarrier);
      // Do an exact check.
      __ Beq(temp, cls.AsRegister<XRegister>(), &done);
      // Otherwise, we need to check that the object's class is a non-primitive array.
      // /* HeapReference<Class> */ temp = temp->component_type_
      GenerateReferenceLoadOneRegister(instruction,
                                       temp_loc,
                                       component_offset,
                                       maybe_temp2_loc,
                                       kWithoutReadBarrier);
      // If the component type is null, jump to the slow path to throw the exception.
      __ Beqz(temp, slow_path->GetEntryLabel());
      // Otherwise, the object is indeed an array, further check that this component
      // type is not a primitive type.
      __ Loadhu(temp, temp, primitive_offset);
      static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
      __ Bnez(temp, slow_path->GetEntryLabel());
      break;
    }

    case TypeCheckKind::kUnresolvedCheck:
      // We always go into the type check slow path for the unresolved check case.
      // We cannot directly call the CheckCast runtime entry point
      // without resorting to a type checking slow path here (i.e. by
      // calling InvokeRuntime directly), as it would require to
      // assign fixed registers for the inputs of this HInstanceOf
      // instruction (following the runtime calling convention), which
      // might be cluttered by the potential first read barrier
      // emission at the beginning of this method.
      __ J(slow_path->GetEntryLabel());
      break;

    case TypeCheckKind::kInterfaceCheck: {
      // Avoid read barriers to improve performance of the fast path. We can not get false
      // positives by doing this. False negatives are handled by the slow path.
      // /* HeapReference<Class> */ temp = obj->klass_
      GenerateReferenceLoadTwoRegisters(instruction,
                                        temp_loc,
                                        obj_loc,
                                        class_offset,
                                        maybe_temp2_loc,
                                        kWithoutReadBarrier);
      // /* HeapReference<Class> */ temp = temp->iftable_
      GenerateReferenceLoadOneRegister(instruction,
                                       temp_loc,
                                       iftable_offset,
                                       maybe_temp2_loc,
                                       kWithoutReadBarrier);
      XRegister temp2 = maybe_temp2_loc.AsRegister<XRegister>();
      XRegister temp3 = maybe_temp3_loc.AsRegister<XRegister>();
      // Load the size of the `IfTable`. The `Class::iftable_` is never null.
      __ Loadw(temp2, temp, array_length_offset);
      // Loop through the iftable and check if any class matches.
      Riscv64Label loop;
      __ Bind(&loop);
      __ Beqz(temp2, slow_path->GetEntryLabel());
      __ Lwu(temp3, temp, object_array_data_offset);
      codegen_->MaybeUnpoisonHeapReference(temp3);
      // Go to next interface.
      __ Addi(temp, temp, 2 * kHeapReferenceSize);
      __ Addi(temp2, temp2, -2);
      // Compare the classes and continue the loop if they do not match.
      __ Bne(temp3, cls.AsRegister<XRegister>(), &loop);
      break;
    }

    case TypeCheckKind::kBitstringCheck: {
      // /* HeapReference<Class> */ temp = obj->klass_
      GenerateReferenceLoadTwoRegisters(instruction,
                                        temp_loc,
                                        obj_loc,
                                        class_offset,
                                        maybe_temp2_loc,
                                        kWithoutReadBarrier);

      GenerateBitstringTypeCheckCompare(instruction, temp);
      __ Bnez(temp, slow_path->GetEntryLabel());
      break;
    }
  }

  __ Bind(&done);
  __ Bind(slow_path->GetExitLabel());
}

void LocationsBuilderRISCV64::VisitClassTableGet(HClassTableGet* instruction) {
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kNoCall);
  locations->SetInAt(0, Location::RequiresRegister());
  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}

void InstructionCodeGeneratorRISCV64::VisitClassTableGet(HClassTableGet* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  XRegister in = locations->InAt(0).AsRegister<XRegister>();
  XRegister out = locations->Out().AsRegister<XRegister>();
  if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) {
    MemberOffset method_offset =
        mirror::Class::EmbeddedVTableEntryOffset(instruction->GetIndex(), kRiscv64PointerSize);
    __ Loadd(out, in, method_offset.SizeValue());
  } else {
    uint32_t method_offset = dchecked_integral_cast<uint32_t>(
        ImTable::OffsetOfElement(instruction->GetIndex(), kRiscv64PointerSize));
    __ Loadd(out, in, mirror::Class::ImtPtrOffset(kRiscv64PointerSize).Uint32Value());
    __ Loadd(out, out, method_offset);
  }
}

static int32_t GetExceptionTlsOffset() {
  return Thread::ExceptionOffset<kRiscv64PointerSize>().Int32Value();
}

void LocationsBuilderRISCV64::VisitClearException(HClearException* instruction) {
  new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kNoCall);
}

void InstructionCodeGeneratorRISCV64::VisitClearException(
    [[maybe_unused]] HClearException* instruction) {
  __ Stored(Zero, TR, GetExceptionTlsOffset());
}

void LocationsBuilderRISCV64::VisitClinitCheck(HClinitCheck* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(
      instruction, LocationSummary::kCallOnSlowPath);
  locations->SetInAt(0, Location::RequiresRegister());
  if (instruction->HasUses()) {
    locations->SetOut(Location::SameAsFirstInput());
  }
  // Rely on the type initialization to save everything we need.
  locations->SetCustomSlowPathCallerSaves(OneRegInReferenceOutSaveEverythingCallerSaves());
}

void InstructionCodeGeneratorRISCV64::VisitClinitCheck(HClinitCheck* instruction) {
  // We assume the class is not null.
  SlowPathCodeRISCV64* slow_path = new (codegen_->GetScopedAllocator()) LoadClassSlowPathRISCV64(
      instruction->GetLoadClass(), instruction);
  codegen_->AddSlowPath(slow_path);
  GenerateClassInitializationCheck(slow_path,
                                   instruction->GetLocations()->InAt(0).AsRegister<XRegister>());
}

void LocationsBuilderRISCV64::VisitCompare(HCompare* instruction) {
  DataType::Type in_type = instruction->InputAt(0)->GetType();

  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);

  switch (in_type) {
    case DataType::Type::kBool:
    case DataType::Type::kUint8:
    case DataType::Type::kInt8:
    case DataType::Type::kUint16:
    case DataType::Type::kInt16:
    case DataType::Type::kInt32:
    case DataType::Type::kInt64:
      locations->SetInAt(0, Location::RequiresRegister());
      locations->SetInAt(1, RegisterOrZeroBitPatternLocation(instruction->InputAt(1)));
      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
      break;

    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64:
      locations->SetInAt(0, Location::RequiresFpuRegister());
      locations->SetInAt(1, Location::RequiresFpuRegister());
      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
      break;

    default:
      LOG(FATAL) << "Unexpected type for compare operation " << in_type;
      UNREACHABLE();
  }
}

void InstructionCodeGeneratorRISCV64::VisitCompare(HCompare* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  XRegister result = locations->Out().AsRegister<XRegister>();
  DataType::Type in_type = instruction->InputAt(0)->GetType();

  //  0 if: left == right
  //  1 if: left  > right
  // -1 if: left  < right
  switch (in_type) {
    case DataType::Type::kBool:
    case DataType::Type::kUint8:
    case DataType::Type::kInt8:
    case DataType::Type::kUint16:
    case DataType::Type::kInt16:
    case DataType::Type::kInt32:
    case DataType::Type::kInt64: {
      XRegister left = locations->InAt(0).AsRegister<XRegister>();
      XRegister right = InputXRegisterOrZero(locations->InAt(1));
      ScratchRegisterScope srs(GetAssembler());
      XRegister tmp = srs.AllocateXRegister();
      __ Slt(tmp, left, right);
      __ Slt(result, right, left);
      __ Sub(result, result, tmp);
      break;
    }

    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64: {
      FRegister left = locations->InAt(0).AsFpuRegister<FRegister>();
      FRegister right = locations->InAt(1).AsFpuRegister<FRegister>();
      ScratchRegisterScope srs(GetAssembler());
      XRegister tmp = srs.AllocateXRegister();
      if (instruction->IsGtBias()) {
        // ((FLE l,r) ^ 1) - (FLT l,r); see `GenerateFpCondition()`.
        FLe(tmp, left, right, in_type);
        FLt(result, left, right, in_type);
        __ Xori(tmp, tmp, 1);
        __ Sub(result, tmp, result);
      } else {
        // ((FLE r,l) - 1) + (FLT r,l); see `GenerateFpCondition()`.
        FLe(tmp, right, left, in_type);
        FLt(result, right, left, in_type);
        __ Addi(tmp, tmp, -1);
        __ Add(result, result, tmp);
      }
      break;
    }

    default:
      LOG(FATAL) << "Unimplemented compare type " << in_type;
  }
}

void LocationsBuilderRISCV64::VisitConstructorFence(HConstructorFence* instruction) {
  instruction->SetLocations(nullptr);
}

void InstructionCodeGeneratorRISCV64::VisitConstructorFence(
    [[maybe_unused]] HConstructorFence* instruction) {
  codegen_->GenerateMemoryBarrier(MemBarrierKind::kStoreStore);
}

void LocationsBuilderRISCV64::VisitCurrentMethod(HCurrentMethod* instruction) {
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kNoCall);
  locations->SetOut(Location::RegisterLocation(kArtMethodRegister));
}

void InstructionCodeGeneratorRISCV64::VisitCurrentMethod(
    [[maybe_unused]] HCurrentMethod* instruction) {
  // Nothing to do, the method is already at its location.
}

void LocationsBuilderRISCV64::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* instruction) {
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kNoCall);
  locations->SetOut(Location::RequiresRegister());
}

void InstructionCodeGeneratorRISCV64::VisitShouldDeoptimizeFlag(
    HShouldDeoptimizeFlag* instruction) {
  __ Loadw(instruction->GetLocations()->Out().AsRegister<XRegister>(),
           SP,
           codegen_->GetStackOffsetOfShouldDeoptimizeFlag());
}

void LocationsBuilderRISCV64::VisitDeoptimize(HDeoptimize* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator())
      LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
  InvokeRuntimeCallingConvention calling_convention;
  RegisterSet caller_saves = RegisterSet::Empty();
  caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
  locations->SetCustomSlowPathCallerSaves(caller_saves);
  if (IsBooleanValueOrMaterializedCondition(instruction->InputAt(0))) {
    locations->SetInAt(0, Location::RequiresRegister());
  }
}

void InstructionCodeGeneratorRISCV64::VisitDeoptimize(HDeoptimize* instruction) {
  SlowPathCodeRISCV64* slow_path =
      deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathRISCV64>(instruction);
  GenerateTestAndBranch(instruction,
                        /* condition_input_index= */ 0,
                        slow_path->GetEntryLabel(),
                        /* false_target= */ nullptr);
}

void LocationsBuilderRISCV64::VisitDiv(HDiv* instruction) {
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kNoCall);
  switch (instruction->GetResultType()) {
    case DataType::Type::kInt32:
    case DataType::Type::kInt64:
      locations->SetInAt(0, Location::RequiresRegister());
      locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
      break;

    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64:
      locations->SetInAt(0, Location::RequiresFpuRegister());
      locations->SetInAt(1, Location::RequiresFpuRegister());
      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
      break;

    default:
      LOG(FATAL) << "Unexpected div type " << instruction->GetResultType();
      UNREACHABLE();
  }
}

void InstructionCodeGeneratorRISCV64::VisitDiv(HDiv* instruction) {
  DataType::Type type = instruction->GetType();
  LocationSummary* locations = instruction->GetLocations();

  switch (type) {
    case DataType::Type::kInt32:
    case DataType::Type::kInt64:
      GenerateDivRemIntegral(instruction);
      break;
    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64: {
      FRegister dst = locations->Out().AsFpuRegister<FRegister>();
      FRegister lhs = locations->InAt(0).AsFpuRegister<FRegister>();
      FRegister rhs = locations->InAt(1).AsFpuRegister<FRegister>();
      FDiv(dst, lhs, rhs, type);
      break;
    }
    default:
      LOG(FATAL) << "Unexpected div type " << type;
      UNREACHABLE();
  }
}

void LocationsBuilderRISCV64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
  LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction);
  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
}

void InstructionCodeGeneratorRISCV64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
  SlowPathCodeRISCV64* slow_path =
      new (codegen_->GetScopedAllocator()) DivZeroCheckSlowPathRISCV64(instruction);
  codegen_->AddSlowPath(slow_path);
  Location value = instruction->GetLocations()->InAt(0);

  DataType::Type type = instruction->GetType();

  if (!DataType::IsIntegralType(type)) {
    LOG(FATAL) << "Unexpected type " << type << " for DivZeroCheck.";
    UNREACHABLE();
  }

  if (value.IsConstant()) {
    int64_t divisor = codegen_->GetInt64ValueOf(value.GetConstant()->AsConstant());
    if (divisor == 0) {
      __ J(slow_path->GetEntryLabel());
    } else {
      // A division by a non-null constant is valid. We don't need to perform
      // any check, so simply fall through.
    }
  } else {
    __ Beqz(value.AsRegister<XRegister>(), slow_path->GetEntryLabel());
  }
}

void LocationsBuilderRISCV64::VisitDoubleConstant(HDoubleConstant* instruction) {
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kNoCall);
  locations->SetOut(Location::ConstantLocation(instruction));
}

void InstructionCodeGeneratorRISCV64::VisitDoubleConstant(
    [[maybe_unused]] HDoubleConstant* instruction) {
  // Will be generated at use site.
}

void LocationsBuilderRISCV64::VisitEqual(HEqual* instruction) {
  HandleCondition(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitEqual(HEqual* instruction) {
  HandleCondition(instruction);
}

void LocationsBuilderRISCV64::VisitExit(HExit* instruction) {
  instruction->SetLocations(nullptr);
}

void InstructionCodeGeneratorRISCV64::VisitExit([[maybe_unused]] HExit* instruction) {}

void LocationsBuilderRISCV64::VisitFloatConstant(HFloatConstant* instruction) {
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kNoCall);
  locations->SetOut(Location::ConstantLocation(instruction));
}

void InstructionCodeGeneratorRISCV64::VisitFloatConstant(
    [[maybe_unused]] HFloatConstant* instruction) {
  // Will be generated at use site.
}

void LocationsBuilderRISCV64::VisitGoto(HGoto* instruction) {
  instruction->SetLocations(nullptr);
}

void InstructionCodeGeneratorRISCV64::VisitGoto(HGoto* instruction) {
  HandleGoto(instruction, instruction->GetSuccessor());
}

void LocationsBuilderRISCV64::VisitGreaterThan(HGreaterThan* instruction) {
  HandleCondition(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitGreaterThan(HGreaterThan* instruction) {
  HandleCondition(instruction);
}

void LocationsBuilderRISCV64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* instruction) {
  HandleCondition(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* instruction) {
  HandleCondition(instruction);
}

void LocationsBuilderRISCV64::VisitIf(HIf* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  if (IsBooleanValueOrMaterializedCondition(instruction->InputAt(0))) {
    locations->SetInAt(0, Location::RequiresRegister());
    if (GetGraph()->IsCompilingBaseline() &&
        codegen_->GetCompilerOptions().ProfileBranches() &&
        !Runtime::Current()->IsAotCompiler()) {
      DCHECK(instruction->InputAt(0)->IsCondition());
      ProfilingInfo* info = GetGraph()->GetProfilingInfo();
      DCHECK(info != nullptr);
      BranchCache* cache = info->GetBranchCache(instruction->GetDexPc());
      if (cache != nullptr) {
        locations->AddTemp(Location::RequiresRegister());
      }
    }
  }
}

void InstructionCodeGeneratorRISCV64::VisitIf(HIf* instruction) {
  HBasicBlock* true_successor = instruction->IfTrueSuccessor();
  HBasicBlock* false_successor = instruction->IfFalseSuccessor();
  Riscv64Label* true_target = codegen_->GoesToNextBlock(instruction->GetBlock(), true_successor)
      ? nullptr
      : codegen_->GetLabelOf(true_successor);
  Riscv64Label* false_target = codegen_->GoesToNextBlock(instruction->GetBlock(), false_successor)
      ? nullptr
      : codegen_->GetLabelOf(false_successor);
  if (IsBooleanValueOrMaterializedCondition(instruction->InputAt(0))) {
    if (GetGraph()->IsCompilingBaseline() &&
        codegen_->GetCompilerOptions().ProfileBranches() &&
        !Runtime::Current()->IsAotCompiler()) {
      DCHECK(instruction->InputAt(0)->IsCondition());
      ProfilingInfo* info = GetGraph()->GetProfilingInfo();
      DCHECK(info != nullptr);
      BranchCache* cache = info->GetBranchCache(instruction->GetDexPc());
      // Currently, not all If branches are profiled.
      if (cache != nullptr) {
        uint64_t address =
            reinterpret_cast64<uint64_t>(cache) + BranchCache::FalseOffset().Int32Value();
        static_assert(
            BranchCache::TrueOffset().Int32Value() - BranchCache::FalseOffset().Int32Value() == 2,
            "Unexpected offsets for BranchCache");
        Riscv64Label done;
        XRegister condition = instruction->GetLocations()->InAt(0).AsRegister<XRegister>();
        XRegister temp = instruction->GetLocations()->GetTemp(0).AsRegister<XRegister>();
        __ LoadConst64(temp, address);
        __ Sh1Add(temp, condition, temp);
        ScratchRegisterScope srs(GetAssembler());
        XRegister counter = srs.AllocateXRegister();
        __ Loadhu(counter, temp, 0);
        __ Addi(counter, counter, 1);
        {
          ScratchRegisterScope srs2(GetAssembler());
          XRegister overflow = srs2.AllocateXRegister();
          __ Srli(overflow, counter, 16);
          __ Bnez(overflow, &done);
        }
        __ Storeh(counter, temp, 0);
        __ Bind(&done);
      }
    }
  }
  GenerateTestAndBranch(instruction, /* condition_input_index= */ 0, true_target, false_target);
}

void LocationsBuilderRISCV64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
  HandleFieldGet(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
  HandleFieldGet(instruction, instruction->GetFieldInfo());
}

void LocationsBuilderRISCV64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
  HandleFieldSet(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
  HandleFieldSet(instruction,
                 instruction->GetFieldInfo(),
                 instruction->GetValueCanBeNull(),
                 instruction->GetWriteBarrierKind());
}

void LocationsBuilderRISCV64::VisitInstanceOf(HInstanceOf* instruction) {
  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
  bool baker_read_barrier_slow_path = false;
  switch (type_check_kind) {
    case TypeCheckKind::kExactCheck:
    case TypeCheckKind::kAbstractClassCheck:
    case TypeCheckKind::kClassHierarchyCheck:
    case TypeCheckKind::kArrayObjectCheck:
    case TypeCheckKind::kInterfaceCheck: {
      bool needs_read_barrier = codegen_->InstanceOfNeedsReadBarrier(instruction);
      call_kind = needs_read_barrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall;
      baker_read_barrier_slow_path = (kUseBakerReadBarrier && needs_read_barrier) &&
                                     (type_check_kind != TypeCheckKind::kInterfaceCheck);
      break;
    }
    case TypeCheckKind::kArrayCheck:
    case TypeCheckKind::kUnresolvedCheck:
      call_kind = LocationSummary::kCallOnSlowPath;
      break;
    case TypeCheckKind::kBitstringCheck:
      break;
  }

  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, call_kind);
  if (baker_read_barrier_slow_path) {
    locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty());  // No caller-save registers.
  }
  locations->SetInAt(0, Location::RequiresRegister());
  if (type_check_kind == TypeCheckKind::kBitstringCheck) {
    locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)));
    locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)));
    locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)));
  } else {
    locations->SetInAt(1, Location::RequiresRegister());
  }
  // The output does overlap inputs.
  // Note that TypeCheckSlowPathRISCV64 uses this register too.
  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
  locations->AddRegisterTemps(
      NumberOfInstanceOfTemps(codegen_->EmitReadBarrier(), type_check_kind));
}

void InstructionCodeGeneratorRISCV64::VisitInstanceOf(HInstanceOf* instruction) {
  TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
  LocationSummary* locations = instruction->GetLocations();
  Location obj_loc = locations->InAt(0);
  XRegister obj = obj_loc.AsRegister<XRegister>();
  Location cls = (type_check_kind == TypeCheckKind::kBitstringCheck)
      ? Location::NoLocation()
      : locations->InAt(1);
  Location out_loc = locations->Out();
  XRegister out = out_loc.AsRegister<XRegister>();
  const size_t num_temps = NumberOfInstanceOfTemps(codegen_->EmitReadBarrier(), type_check_kind);
  DCHECK_LE(num_temps, 1u);
  Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation();
  const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
  const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
  const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
  const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
  const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value();
  const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value();
  const uint32_t object_array_data_offset =
      mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value();
  Riscv64Label done;
  SlowPathCodeRISCV64* slow_path = nullptr;

  // Return 0 if `obj` is null.
  // Avoid this check if we know `obj` is not null.
  if (instruction->MustDoNullCheck()) {
    __ Mv(out, Zero);
    __ Beqz(obj, &done);
  }

  switch (type_check_kind) {
    case TypeCheckKind::kExactCheck: {
      ReadBarrierOption read_barrier_option =
          codegen_->ReadBarrierOptionForInstanceOf(instruction);
      // /* HeapReference<Class> */ out = obj->klass_
      GenerateReferenceLoadTwoRegisters(
          instruction, out_loc, obj_loc, class_offset, maybe_temp_loc, read_barrier_option);
      // Classes must be equal for the instanceof to succeed.
      __ Xor(out, out, cls.AsRegister<XRegister>());
      __ Seqz(out, out);
      break;
    }

    case TypeCheckKind::kAbstractClassCheck: {
      ReadBarrierOption read_barrier_option =
          codegen_->ReadBarrierOptionForInstanceOf(instruction);
      // /* HeapReference<Class> */ out = obj->klass_
      GenerateReferenceLoadTwoRegisters(
          instruction, out_loc, obj_loc, class_offset, maybe_temp_loc, read_barrier_option);
      // If the class is abstract, we eagerly fetch the super class of the
      // object to avoid doing a comparison we know will fail.
      Riscv64Label loop;
      __ Bind(&loop);
      // /* HeapReference<Class> */ out = out->super_class_
      GenerateReferenceLoadOneRegister(
          instruction, out_loc, super_offset, maybe_temp_loc, read_barrier_option);
      // If `out` is null, we use it for the result, and jump to `done`.
      __ Beqz(out, &done);
      __ Bne(out, cls.AsRegister<XRegister>(), &loop);
      __ LoadConst32(out, 1);
      break;
    }

    case TypeCheckKind::kClassHierarchyCheck: {
      ReadBarrierOption read_barrier_option =
          codegen_->ReadBarrierOptionForInstanceOf(instruction);
      // /* HeapReference<Class> */ out = obj->klass_
      GenerateReferenceLoadTwoRegisters(
          instruction, out_loc, obj_loc, class_offset, maybe_temp_loc, read_barrier_option);
      // Walk over the class hierarchy to find a match.
      Riscv64Label loop, success;
      __ Bind(&loop);
      __ Beq(out, cls.AsRegister<XRegister>(), &success);
      // /* HeapReference<Class> */ out = out->super_class_
      GenerateReferenceLoadOneRegister(
          instruction, out_loc, super_offset, maybe_temp_loc, read_barrier_option);
      __ Bnez(out, &loop);
      // If `out` is null, we use it for the result, and jump to `done`.
      __ J(&done);
      __ Bind(&success);
      __ LoadConst32(out, 1);
      break;
    }

    case TypeCheckKind::kArrayObjectCheck: {
      ReadBarrierOption read_barrier_option =
          codegen_->ReadBarrierOptionForInstanceOf(instruction);
      // FIXME(riscv64): We currently have marking entrypoints for 29 registers.
      // We need to either store entrypoint for register `N` in entry `N-A` where
      // `A` can be up to 5 (Zero, RA, SP, GP, TP are not valid registers for
      // marking), or define two more entrypoints, or request an additional temp
      // from the register allocator instead of using a scratch register.
      ScratchRegisterScope srs(GetAssembler());
      Location tmp = Location::RegisterLocation(srs.AllocateXRegister());
      // /* HeapReference<Class> */ tmp = obj->klass_
      GenerateReferenceLoadTwoRegisters(
          instruction, tmp, obj_loc, class_offset, maybe_temp_loc, read_barrier_option);
      // Do an exact check.
      __ LoadConst32(out, 1);
      __ Beq(tmp.AsRegister<XRegister>(), cls.AsRegister<XRegister>(), &done);
      // Otherwise, we need to check that the object's class is a non-primitive array.
      // /* HeapReference<Class> */ out = out->component_type_
      GenerateReferenceLoadTwoRegisters(
          instruction, out_loc, tmp, component_offset, maybe_temp_loc, read_barrier_option);
      // If `out` is null, we use it for the result, and jump to `done`.
      __ Beqz(out, &done);
      __ Loadhu(out, out, primitive_offset);
      static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
      __ Seqz(out, out);
      break;
    }

    case TypeCheckKind::kArrayCheck: {
      // No read barrier since the slow path will retry upon failure.
      // /* HeapReference<Class> */ out = obj->klass_
      GenerateReferenceLoadTwoRegisters(
          instruction, out_loc, obj_loc, class_offset, maybe_temp_loc, kWithoutReadBarrier);
      DCHECK(locations->OnlyCallsOnSlowPath());
      slow_path = new (codegen_->GetScopedAllocator())
          TypeCheckSlowPathRISCV64(instruction, /* is_fatal= */ false);
      codegen_->AddSlowPath(slow_path);
      __ Bne(out, cls.AsRegister<XRegister>(), slow_path->GetEntryLabel());
      __ LoadConst32(out, 1);
      break;
    }

    case TypeCheckKind::kInterfaceCheck: {
      if (codegen_->InstanceOfNeedsReadBarrier(instruction)) {
        DCHECK(locations->OnlyCallsOnSlowPath());
        slow_path = new (codegen_->GetScopedAllocator()) TypeCheckSlowPathRISCV64(
            instruction, /* is_fatal= */ false);
        codegen_->AddSlowPath(slow_path);
        if (codegen_->EmitNonBakerReadBarrier()) {
          __ J(slow_path->GetEntryLabel());
          break;
        }
        // For Baker read barrier, take the slow path while marking.
        __ Loadw(out, TR, Thread::IsGcMarkingOffset<kRiscv64PointerSize>().Int32Value());
        __ Bnez(out, slow_path->GetEntryLabel());
      }

      // Fast-path without read barriers.
      ScratchRegisterScope srs(GetAssembler());
      XRegister temp = srs.AllocateXRegister();
      // /* HeapReference<Class> */ temp = obj->klass_
      __ Loadwu(temp, obj, class_offset);
      codegen_->MaybeUnpoisonHeapReference(temp);
      // /* HeapReference<Class> */ temp = temp->iftable_
      __ Loadwu(temp, temp, iftable_offset);
      codegen_->MaybeUnpoisonHeapReference(temp);
      // Load the size of the `IfTable`. The `Class::iftable_` is never null.
      __ Loadw(out, temp, array_length_offset);
      // Loop through the `IfTable` and check if any class matches.
      Riscv64Label loop;
      XRegister temp2 = srs.AllocateXRegister();
      __ Bind(&loop);
      __ Beqz(out, &done);  // If taken, the result in `out` is already 0 (false).
      __ Loadwu(temp2, temp, object_array_data_offset);
      codegen_->MaybeUnpoisonHeapReference(temp2);
      // Go to next interface.
      __ Addi(temp, temp, 2 * kHeapReferenceSize);
      __ Addi(out, out, -2);
      // Compare the classes and continue the loop if they do not match.
      __ Bne(cls.AsRegister<XRegister>(), temp2, &loop);
      __ LoadConst32(out, 1);
      break;
    }

    case TypeCheckKind::kUnresolvedCheck: {
      // Note that we indeed only call on slow path, but we always go
      // into the slow path for the unresolved check case.
      //
      // We cannot directly call the InstanceofNonTrivial runtime
      // entry point without resorting to a type checking slow path
      // here (i.e. by calling InvokeRuntime directly), as it would
      // require to assign fixed registers for the inputs of this
      // HInstanceOf instruction (following the runtime calling
      // convention), which might be cluttered by the potential first
      // read barrier emission at the beginning of this method.
      //
      // TODO: Introduce a new runtime entry point taking the object
      // to test (instead of its class) as argument, and let it deal
      // with the read barrier issues. This will let us refactor this
      // case of the `switch` code as it was previously (with a direct
      // call to the runtime not using a type checking slow path).
      // This should also be beneficial for the other cases above.
      DCHECK(locations->OnlyCallsOnSlowPath());
      slow_path = new (codegen_->GetScopedAllocator()) TypeCheckSlowPathRISCV64(
          instruction, /* is_fatal= */ false);
      codegen_->AddSlowPath(slow_path);
      __ J(slow_path->GetEntryLabel());
      break;
    }

    case TypeCheckKind::kBitstringCheck: {
      // /* HeapReference<Class> */ temp = obj->klass_
      GenerateReferenceLoadTwoRegisters(
          instruction, out_loc, obj_loc, class_offset, maybe_temp_loc, kWithoutReadBarrier);

      GenerateBitstringTypeCheckCompare(instruction, out);
      __ Beqz(out, out);
      break;
    }
  }

  __ Bind(&done);

  if (slow_path != nullptr) {
    __ Bind(slow_path->GetExitLabel());
  }
}

void LocationsBuilderRISCV64::VisitIntConstant(HIntConstant* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  locations->SetOut(Location::ConstantLocation(instruction));
}

void InstructionCodeGeneratorRISCV64::VisitIntConstant([[maybe_unused]] HIntConstant* instruction) {
  // Will be generated at use site.
}

void LocationsBuilderRISCV64::VisitIntermediateAddress(HIntermediateAddress* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitIntermediateAddress(HIntermediateAddress* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitInvokeUnresolved(HInvokeUnresolved* instruction) {
  // The trampoline uses the same calling convention as dex calling conventions, except
  // instead of loading arg0/A0 with the target Method*, arg0/A0 will contain the method_idx.
  HandleInvoke(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitInvokeUnresolved(HInvokeUnresolved* instruction) {
  codegen_->GenerateInvokeUnresolvedRuntimeCall(instruction);
}

void LocationsBuilderRISCV64::VisitInvokeInterface(HInvokeInterface* instruction) {
  HandleInvoke(instruction);
  // Use T0 as the hidden argument for `art_quick_imt_conflict_trampoline`.
  if (instruction->GetHiddenArgumentLoadKind() == MethodLoadKind::kRecursive) {
    instruction->GetLocations()->SetInAt(instruction->GetNumberOfArguments() - 1,
                                         Location::RegisterLocation(T0));
  } else {
    instruction->GetLocations()->AddTemp(Location::RegisterLocation(T0));
  }
}

void InstructionCodeGeneratorRISCV64::VisitInvokeInterface(HInvokeInterface* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  XRegister temp = locations->GetTemp(0).AsRegister<XRegister>();
  XRegister receiver = locations->InAt(0).AsRegister<XRegister>();
  int32_t class_offset = mirror::Object::ClassOffset().Int32Value();
  Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kRiscv64PointerSize);

  // /* HeapReference<Class> */ temp = receiver->klass_
  __ Loadwu(temp, receiver, class_offset);
  codegen_->MaybeRecordImplicitNullCheck(instruction);
  // Instead of simply (possibly) unpoisoning `temp` here, we should
  // emit a read barrier for the previous class reference load.
  // However this is not required in practice, as this is an
  // intermediate/temporary reference and because the current
  // concurrent copying collector keeps the from-space memory
  // intact/accessible until the end of the marking phase (the
  // concurrent copying collector may not in the future).
  codegen_->MaybeUnpoisonHeapReference(temp);

  // If we're compiling baseline, update the inline cache.
  codegen_->MaybeGenerateInlineCacheCheck(instruction, temp);

  // The register T0 is required to be used for the hidden argument in
  // `art_quick_imt_conflict_trampoline`.
  if (instruction->GetHiddenArgumentLoadKind() != MethodLoadKind::kRecursive &&
      instruction->GetHiddenArgumentLoadKind() != MethodLoadKind::kRuntimeCall) {
    Location hidden_reg = instruction->GetLocations()->GetTemp(1);
    // Load the resolved interface method in the hidden argument register T0.
    DCHECK_EQ(T0, hidden_reg.AsRegister<XRegister>());
    codegen_->LoadMethod(instruction->GetHiddenArgumentLoadKind(), hidden_reg, instruction);
  }

  __ Loadd(temp, temp, mirror::Class::ImtPtrOffset(kRiscv64PointerSize).Uint32Value());
  uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement(
      instruction->GetImtIndex(), kRiscv64PointerSize));
  // temp = temp->GetImtEntryAt(method_offset);
  __ Loadd(temp, temp, method_offset);
  if (instruction->GetHiddenArgumentLoadKind() == MethodLoadKind::kRuntimeCall) {
    // We pass the method from the IMT in case of a conflict. This will ensure
    // we go into the runtime to resolve the actual method.
    Location hidden_reg = instruction->GetLocations()->GetTemp(1);
    DCHECK_EQ(T0, hidden_reg.AsRegister<XRegister>());
    __ Mv(hidden_reg.AsRegister<XRegister>(), temp);
  }
  // RA = temp->GetEntryPoint();
  __ Loadd(RA, temp, entry_point.Int32Value());

  // RA();
  __ Jalr(RA);
  DCHECK(!codegen_->IsLeafMethod());
  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
}

void LocationsBuilderRISCV64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* instruction) {
  // Explicit clinit checks triggered by static invokes must have been pruned by
  // art::PrepareForRegisterAllocation.
  DCHECK(!instruction->IsStaticWithExplicitClinitCheck());

  IntrinsicLocationsBuilderRISCV64 intrinsic(GetGraph()->GetAllocator(), codegen_);
  if (intrinsic.TryDispatch(instruction)) {
    return;
  }

  if (instruction->GetCodePtrLocation() == CodePtrLocation::kCallCriticalNative) {
    CriticalNativeCallingConventionVisitorRiscv64 calling_convention_visitor(
        /*for_register_allocation=*/ true);
    CodeGenerator::CreateCommonInvokeLocationSummary(instruction, &calling_convention_visitor);
  } else {
    HandleInvoke(instruction);
  }
}

static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorRISCV64* codegen) {
  if (invoke->GetLocations()->Intrinsified()) {
    IntrinsicCodeGeneratorRISCV64 intrinsic(codegen);
    intrinsic.Dispatch(invoke);
    return true;
  }
  return false;
}

void InstructionCodeGeneratorRISCV64::VisitInvokeStaticOrDirect(
    HInvokeStaticOrDirect* instruction) {
  // Explicit clinit checks triggered by static invokes must have been pruned by
  // art::PrepareForRegisterAllocation.
  DCHECK(!instruction->IsStaticWithExplicitClinitCheck());

  if (TryGenerateIntrinsicCode(instruction, codegen_)) {
    return;
  }

  LocationSummary* locations = instruction->GetLocations();
  codegen_->GenerateStaticOrDirectCall(
      instruction, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
}

void LocationsBuilderRISCV64::VisitInvokeVirtual(HInvokeVirtual* instruction) {
  IntrinsicLocationsBuilderRISCV64 intrinsic(GetGraph()->GetAllocator(), codegen_);
  if (intrinsic.TryDispatch(instruction)) {
    return;
  }

  HandleInvoke(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitInvokeVirtual(HInvokeVirtual* instruction) {
  if (TryGenerateIntrinsicCode(instruction, codegen_)) {
    return;
  }

  codegen_->GenerateVirtualCall(instruction, instruction->GetLocations()->GetTemp(0));
  DCHECK(!codegen_->IsLeafMethod());
}

void LocationsBuilderRISCV64::VisitInvokePolymorphic(HInvokePolymorphic* instruction) {
  IntrinsicLocationsBuilderRISCV64 intrinsic(GetGraph()->GetAllocator(), codegen_);
  if (intrinsic.TryDispatch(instruction)) {
    return;
  }
  HandleInvoke(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitInvokePolymorphic(HInvokePolymorphic* instruction) {
  if (TryGenerateIntrinsicCode(instruction, codegen_)) {
    return;
  }
  codegen_->GenerateInvokePolymorphicCall(instruction);
}

void LocationsBuilderRISCV64::VisitInvokeCustom(HInvokeCustom* instruction) {
  HandleInvoke(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitInvokeCustom(HInvokeCustom* instruction) {
  codegen_->GenerateInvokeCustomCall(instruction);
}

void LocationsBuilderRISCV64::VisitLessThan(HLessThan* instruction) {
  HandleCondition(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitLessThan(HLessThan* instruction) {
  HandleCondition(instruction);
}

void LocationsBuilderRISCV64::VisitLessThanOrEqual(HLessThanOrEqual* instruction) {
  HandleCondition(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitLessThanOrEqual(HLessThanOrEqual* instruction) {
  HandleCondition(instruction);
}

void LocationsBuilderRISCV64::VisitLoadClass(HLoadClass* instruction) {
  HLoadClass::LoadKind load_kind = instruction->GetLoadKind();
  if (load_kind == HLoadClass::LoadKind::kRuntimeCall) {
    InvokeRuntimeCallingConvention calling_convention;
    Location loc = Location::RegisterLocation(calling_convention.GetRegisterAt(0));
    DCHECK_EQ(DataType::Type::kReference, instruction->GetType());
    DCHECK(loc.Equals(calling_convention.GetReturnLocation(DataType::Type::kReference)));
    CodeGenerator::CreateLoadClassRuntimeCallLocationSummary(instruction, loc, loc);
    return;
  }
  DCHECK_EQ(instruction->NeedsAccessCheck(),
            load_kind == HLoadClass::LoadKind::kBssEntryPublic ||
                load_kind == HLoadClass::LoadKind::kBssEntryPackage);

  const bool requires_read_barrier = !instruction->IsInImage() && codegen_->EmitReadBarrier();
  LocationSummary::CallKind call_kind = (instruction->NeedsEnvironment() || requires_read_barrier)
      ? LocationSummary::kCallOnSlowPath
      : LocationSummary::kNoCall;
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, call_kind);
  if (kUseBakerReadBarrier && requires_read_barrier && !instruction->NeedsEnvironment()) {
    locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty());  // No caller-save registers.
  }
  if (load_kind == HLoadClass::LoadKind::kReferrersClass) {
    locations->SetInAt(0, Location::RequiresRegister());
  }
  locations->SetOut(Location::RequiresRegister());
  if (load_kind == HLoadClass::LoadKind::kBssEntry ||
      load_kind == HLoadClass::LoadKind::kBssEntryPublic ||
      load_kind == HLoadClass::LoadKind::kBssEntryPackage) {
    if (codegen_->EmitNonBakerReadBarrier()) {
      // For non-Baker read barriers we have a temp-clobbering call.
    } else {
      // Rely on the type resolution or initialization and marking to save everything we need.
      locations->SetCustomSlowPathCallerSaves(OneRegInReferenceOutSaveEverythingCallerSaves());
    }
  }
}

// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not
// move.
void InstructionCodeGeneratorRISCV64::VisitLoadClass(HLoadClass* instruction)
    NO_THREAD_SAFETY_ANALYSIS {
  HLoadClass::LoadKind load_kind = instruction->GetLoadKind();
  if (load_kind == HLoadClass::LoadKind::kRuntimeCall) {
    codegen_->GenerateLoadClassRuntimeCall(instruction);
    return;
  }
  DCHECK_EQ(instruction->NeedsAccessCheck(),
            load_kind == HLoadClass::LoadKind::kBssEntryPublic ||
                load_kind == HLoadClass::LoadKind::kBssEntryPackage);

  LocationSummary* locations = instruction->GetLocations();
  Location out_loc = locations->Out();
  XRegister out = out_loc.AsRegister<XRegister>();
  const ReadBarrierOption read_barrier_option =
      instruction->IsInImage() ? kWithoutReadBarrier : codegen_->GetCompilerReadBarrierOption();
  bool generate_null_check = false;
  switch (load_kind) {
    case HLoadClass::LoadKind::kReferrersClass: {
      DCHECK(!instruction->CanCallRuntime());
      DCHECK(!instruction->MustGenerateClinitCheck());
      // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
      XRegister current_method = locations->InAt(0).AsRegister<XRegister>();
      codegen_->GenerateGcRootFieldLoad(instruction,
                                        out_loc,
                                        current_method,
                                        ArtMethod::DeclaringClassOffset().Int32Value(),
                                        read_barrier_option);
      break;
    }
    case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: {
      DCHECK(codegen_->GetCompilerOptions().IsBootImage() ||
             codegen_->GetCompilerOptions().IsBootImageExtension());
      DCHECK_EQ(read_barrier_option, kWithoutReadBarrier);
      CodeGeneratorRISCV64::PcRelativePatchInfo* info_high =
          codegen_->NewBootImageTypePatch(instruction->GetDexFile(), instruction->GetTypeIndex());
      codegen_->EmitPcRelativeAuipcPlaceholder(info_high, out);
      CodeGeneratorRISCV64::PcRelativePatchInfo* info_low =
          codegen_->NewBootImageTypePatch(
              instruction->GetDexFile(), instruction->GetTypeIndex(), info_high);
      codegen_->EmitPcRelativeAddiPlaceholder(info_low, out, out);
      break;
    }
    case HLoadClass::LoadKind::kBootImageRelRo: {
      DCHECK(!codegen_->GetCompilerOptions().IsBootImage());
      uint32_t boot_image_offset = codegen_->GetBootImageOffset(instruction);
      codegen_->LoadBootImageRelRoEntry(out, boot_image_offset);
      break;
    }
    case HLoadClass::LoadKind::kAppImageRelRo: {
      DCHECK(codegen_->GetCompilerOptions().IsAppImage());
      DCHECK_EQ(read_barrier_option, kWithoutReadBarrier);
      CodeGeneratorRISCV64::PcRelativePatchInfo* info_high =
          codegen_->NewAppImageTypePatch(instruction->GetDexFile(), instruction->GetTypeIndex());
      codegen_->EmitPcRelativeAuipcPlaceholder(info_high, out);
      CodeGeneratorRISCV64::PcRelativePatchInfo* info_low =
          codegen_->NewAppImageTypePatch(
              instruction->GetDexFile(), instruction->GetTypeIndex(), info_high);
      codegen_->EmitPcRelativeLwuPlaceholder(info_low, out, out);
      break;
    }
    case HLoadClass::LoadKind::kBssEntry:
    case HLoadClass::LoadKind::kBssEntryPublic:
    case HLoadClass::LoadKind::kBssEntryPackage: {
      CodeGeneratorRISCV64::PcRelativePatchInfo* bss_info_high =
          codegen_->NewTypeBssEntryPatch(instruction);
      codegen_->EmitPcRelativeAuipcPlaceholder(bss_info_high, out);
      CodeGeneratorRISCV64::PcRelativePatchInfo* info_low = codegen_->NewTypeBssEntryPatch(
          instruction, bss_info_high);
      codegen_->GenerateGcRootFieldLoad(instruction,
                                        out_loc,
                                        out,
                                        /* offset= */ kLinkTimeOffsetPlaceholderLow,
                                        read_barrier_option,
                                        &info_low->label);
      generate_null_check = true;
      break;
    }
    case HLoadClass::LoadKind::kJitBootImageAddress: {
      DCHECK_EQ(read_barrier_option, kWithoutReadBarrier);
      uint32_t address = reinterpret_cast32<uint32_t>(instruction->GetClass().Get());
      DCHECK_NE(address, 0u);
      __ Loadwu(out, codegen_->DeduplicateBootImageAddressLiteral(address));
      break;
    }
    case HLoadClass::LoadKind::kJitTableAddress:
      __ Loadwu(out, codegen_->DeduplicateJitClassLiteral(instruction->GetDexFile(),
                                                          instruction->GetTypeIndex(),
                                                          instruction->GetClass()));
      codegen_->GenerateGcRootFieldLoad(
          instruction, out_loc, out, /* offset= */ 0, read_barrier_option);
      break;
    case HLoadClass::LoadKind::kRuntimeCall:
    case HLoadClass::LoadKind::kInvalid:
      LOG(FATAL) << "UNREACHABLE";
      UNREACHABLE();
  }

  if (generate_null_check || instruction->MustGenerateClinitCheck()) {
    DCHECK(instruction->CanCallRuntime());
    SlowPathCodeRISCV64* slow_path =
        new (codegen_->GetScopedAllocator()) LoadClassSlowPathRISCV64(instruction, instruction);
    codegen_->AddSlowPath(slow_path);
    if (generate_null_check) {
      __ Beqz(out, slow_path->GetEntryLabel());
    }
    if (instruction->MustGenerateClinitCheck()) {
      GenerateClassInitializationCheck(slow_path, out);
    } else {
      __ Bind(slow_path->GetExitLabel());
    }
  }
}

void LocationsBuilderRISCV64::VisitLoadException(HLoadException* instruction) {
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kNoCall);
  locations->SetOut(Location::RequiresRegister());
}

void InstructionCodeGeneratorRISCV64::VisitLoadException(HLoadException* instruction) {
  XRegister out = instruction->GetLocations()->Out().AsRegister<XRegister>();
  __ Loadwu(out, TR, GetExceptionTlsOffset());
}

void LocationsBuilderRISCV64::VisitLoadMethodHandle(HLoadMethodHandle* instruction) {
  InvokeRuntimeCallingConvention calling_convention;
  Location loc = Location::RegisterLocation(calling_convention.GetRegisterAt(0));
  CodeGenerator::CreateLoadMethodHandleRuntimeCallLocationSummary(instruction, loc, loc);
}

void InstructionCodeGeneratorRISCV64::VisitLoadMethodHandle(HLoadMethodHandle* instruction) {
  codegen_->GenerateLoadMethodHandleRuntimeCall(instruction);
}

void LocationsBuilderRISCV64::VisitLoadMethodType(HLoadMethodType* instruction) {
  InvokeRuntimeCallingConvention calling_convention;
  Location loc = Location::RegisterLocation(calling_convention.GetRegisterAt(0));
  CodeGenerator::CreateLoadMethodTypeRuntimeCallLocationSummary(instruction, loc, loc);
}

void InstructionCodeGeneratorRISCV64::VisitLoadMethodType(HLoadMethodType* instruction) {
  codegen_->GenerateLoadMethodTypeRuntimeCall(instruction);
}

void LocationsBuilderRISCV64::VisitLoadString(HLoadString* instruction) {
  HLoadString::LoadKind load_kind = instruction->GetLoadKind();
  LocationSummary::CallKind call_kind = codegen_->GetLoadStringCallKind(instruction);
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, call_kind);
  if (load_kind == HLoadString::LoadKind::kRuntimeCall) {
    InvokeRuntimeCallingConvention calling_convention;
    DCHECK_EQ(DataType::Type::kReference, instruction->GetType());
    locations->SetOut(calling_convention.GetReturnLocation(DataType::Type::kReference));
  } else {
    locations->SetOut(Location::RequiresRegister());
    if (load_kind == HLoadString::LoadKind::kBssEntry) {
      if (codegen_->EmitNonBakerReadBarrier()) {
        // For non-Baker read barriers we have a temp-clobbering call.
      } else {
        // Rely on the pResolveString and marking to save everything we need.
        locations->SetCustomSlowPathCallerSaves(OneRegInReferenceOutSaveEverythingCallerSaves());
      }
    }
  }
}

// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not
// move.
void InstructionCodeGeneratorRISCV64::VisitLoadString(HLoadString* instruction)
    NO_THREAD_SAFETY_ANALYSIS {
  HLoadString::LoadKind load_kind = instruction->GetLoadKind();
  LocationSummary* locations = instruction->GetLocations();
  Location out_loc = locations->Out();
  XRegister out = out_loc.AsRegister<XRegister>();

  switch (load_kind) {
    case HLoadString::LoadKind::kBootImageLinkTimePcRelative: {
      DCHECK(codegen_->GetCompilerOptions().IsBootImage() ||
             codegen_->GetCompilerOptions().IsBootImageExtension());
      CodeGeneratorRISCV64::PcRelativePatchInfo* info_high = codegen_->NewBootImageStringPatch(
          instruction->GetDexFile(), instruction->GetStringIndex());
      codegen_->EmitPcRelativeAuipcPlaceholder(info_high, out);
      CodeGeneratorRISCV64::PcRelativePatchInfo* info_low = codegen_->NewBootImageStringPatch(
          instruction->GetDexFile(), instruction->GetStringIndex(), info_high);
      codegen_->EmitPcRelativeAddiPlaceholder(info_low, out, out);
      return;
    }
    case HLoadString::LoadKind::kBootImageRelRo: {
      DCHECK(!codegen_->GetCompilerOptions().IsBootImage());
      uint32_t boot_image_offset = codegen_->GetBootImageOffset(instruction);
      codegen_->LoadBootImageRelRoEntry(out, boot_image_offset);
      return;
    }
    case HLoadString::LoadKind::kBssEntry: {
      CodeGeneratorRISCV64::PcRelativePatchInfo* info_high = codegen_->NewStringBssEntryPatch(
          instruction->GetDexFile(), instruction->GetStringIndex());
      codegen_->EmitPcRelativeAuipcPlaceholder(info_high, out);
      CodeGeneratorRISCV64::PcRelativePatchInfo* info_low = codegen_->NewStringBssEntryPatch(
          instruction->GetDexFile(), instruction->GetStringIndex(), info_high);
      codegen_->GenerateGcRootFieldLoad(instruction,
                                        out_loc,
                                        out,
                                        /* offset= */ kLinkTimeOffsetPlaceholderLow,
                                        codegen_->GetCompilerReadBarrierOption(),
                                        &info_low->label);
      SlowPathCodeRISCV64* slow_path =
          new (codegen_->GetScopedAllocator()) LoadStringSlowPathRISCV64(instruction);
      codegen_->AddSlowPath(slow_path);
      __ Beqz(out, slow_path->GetEntryLabel());
      __ Bind(slow_path->GetExitLabel());
      return;
    }
    case HLoadString::LoadKind::kJitBootImageAddress: {
      uint32_t address = reinterpret_cast32<uint32_t>(instruction->GetString().Get());
      DCHECK_NE(address, 0u);
      __ Loadwu(out, codegen_->DeduplicateBootImageAddressLiteral(address));
      return;
    }
    case HLoadString::LoadKind::kJitTableAddress:
      __ Loadwu(
          out,
          codegen_->DeduplicateJitStringLiteral(
              instruction->GetDexFile(), instruction->GetStringIndex(), instruction->GetString()));
      codegen_->GenerateGcRootFieldLoad(
          instruction, out_loc, out, 0, codegen_->GetCompilerReadBarrierOption());
      return;
    default:
      break;
  }

  DCHECK(load_kind == HLoadString::LoadKind::kRuntimeCall);
  InvokeRuntimeCallingConvention calling_convention;
  DCHECK(calling_convention.GetReturnLocation(DataType::Type::kReference).Equals(out_loc));
  __ LoadConst32(calling_convention.GetRegisterAt(0), instruction->GetStringIndex().index_);
  codegen_->InvokeRuntime(kQuickResolveString, instruction, instruction->GetDexPc());
  CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
}

void LocationsBuilderRISCV64::VisitLongConstant(HLongConstant* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  locations->SetOut(Location::ConstantLocation(instruction));
}

void InstructionCodeGeneratorRISCV64::VisitLongConstant(
    [[maybe_unused]] HLongConstant* instruction) {
  // Will be generated at use site.
}

void LocationsBuilderRISCV64::VisitMax(HMax* instruction) {
  HandleBinaryOp(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitMax(HMax* instruction) {
  HandleBinaryOp(instruction);
}

void LocationsBuilderRISCV64::VisitMemoryBarrier(HMemoryBarrier* instruction) {
  instruction->SetLocations(nullptr);
}

void InstructionCodeGeneratorRISCV64::VisitMemoryBarrier(HMemoryBarrier* instruction) {
  codegen_->GenerateMemoryBarrier(instruction->GetBarrierKind());
}

void LocationsBuilderRISCV64::VisitMethodEntryHook(HMethodEntryHook* instruction) {
  new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
}

void InstructionCodeGeneratorRISCV64::VisitMethodEntryHook(HMethodEntryHook* instruction) {
  DCHECK(codegen_->GetCompilerOptions().IsJitCompiler() && GetGraph()->IsDebuggable());
  DCHECK(codegen_->RequiresCurrentMethod());
  GenerateMethodEntryExitHook(instruction);
}

void LocationsBuilderRISCV64::VisitMethodExitHook(HMethodExitHook* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator())
      LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
  DataType::Type return_type = instruction->InputAt(0)->GetType();
  locations->SetInAt(0, Riscv64ReturnLocation(return_type));
}

void InstructionCodeGeneratorRISCV64::VisitMethodExitHook(HMethodExitHook* instruction) {
  DCHECK(codegen_->GetCompilerOptions().IsJitCompiler() && GetGraph()->IsDebuggable());
  DCHECK(codegen_->RequiresCurrentMethod());
  GenerateMethodEntryExitHook(instruction);
}

void LocationsBuilderRISCV64::VisitMin(HMin* instruction) {
  HandleBinaryOp(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitMin(HMin* instruction) {
  HandleBinaryOp(instruction);
}

void LocationsBuilderRISCV64::VisitMonitorOperation(HMonitorOperation* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(
      instruction, LocationSummary::kCallOnMainOnly);
  InvokeRuntimeCallingConvention calling_convention;
  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
}

void InstructionCodeGeneratorRISCV64::VisitMonitorOperation(HMonitorOperation* instruction) {
  codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject,
                          instruction,
                          instruction->GetDexPc());
  if (instruction->IsEnter()) {
    CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
  } else {
    CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>();
  }
}

void LocationsBuilderRISCV64::VisitMul(HMul* instruction) {
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kNoCall);
  switch (instruction->GetResultType()) {
    case DataType::Type::kInt32:
    case DataType::Type::kInt64:
      locations->SetInAt(0, Location::RequiresRegister());
      locations->SetInAt(1, Location::RequiresRegister());
      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
      break;

    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64:
      locations->SetInAt(0, Location::RequiresFpuRegister());
      locations->SetInAt(1, Location::RequiresFpuRegister());
      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
      break;

    default:
      LOG(FATAL) << "Unexpected mul type " << instruction->GetResultType();
  }
}

void InstructionCodeGeneratorRISCV64::VisitMul(HMul* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  switch (instruction->GetResultType()) {
    case DataType::Type::kInt32:
      __ Mulw(locations->Out().AsRegister<XRegister>(),
              locations->InAt(0).AsRegister<XRegister>(),
              locations->InAt(1).AsRegister<XRegister>());
      break;

    case DataType::Type::kInt64:
      __ Mul(locations->Out().AsRegister<XRegister>(),
             locations->InAt(0).AsRegister<XRegister>(),
             locations->InAt(1).AsRegister<XRegister>());
      break;

    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64:
      FMul(locations->Out().AsFpuRegister<FRegister>(),
           locations->InAt(0).AsFpuRegister<FRegister>(),
           locations->InAt(1).AsFpuRegister<FRegister>(),
           instruction->GetResultType());
      break;

    default:
      LOG(FATAL) << "Unexpected mul type " << instruction->GetResultType();
  }
}

void LocationsBuilderRISCV64::VisitNeg(HNeg* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  switch (instruction->GetResultType()) {
    case DataType::Type::kInt32:
    case DataType::Type::kInt64:
      locations->SetInAt(0, Location::RequiresRegister());
      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
      break;

    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64:
      locations->SetInAt(0, Location::RequiresFpuRegister());
      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
      break;

    default:
      LOG(FATAL) << "Unexpected neg type " << instruction->GetResultType();
      UNREACHABLE();
  }
}

void InstructionCodeGeneratorRISCV64::VisitNeg(HNeg* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  switch (instruction->GetResultType()) {
    case DataType::Type::kInt32:
      __ NegW(locations->Out().AsRegister<XRegister>(), locations->InAt(0).AsRegister<XRegister>());
      break;

    case DataType::Type::kInt64:
      __ Neg(locations->Out().AsRegister<XRegister>(), locations->InAt(0).AsRegister<XRegister>());
      break;

    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64:
      FNeg(locations->Out().AsFpuRegister<FRegister>(),
           locations->InAt(0).AsFpuRegister<FRegister>(),
           instruction->GetResultType());
      break;

    default:
      LOG(FATAL) << "Unexpected neg type " << instruction->GetResultType();
      UNREACHABLE();
  }
}

void LocationsBuilderRISCV64::VisitNewArray(HNewArray* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator())
      LocationSummary(instruction, LocationSummary::kCallOnMainOnly);
  InvokeRuntimeCallingConvention calling_convention;
  locations->SetOut(calling_convention.GetReturnLocation(DataType::Type::kReference));
  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
}

void InstructionCodeGeneratorRISCV64::VisitNewArray(HNewArray* instruction) {
  QuickEntrypointEnum entrypoint = CodeGenerator::GetArrayAllocationEntrypoint(instruction);
  codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc());
  CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>();
  DCHECK(!codegen_->IsLeafMethod());
}

void LocationsBuilderRISCV64::VisitNewInstance(HNewInstance* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(
      instruction, LocationSummary::kCallOnMainOnly);
  InvokeRuntimeCallingConvention calling_convention;
  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
  locations->SetOut(calling_convention.GetReturnLocation(DataType::Type::kReference));
}

void InstructionCodeGeneratorRISCV64::VisitNewInstance(HNewInstance* instruction) {
  codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc());
  CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>();
}

void LocationsBuilderRISCV64::VisitNop(HNop* instruction) {
  new (GetGraph()->GetAllocator()) LocationSummary(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitNop([[maybe_unused]] HNop* instruction) {
  // The environment recording already happened in CodeGenerator::Compile.
}

void LocationsBuilderRISCV64::VisitNot(HNot* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  locations->SetInAt(0, Location::RequiresRegister());
  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}

void InstructionCodeGeneratorRISCV64::VisitNot(HNot* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  switch (instruction->GetResultType()) {
    case DataType::Type::kInt32:
    case DataType::Type::kInt64:
      __ Not(locations->Out().AsRegister<XRegister>(), locations->InAt(0).AsRegister<XRegister>());
      break;

    default:
      LOG(FATAL) << "Unexpected type for not operation " << instruction->GetResultType();
      UNREACHABLE();
  }
}

void LocationsBuilderRISCV64::VisitNotEqual(HNotEqual* instruction) {
  HandleCondition(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitNotEqual(HNotEqual* instruction) {
  HandleCondition(instruction);
}

void LocationsBuilderRISCV64::VisitNullConstant(HNullConstant* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  locations->SetOut(Location::ConstantLocation(instruction));
}

void InstructionCodeGeneratorRISCV64::VisitNullConstant(
    [[maybe_unused]] HNullConstant* instruction) {
  // Will be generated at use site.
}

void LocationsBuilderRISCV64::VisitNullCheck(HNullCheck* instruction) {
  LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction);
  locations->SetInAt(0, Location::RequiresRegister());
}

void InstructionCodeGeneratorRISCV64::VisitNullCheck(HNullCheck* instruction) {
  codegen_->GenerateNullCheck(instruction);
}

void LocationsBuilderRISCV64::VisitOr(HOr* instruction) {
  HandleBinaryOp(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitOr(HOr* instruction) {
  HandleBinaryOp(instruction);
}

void LocationsBuilderRISCV64::VisitPackedSwitch(HPackedSwitch* instruction) {
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kNoCall);
  locations->SetInAt(0, Location::RequiresRegister());
}

void InstructionCodeGeneratorRISCV64::VisitPackedSwitch(HPackedSwitch* instruction) {
  int32_t lower_bound = instruction->GetStartValue();
  uint32_t num_entries = instruction->GetNumEntries();
  LocationSummary* locations = instruction->GetLocations();
  XRegister value = locations->InAt(0).AsRegister<XRegister>();
  HBasicBlock* switch_block = instruction->GetBlock();
  HBasicBlock* default_block = instruction->GetDefaultBlock();

  // Prepare a temporary register and an adjusted zero-based value.
  ScratchRegisterScope srs(GetAssembler());
  XRegister temp = srs.AllocateXRegister();
  XRegister adjusted = value;
  if (lower_bound != 0) {
    adjusted = temp;
    __ AddConst32(temp, value, -lower_bound);
  }

  // Jump to the default block if the index is out of the packed switch value range.
  // Note: We could save one instruction for `num_entries == 1` with BNEZ but the
  // `HInstructionBuilder` transforms that case to an `HIf`, so let's keep the code simple.
  CHECK_NE(num_entries, 0u);  // `HInstructionBuilder` creates a `HGoto` for empty packed-switch.
  {
    ScratchRegisterScope srs2(GetAssembler());
    XRegister temp2 = srs2.AllocateXRegister();
    __ LoadConst32(temp2, num_entries);
    __ Bgeu(adjusted, temp2, codegen_->GetLabelOf(default_block));  // Can clobber `TMP` if taken.
  }

  if (num_entries >= kPackedSwitchCompareJumpThreshold) {
    GenTableBasedPackedSwitch(adjusted, temp, num_entries, switch_block);
  } else {
    GenPackedSwitchWithCompares(adjusted, temp, num_entries, switch_block);
  }
}

void LocationsBuilderRISCV64::VisitParallelMove([[maybe_unused]] HParallelMove* instruction) {
  LOG(FATAL) << "Unreachable";
}

void InstructionCodeGeneratorRISCV64::VisitParallelMove(HParallelMove* instruction) {
  if (instruction->GetNext()->IsSuspendCheck() &&
      instruction->GetBlock()->GetLoopInformation() != nullptr) {
    HSuspendCheck* suspend_check = instruction->GetNext()->AsSuspendCheck();
    // The back edge will generate the suspend check.
    codegen_->ClearSpillSlotsFromLoopPhisInStackMap(suspend_check, instruction);
  }

  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
}

void LocationsBuilderRISCV64::VisitParameterValue(HParameterValue* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
  if (location.IsStackSlot()) {
    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
  } else if (location.IsDoubleStackSlot()) {
    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
  }
  locations->SetOut(location);
}

void InstructionCodeGeneratorRISCV64::VisitParameterValue(
    [[maybe_unused]] HParameterValue* instruction) {
  // Nothing to do, the parameter is already at its location.
}

void LocationsBuilderRISCV64::VisitPhi(HPhi* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) {
    locations->SetInAt(i, Location::Any());
  }
  locations->SetOut(Location::Any());
}

void InstructionCodeGeneratorRISCV64::VisitPhi([[maybe_unused]] HPhi* instruction) {
  LOG(FATAL) << "Unreachable";
}

void LocationsBuilderRISCV64::VisitRem(HRem* instruction) {
  DataType::Type type = instruction->GetResultType();
  LocationSummary::CallKind call_kind =
      DataType::IsFloatingPointType(type) ? LocationSummary::kCallOnMainOnly
                                          : LocationSummary::kNoCall;
  LocationSummary* locations =
      new (GetGraph()->GetAllocator()) LocationSummary(instruction, call_kind);

  switch (type) {
    case DataType::Type::kInt32:
    case DataType::Type::kInt64:
      locations->SetInAt(0, Location::RequiresRegister());
      locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
      break;

    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64: {
      InvokeRuntimeCallingConvention calling_convention;
      locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0)));
      locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1)));
      locations->SetOut(calling_convention.GetReturnLocation(type));
      break;
    }

    default:
      LOG(FATAL) << "Unexpected rem type " << type;
      UNREACHABLE();
  }
}

void InstructionCodeGeneratorRISCV64::VisitRem(HRem* instruction) {
  DataType::Type type = instruction->GetType();

  switch (type) {
    case DataType::Type::kInt32:
    case DataType::Type::kInt64:
      GenerateDivRemIntegral(instruction);
      break;

    case DataType::Type::kFloat32:
    case DataType::Type::kFloat64: {
      QuickEntrypointEnum entrypoint =
          (type == DataType::Type::kFloat32) ? kQuickFmodf : kQuickFmod;
      codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc());
      if (type == DataType::Type::kFloat32) {
        CheckEntrypointTypes<kQuickFmodf, float, float, float>();
      } else {
        CheckEntrypointTypes<kQuickFmod, double, double, double>();
      }
      break;
    }
    default:
      LOG(FATAL) << "Unexpected rem type " << type;
      UNREACHABLE();
  }
}

void LocationsBuilderRISCV64::VisitReturn(HReturn* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  DataType::Type return_type = instruction->InputAt(0)->GetType();
  DCHECK_NE(return_type, DataType::Type::kVoid);
  locations->SetInAt(0, Riscv64ReturnLocation(return_type));
}

void InstructionCodeGeneratorRISCV64::VisitReturn(HReturn* instruction) {
  if (GetGraph()->IsCompilingOsr()) {
    // To simplify callers of an OSR method, we put a floating point return value
    // in both floating point and core return registers.
    DataType::Type type = instruction->InputAt(0)->GetType();
    if (DataType::IsFloatingPointType(type)) {
      FMvX(A0, FA0, type);
    }
  }
  codegen_->GenerateFrameExit();
}

void LocationsBuilderRISCV64::VisitReturnVoid(HReturnVoid* instruction) {
  instruction->SetLocations(nullptr);
}

void InstructionCodeGeneratorRISCV64::VisitReturnVoid([[maybe_unused]] HReturnVoid* instruction) {
  codegen_->GenerateFrameExit();
}

void LocationsBuilderRISCV64::VisitRor(HRor* instruction) {
  HandleShift(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitRor(HRor* instruction) {
  HandleShift(instruction);
}

void LocationsBuilderRISCV64::VisitShl(HShl* instruction) {
  HandleShift(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitShl(HShl* instruction) {
  HandleShift(instruction);
}

void LocationsBuilderRISCV64::VisitShr(HShr* instruction) {
  HandleShift(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitShr(HShr* instruction) {
  HandleShift(instruction);
}

void LocationsBuilderRISCV64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
  HandleFieldGet(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
  HandleFieldGet(instruction, instruction->GetFieldInfo());
}

void LocationsBuilderRISCV64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
  HandleFieldSet(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
  HandleFieldSet(instruction,
                 instruction->GetFieldInfo(),
                 instruction->GetValueCanBeNull(),
                 instruction->GetWriteBarrierKind());
}

void LocationsBuilderRISCV64::VisitStringBuilderAppend(HStringBuilderAppend* instruction) {
  codegen_->CreateStringBuilderAppendLocations(instruction, Location::RegisterLocation(A0));
}

void InstructionCodeGeneratorRISCV64::VisitStringBuilderAppend(HStringBuilderAppend* instruction) {
  __ LoadConst32(A0, instruction->GetFormat()->GetValue());
  codegen_->InvokeRuntime(kQuickStringBuilderAppend, instruction, instruction->GetDexPc());
}

void LocationsBuilderRISCV64::VisitUnresolvedInstanceFieldGet(
    HUnresolvedInstanceFieldGet* instruction) {
  FieldAccessCallingConventionRISCV64 calling_convention;
  codegen_->CreateUnresolvedFieldLocationSummary(
      instruction, instruction->GetFieldType(), calling_convention);
}

void InstructionCodeGeneratorRISCV64::VisitUnresolvedInstanceFieldGet(
    HUnresolvedInstanceFieldGet* instruction) {
  FieldAccessCallingConventionRISCV64 calling_convention;
  codegen_->GenerateUnresolvedFieldAccess(instruction,
                                          instruction->GetFieldType(),
                                          instruction->GetFieldIndex(),
                                          instruction->GetDexPc(),
                                          calling_convention);
}

void LocationsBuilderRISCV64::VisitUnresolvedInstanceFieldSet(
    HUnresolvedInstanceFieldSet* instruction) {
  FieldAccessCallingConventionRISCV64 calling_convention;
  codegen_->CreateUnresolvedFieldLocationSummary(
      instruction, instruction->GetFieldType(), calling_convention);
}

void InstructionCodeGeneratorRISCV64::VisitUnresolvedInstanceFieldSet(
    HUnresolvedInstanceFieldSet* instruction) {
  FieldAccessCallingConventionRISCV64 calling_convention;
  codegen_->GenerateUnresolvedFieldAccess(instruction,
                                          instruction->GetFieldType(),
                                          instruction->GetFieldIndex(),
                                          instruction->GetDexPc(),
                                          calling_convention);
}

void LocationsBuilderRISCV64::VisitUnresolvedStaticFieldGet(
    HUnresolvedStaticFieldGet* instruction) {
  FieldAccessCallingConventionRISCV64 calling_convention;
  codegen_->CreateUnresolvedFieldLocationSummary(
      instruction, instruction->GetFieldType(), calling_convention);
}

void InstructionCodeGeneratorRISCV64::VisitUnresolvedStaticFieldGet(
    HUnresolvedStaticFieldGet* instruction) {
  FieldAccessCallingConventionRISCV64 calling_convention;
  codegen_->GenerateUnresolvedFieldAccess(instruction,
                                          instruction->GetFieldType(),
                                          instruction->GetFieldIndex(),
                                          instruction->GetDexPc(),
                                          calling_convention);
}

void LocationsBuilderRISCV64::VisitUnresolvedStaticFieldSet(
    HUnresolvedStaticFieldSet* instruction) {
  FieldAccessCallingConventionRISCV64 calling_convention;
  codegen_->CreateUnresolvedFieldLocationSummary(
      instruction, instruction->GetFieldType(), calling_convention);
}

void InstructionCodeGeneratorRISCV64::VisitUnresolvedStaticFieldSet(
    HUnresolvedStaticFieldSet* instruction) {
  FieldAccessCallingConventionRISCV64 calling_convention;
  codegen_->GenerateUnresolvedFieldAccess(instruction,
                                          instruction->GetFieldType(),
                                          instruction->GetFieldIndex(),
                                          instruction->GetDexPc(),
                                          calling_convention);
}

void LocationsBuilderRISCV64::VisitSelect(HSelect* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  if (DataType::IsFloatingPointType(instruction->GetType())) {
    locations->SetInAt(0, FpuRegisterOrZeroBitPatternLocation(instruction->GetFalseValue()));
    locations->SetInAt(1, FpuRegisterOrZeroBitPatternLocation(instruction->GetTrueValue()));
    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
    if (!locations->InAt(0).IsConstant() && !locations->InAt(1).IsConstant()) {
      locations->AddTemp(Location::RequiresRegister());
    }
  }  else {
    locations->SetInAt(0, RegisterOrZeroBitPatternLocation(instruction->GetFalseValue()));
    locations->SetInAt(1, RegisterOrZeroBitPatternLocation(instruction->GetTrueValue()));
    locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
  }

  if (IsBooleanValueOrMaterializedCondition(instruction->GetCondition())) {
    locations->SetInAt(2, Location::RequiresRegister());
  }
}

void InstructionCodeGeneratorRISCV64::VisitSelect(HSelect* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  HInstruction* cond = instruction->GetCondition();
  ScratchRegisterScope srs(GetAssembler());
  XRegister tmp = srs.AllocateXRegister();
  if (!IsBooleanValueOrMaterializedCondition(cond)) {
    DataType::Type cond_type = cond->InputAt(0)->GetType();
    IfCondition if_cond = cond->AsCondition()->GetCondition();
    if (DataType::IsFloatingPointType(cond_type)) {
      GenerateFpCondition(if_cond,
                          cond->AsCondition()->IsGtBias(),
                          cond_type,
                          cond->GetLocations(),
                          /*label=*/ nullptr,
                          tmp,
                          /*to_all_bits=*/ true);
    } else {
      GenerateIntLongCondition(if_cond, cond->GetLocations(), tmp, /*to_all_bits=*/ true);
    }
  } else {
    // TODO(riscv64): Remove the normalizing SNEZ when we can ensure that booleans
    // have only values 0 and 1. b/279302742
    __ Snez(tmp, locations->InAt(2).AsRegister<XRegister>());
    __ Neg(tmp, tmp);
  }

  XRegister true_reg, false_reg, xor_reg, out_reg;
  DataType::Type type = instruction->GetType();
  if (DataType::IsFloatingPointType(type)) {
    if (locations->InAt(0).IsConstant()) {
      DCHECK(locations->InAt(0).GetConstant()->IsZeroBitPattern());
      false_reg = Zero;
    } else {
      false_reg = srs.AllocateXRegister();
      FMvX(false_reg, locations->InAt(0).AsFpuRegister<FRegister>(), type);
    }
    if (locations->InAt(1).IsConstant()) {
      DCHECK(locations->InAt(1).GetConstant()->IsZeroBitPattern());
      true_reg = Zero;
    } else {
      true_reg = (false_reg == Zero) ? srs.AllocateXRegister()
                                     : locations->GetTemp(0).AsRegister<XRegister>();
      FMvX(true_reg, locations->InAt(1).AsFpuRegister<FRegister>(), type);
    }
    // We can clobber the "true value" with the XOR result.
    // Note: The XOR is not emitted if `true_reg == Zero`, see below.
    xor_reg = true_reg;
    out_reg = tmp;
  } else {
    false_reg = InputXRegisterOrZero(locations->InAt(0));
    true_reg = InputXRegisterOrZero(locations->InAt(1));
    xor_reg = srs.AllocateXRegister();
    out_reg = locations->Out().AsRegister<XRegister>();
  }

  // We use a branch-free implementation of `HSelect`.
  // With `tmp` initialized to 0 for `false` and -1 for `true`:
  //     xor xor_reg, false_reg, true_reg
  //     and tmp, tmp, xor_reg
  //     xor out_reg, tmp, false_reg
  if (false_reg == Zero) {
    xor_reg = true_reg;
  } else if (true_reg == Zero) {
    xor_reg = false_reg;
  } else {
    DCHECK_NE(xor_reg, Zero);
    __ Xor(xor_reg, false_reg, true_reg);
  }
  __ And(tmp, tmp, xor_reg);
  __ Xor(out_reg, tmp, false_reg);

  if (type == DataType::Type::kFloat64) {
    __ FMvDX(locations->Out().AsFpuRegister<FRegister>(), out_reg);
  } else if (type == DataType::Type::kFloat32) {
    __ FMvWX(locations->Out().AsFpuRegister<FRegister>(), out_reg);
  }
}

void LocationsBuilderRISCV64::VisitSub(HSub* instruction) {
  HandleBinaryOp(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitSub(HSub* instruction) {
  HandleBinaryOp(instruction);
}

void LocationsBuilderRISCV64::VisitSuspendCheck(HSuspendCheck* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator())
      LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
  // In suspend check slow path, usually there are no caller-save registers at all.
  // If SIMD instructions are present, however, we force spilling all live SIMD
  // registers in full width (since the runtime only saves/restores lower part).
  locations->SetCustomSlowPathCallerSaves(GetGraph()->HasSIMD() ? RegisterSet::AllFpu() :
                                                                  RegisterSet::Empty());
}

void InstructionCodeGeneratorRISCV64::VisitSuspendCheck(HSuspendCheck* instruction) {
  HBasicBlock* block = instruction->GetBlock();
  if (block->GetLoopInformation() != nullptr) {
    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
    // The back edge will generate the suspend check.
    return;
  }
  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
    // The goto will generate the suspend check.
    return;
  }
  GenerateSuspendCheck(instruction, nullptr);
}

void LocationsBuilderRISCV64::VisitThrow(HThrow* instruction) {
  LocationSummary* locations = new (GetGraph()->GetAllocator())
      LocationSummary(instruction, LocationSummary::kCallOnMainOnly);
  InvokeRuntimeCallingConvention calling_convention;
  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
}

void InstructionCodeGeneratorRISCV64::VisitThrow(HThrow* instruction) {
  codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc());
  CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
}

void LocationsBuilderRISCV64::VisitTryBoundary(HTryBoundary* instruction) {
  instruction->SetLocations(nullptr);
}

void InstructionCodeGeneratorRISCV64::VisitTryBoundary(HTryBoundary* instruction) {
  HBasicBlock* successor = instruction->GetNormalFlowSuccessor();
  if (!successor->IsExitBlock()) {
    HandleGoto(instruction, successor);
  }
}

void LocationsBuilderRISCV64::VisitTypeConversion(HTypeConversion* instruction) {
  DataType::Type input_type = instruction->GetInputType();
  DataType::Type result_type = instruction->GetResultType();
  DCHECK(!DataType::IsTypeConversionImplicit(input_type, result_type))
      << input_type << " -> " << result_type;

  if ((input_type == DataType::Type::kReference) || (input_type == DataType::Type::kVoid) ||
      (result_type == DataType::Type::kReference) || (result_type == DataType::Type::kVoid)) {
    LOG(FATAL) << "Unexpected type conversion from " << input_type << " to " << result_type;
  }

  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);

  if (DataType::IsFloatingPointType(input_type)) {
    locations->SetInAt(0, Location::RequiresFpuRegister());
  } else {
    locations->SetInAt(0, Location::RequiresRegister());
  }

  if (DataType::IsFloatingPointType(result_type)) {
    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
  } else {
    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
  }
}

void InstructionCodeGeneratorRISCV64::VisitTypeConversion(HTypeConversion* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  DataType::Type result_type = instruction->GetResultType();
  DataType::Type input_type = instruction->GetInputType();

  DCHECK(!DataType::IsTypeConversionImplicit(input_type, result_type))
      << input_type << " -> " << result_type;

  if (DataType::IsIntegralType(result_type) && DataType::IsIntegralType(input_type)) {
    XRegister dst = locations->Out().AsRegister<XRegister>();
    XRegister src = locations->InAt(0).AsRegister<XRegister>();
    switch (result_type) {
      case DataType::Type::kUint8:
        __ ZextB(dst, src);
        break;
      case DataType::Type::kInt8:
        __ SextB(dst, src);
        break;
      case DataType::Type::kUint16:
        __ ZextH(dst, src);
        break;
      case DataType::Type::kInt16:
        __ SextH(dst, src);
        break;
      case DataType::Type::kInt32:
      case DataType::Type::kInt64:
        // Sign-extend 32-bit int into bits 32 through 63 for int-to-long and long-to-int
        // conversions, except when the input and output registers are the same and we are not
        // converting longs to shorter types. In these cases, do nothing.
        if ((input_type == DataType::Type::kInt64) || (dst != src)) {
          __ Addiw(dst, src, 0);
        }
        break;

      default:
        LOG(FATAL) << "Unexpected type conversion from " << input_type
                   << " to " << result_type;
        UNREACHABLE();
    }
  } else if (DataType::IsFloatingPointType(result_type) && DataType::IsIntegralType(input_type)) {
    FRegister dst = locations->Out().AsFpuRegister<FRegister>();
    XRegister src = locations->InAt(0).AsRegister<XRegister>();
    if (input_type == DataType::Type::kInt64) {
      if (result_type == DataType::Type::kFloat32) {
        __ FCvtSL(dst, src, FPRoundingMode::kRNE);
      } else {
        __ FCvtDL(dst, src, FPRoundingMode::kRNE);
      }
    } else {
      if (result_type == DataType::Type::kFloat32) {
        __ FCvtSW(dst, src, FPRoundingMode::kRNE);
      } else {
        __ FCvtDW(dst, src);  // No rounding.
      }
    }
  } else if (DataType::IsIntegralType(result_type) && DataType::IsFloatingPointType(input_type)) {
    CHECK(result_type == DataType::Type::kInt32 || result_type == DataType::Type::kInt64);
    XRegister dst = locations->Out().AsRegister<XRegister>();
    FRegister src = locations->InAt(0).AsFpuRegister<FRegister>();
    if (result_type == DataType::Type::kInt64) {
      if (input_type == DataType::Type::kFloat32) {
        __ FCvtLS(dst, src, FPRoundingMode::kRTZ);
      } else {
        __ FCvtLD(dst, src, FPRoundingMode::kRTZ);
      }
    } else {
      if (input_type == DataType::Type::kFloat32) {
        __ FCvtWS(dst, src, FPRoundingMode::kRTZ);
      } else {
        __ FCvtWD(dst, src, FPRoundingMode::kRTZ);
      }
    }
    // For NaN inputs we need to return 0.
    ScratchRegisterScope srs(GetAssembler());
    XRegister tmp = srs.AllocateXRegister();
    FClass(tmp, src, input_type);
    __ Sltiu(tmp, tmp, kFClassNaNMinValue);  // 0 for NaN, 1 otherwise.
    __ Neg(tmp, tmp);  // 0 for NaN, -1 otherwise.
    __ And(dst, dst, tmp);  // Cleared for NaN.
  } else if (DataType::IsFloatingPointType(result_type) &&
             DataType::IsFloatingPointType(input_type)) {
    FRegister dst = locations->Out().AsFpuRegister<FRegister>();
    FRegister src = locations->InAt(0).AsFpuRegister<FRegister>();
    if (result_type == DataType::Type::kFloat32) {
      __ FCvtSD(dst, src);
    } else {
      __ FCvtDS(dst, src);
    }
  } else {
    LOG(FATAL) << "Unexpected or unimplemented type conversion from " << input_type
                << " to " << result_type;
    UNREACHABLE();
  }
}

void LocationsBuilderRISCV64::VisitUShr(HUShr* instruction) {
  HandleShift(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitUShr(HUShr* instruction) {
  HandleShift(instruction);
}

void LocationsBuilderRISCV64::VisitXor(HXor* instruction) {
  HandleBinaryOp(instruction);
}

void InstructionCodeGeneratorRISCV64::VisitXor(HXor* instruction) {
  HandleBinaryOp(instruction);
}

void LocationsBuilderRISCV64::VisitRiscv64ShiftAdd(HRiscv64ShiftAdd* instruction) {
  DCHECK(codegen_->GetInstructionSetFeatures().HasZba());
  DCHECK_EQ(instruction->GetType(), DataType::Type::kInt64)
      << "Unexpected ShiftAdd type: " << instruction->GetType();

  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  locations->SetInAt(0, Location::RequiresRegister());
  locations->SetInAt(1, Location::RequiresRegister());
  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}

void InstructionCodeGeneratorRISCV64::VisitRiscv64ShiftAdd(HRiscv64ShiftAdd* instruction) {
  DCHECK_EQ(instruction->GetType(), DataType::Type::kInt64)
      << "Unexpected ShiftAdd type: " << instruction->GetType();
  LocationSummary* locations = instruction->GetLocations();
  XRegister first = locations->InAt(0).AsRegister<XRegister>();
  XRegister second = locations->InAt(1).AsRegister<XRegister>();
  XRegister dest = locations->Out().AsRegister<XRegister>();

  switch (instruction->GetDistance()) {
    case 1:
      __ Sh1Add(dest, first, second);
      break;
    case 2:
      __ Sh2Add(dest, first, second);
      break;
    case 3:
      __ Sh3Add(dest, first, second);
      break;
    default:
      LOG(FATAL) << "Unexpected distance of ShiftAdd: " << instruction->GetDistance();
      UNREACHABLE();
  }
}

void LocationsBuilderRISCV64::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) {
  DCHECK(codegen_->GetInstructionSetFeatures().HasZbb());
  DCHECK(DataType::IsIntegralType(instruction->GetType())) << instruction->GetType();

  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
  locations->SetInAt(0, Location::RequiresRegister());
  locations->SetInAt(1, Location::RequiresRegister());
  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}

void InstructionCodeGeneratorRISCV64::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) {
  LocationSummary* locations = instruction->GetLocations();
  XRegister lhs = locations->InAt(0).AsRegister<XRegister>();
  XRegister rhs = locations->InAt(1).AsRegister<XRegister>();
  XRegister dst = locations->Out().AsRegister<XRegister>();

  switch (instruction->GetOpKind()) {
    case HInstruction::kAnd:
      __ Andn(dst, lhs, rhs);
      break;
    case HInstruction::kOr:
      __ Orn(dst, lhs, rhs);
      break;
    case HInstruction::kXor:
      __ Xnor(dst, lhs, rhs);
      break;
    default:
      LOG(FATAL) << "Unreachable";
  }
}

void LocationsBuilderRISCV64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecExtractScalar(HVecExtractScalar* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecExtractScalar(HVecExtractScalar* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecReduce(HVecReduce* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecReduce(HVecReduce* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecCnv(HVecCnv* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecCnv(HVecCnv* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecNeg(HVecNeg* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecNeg(HVecNeg* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecAbs(HVecAbs* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecAbs(HVecAbs* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecNot(HVecNot* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecNot(HVecNot* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecAdd(HVecAdd* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecAdd(HVecAdd* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecSub(HVecSub* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecSub(HVecSub* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecMul(HVecMul* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecMul(HVecMul* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecDiv(HVecDiv* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecDiv(HVecDiv* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecMin(HVecMin* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecMin(HVecMin* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecMax(HVecMax* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecMax(HVecMax* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecAnd(HVecAnd* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecAnd(HVecAnd* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecAndNot(HVecAndNot* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecAndNot(HVecAndNot* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecOr(HVecOr* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecOr(HVecOr* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecXor(HVecXor* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecXor(HVecXor* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecSaturationAdd(HVecSaturationAdd* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecSaturationAdd(HVecSaturationAdd* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecSaturationSub(HVecSaturationSub* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecSaturationSub(HVecSaturationSub* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecShl(HVecShl* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecShl(HVecShl* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecShr(HVecShr* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecShr(HVecShr* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecUShr(HVecUShr* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecUShr(HVecUShr* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecSetScalars(HVecSetScalars* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecSetScalars(HVecSetScalars* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecMultiplyAccumulate(
    HVecMultiplyAccumulate* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecSADAccumulate(HVecSADAccumulate* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecSADAccumulate(HVecSADAccumulate* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecDotProd(HVecDotProd* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecDotProd(HVecDotProd* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecLoad(HVecLoad* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecLoad(HVecLoad* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecStore(HVecStore* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecStore(HVecStore* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecPredSetAll(HVecPredSetAll* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecPredSetAll(HVecPredSetAll* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecPredWhile(HVecPredWhile* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecPredWhile(HVecPredWhile* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecPredToBoolean(HVecPredToBoolean* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecPredToBoolean(HVecPredToBoolean* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecCondition(HVecCondition* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecCondition(HVecCondition* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void LocationsBuilderRISCV64::VisitVecPredNot(HVecPredNot* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

void InstructionCodeGeneratorRISCV64::VisitVecPredNot(HVecPredNot* instruction) {
  UNUSED(instruction);
  LOG(FATAL) << "Unimplemented";
}

namespace detail {

// Mark which intrinsics we don't have handcrafted code for.
template <Intrinsics T>
struct IsUnimplemented {
  bool is_unimplemented = false;
};

#define TRUE_OVERRIDE(Name)                     \
  template <>                                   \
  struct IsUnimplemented<Intrinsics::k##Name> { \
    bool is_unimplemented = true;               \
  };
UNIMPLEMENTED_INTRINSIC_LIST_RISCV64(TRUE_OVERRIDE)
#undef TRUE_OVERRIDE

static constexpr bool kIsIntrinsicUnimplemented[] = {
    false,  // kNone
#define IS_UNIMPLEMENTED(Intrinsic, ...) \
    IsUnimplemented<Intrinsics::k##Intrinsic>().is_unimplemented,
    ART_INTRINSICS_LIST(IS_UNIMPLEMENTED)
#undef IS_UNIMPLEMENTED
};

}  // namespace detail

CodeGeneratorRISCV64::CodeGeneratorRISCV64(HGraph* graph,
                                           const CompilerOptions& compiler_options,
                                           OptimizingCompilerStats* stats)
    : CodeGenerator(graph,
                    kNumberOfXRegisters,
                    kNumberOfFRegisters,
                    /*number_of_register_pairs=*/ 0u,
                    ComputeRegisterMask(kCoreCalleeSaves, arraysize(kCoreCalleeSaves)),
                    ComputeRegisterMask(kFpuCalleeSaves, arraysize(kFpuCalleeSaves)),
                    compiler_options,
                    stats,
                    ArrayRef<const bool>(detail::kIsIntrinsicUnimplemented)),
      assembler_(graph->GetAllocator(),
                 compiler_options.GetInstructionSetFeatures()->AsRiscv64InstructionSetFeatures()),
      location_builder_(graph, this),
      instruction_visitor_(graph, this),
      block_labels_(nullptr),
      move_resolver_(graph->GetAllocator(), this),
      uint32_literals_(std::less<uint32_t>(),
                       graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      uint64_literals_(std::less<uint64_t>(),
                       graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      boot_image_method_patches_(graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      method_bss_entry_patches_(graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      boot_image_type_patches_(graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      app_image_type_patches_(graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      type_bss_entry_patches_(graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      public_type_bss_entry_patches_(graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      package_type_bss_entry_patches_(graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      boot_image_string_patches_(graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      string_bss_entry_patches_(graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      boot_image_jni_entrypoint_patches_(graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      boot_image_other_patches_(graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      jit_string_patches_(StringReferenceValueComparator(),
                          graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)),
      jit_class_patches_(TypeReferenceValueComparator(),
                         graph->GetAllocator()->Adapter(kArenaAllocCodeGenerator)) {
  // Always mark the RA register to be saved.
  AddAllocatedRegister(Location::RegisterLocation(RA));
}

void CodeGeneratorRISCV64::MaybeIncrementHotness(HSuspendCheck* suspend_check,
                                                 bool is_frame_entry) {
  if (GetCompilerOptions().CountHotnessInCompiledCode()) {
    ScratchRegisterScope srs(GetAssembler());
    XRegister method = is_frame_entry ? kArtMethodRegister : srs.AllocateXRegister();
    if (!is_frame_entry) {
      __ Loadd(method, SP, 0);
    }
    XRegister counter = srs.AllocateXRegister();
    __ Loadhu(counter, method, ArtMethod::HotnessCountOffset().Int32Value());
    Riscv64Label done;
    DCHECK_EQ(0u, interpreter::kNterpHotnessValue);
    __ Beqz(counter, &done);  // Can clobber `TMP` if taken.
    __ Addi(counter, counter, -1);
    // We may not have another scratch register available for `Storeh`()`,
    // so we must use the `Sh()` function directly.
    static_assert(IsInt<12>(ArtMethod::HotnessCountOffset().Int32Value()));
    __ Sh(counter, method, ArtMethod::HotnessCountOffset().Int32Value());
    __ Bind(&done);
  }

  if (GetGraph()->IsCompilingBaseline() &&
      GetGraph()->IsUsefulOptimizing() &&
      !Runtime::Current()->IsAotCompiler()) {
    ProfilingInfo* info = GetGraph()->GetProfilingInfo();
    DCHECK(info != nullptr);
    DCHECK(!HasEmptyFrame());
    uint64_t address = reinterpret_cast64<uint64_t>(info) +
                       ProfilingInfo::BaselineHotnessCountOffset().SizeValue();
    auto [base_address, imm12] = SplitJitAddress(address);
    ScratchRegisterScope srs(GetAssembler());
    XRegister counter = srs.AllocateXRegister();
    XRegister tmp = RA;
    __ LoadConst64(tmp, base_address);
    SlowPathCodeRISCV64* slow_path =
        new (GetScopedAllocator()) CompileOptimizedSlowPathRISCV64(suspend_check, tmp, imm12);
    AddSlowPath(slow_path);
    __ Lhu(counter, tmp, imm12);
    __ Beqz(counter, slow_path->GetEntryLabel());  // Can clobber `TMP` if taken.
    __ Addi(counter, counter, -1);
    __ Sh(counter, tmp, imm12);
    __ Bind(slow_path->GetExitLabel());
  }
}

bool CodeGeneratorRISCV64::CanUseImplicitSuspendCheck() const {
  // TODO(riscv64): Implement implicit suspend checks to reduce code size.
  return false;
}

void CodeGeneratorRISCV64::GenerateMemoryBarrier(MemBarrierKind kind) {
  switch (kind) {
    case MemBarrierKind::kAnyAny:
      __ Fence(/*pred=*/ kFenceRead | kFenceWrite, /*succ=*/ kFenceRead | kFenceWrite);
      break;
    case MemBarrierKind::kAnyStore:
      __ Fence(/*pred=*/ kFenceRead | kFenceWrite, /*succ=*/ kFenceWrite);
      break;
    case MemBarrierKind::kLoadAny:
      __ Fence(/*pred=*/ kFenceRead, /*succ=*/ kFenceRead | kFenceWrite);
      break;
    case MemBarrierKind::kStoreStore:
      __ Fence(/*pred=*/ kFenceWrite, /*succ=*/ kFenceWrite);
      break;

    default:
      LOG(FATAL) << "Unexpected memory barrier " << kind;
      UNREACHABLE();
  }
}

void CodeGeneratorRISCV64::GenerateFrameEntry() {
  // Check if we need to generate the clinit check. We will jump to the
  // resolution stub if the class is not initialized and the executing thread is
  // not the thread initializing it.
  // We do this before constructing the frame to get the correct stack trace if
  // an exception is thrown.
  if (GetCompilerOptions().ShouldCompileWithClinitCheck(GetGraph()->GetArtMethod())) {
    Riscv64Label resolution;
    Riscv64Label memory_barrier;

    ScratchRegisterScope srs(GetAssembler());
    XRegister tmp = srs.AllocateXRegister();
    XRegister tmp2 = srs.AllocateXRegister();

    // We don't emit a read barrier here to save on code size. We rely on the
    // resolution trampoline to do a clinit check before re-entering this code.
    __ Loadwu(tmp2, kArtMethodRegister, ArtMethod::DeclaringClassOffset().Int32Value());

    // We shall load the full 32-bit status word with sign-extension and compare as unsigned
    // to sign-extended shifted status values. This yields the same comparison as loading and
    // materializing unsigned but the constant is materialized with a single LUI instruction.
    __ Loadw(tmp, tmp2, mirror::Class::StatusOffset().SizeValue());  // Sign-extended.

    // Check if we're visibly initialized.
    __ Li(tmp2, ShiftedSignExtendedClassStatusValue<ClassStatus::kVisiblyInitialized>());
    __ Bgeu(tmp, tmp2, &frame_entry_label_);  // Can clobber `TMP` if taken.

    // Check if we're initialized and jump to code that does a memory barrier if so.
    __ Li(tmp2, ShiftedSignExtendedClassStatusValue<ClassStatus::kInitialized>());
    __ Bgeu(tmp, tmp2, &memory_barrier);  // Can clobber `TMP` if taken.

    // Check if we're initializing and the thread initializing is the one
    // executing the code.
    __ Li(tmp2, ShiftedSignExtendedClassStatusValue<ClassStatus::kInitializing>());
    __ Bltu(tmp, tmp2, &resolution);  // Can clobber `TMP` if taken.

    __ Loadwu(tmp2, kArtMethodRegister, ArtMethod::DeclaringClassOffset().Int32Value());
    __ Loadw(tmp, tmp2, mirror::Class::ClinitThreadIdOffset().Int32Value());
    __ Loadw(tmp2, TR, Thread::TidOffset<kRiscv64PointerSize>().Int32Value());
    __ Beq(tmp, tmp2, &frame_entry_label_);
    __ Bind(&resolution);

    // Jump to the resolution stub.
    ThreadOffset64 entrypoint_offset =
        GetThreadOffset<kRiscv64PointerSize>(kQuickQuickResolutionTrampoline);
    __ Loadd(tmp, TR, entrypoint_offset.Int32Value());
    __ Jr(tmp);

    __ Bind(&memory_barrier);
    GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
  }
  __ Bind(&frame_entry_label_);

  bool do_overflow_check =
      FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kRiscv64) || !IsLeafMethod();

  if (do_overflow_check) {
    DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
    __ Loadw(
        Zero, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(InstructionSet::kRiscv64)));
    RecordPcInfo(nullptr, 0);
  }

  if (!HasEmptyFrame()) {
    // Make sure the frame size isn't unreasonably large.
    DCHECK_LE(GetFrameSize(), GetMaximumFrameSize());

    // Spill callee-saved registers.

    uint32_t frame_size = GetFrameSize();

    IncreaseFrame(frame_size);

    uint32_t offset = frame_size;
    for (size_t i = arraysize(kCoreCalleeSaves); i != 0; ) {
      --i;
      XRegister reg = kCoreCalleeSaves[i];
      if (allocated_registers_.ContainsCoreRegister(reg)) {
        offset -= kRiscv64DoublewordSize;
        __ Stored(reg, SP, offset);
        __ cfi().RelOffset(dwarf::Reg::Riscv64Core(reg), offset);
      }
    }

    for (size_t i = arraysize(kFpuCalleeSaves); i != 0; ) {
      --i;
      FRegister reg = kFpuCalleeSaves[i];
      if (allocated_registers_.ContainsFloatingPointRegister(reg)) {
        offset -= kRiscv64DoublewordSize;
        __ FStored(reg, SP, offset);
        __ cfi().RelOffset(dwarf::Reg::Riscv64Fp(reg), offset);
      }
    }

    // Save the current method if we need it. Note that we do not
    // do this in HCurrentMethod, as the instruction might have been removed
    // in the SSA graph.
    if (RequiresCurrentMethod()) {
      __ Stored(kArtMethodRegister, SP, 0);
    }

    if (GetGraph()->HasShouldDeoptimizeFlag()) {
      // Initialize should_deoptimize flag to 0.
      __ Storew(Zero, SP, GetStackOffsetOfShouldDeoptimizeFlag());
    }
  }
  MaybeIncrementHotness(/* suspend_check= */ nullptr, /*is_frame_entry=*/ true);
}

void CodeGeneratorRISCV64::GenerateFrameExit() {
  __ cfi().RememberState();

  if (!HasEmptyFrame()) {
    // Restore callee-saved registers.

    // For better instruction scheduling restore RA before other registers.
    uint32_t offset = GetFrameSize();
    for (size_t i = arraysize(kCoreCalleeSaves); i != 0; ) {
      --i;
      XRegister reg = kCoreCalleeSaves[i];
      if (allocated_registers_.ContainsCoreRegister(reg)) {
        offset -= kRiscv64DoublewordSize;
        __ Loadd(reg, SP, offset);
        __ cfi().Restore(dwarf::Reg::Riscv64Core(reg));
      }
    }

    for (size_t i = arraysize(kFpuCalleeSaves); i != 0; ) {
      --i;
      FRegister reg = kFpuCalleeSaves[i];
      if (allocated_registers_.ContainsFloatingPointRegister(reg)) {
        offset -= kRiscv64DoublewordSize;
        __ FLoadd(reg, SP, offset);
        __ cfi().Restore(dwarf::Reg::Riscv64Fp(reg));
      }
    }

    DecreaseFrame(GetFrameSize());
  }

  __ Jr(RA);

  __ cfi().RestoreState();
  __ cfi().DefCFAOffset(GetFrameSize());
}

void CodeGeneratorRISCV64::Bind(HBasicBlock* block) { __ Bind(GetLabelOf(block)); }

void CodeGeneratorRISCV64::MoveConstant(Location destination, int32_t value) {
  DCHECK(destination.IsRegister());
  __ LoadConst32(destination.AsRegister<XRegister>(), value);
}

void CodeGeneratorRISCV64::MoveLocation(Location destination,
                                        Location source,
                                        DataType::Type dst_type) {
  if (source.Equals(destination)) {
    return;
  }

  // A valid move type can always be inferred from the destination and source locations.
  // When moving from and to a register, the `dst_type` can be used to generate 32-bit instead
  // of 64-bit moves but it's generally OK to use 64-bit moves for 32-bit values in registers.
  bool unspecified_type = (dst_type == DataType::Type::kVoid);
  // TODO(riscv64): Is the destination type known in all cases?
  // TODO(riscv64): Can unspecified `dst_type` move 32-bit GPR to FPR without NaN-boxing?
  CHECK(!unspecified_type);

  if (destination.IsRegister() || destination.IsFpuRegister()) {
    if (unspecified_type) {
      HConstant* src_cst = source.IsConstant() ? source.GetConstant() : nullptr;
      if (source.IsStackSlot() ||
          (src_cst != nullptr &&
           (src_cst->IsIntConstant() || src_cst->IsFloatConstant() || src_cst->IsNullConstant()))) {
        // For stack slots and 32-bit constants, a 32-bit type is appropriate.
        dst_type = destination.IsRegister() ? DataType::Type::kInt32 : DataType::Type::kFloat32;
      } else {
        // If the source is a double stack slot or a 64-bit constant, a 64-bit type
        // is appropriate. Else the source is a register, and since the type has not
        // been specified, we chose a 64-bit type to force a 64-bit move.
        dst_type = destination.IsRegister() ? DataType::Type::kInt64 : DataType::Type::kFloat64;
      }
    }
    DCHECK((destination.IsFpuRegister() && DataType::IsFloatingPointType(dst_type)) ||
           (destination.IsRegister() && !DataType::IsFloatingPointType(dst_type)));

    if (source.IsStackSlot() || source.IsDoubleStackSlot()) {
      // Move to GPR/FPR from stack
      if (DataType::IsFloatingPointType(dst_type)) {
        if (DataType::Is64BitType(dst_type)) {
          __ FLoadd(destination.AsFpuRegister<FRegister>(), SP, source.GetStackIndex());
        } else {
          __ FLoadw(destination.AsFpuRegister<FRegister>(), SP, source.GetStackIndex());
        }
      } else {
        if (DataType::Is64BitType(dst_type)) {
          __ Loadd(destination.AsRegister<XRegister>(), SP, source.GetStackIndex());
        } else if (dst_type == DataType::Type::kReference) {
          __ Loadwu(destination.AsRegister<XRegister>(), SP, source.GetStackIndex());
        } else {
          __ Loadw(destination.AsRegister<XRegister>(), SP, source.GetStackIndex());
        }
      }
    } else if (source.IsConstant()) {
      // Move to GPR/FPR from constant
      // TODO(riscv64): Consider using literals for difficult-to-materialize 64-bit constants.
      int64_t value = GetInt64ValueOf(source.GetConstant()->AsConstant());
      ScratchRegisterScope srs(GetAssembler());
      XRegister gpr = DataType::IsFloatingPointType(dst_type)
          ? srs.AllocateXRegister()
          : destination.AsRegister<XRegister>();
      if (DataType::IsFloatingPointType(dst_type) && value == 0) {
        gpr = Zero;  // Note: The scratch register allocated above shall not be used.
      } else {
        // Note: For `float` we load the sign-extended value here as it can sometimes yield
        // a shorter instruction sequence. The higher 32 bits shall be ignored during the
        // transfer to FP reg and the result shall be correctly NaN-boxed.
        __ LoadConst64(gpr, value);
      }
      if (dst_type == DataType::Type::kFloat32) {
        __ FMvWX(destination.AsFpuRegister<FRegister>(), gpr);
      } else if (dst_type == DataType::Type::kFloat64) {
        __ FMvDX(destination.AsFpuRegister<FRegister>(), gpr);
      }
    } else if (source.IsRegister()) {
      if (destination.IsRegister()) {
        // Move to GPR from GPR
        __ Mv(destination.AsRegister<XRegister>(), source.AsRegister<XRegister>());
      } else {
        DCHECK(destination.IsFpuRegister());
        if (DataType::Is64BitType(dst_type)) {
          __ FMvDX(destination.AsFpuRegister<FRegister>(), source.AsRegister<XRegister>());
        } else {
          __ FMvWX(destination.AsFpuRegister<FRegister>(), source.AsRegister<XRegister>());
        }
      }
    } else if (source.IsFpuRegister()) {
      if (destination.IsFpuRegister()) {
        if (GetGraph()->HasSIMD()) {
          LOG(FATAL) << "Vector extension is unsupported";
          UNREACHABLE();
        } else {
          // Move to FPR from FPR
          if (dst_type == DataType::Type::kFloat32) {
            __ FMvS(destination.AsFpuRegister<FRegister>(), source.AsFpuRegister<FRegister>());
          } else {
            DCHECK_EQ(dst_type, DataType::Type::kFloat64);
            __ FMvD(destination.AsFpuRegister<FRegister>(), source.AsFpuRegister<FRegister>());
          }
        }
      } else {
        DCHECK(destination.IsRegister());
        if (DataType::Is64BitType(dst_type)) {
          __ FMvXD(destination.AsRegister<XRegister>(), source.AsFpuRegister<FRegister>());
        } else {
          __ FMvXW(destination.AsRegister<XRegister>(), source.AsFpuRegister<FRegister>());
        }
      }
    }
  } else if (destination.IsSIMDStackSlot()) {
    LOG(FATAL) << "SIMD is unsupported";
    UNREACHABLE();
  } else {  // The destination is not a register. It must be a stack slot.
    DCHECK(destination.IsStackSlot() || destination.IsDoubleStackSlot());
    if (source.IsRegister() || source.IsFpuRegister()) {
      if (unspecified_type) {
        if (source.IsRegister()) {
          dst_type = destination.IsStackSlot() ? DataType::Type::kInt32 : DataType::Type::kInt64;
        } else {
          dst_type =
              destination.IsStackSlot() ? DataType::Type::kFloat32 : DataType::Type::kFloat64;
        }
      }
      DCHECK_EQ(source.IsFpuRegister(), DataType::IsFloatingPointType(dst_type));
      // For direct @CriticalNative calls, we need to sign-extend narrow integral args
      // to 64 bits, so widening integral values is allowed. Narrowing is forbidden.
      DCHECK_IMPLIES(DataType::IsFloatingPointType(dst_type) || destination.IsStackSlot(),
                     destination.IsDoubleStackSlot() == DataType::Is64BitType(dst_type));
      // Move to stack from GPR/FPR
      if (destination.IsDoubleStackSlot()) {
        if (source.IsRegister()) {
          __ Stored(source.AsRegister<XRegister>(), SP, destination.GetStackIndex());
        } else {
          __ FStored(source.AsFpuRegister<FRegister>(), SP, destination.GetStackIndex());
        }
      } else {
        if (source.IsRegister()) {
          __ Storew(source.AsRegister<XRegister>(), SP, destination.GetStackIndex());
        } else {
          __ FStorew(source.AsFpuRegister<FRegister>(), SP, destination.GetStackIndex());
        }
      }
    } else if (source.IsConstant()) {
      // Move to stack from constant
      int64_t value = GetInt64ValueOf(source.GetConstant());
      ScratchRegisterScope srs(GetAssembler());
      XRegister gpr = (value != 0) ? srs.AllocateXRegister() : Zero;
      if (value != 0) {
        __ LoadConst64(gpr, value);
      }
      if (destination.IsStackSlot()) {
        __ Storew(gpr, SP, destination.GetStackIndex());
      } else {
        DCHECK(destination.IsDoubleStackSlot());
        __ Stored(gpr, SP, destination.GetStackIndex());
      }
    } else {
      DCHECK(source.IsStackSlot() || source.IsDoubleStackSlot());
      // For direct @CriticalNative calls, we need to sign-extend narrow integral args
      // to 64 bits, so widening move is allowed. Narrowing move is forbidden.
      DCHECK_IMPLIES(destination.IsStackSlot(), source.IsStackSlot());
      // Move to stack from stack
      ScratchRegisterScope srs(GetAssembler());
      XRegister tmp = srs.AllocateXRegister();
      if (source.IsStackSlot()) {
        __ Loadw(tmp, SP, source.GetStackIndex());
      } else {
        __ Loadd(tmp, SP, source.GetStackIndex());
      }
      if (destination.IsStackSlot()) {
        __ Storew(tmp, SP, destination.GetStackIndex());
      } else {
        __ Stored(tmp, SP, destination.GetStackIndex());
      }
    }
  }
}

void CodeGeneratorRISCV64::AddLocationAsTemp(Location location, LocationSummary* locations) {
  if (location.IsRegister()) {
    locations->AddTemp(location);
  } else {
    UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
  }
}

void CodeGeneratorRISCV64::SetupBlockedRegisters() const {
  // ZERO, GP, SP, RA, TP and TR(S1) are reserved and can't be allocated.
  blocked_core_registers_[Zero] = true;
  blocked_core_registers_[GP] = true;
  blocked_core_registers_[SP] = true;
  blocked_core_registers_[RA] = true;
  blocked_core_registers_[TP] = true;
  blocked_core_registers_[TR] = true;  // ART Thread register.

  // TMP(T6), TMP2(T5) and FTMP(FT11) are used as temporary/scratch registers.
  blocked_core_registers_[TMP] = true;
  blocked_core_registers_[TMP2] = true;
  blocked_fpu_registers_[FTMP] = true;

  if (GetGraph()->IsDebuggable()) {
    // Stubs do not save callee-save floating point registers. If the graph
    // is debuggable, we need to deal with these registers differently. For
    // now, just block them.
    for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) {
      blocked_fpu_registers_[kFpuCalleeSaves[i]] = true;
    }
  }
}

size_t CodeGeneratorRISCV64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
  __ Stored(XRegister(reg_id), SP, stack_index);
  return kRiscv64DoublewordSize;
}

size_t CodeGeneratorRISCV64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
  __ Loadd(XRegister(reg_id), SP, stack_index);
  return kRiscv64DoublewordSize;
}

size_t CodeGeneratorRISCV64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
  if (GetGraph()->HasSIMD()) {
    // TODO(riscv64): RISC-V vector extension.
    UNIMPLEMENTED(FATAL) << "Vector extension is unsupported";
    UNREACHABLE();
  }
  __ FStored(FRegister(reg_id), SP, stack_index);
  return kRiscv64FloatRegSizeInBytes;
}

size_t CodeGeneratorRISCV64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
  if (GetGraph()->HasSIMD()) {
    // TODO(riscv64): RISC-V vector extension.
    UNIMPLEMENTED(FATAL) << "Vector extension is unsupported";
    UNREACHABLE();
  }
  __ FLoadd(FRegister(reg_id), SP, stack_index);
  return kRiscv64FloatRegSizeInBytes;
}

void CodeGeneratorRISCV64::DumpCoreRegister(std::ostream& stream, int reg) const {
  stream << XRegister(reg);
}

void CodeGeneratorRISCV64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
  stream << FRegister(reg);
}

const Riscv64InstructionSetFeatures& CodeGeneratorRISCV64::GetInstructionSetFeatures() const {
  return *GetCompilerOptions().GetInstructionSetFeatures()->AsRiscv64InstructionSetFeatures();
}

void CodeGeneratorRISCV64::Finalize() {
  // Ensure that we fix up branches and literal loads and emit the literal pool.
  __ FinalizeCode();

  // Adjust native pc offsets in stack maps.
  StackMapStream* stack_map_stream = GetStackMapStream();
  for (size_t i = 0, num = stack_map_stream->GetNumberOfStackMaps(); i != num; ++i) {
    uint32_t old_position = stack_map_stream->GetStackMapNativePcOffset(i);
    uint32_t new_position = __ GetAdjustedPosition(old_position);
    DCHECK_GE(new_position, old_position);
    stack_map_stream->SetStackMapNativePcOffset(i, new_position);
  }

  // Adjust pc offsets for the disassembly information.
  if (disasm_info_ != nullptr) {
    GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval();
    frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start);
    frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end);
    for (auto& entry : *disasm_info_->GetInstructionIntervals()) {
      entry.second.start = __ GetAdjustedPosition(entry.second.start);
      entry.second.end = __ GetAdjustedPosition(entry.second.end);
    }
    for (auto& entry : *disasm_info_->GetSlowPathIntervals()) {
      entry.code_interval.start = __ GetAdjustedPosition(entry.code_interval.start);
      entry.code_interval.end = __ GetAdjustedPosition(entry.code_interval.end);
    }
  }
}

// Generate code to invoke a runtime entry point.
void CodeGeneratorRISCV64::InvokeRuntime(QuickEntrypointEnum entrypoint,
                                         HInstruction* instruction,
                                         uint32_t dex_pc,
                                         SlowPathCode* slow_path) {
  ValidateInvokeRuntime(entrypoint, instruction, slow_path);

  ThreadOffset64 entrypoint_offset = GetThreadOffset<kRiscv64PointerSize>(entrypoint);

  // TODO(riscv64): Reduce code size for AOT by using shared trampolines for slow path
  // runtime calls across the entire oat file.
  __ Loadd(RA, TR, entrypoint_offset.Int32Value());
  __ Jalr(RA);
  if (EntrypointRequiresStackMap(entrypoint)) {
    RecordPcInfo(instruction, dex_pc, slow_path);
  }
}

// Generate code to invoke a runtime entry point, but do not record
// PC-related information in a stack map.
void CodeGeneratorRISCV64::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset,
                                                               HInstruction* instruction,
                                                               SlowPathCode* slow_path) {
  ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path);
  __ Loadd(RA, TR, entry_point_offset);
  __ Jalr(RA);
}

void CodeGeneratorRISCV64::IncreaseFrame(size_t adjustment) {
  int32_t adjustment32 = dchecked_integral_cast<int32_t>(adjustment);
  __ AddConst64(SP, SP, -adjustment32);
  GetAssembler()->cfi().AdjustCFAOffset(adjustment32);
}

void CodeGeneratorRISCV64::DecreaseFrame(size_t adjustment) {
  int32_t adjustment32 = dchecked_integral_cast<int32_t>(adjustment);
  __ AddConst64(SP, SP, adjustment32);
  GetAssembler()->cfi().AdjustCFAOffset(-adjustment32);
}

void CodeGeneratorRISCV64::GenerateNop() {
  __ Nop();
}

void CodeGeneratorRISCV64::GenerateImplicitNullCheck(HNullCheck* instruction) {
  if (CanMoveNullCheckToUser(instruction)) {
    return;
  }
  Location obj = instruction->GetLocations()->InAt(0);

  __ Lw(Zero, obj.AsRegister<XRegister>(), 0);
  RecordPcInfo(instruction, instruction->GetDexPc());
}

void CodeGeneratorRISCV64::GenerateExplicitNullCheck(HNullCheck* instruction) {
  SlowPathCodeRISCV64* slow_path = new (GetScopedAllocator()) NullCheckSlowPathRISCV64(instruction);
  AddSlowPath(slow_path);

  Location obj = instruction->GetLocations()->InAt(0);

  __ Beqz(obj.AsRegister<XRegister>(), slow_path->GetEntryLabel());
}

HLoadString::LoadKind CodeGeneratorRISCV64::GetSupportedLoadStringKind(
    HLoadString::LoadKind desired_string_load_kind) {
  switch (desired_string_load_kind) {
    case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
    case HLoadString::LoadKind::kBootImageRelRo:
    case HLoadString::LoadKind::kBssEntry:
      DCHECK(!Runtime::Current()->UseJitCompilation());
      break;
    case HLoadString::LoadKind::kJitBootImageAddress:
    case HLoadString::LoadKind::kJitTableAddress:
      DCHECK(Runtime::Current()->UseJitCompilation());
      break;
    case HLoadString::LoadKind::kRuntimeCall:
      break;
  }
  return desired_string_load_kind;
}

HLoadClass::LoadKind CodeGeneratorRISCV64::GetSupportedLoadClassKind(
    HLoadClass::LoadKind desired_class_load_kind) {
  switch (desired_class_load_kind) {
    case HLoadClass::LoadKind::kInvalid:
      LOG(FATAL) << "UNREACHABLE";
      UNREACHABLE();
    case HLoadClass::LoadKind::kReferrersClass:
      break;
    case HLoadClass::LoadKind::kBootImageLinkTimePcRelative:
    case HLoadClass::LoadKind::kBootImageRelRo:
    case HLoadClass::LoadKind::kAppImageRelRo:
    case HLoadClass::LoadKind::kBssEntry:
    case HLoadClass::LoadKind::kBssEntryPublic:
    case HLoadClass::LoadKind::kBssEntryPackage:
      DCHECK(!Runtime::Current()->UseJitCompilation());
      break;
    case HLoadClass::LoadKind::kJitBootImageAddress:
    case HLoadClass::LoadKind::kJitTableAddress:
      DCHECK(Runtime::Current()->UseJitCompilation());
      break;
    case HLoadClass::LoadKind::kRuntimeCall:
      break;
  }
  return desired_class_load_kind;
}

HInvokeStaticOrDirect::DispatchInfo CodeGeneratorRISCV64::GetSupportedInvokeStaticOrDirectDispatch(
    const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, ArtMethod* method) {
  UNUSED(method);
  // On RISCV64 we support all dispatch types.
  return desired_dispatch_info;
}

CodeGeneratorRISCV64::PcRelativePatchInfo* CodeGeneratorRISCV64::NewBootImageIntrinsicPatch(
    uint32_t intrinsic_data, const PcRelativePatchInfo* info_high) {
  return NewPcRelativePatch(
      /* dex_file= */ nullptr, intrinsic_data, info_high, &boot_image_other_patches_);
}

CodeGeneratorRISCV64::PcRelativePatchInfo* CodeGeneratorRISCV64::NewBootImageRelRoPatch(
    uint32_t boot_image_offset, const PcRelativePatchInfo* info_high) {
  return NewPcRelativePatch(
      /* dex_file= */ nullptr, boot_image_offset, info_high, &boot_image_other_patches_);
}

CodeGeneratorRISCV64::PcRelativePatchInfo* CodeGeneratorRISCV64::NewBootImageMethodPatch(
    MethodReference target_method, const PcRelativePatchInfo* info_high) {
  return NewPcRelativePatch(
      target_method.dex_file, target_method.index, info_high, &boot_image_method_patches_);
}

CodeGeneratorRISCV64::PcRelativePatchInfo* CodeGeneratorRISCV64::NewMethodBssEntryPatch(
    MethodReference target_method, const PcRelativePatchInfo* info_high) {
  return NewPcRelativePatch(
      target_method.dex_file, target_method.index, info_high, &method_bss_entry_patches_);
}

CodeGeneratorRISCV64::PcRelativePatchInfo* CodeGeneratorRISCV64::NewBootImageTypePatch(
    const DexFile& dex_file, dex::TypeIndex type_index, const PcRelativePatchInfo* info_high) {
  return NewPcRelativePatch(&dex_file, type_index.index_, info_high, &boot_image_type_patches_);
}

CodeGeneratorRISCV64::PcRelativePatchInfo* CodeGeneratorRISCV64::NewAppImageTypePatch(
    const DexFile& dex_file, dex::TypeIndex type_index, const PcRelativePatchInfo* info_high) {
  return NewPcRelativePatch(&dex_file, type_index.index_, info_high, &app_image_type_patches_);
}

CodeGeneratorRISCV64::PcRelativePatchInfo* CodeGeneratorRISCV64::NewBootImageJniEntrypointPatch(
    MethodReference target_method, const PcRelativePatchInfo* info_high) {
  return NewPcRelativePatch(
      target_method.dex_file, target_method.index, info_high, &boot_image_jni_entrypoint_patches_);
}

CodeGeneratorRISCV64::PcRelativePatchInfo* CodeGeneratorRISCV64::NewTypeBssEntryPatch(
    HLoadClass* load_class,
    const PcRelativePatchInfo* info_high) {
  const DexFile& dex_file = load_class->GetDexFile();
  dex::TypeIndex type_index = load_class->GetTypeIndex();
  ArenaDeque<PcRelativePatchInfo>* patches = nullptr;
  switch (load_class->GetLoadKind()) {
    case HLoadClass::LoadKind::kBssEntry:
      patches = &type_bss_entry_patches_;
      break;
    case HLoadClass::LoadKind::kBssEntryPublic:
      patches = &public_type_bss_entry_patches_;
      break;
    case HLoadClass::LoadKind::kBssEntryPackage:
      patches = &package_type_bss_entry_patches_;
      break;
    default:
      LOG(FATAL) << "Unexpected load kind: " << load_class->GetLoadKind();
      UNREACHABLE();
  }
  return NewPcRelativePatch(&dex_file, type_index.index_, info_high, patches);
}

CodeGeneratorRISCV64::PcRelativePatchInfo* CodeGeneratorRISCV64::NewBootImageStringPatch(
    const DexFile& dex_file, dex::StringIndex string_index, const PcRelativePatchInfo* info_high) {
  return NewPcRelativePatch(&dex_file, string_index.index_, info_high, &boot_image_string_patches_);
}

CodeGeneratorRISCV64::PcRelativePatchInfo* CodeGeneratorRISCV64::NewStringBssEntryPatch(
    const DexFile& dex_file, dex::StringIndex string_index, const PcRelativePatchInfo* info_high) {
  return NewPcRelativePatch(&dex_file, string_index.index_, info_high, &string_bss_entry_patches_);
}

CodeGeneratorRISCV64::PcRelativePatchInfo* CodeGeneratorRISCV64::NewPcRelativePatch(
    const DexFile* dex_file,
    uint32_t offset_or_index,
    const PcRelativePatchInfo* info_high,
    ArenaDeque<PcRelativePatchInfo>* patches) {
  patches->emplace_back(dex_file, offset_or_index, info_high);
  return &patches->back();
}

Literal* CodeGeneratorRISCV64::DeduplicateUint32Literal(uint32_t value) {
  return uint32_literals_.GetOrCreate(value,
                                      [this, value]() { return __ NewLiteral<uint32_t>(value); });
}

Literal* CodeGeneratorRISCV64::DeduplicateUint64Literal(uint64_t value) {
  return uint64_literals_.GetOrCreate(value,
                                      [this, value]() { return __ NewLiteral<uint64_t>(value); });
}

Literal* CodeGeneratorRISCV64::DeduplicateBootImageAddressLiteral(uint64_t address) {
  return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address));
}

Literal* CodeGeneratorRISCV64::DeduplicateJitStringLiteral(const DexFile& dex_file,
                                                           dex::StringIndex string_index,
                                                           Handle<mirror::String> handle) {
  ReserveJitStringRoot(StringReference(&dex_file, string_index), handle);
  return jit_string_patches_.GetOrCreate(
      StringReference(&dex_file, string_index),
      [this]() { return __ NewLiteral<uint32_t>(/* value= */ 0u); });
}

Literal* CodeGeneratorRISCV64::DeduplicateJitClassLiteral(const DexFile& dex_file,
                                                          dex::TypeIndex type_index,
                                                          Handle<mirror::Class> handle) {
  ReserveJitClassRoot(TypeReference(&dex_file, type_index), handle);
  return jit_class_patches_.GetOrCreate(
      TypeReference(&dex_file, type_index),
      [this]() { return __ NewLiteral<uint32_t>(/* value= */ 0u); });
}

void CodeGeneratorRISCV64::PatchJitRootUse(uint8_t* code,
                                          const uint8_t* roots_data,
                                          const Literal* literal,
                                          uint64_t index_in_table) const {
  uint32_t literal_offset = GetAssembler().GetLabelLocation(literal->GetLabel());
  uintptr_t address =
      reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>);
  reinterpret_cast<uint32_t*>(code + literal_offset)[0] = dchecked_integral_cast<uint32_t>(address);
}

void CodeGeneratorRISCV64::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) {
  for (const auto& entry : jit_string_patches_) {
    const StringReference& string_reference = entry.first;
    Literal* table_entry_literal = entry.second;
    uint64_t index_in_table = GetJitStringRootIndex(string_reference);
    PatchJitRootUse(code, roots_data, table_entry_literal, index_in_table);
  }
  for (const auto& entry : jit_class_patches_) {
    const TypeReference& type_reference = entry.first;
    Literal* table_entry_literal = entry.second;
    uint64_t index_in_table = GetJitClassRootIndex(type_reference);
    PatchJitRootUse(code, roots_data, table_entry_literal, index_in_table);
  }
}

void CodeGeneratorRISCV64::EmitPcRelativeAuipcPlaceholder(PcRelativePatchInfo* info_high,
                                                          XRegister out) {
  DCHECK(info_high->pc_insn_label == &info_high->label);
  __ Bind(&info_high->label);
  __ Auipc(out, /*imm20=*/ kLinkTimeOffsetPlaceholderHigh);
}

void CodeGeneratorRISCV64::EmitPcRelativeAddiPlaceholder(PcRelativePatchInfo* info_low,
                                                         XRegister rd,
                                                         XRegister rs1) {
  DCHECK(info_low->pc_insn_label != &info_low->label);
  __ Bind(&info_low->label);
  __ Addi(rd, rs1, /*imm12=*/ kLinkTimeOffsetPlaceholderLow);
}

void CodeGeneratorRISCV64::EmitPcRelativeLwuPlaceholder(PcRelativePatchInfo* info_low,
                                                        XRegister rd,
                                                        XRegister rs1) {
  DCHECK(info_low->pc_insn_label != &info_low->label);
  __ Bind(&info_low->label);
  __ Lwu(rd, rs1, /*offset=*/ kLinkTimeOffsetPlaceholderLow);
}

void CodeGeneratorRISCV64::EmitPcRelativeLdPlaceholder(PcRelativePatchInfo* info_low,
                                                       XRegister rd,
                                                       XRegister rs1) {
  DCHECK(info_low->pc_insn_label != &info_low->label);
  __ Bind(&info_low->label);
  __ Ld(rd, rs1, /*offset=*/ kLinkTimeOffsetPlaceholderLow);
}

template <linker::LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)>
inline void CodeGeneratorRISCV64::EmitPcRelativeLinkerPatches(
    const ArenaDeque<PcRelativePatchInfo>& infos,
    ArenaVector<linker::LinkerPatch>* linker_patches) {
  for (const PcRelativePatchInfo& info : infos) {
    linker_patches->push_back(Factory(__ GetLabelLocation(&info.label),
                                      info.target_dex_file,
                                      __ GetLabelLocation(info.pc_insn_label),
                                      info.offset_or_index));
  }
}

template <linker::LinkerPatch (*Factory)(size_t, uint32_t, uint32_t)>
linker::LinkerPatch NoDexFileAdapter(size_t literal_offset,
                                     const DexFile* target_dex_file,
                                     uint32_t pc_insn_offset,
                                     uint32_t boot_image_offset) {
  DCHECK(target_dex_file == nullptr);  // Unused for these patches, should be null.
  return Factory(literal_offset, pc_insn_offset, boot_image_offset);
}

void CodeGeneratorRISCV64::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) {
  DCHECK(linker_patches->empty());
  size_t size =
      boot_image_method_patches_.size() +
      method_bss_entry_patches_.size() +
      boot_image_type_patches_.size() +
      app_image_type_patches_.size() +
      type_bss_entry_patches_.size() +
      public_type_bss_entry_patches_.size() +
      package_type_bss_entry_patches_.size() +
      boot_image_string_patches_.size() +
      string_bss_entry_patches_.size() +
      boot_image_jni_entrypoint_patches_.size() +
      boot_image_other_patches_.size();
  linker_patches->reserve(size);
  if (GetCompilerOptions().IsBootImage() || GetCompilerOptions().IsBootImageExtension()) {
    EmitPcRelativeLinkerPatches<linker::LinkerPatch::RelativeMethodPatch>(
        boot_image_method_patches_, linker_patches);
    EmitPcRelativeLinkerPatches<linker::LinkerPatch::RelativeTypePatch>(
        boot_image_type_patches_, linker_patches);
    EmitPcRelativeLinkerPatches<linker::LinkerPatch::RelativeStringPatch>(
        boot_image_string_patches_, linker_patches);
  } else {
    DCHECK(boot_image_method_patches_.empty());
    DCHECK(boot_image_type_patches_.empty());
    DCHECK(boot_image_string_patches_.empty());
  }
  DCHECK_IMPLIES(!GetCompilerOptions().IsAppImage(), app_image_type_patches_.empty());
  if (GetCompilerOptions().IsBootImage()) {
    EmitPcRelativeLinkerPatches<NoDexFileAdapter<linker::LinkerPatch::IntrinsicReferencePatch>>(
        boot_image_other_patches_, linker_patches);
  } else {
    EmitPcRelativeLinkerPatches<NoDexFileAdapter<linker::LinkerPatch::BootImageRelRoPatch>>(
        boot_image_other_patches_, linker_patches);
    EmitPcRelativeLinkerPatches<linker::LinkerPatch::TypeAppImageRelRoPatch>(
        app_image_type_patches_, linker_patches);
  }
  EmitPcRelativeLinkerPatches<linker::LinkerPatch::MethodBssEntryPatch>(
      method_bss_entry_patches_, linker_patches);
  EmitPcRelativeLinkerPatches<linker::LinkerPatch::TypeBssEntryPatch>(
      type_bss_entry_patches_, linker_patches);
  EmitPcRelativeLinkerPatches<linker::LinkerPatch::PublicTypeBssEntryPatch>(
      public_type_bss_entry_patches_, linker_patches);
  EmitPcRelativeLinkerPatches<linker::LinkerPatch::PackageTypeBssEntryPatch>(
      package_type_bss_entry_patches_, linker_patches);
  EmitPcRelativeLinkerPatches<linker::LinkerPatch::StringBssEntryPatch>(
      string_bss_entry_patches_, linker_patches);
  EmitPcRelativeLinkerPatches<linker::LinkerPatch::RelativeJniEntrypointPatch>(
      boot_image_jni_entrypoint_patches_, linker_patches);
  DCHECK_EQ(size, linker_patches->size());
}

void CodeGeneratorRISCV64::LoadTypeForBootImageIntrinsic(XRegister dest,
                                                         TypeReference target_type) {
  // Load the type the same way as for HLoadClass::LoadKind::kBootImageLinkTimePcRelative.
  DCHECK(GetCompilerOptions().IsBootImage() || GetCompilerOptions().IsBootImageExtension());
  PcRelativePatchInfo* info_high =
      NewBootImageTypePatch(*target_type.dex_file, target_type.TypeIndex());
  EmitPcRelativeAuipcPlaceholder(info_high, dest);
  PcRelativePatchInfo* info_low =
      NewBootImageTypePatch(*target_type.dex_file, target_type.TypeIndex(), info_high);
  EmitPcRelativeAddiPlaceholder(info_low, dest, dest);
}

void CodeGeneratorRISCV64::LoadBootImageRelRoEntry(XRegister dest, uint32_t boot_image_offset) {
  PcRelativePatchInfo* info_high = NewBootImageRelRoPatch(boot_image_offset);
  EmitPcRelativeAuipcPlaceholder(info_high, dest);
  PcRelativePatchInfo* info_low = NewBootImageRelRoPatch(boot_image_offset, info_high);
  // Note: Boot image is in the low 4GiB and the entry is always 32-bit, so emit a 32-bit load.
  EmitPcRelativeLwuPlaceholder(info_low, dest, dest);
}

void CodeGeneratorRISCV64::LoadBootImageAddress(XRegister dest, uint32_t boot_image_reference) {
  if (GetCompilerOptions().IsBootImage()) {
    PcRelativePatchInfo* info_high = NewBootImageIntrinsicPatch(boot_image_reference);
    EmitPcRelativeAuipcPlaceholder(info_high, dest);
    PcRelativePatchInfo* info_low = NewBootImageIntrinsicPatch(boot_image_reference, info_high);
    EmitPcRelativeAddiPlaceholder(info_low, dest, dest);
  } else if (GetCompilerOptions().GetCompilePic()) {
    LoadBootImageRelRoEntry(dest, boot_image_reference);
  } else {
    DCHECK(GetCompilerOptions().IsJitCompiler());
    gc::Heap* heap = Runtime::Current()->GetHeap();
    DCHECK(!heap->GetBootImageSpaces().empty());
    const uint8_t* address = heap->GetBootImageSpaces()[0]->Begin() + boot_image_reference;
    // Note: Boot image is in the low 4GiB (usually the low 2GiB, requiring just LUI+ADDI).
    // We may not have an available scratch register for `LoadConst64()` but it never
    // emits better code than `Li()` for 32-bit unsigned constants anyway.
    __ Li(dest, reinterpret_cast32<uint32_t>(address));
  }
}

void CodeGeneratorRISCV64::LoadIntrinsicDeclaringClass(XRegister dest, HInvoke* invoke) {
  DCHECK_NE(invoke->GetIntrinsic(), Intrinsics::kNone);
  if (GetCompilerOptions().IsBootImage()) {
    MethodReference target_method = invoke->GetResolvedMethodReference();
    dex::TypeIndex type_idx = target_method.dex_file->GetMethodId(target_method.index).class_idx_;
    LoadTypeForBootImageIntrinsic(dest, TypeReference(target_method.dex_file, type_idx));
  } else {
    uint32_t boot_image_offset = GetBootImageOffsetOfIntrinsicDeclaringClass(invoke);
    LoadBootImageAddress(dest, boot_image_offset);
  }
}

void CodeGeneratorRISCV64::LoadClassRootForIntrinsic(XRegister dest, ClassRoot class_root) {
  if (GetCompilerOptions().IsBootImage()) {
    ScopedObjectAccess soa(Thread::Current());
    ObjPtr<mirror::Class> klass = GetClassRoot(class_root);
    TypeReference target_type(&klass->GetDexFile(), klass->GetDexTypeIndex());
    LoadTypeForBootImageIntrinsic(dest, target_type);
  } else {
    uint32_t boot_image_offset = GetBootImageOffset(class_root);
    LoadBootImageAddress(dest, boot_image_offset);
  }
}

void CodeGeneratorRISCV64::LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke) {
  switch (load_kind) {
    case MethodLoadKind::kBootImageLinkTimePcRelative: {
      DCHECK(GetCompilerOptions().IsBootImage() || GetCompilerOptions().IsBootImageExtension());
      CodeGeneratorRISCV64::PcRelativePatchInfo* info_high =
          NewBootImageMethodPatch(invoke->GetResolvedMethodReference());
      EmitPcRelativeAuipcPlaceholder(info_high, temp.AsRegister<XRegister>());
      CodeGeneratorRISCV64::PcRelativePatchInfo* info_low =
          NewBootImageMethodPatch(invoke->GetResolvedMethodReference(), info_high);
      EmitPcRelativeAddiPlaceholder(
          info_low, temp.AsRegister<XRegister>(), temp.AsRegister<XRegister>());
      break;
    }
    case MethodLoadKind::kBootImageRelRo: {
      uint32_t boot_image_offset = GetBootImageOffset(invoke);
      LoadBootImageRelRoEntry(temp.AsRegister<XRegister>(), boot_image_offset);
      break;
    }
    case MethodLoadKind::kBssEntry: {
      PcRelativePatchInfo* info_high = NewMethodBssEntryPatch(invoke->GetMethodReference());
      EmitPcRelativeAuipcPlaceholder(info_high, temp.AsRegister<XRegister>());
      PcRelativePatchInfo* info_low =
          NewMethodBssEntryPatch(invoke->GetMethodReference(), info_high);
      EmitPcRelativeLdPlaceholder(
          info_low, temp.AsRegister<XRegister>(), temp.AsRegister<XRegister>());
      break;
    }
    case MethodLoadKind::kJitDirectAddress: {
      __ LoadConst64(temp.AsRegister<XRegister>(),
                     reinterpret_cast<uint64_t>(invoke->GetResolvedMethod()));
      break;
    }
    case MethodLoadKind::kRuntimeCall: {
      // Test situation, don't do anything.
      break;
    }
    default: {
      LOG(FATAL) << "Load kind should have already been handled " << load_kind;
      UNREACHABLE();
    }
  }
}

void CodeGeneratorRISCV64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke,
                                                      Location temp,
                                                      SlowPathCode* slow_path) {
  // All registers are assumed to be correctly set up per the calling convention.
  Location callee_method = temp;  // For all kinds except kRecursive, callee will be in temp.

  switch (invoke->GetMethodLoadKind()) {
    case MethodLoadKind::kStringInit: {
      // temp = thread->string_init_entrypoint
      uint32_t offset =
          GetThreadOffset<kRiscv64PointerSize>(invoke->GetStringInitEntryPoint()).Int32Value();
      __ Loadd(temp.AsRegister<XRegister>(), TR, offset);
      break;
    }
    case MethodLoadKind::kRecursive:
      callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodIndex());
      break;
    case MethodLoadKind::kRuntimeCall:
      GenerateInvokeStaticOrDirectRuntimeCall(invoke, temp, slow_path);
      return;  // No code pointer retrieval; the runtime performs the call directly.
    case MethodLoadKind::kBootImageLinkTimePcRelative:
      DCHECK(GetCompilerOptions().IsBootImage() || GetCompilerOptions().IsBootImageExtension());
      if (invoke->GetCodePtrLocation() == CodePtrLocation::kCallCriticalNative) {
        // Do not materialize the method pointer, load directly the entrypoint.
        CodeGeneratorRISCV64::PcRelativePatchInfo* info_high =
            NewBootImageJniEntrypointPatch(invoke->GetResolvedMethodReference());
        EmitPcRelativeAuipcPlaceholder(info_high, RA);
        CodeGeneratorRISCV64::PcRelativePatchInfo* info_low =
            NewBootImageJniEntrypointPatch(invoke->GetResolvedMethodReference(), info_high);
        EmitPcRelativeLdPlaceholder(info_low, RA, RA);
        break;
      }
      FALLTHROUGH_INTENDED;
    default:
      LoadMethod(invoke->GetMethodLoadKind(), temp, invoke);
      break;
  }

  switch (invoke->GetCodePtrLocation()) {
    case CodePtrLocation::kCallSelf:
      DCHECK(!GetGraph()->HasShouldDeoptimizeFlag());
      __ Jal(&frame_entry_label_);
      RecordPcInfo(invoke, invoke->GetDexPc(), slow_path);
      break;
    case CodePtrLocation::kCallArtMethod:
      // RA = callee_method->entry_point_from_quick_compiled_code_;
      __ Loadd(RA,
               callee_method.AsRegister<XRegister>(),
               ArtMethod::EntryPointFromQuickCompiledCodeOffset(kRiscv64PointerSize).Int32Value());
      // RA()
      __ Jalr(RA);
      RecordPcInfo(invoke, invoke->GetDexPc(), slow_path);
      break;
    case CodePtrLocation::kCallCriticalNative: {
      size_t out_frame_size =
          PrepareCriticalNativeCall<CriticalNativeCallingConventionVisitorRiscv64,
                                    kNativeStackAlignment,
                                    GetCriticalNativeDirectCallFrameSize>(invoke);
      if (invoke->GetMethodLoadKind() == MethodLoadKind::kBootImageLinkTimePcRelative) {
        // Entrypoint is already loaded in RA.
      } else {
        // RA = callee_method->ptr_sized_fields_.data_;  // EntryPointFromJni
        MemberOffset offset = ArtMethod::EntryPointFromJniOffset(kRiscv64PointerSize);
        __ Loadd(RA, callee_method.AsRegister<XRegister>(), offset.Int32Value());
      }
      __ Jalr(RA);
      RecordPcInfo(invoke, invoke->GetDexPc(), slow_path);
      // The result is returned the same way in native ABI and managed ABI. No result conversion is
      // needed, see comments in `Riscv64JniCallingConvention::RequiresSmallResultTypeExtension()`.
      if (out_frame_size != 0u) {
        DecreaseFrame(out_frame_size);
      }
      break;
    }
  }

  DCHECK(!IsLeafMethod());
}

void CodeGeneratorRISCV64::MaybeGenerateInlineCacheCheck(HInstruction* instruction,
                                                         XRegister klass) {
  if (ProfilingInfoBuilder::IsInlineCacheUseful(instruction->AsInvoke(), this)) {
    ProfilingInfo* info = GetGraph()->GetProfilingInfo();
    DCHECK(info != nullptr);
    InlineCache* cache = ProfilingInfoBuilder::GetInlineCache(
        info, GetCompilerOptions(), instruction->AsInvoke());
    if (cache != nullptr) {
      uint64_t address = reinterpret_cast64<uint64_t>(cache);
      Riscv64Label done;
      // The `art_quick_update_inline_cache` expects the inline cache in T5.
      XRegister ic_reg = T5;
      ScratchRegisterScope srs(GetAssembler());
      DCHECK_EQ(srs.AvailableXRegisters(), 2u);
      srs.ExcludeXRegister(ic_reg);
      DCHECK_EQ(srs.AvailableXRegisters(), 1u);
      __ LoadConst64(ic_reg, address);
      {
        ScratchRegisterScope srs2(GetAssembler());
        XRegister tmp = srs2.AllocateXRegister();
        __ Loadd(tmp, ic_reg, InlineCache::ClassesOffset().Int32Value());
        // Fast path for a monomorphic cache.
        __ Beq(klass, tmp, &done);
      }
      InvokeRuntime(kQuickUpdateInlineCache, instruction, instruction->GetDexPc());
      __ Bind(&done);
    } else {
      // This is unexpected, but we don't guarantee stable compilation across
      // JIT runs so just warn about it.
      ScopedObjectAccess soa(Thread::Current());
      LOG(WARNING) << "Missing inline cache for " << GetGraph()->GetArtMethod()->PrettyMethod();
    }
  }
}

void CodeGeneratorRISCV64::GenerateVirtualCall(HInvokeVirtual* invoke,
                                               Location temp_location,
                                               SlowPathCode* slow_path) {
  // Use the calling convention instead of the location of the receiver, as
  // intrinsics may have put the receiver in a different register. In the intrinsics
  // slow path, the arguments have been moved to the right place, so here we are
  // guaranteed that the receiver is the first register of the calling convention.
  InvokeDexCallingConvention calling_convention;
  XRegister receiver = calling_convention.GetRegisterAt(0);
  XRegister temp = temp_location.AsRegister<XRegister>();
  MemberOffset method_offset =
      mirror::Class::EmbeddedVTableEntryOffset(invoke->GetVTableIndex(), kRiscv64PointerSize);
  MemberOffset class_offset = mirror::Object::ClassOffset();
  Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kRiscv64PointerSize);

  // temp = object->GetClass();
  __ Loadwu(temp, receiver, class_offset.Int32Value());
  MaybeRecordImplicitNullCheck(invoke);
  // Instead of simply (possibly) unpoisoning `temp` here, we should
  // emit a read barrier for the previous class reference load.
  // However this is not required in practice, as this is an
  // intermediate/temporary reference and because the current
  // concurrent copying collector keeps the from-space memory
  // intact/accessible until the end of the marking phase (the
  // concurrent copying collector may not in the future).
  MaybeUnpoisonHeapReference(temp);

  // If we're compiling baseline, update the inline cache.
  MaybeGenerateInlineCacheCheck(invoke, temp);

  // temp = temp->GetMethodAt(method_offset);
  __ Loadd(temp, temp, method_offset.Int32Value());
  // RA = temp->GetEntryPoint();
  __ Loadd(RA, temp, entry_point.Int32Value());
  // RA();
  __ Jalr(RA);
  RecordPcInfo(invoke, invoke->GetDexPc(), slow_path);
}

void CodeGeneratorRISCV64::MoveFromReturnRegister(Location trg, DataType::Type type) {
  if (!trg.IsValid()) {
    DCHECK_EQ(type, DataType::Type::kVoid);
    return;
  }

  DCHECK_NE(type, DataType::Type::kVoid);

  if (DataType::IsIntegralType(type) || type == DataType::Type::kReference) {
    XRegister trg_reg = trg.AsRegister<XRegister>();
    XRegister res_reg = Riscv64ReturnLocation(type).AsRegister<XRegister>();
    if (trg_reg != res_reg) {
      __ Mv(trg_reg, res_reg);
    }
  } else {
    FRegister trg_reg = trg.AsFpuRegister<FRegister>();
    FRegister res_reg = Riscv64ReturnLocation(type).AsFpuRegister<FRegister>();
    if (trg_reg != res_reg) {
      __ FMvD(trg_reg, res_reg);  // 64-bit move is OK also for `float`.
    }
  }
}

void CodeGeneratorRISCV64::PoisonHeapReference(XRegister reg) {
  __ Sub(reg, Zero, reg);  // Negate the ref.
  __ ZextW(reg, reg);      // Zero-extend the 32-bit ref.
}

void CodeGeneratorRISCV64::UnpoisonHeapReference(XRegister reg) {
  __ Sub(reg, Zero, reg);  // Negate the ref.
  __ ZextW(reg, reg);      // Zero-extend the 32-bit ref.
}

void CodeGeneratorRISCV64::MaybePoisonHeapReference(XRegister reg) {
  if (kPoisonHeapReferences) {
    PoisonHeapReference(reg);
  }
}

void CodeGeneratorRISCV64::MaybeUnpoisonHeapReference(XRegister reg) {
  if (kPoisonHeapReferences) {
    UnpoisonHeapReference(reg);
  }
}

void CodeGeneratorRISCV64::SwapLocations(Location loc1, Location loc2, DataType::Type type) {
  DCHECK(!loc1.IsConstant());
  DCHECK(!loc2.IsConstant());

  if (loc1.Equals(loc2)) {
    return;
  }

  bool is_slot1 = loc1.IsStackSlot() || loc1.IsDoubleStackSlot();
  bool is_slot2 = loc2.IsStackSlot() || loc2.IsDoubleStackSlot();
  bool is_simd1 = loc1.IsSIMDStackSlot();
  bool is_simd2 = loc2.IsSIMDStackSlot();
  bool is_fp_reg1 = loc1.IsFpuRegister();
  bool is_fp_reg2 = loc2.IsFpuRegister();

  if ((is_slot1 != is_slot2) ||
      (loc2.IsRegister() && loc1.IsRegister()) ||
      (is_fp_reg2 && is_fp_reg1)) {
    if ((is_fp_reg2 && is_fp_reg1) && GetGraph()->HasSIMD()) {
      LOG(FATAL) << "Unsupported";
      UNREACHABLE();
    }
    ScratchRegisterScope srs(GetAssembler());
    Location tmp = (is_fp_reg2 || is_fp_reg1)
        ? Location::FpuRegisterLocation(srs.AllocateFRegister())
        : Location::RegisterLocation(srs.AllocateXRegister());
    MoveLocation(tmp, loc1, type);
    MoveLocation(loc1, loc2, type);
    MoveLocation(loc2, tmp, type);
  } else if (is_slot1 && is_slot2) {
    move_resolver_.Exchange(loc1.GetStackIndex(), loc2.GetStackIndex(), loc1.IsDoubleStackSlot());
  } else if (is_simd1 && is_simd2) {
    // TODO(riscv64): Add VECTOR/SIMD later.
    UNIMPLEMENTED(FATAL) << "Vector extension is unsupported";
  } else if ((is_fp_reg1 && is_simd2) || (is_fp_reg2 && is_simd1)) {
    // TODO(riscv64): Add VECTOR/SIMD later.
    UNIMPLEMENTED(FATAL) << "Vector extension is unsupported";
  } else {
    LOG(FATAL) << "Unimplemented swap between locations " << loc1 << " and " << loc2;
  }
}

}  // namespace riscv64
}  // namespace art