/* -*- mesa-c++  -*-
 * Copyright 2022 Collabora LTD
 * Author: Gert Wollny <gert.wollny@collabora.com>
 * SPDX-License-Identifier: MIT
 */

#include "sfn_assembler.h"

#include "../eg_sq.h"
#include "../r600_asm.h"

#include "sfn_callstack.h"
#include "sfn_conditionaljumptracker.h"
#include "sfn_debug.h"
#include "sfn_instr_alugroup.h"
#include "sfn_instr_controlflow.h"
#include "sfn_instr_export.h"
#include "sfn_instr_fetch.h"
#include "sfn_instr_mem.h"
#include "sfn_instr_tex.h"

namespace r600 {
Assembler::Assembler(r600_shader *sh, const r600_shader_key& key):
    m_sh(sh),
    m_key(key)
{
}

extern const std::map<ESDOp, int> ds_opcode_map;

class AssamblerVisitor : public ConstInstrVisitor {
public:
   AssamblerVisitor(r600_shader *sh, const r600_shader_key& key, bool legacy_math_rules);

   void visit(const AluInstr& instr) override;
   void visit(const AluGroup& instr) override;
   void visit(const TexInstr& instr) override;
   void visit(const ExportInstr& instr) override;
   void visit(const FetchInstr& instr) override;
   void visit(const Block& instr) override;
   void visit(const IfInstr& instr) override;
   void visit(const ControlFlowInstr& instr) override;
   void visit(const ScratchIOInstr& instr) override;
   void visit(const StreamOutInstr& instr) override;
   void visit(const MemRingOutInstr& instr) override;
   void visit(const EmitVertexInstr& instr) override;
   void visit(const GDSInstr& instr) override;
   void visit(const WriteTFInstr& instr) override;
   void visit(const LDSAtomicInstr& instr) override;
   void visit(const LDSReadInstr& instr) override;
   void visit(const RatInstr& instr) override;

   void finalize();

   const uint32_t sf_vtx = 1;
   const uint32_t sf_tex = 2;
   const uint32_t sf_alu = 4;
   const uint32_t sf_addr_register = 8;
   const uint32_t sf_all = 0xf;

   void clear_states(const uint32_t& states);
   bool copy_dst(r600_bytecode_alu_dst& dst, const Register& d, bool write);
   PVirtualValue copy_src(r600_bytecode_alu_src& src, const VirtualValue& s);

   EBufferIndexMode emit_index_reg(const VirtualValue& addr, unsigned idx);

   void emit_endif();
   void emit_else();
   void emit_loop_begin(bool vpm);
   void emit_loop_end();
   void emit_loop_break();
   void emit_loop_cont();
   void emit_alu_push_before();

   void emit_alu_op(const AluInstr& ai);
   void emit_lds_op(const AluInstr& lds);

   auto translate_for_mathrules(EAluOp op) -> EAluOp;

   void emit_wait_ack();

   /* Start initialized in constructor */
   const r600_shader_key& m_key;
   r600_shader *m_shader;
   r600_bytecode *m_bc;

   ConditionalJumpTracker m_jump_tracker;
   CallStack m_callstack;
   bool ps_alpha_to_one;
   /* End initialized in constructor */

   std::set<uint32_t> m_nliterals_in_group;
   std::set<int> vtx_fetch_results;
   std::set<int> tex_fetch_results;

   const VirtualValue *m_last_addr{nullptr};

   unsigned m_max_color_exports{0};
   int m_loop_nesting{0};

   bool m_ack_suggested{false};
   bool m_has_param_output{false};
   bool m_has_pos_output{false};
   bool m_last_op_was_barrier{false};
   bool m_result{true};
   bool m_legacy_math_rules{false};
   bool m_require_alu_extended{false};
};

bool
Assembler::lower(Shader *shader)
{
   AssamblerVisitor ass(m_sh, m_key, shader->has_flag(Shader::sh_legacy_math_rules));

   auto& blocks = shader->func();
   for (auto b : blocks) {
      b->accept(ass);
      if (!ass.m_result)
         return false;
   }

   ass.finalize();

   return ass.m_result;
}

AssamblerVisitor::AssamblerVisitor(r600_shader *sh, const r600_shader_key& key,
                                   bool legacy_math_rules):
    m_key(key),
    m_shader(sh),

    m_bc(&sh->bc),
    m_callstack(sh->bc),
    ps_alpha_to_one(key.ps.alpha_to_one),
    m_legacy_math_rules(legacy_math_rules)
{
   if (m_shader->processor_type == MESA_SHADER_FRAGMENT)
      m_max_color_exports = MAX2(m_key.ps.nr_cbufs, 1);

   if (m_shader->processor_type == MESA_SHADER_VERTEX && m_shader->ninput > 0)
      r600_bytecode_add_cfinst(m_bc, CF_OP_CALL_FS);
}

void
AssamblerVisitor::finalize()
{
   const struct cf_op_info *last = nullptr;

   if (m_bc->cf_last)
      last = r600_isa_cf(m_bc->cf_last->op);

   /* alu clause instructions don't have EOP bit, so add NOP */
   if (m_shader->bc.gfx_level < CAYMAN &&
       (!last || last->flags & CF_ALU || m_bc->cf_last->op == CF_OP_LOOP_END ||
        m_bc->cf_last->op == CF_OP_POP))
      r600_bytecode_add_cfinst(m_bc, CF_OP_NOP);

   /* A fetch shader only can't be EOP (results in hang), but we can replace
    * it by a NOP */
   else if (last && m_bc->cf_last->op == CF_OP_CALL_FS)
      m_bc->cf_last->op = CF_OP_NOP;

   if (m_shader->bc.gfx_level != CAYMAN)
      m_bc->cf_last->end_of_program = 1;
   else
      cm_bytecode_add_cf_end(m_bc);
}

extern const std::map<EAluOp, int> opcode_map;

void
AssamblerVisitor::visit(const AluInstr& ai)
{
   assert(vtx_fetch_results.empty());
   assert(tex_fetch_results.empty());

   if (unlikely(ai.has_alu_flag(alu_is_lds)))
      emit_lds_op(ai);
   else
      emit_alu_op(ai);
}

void
AssamblerVisitor::emit_lds_op(const AluInstr& lds)
{
   struct r600_bytecode_alu alu;
   memset(&alu, 0, sizeof(alu));

   alu.is_lds_idx_op = true;
   alu.op = lds.lds_opcode();

   bool has_lds_fetch = false;
   switch (alu.op) {
   case LDS_WRITE:
      alu.op = LDS_OP2_LDS_WRITE;
      break;
   case LDS_WRITE_REL:
      alu.op = LDS_OP3_LDS_WRITE_REL;
      alu.lds_idx = 1;
      break;
   case DS_OP_READ_RET:
      alu.op = LDS_OP1_LDS_READ_RET;
      FALLTHROUGH;
   case LDS_ADD_RET:
   case LDS_AND_RET:
   case LDS_OR_RET:
   case LDS_MAX_INT_RET:
   case LDS_MAX_UINT_RET:
   case LDS_MIN_INT_RET:
   case LDS_MIN_UINT_RET:
   case LDS_XOR_RET:
   case LDS_XCHG_RET:
   case LDS_CMP_XCHG_RET:
      has_lds_fetch = true;
      break;
   case LDS_ADD:
   case LDS_AND:
   case LDS_OR:
   case LDS_MAX_INT:
   case LDS_MAX_UINT:
   case LDS_MIN_INT:
   case LDS_MIN_UINT:
   case LDS_XOR:
      break;
   default:
      std::cerr << "\n R600: error op: " << lds << "\n";
      UNREACHABLE("Unhandled LDS op");
   }

   copy_src(alu.src[0], lds.src(0));

   if (lds.n_sources() > 1)
      copy_src(alu.src[1], lds.src(1));
   else
      alu.src[1].sel = V_SQ_ALU_SRC_0;

   if (lds.n_sources() > 2)
      copy_src(alu.src[2], lds.src(2));
   else
      alu.src[2].sel = V_SQ_ALU_SRC_0;

   alu.last = lds.has_alu_flag(alu_last_instr);

   int r = r600_bytecode_add_alu(m_bc, &alu);
   if (has_lds_fetch)
      m_bc->cf_last->nlds_read++;

   if (r)
      m_result = false;
}

auto AssamblerVisitor::translate_for_mathrules(EAluOp op) -> EAluOp
{
   switch (op) {
   case op2_dot_ieee: return op2_dot;
   case op2_dot4_ieee: return op2_dot4;
   case op2_mul_ieee: return op2_mul;
   case op3_muladd_ieee : return op2_mul_ieee;
   default:
      return op;
   }
}

void
AssamblerVisitor::emit_alu_op(const AluInstr& ai)
{
   sfn_log << SfnLog::assembly << "Emit ALU op " << ai << "\n";

   struct r600_bytecode_alu alu;
   memset(&alu, 0, sizeof(alu));

   auto opcode = ai.opcode();

   if (unlikely(ai.opcode() == op1_mova_int &&
                (m_bc->gfx_level < CAYMAN || alu.dst.sel == 0))) {
      m_last_addr = ai.psrc(0);
      m_bc->ar_chan = m_last_addr->chan();
      m_bc->ar_reg = m_last_addr->sel();
   }

   if (m_legacy_math_rules)
       opcode = translate_for_mathrules(opcode);

   auto hw_opcode = opcode_map.find(opcode);

   if (hw_opcode == opcode_map.end()) {
      std::cerr << "Opcode not handled for " << ai << "\n";
      m_result = false;
      return;
   }

   // skip multiple barriers
   if (m_last_op_was_barrier && opcode == op0_group_barrier)
      return;

   m_last_op_was_barrier = opcode == op0_group_barrier;

   alu.op = hw_opcode->second;

   auto dst = ai.dest();
   if (dst) {
      if (ai.opcode() != op1_mova_int) {
         if (!copy_dst(alu.dst, *dst, ai.has_alu_flag(alu_write))) {
            m_result = false;
            return;
         }

         alu.dst.write = ai.has_alu_flag(alu_write);
         alu.dst.rel = dst->addr() ? 1 : 0;
      } else if (m_bc->gfx_level == CAYMAN && ai.dest()->sel() > 0) {
         alu.dst.sel = ai.dest()->sel() + 1;
      }
   } else {
      alu.dst.chan = ai.dest_chan();
   }

   alu.dst.clamp = ai.has_alu_flag(alu_dst_clamp);
   alu.is_op3 = ai.n_sources() == 3;

   if (!alu.is_op3)
      alu.omod = ai.output_modifier();

   EBufferIndexMode kcache_index_mode = bim_none;
   PVirtualValue buffer_offset = nullptr;

   for (unsigned i = 0; i < ai.n_sources(); ++i) {
      buffer_offset = copy_src(alu.src[i], ai.src(i));
      alu.src[i].neg = ai.has_source_mod(i, AluInstr::mod_neg);
      if (!alu.is_op3)
         alu.src[i].abs = ai.has_source_mod(i, AluInstr::mod_abs);

      if (buffer_offset && kcache_index_mode == bim_none) {
         auto idx_reg = buffer_offset->as_register();
         if (idx_reg && idx_reg->has_flag(Register::addr_or_idx)) {
            switch (idx_reg->sel()) {
            case 1: kcache_index_mode = bim_zero; break;
            case 2: kcache_index_mode = bim_one; break;
            default:
               UNREACHABLE("Unsupported index mode");
            }
         } else {
            kcache_index_mode = bim_zero;
         }
         alu.src[i].kc_rel = kcache_index_mode;
      }

      if (ai.has_lds_queue_read()) {
         assert(m_bc->cf_last->nlds_read > 0);
         m_bc->cf_last->nlds_read--;
      }
   }

   if (ai.bank_swizzle() != alu_vec_unknown)
      alu.bank_swizzle_force = ai.bank_swizzle();

   alu.last = ai.has_alu_flag(alu_last_instr);
   alu.execute_mask = ai.has_alu_flag(alu_update_exec);

   /* If the destination register is equal to the last loaded address register
    * then clear the latter one, because the values will no longer be
    * identical */
   if (m_last_addr)
      sfn_log << SfnLog::assembly << "  Current address register is " << *m_last_addr
              << "\n";

   if (dst)
      sfn_log << SfnLog::assembly << "  Current dst register is " << *dst << "\n";

   /*
   auto cf_op = ai.cf_type();

   unsigned type = 0;
   switch (cf_op) {
   case cf_alu:
   case cf_alu_push_before:
      type = CF_OP_ALU;
      break;
   case cf_alu_pop_after:
      type = CF_OP_ALU_POP_AFTER;
      break;
   case cf_alu_pop2_after:
      type = CF_OP_ALU_POP2_AFTER;
      break;
   case cf_alu_break:
      type = CF_OP_ALU_BREAK;
      break;
   case cf_alu_else_after:
      type = CF_OP_ALU_ELSE_AFTER;
      break;
   case cf_alu_continue:
      type = CF_OP_ALU_CONTINUE;
      break;
   case cf_alu_extended:
      type = CF_OP_ALU_EXT;
      break;
   default:
      assert(0 && "cf_alu_undefined should have been replaced");
   }
*/
   if (alu.last)
      m_nliterals_in_group.clear();

   m_result = !r600_bytecode_add_alu(m_bc, &alu);

   if (unlikely(ai.opcode() == op1_mova_int)) {
      if (m_bc->gfx_level < CAYMAN || alu.dst.sel == 0) {
         m_bc->ar_loaded = 1;
      } else if (m_bc->gfx_level == CAYMAN) {
         int idx = alu.dst.sel - 2;
         m_bc->index_loaded[idx] = 1;
         m_bc->index_reg[idx] = -1;
      }
   }

   if (alu.dst.sel >= g_clause_local_start && alu.dst.sel < g_clause_local_end) {
      int clidx = 4 * (alu.dst.sel - g_clause_local_start) + alu.dst.chan;
      m_bc->cf_last->clause_local_written |= 1 << clidx;
   }

   if (ai.opcode() == op1_set_cf_idx0) {
      m_bc->index_loaded[0] = 1;
      m_bc->index_reg[0] = -1;
   }

   if (ai.opcode() == op1_set_cf_idx1) {
      m_bc->index_loaded[1] = 1;
      m_bc->index_reg[1] = -1;
   }
}

void
AssamblerVisitor::visit(const AluGroup& group)
{
   clear_states(sf_vtx | sf_tex);

   if (group.slots() == 0)
      return;

   static const unsigned slot_limit = 256;

   if (m_bc->cf_last && !m_bc->force_add_cf) {
      if (group.has_lds_group_start()) {
         if (m_bc->cf_last->ndw + 2 * (*group.begin())->required_slots() > slot_limit) {
            assert(m_bc->cf_last->nlds_read == 0);
            assert(0 && "Not allowed to start new alu group here");
            m_bc->force_add_cf = 1;
            m_last_addr = nullptr;
         }
      } else {
         if (m_bc->cf_last->ndw + 2 * group.slots() > slot_limit) {
            std::cerr << "m_bc->cf_last->ndw = " << m_bc->cf_last->ndw
                      << " group.slots() = " << group.slots()
                      << " -> " << m_bc->cf_last->ndw + 2 * group.slots()
                      << "> slot_limit = " << slot_limit << "\n";
            assert(m_bc->cf_last->nlds_read == 0);
            assert(0 && "Not allowed to start new alu group here");
            m_bc->force_add_cf = 1;
            m_last_addr = nullptr;
         } else {
            auto instr = *group.begin();
            if (instr && !instr->has_alu_flag(alu_is_lds) &&
                instr->opcode() == op0_group_barrier && m_bc->cf_last->ndw + 14 > slot_limit) {
               assert(0 && "Not allowed to start new alu group here");
               assert(m_bc->cf_last->nlds_read == 0);
               m_bc->force_add_cf = 1;
               m_last_addr = nullptr;
            }
         }
      }
   }

   auto [addr, is_index] = group.addr();
   assert(!addr || addr->has_flag(Register::addr_or_idx));

   for (auto& i : group) {
      if (i)
         i->accept(*this);
   }
}

void
AssamblerVisitor::visit(const TexInstr& tex_instr)
{
   clear_states(sf_vtx | sf_alu);

   if (tex_fetch_results.find(tex_instr.src().sel()) != tex_fetch_results.end()) {
      m_bc->force_add_cf = 1;
      tex_fetch_results.clear();
   }

   r600_bytecode_tex tex;
   memset(&tex, 0, sizeof(struct r600_bytecode_tex));
   tex.op = tex_instr.opcode();
   tex.sampler_id = tex_instr.sampler_id();
   tex.resource_id = tex_instr.resource_id();
   tex.src_gpr = tex_instr.src().sel();
   tex.dst_gpr = tex_instr.dst().sel();
   tex.dst_sel_x = tex_instr.dest_swizzle(0);
   tex.dst_sel_y = tex_instr.dest_swizzle(1);
   tex.dst_sel_z = tex_instr.dest_swizzle(2);
   tex.dst_sel_w = tex_instr.dest_swizzle(3);
   tex.src_sel_x = tex_instr.src()[0]->chan();
   tex.src_sel_y = tex_instr.src()[1]->chan();
   tex.src_sel_z = tex_instr.src()[2]->chan();
   tex.src_sel_w = tex_instr.src()[3]->chan();
   tex.coord_type_x = !tex_instr.has_tex_flag(TexInstr::x_unnormalized);
   tex.coord_type_y = !tex_instr.has_tex_flag(TexInstr::y_unnormalized);
   tex.coord_type_z = !tex_instr.has_tex_flag(TexInstr::z_unnormalized);
   tex.coord_type_w = !tex_instr.has_tex_flag(TexInstr::w_unnormalized);
   tex.offset_x = tex_instr.get_offset(0);
   tex.offset_y = tex_instr.get_offset(1);
   tex.offset_z = tex_instr.get_offset(2);
   tex.resource_index_mode = tex_instr.resource_index_mode();
   tex.sampler_index_mode = tex_instr.sampler_index_mode();

   if (tex.dst_sel_x < 4 && tex.dst_sel_y < 4 && tex.dst_sel_z < 4 && tex.dst_sel_w < 4)
      tex_fetch_results.insert(tex.dst_gpr);

   if (tex_instr.opcode() == TexInstr::get_gradient_h ||
       tex_instr.opcode() == TexInstr::get_gradient_v)
      tex.inst_mod = tex_instr.has_tex_flag(TexInstr::grad_fine) ? 1 : 0;
   else
      tex.inst_mod = tex_instr.inst_mode();
   if (r600_bytecode_add_tex(m_bc, &tex)) {
      R600_ASM_ERR("shader_from_nir: Error creating tex assembly instruction\n");
      m_result = false;
   }
}

void
AssamblerVisitor::visit(const ExportInstr& exi)
{
   const auto& value = exi.value();

   r600_bytecode_output output;
   memset(&output, 0, sizeof(output));

   output.gpr = value.sel();
   output.elem_size = 3;
   output.swizzle_x = value[0]->chan();
   output.swizzle_y = value[1]->chan();
   output.swizzle_z = value[2]->chan();
   output.burst_count = 1;
   output.op = exi.is_last_export() ? CF_OP_EXPORT_DONE : CF_OP_EXPORT;
   output.type = exi.export_type();

   clear_states(sf_all);
   switch (exi.export_type()) {
   case ExportInstr::pixel:
      output.swizzle_w = ps_alpha_to_one ? 5 : exi.value()[3]->chan();
      output.array_base = exi.location();
      break;
   case ExportInstr::pos:
      output.swizzle_w = exi.value()[3]->chan();
      output.array_base = 60 + exi.location();
      break;
   case ExportInstr::param:
      output.swizzle_w = exi.value()[3]->chan();
      output.array_base = exi.location();
      break;
   default:
      R600_ASM_ERR("shader_from_nir: export %d type not yet supported\n",
                   exi.export_type());
      m_result = false;
   }

   /* If all register elements pinned to fixed values
    * we can override the gpr (the register allocator doesn't see
    * this because it doesn't take these channels into account. */
   if (output.swizzle_x > 3 && output.swizzle_y > 3 && output.swizzle_z > 3 &&
       output.swizzle_w > 3)
      output.gpr = 0;

   int r = 0;
   if ((r = r600_bytecode_add_output(m_bc, &output))) {
      R600_ASM_ERR("Error adding export at location %d : err: %d\n", exi.location(), r);
      m_result = false;
   }
}

void
AssamblerVisitor::visit(const ScratchIOInstr& instr)
{
   clear_states(sf_all);

   struct r600_bytecode_output cf;

   memset(&cf, 0, sizeof(struct r600_bytecode_output));

   cf.op = CF_OP_MEM_SCRATCH;
   cf.elem_size = 3;
   cf.gpr = instr.value().sel();
   cf.mark = !instr.is_read();
   cf.comp_mask = instr.is_read() ? 0xf : instr.write_mask();
   cf.swizzle_x = 0;
   cf.swizzle_y = 1;
   cf.swizzle_z = 2;
   cf.swizzle_w = 3;
   cf.burst_count = 1;

   assert(!instr.is_read() || m_bc->gfx_level < R700);

   if (instr.address()) {
      cf.type = instr.is_read() || m_bc->gfx_level > R600 ? 3 : 1;
      cf.index_gpr = instr.address()->sel();

      /* The docu seems to be wrong here: In indirect addressing the
       * address_base seems to be the array_size */
      cf.array_size = instr.array_size();
   } else {
      cf.type = instr.is_read() || m_bc->gfx_level > R600 ? 2 : 0;
      cf.array_base = instr.location();
   }

   if (r600_bytecode_add_output(m_bc, &cf)) {
      R600_ASM_ERR("shader_from_nir: Error creating SCRATCH_WR assembly instruction\n");
      m_result = false;
   }
}

void
AssamblerVisitor::visit(const StreamOutInstr& instr)
{
   struct r600_bytecode_output output;
   memset(&output, 0, sizeof(struct r600_bytecode_output));

   output.gpr = instr.value().sel();
   output.elem_size = instr.element_size();
   output.array_base = instr.array_base();
   output.type = V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_WRITE;
   output.burst_count = instr.burst_count();
   output.array_size = instr.array_size();
   output.comp_mask = instr.comp_mask();
   output.op = instr.op(m_shader->bc.gfx_level);

   if (r600_bytecode_add_output(m_bc, &output)) {
      R600_ASM_ERR("shader_from_nir: Error creating stream output instruction\n");
      m_result = false;
   }
}

void
AssamblerVisitor::visit(const MemRingOutInstr& instr)
{
   struct r600_bytecode_output output;
   memset(&output, 0, sizeof(struct r600_bytecode_output));

   output.gpr = instr.value().sel();
   output.type = instr.type();
   output.elem_size = 3;
   output.comp_mask = 0xf;
   output.burst_count = 1;
   output.op = instr.op();
   if (instr.type() == MemRingOutInstr::mem_write_ind ||
       instr.type() == MemRingOutInstr::mem_write_ind_ack) {
      output.index_gpr = instr.index_reg();
      output.array_size = 0xfff;
   }
   output.array_base = instr.array_base();

   if (r600_bytecode_add_output(m_bc, &output)) {
      R600_ASM_ERR("shader_from_nir: Error creating mem ring write instruction\n");
      m_result = false;
   }
}

void
AssamblerVisitor::visit(const EmitVertexInstr& instr)
{
   int r = r600_bytecode_add_cfinst(m_bc, instr.op());
   if (!r)
      m_bc->cf_last->count = instr.stream();
   else
      m_result = false;
   assert(m_bc->cf_last->count < 4);
}

void
AssamblerVisitor::visit(const FetchInstr& fetch_instr)
{
   bool use_tc =
      fetch_instr.has_fetch_flag(FetchInstr::use_tc) || (m_bc->gfx_level == CAYMAN);

   auto clear_flags = use_tc ? sf_vtx : sf_tex;

   clear_states(clear_flags | sf_alu);

   if (fetch_instr.has_fetch_flag(FetchInstr::wait_ack))
      emit_wait_ack();


   if (!use_tc &&
       vtx_fetch_results.find(fetch_instr.src().sel()) != vtx_fetch_results.end()) {
      m_bc->force_add_cf = 1;
      vtx_fetch_results.clear();
   }

   if (fetch_instr.has_fetch_flag(FetchInstr::use_tc) &&
       tex_fetch_results.find(fetch_instr.src().sel()) != tex_fetch_results.end()) {
      m_bc->force_add_cf = 1;
      tex_fetch_results.clear();
   }

   if (use_tc)
      tex_fetch_results.insert(fetch_instr.dst().sel());
   else
      vtx_fetch_results.insert(fetch_instr.dst().sel());

   struct r600_bytecode_vtx vtx;
   memset(&vtx, 0, sizeof(vtx));
   vtx.op = fetch_instr.opcode();
   vtx.buffer_id = fetch_instr.resource_id();
   vtx.fetch_type = fetch_instr.fetch_type();
   vtx.src_gpr = fetch_instr.src().sel();
   vtx.src_sel_x = fetch_instr.src().chan();
   vtx.mega_fetch_count = fetch_instr.mega_fetch_count();
   vtx.dst_gpr = fetch_instr.dst().sel();
   vtx.dst_sel_x = fetch_instr.dest_swizzle(0); /* SEL_X */
   vtx.dst_sel_y = fetch_instr.dest_swizzle(1); /* SEL_Y */
   vtx.dst_sel_z = fetch_instr.dest_swizzle(2); /* SEL_Z */
   vtx.dst_sel_w = fetch_instr.dest_swizzle(3); /* SEL_W */
   vtx.use_const_fields = fetch_instr.has_fetch_flag(FetchInstr::use_const_field);
   vtx.data_format = fetch_instr.data_format();
   vtx.num_format_all = fetch_instr.num_format(); /* NUM_FORMAT_SCALED */
   vtx.format_comp_all = fetch_instr.has_fetch_flag(FetchInstr::format_comp_signed);
   vtx.endian = fetch_instr.endian_swap();
   vtx.buffer_index_mode = fetch_instr.resource_index_mode();
   vtx.offset = fetch_instr.src_offset();
   vtx.indexed = fetch_instr.has_fetch_flag(FetchInstr::indexed);
   vtx.uncached = fetch_instr.has_fetch_flag(FetchInstr::uncached);
   vtx.elem_size = fetch_instr.elm_size();
   vtx.array_base = fetch_instr.array_base();
   vtx.array_size = fetch_instr.array_size();
   vtx.srf_mode_all = fetch_instr.has_fetch_flag(FetchInstr::srf_mode);

   if (fetch_instr.has_fetch_flag(FetchInstr::use_tc)) {
      if ((r600_bytecode_add_vtx_tc(m_bc, &vtx))) {
         R600_ASM_ERR("shader_from_nir: Error creating tex assembly instruction\n");
         m_result = false;
      }

   } else {
      if ((r600_bytecode_add_vtx(m_bc, &vtx))) {
         R600_ASM_ERR("shader_from_nir: Error creating tex assembly instruction\n");
         m_result = false;
      }
   }

   m_bc->cf_last->vpm =
      (m_bc->type == MESA_SHADER_FRAGMENT) && fetch_instr.has_fetch_flag(FetchInstr::vpm);
   m_bc->cf_last->barrier = 1;
}

void
AssamblerVisitor::visit(const WriteTFInstr& instr)
{
   struct r600_bytecode_gds gds;

   auto& value = instr.value();

   memset(&gds, 0, sizeof(struct r600_bytecode_gds));
   gds.src_gpr = value.sel();
   gds.src_sel_x = value[0]->chan();
   gds.src_sel_y = value[1]->chan();
   gds.src_sel_z = 4;
   gds.dst_sel_x = 7;
   gds.dst_sel_y = 7;
   gds.dst_sel_z = 7;
   gds.dst_sel_w = 7;
   gds.op = FETCH_OP_TF_WRITE;

   if (r600_bytecode_add_gds(m_bc, &gds) != 0) {
      m_result = false;
      return;
   }

   if (value[2]->chan() != 7) {
      memset(&gds, 0, sizeof(struct r600_bytecode_gds));
      gds.src_gpr = value.sel();
      gds.src_sel_x = value[2]->chan();
      gds.src_sel_y = value[3]->chan();
      gds.src_sel_z = 4;
      gds.dst_sel_x = 7;
      gds.dst_sel_y = 7;
      gds.dst_sel_z = 7;
      gds.dst_sel_w = 7;
      gds.op = FETCH_OP_TF_WRITE;

      if (r600_bytecode_add_gds(m_bc, &gds)) {
         m_result = false;
         return;
      }
   }
}

void
AssamblerVisitor::visit(const RatInstr& instr)
{
   struct r600_bytecode_gds gds;

   /* The instruction writes to the retuen buffer location, and
    * the value will actually be read back, so make sure all previous writes
    * have been finished */
   if (m_ack_suggested /*&& instr.has_instr_flag(Instr::ack_rat_return_write)*/)
      emit_wait_ack();

   int rat_idx = instr.resource_id();

   memset(&gds, 0, sizeof(struct r600_bytecode_gds));

   r600_bytecode_add_cfinst(m_bc, instr.cf_opcode());
   auto cf = m_bc->cf_last;
   cf->rat.id = rat_idx + m_shader->rat_base;
   cf->rat.inst = instr.rat_op();
   cf->rat.index_mode = instr.resource_index_mode();
   cf->output.type = instr.need_ack() ? 3 : 1;
   cf->output.gpr = instr.data_gpr();
   cf->output.index_gpr = instr.index_gpr();
   cf->output.comp_mask = instr.comp_mask();
   cf->output.burst_count = instr.burst_count();
   assert(instr.data_swz(0) == PIPE_SWIZZLE_X);
   if (cf->rat.inst != RatInstr::STORE_TYPED) {
      assert(instr.data_swz(1) == PIPE_SWIZZLE_Y ||
             instr.data_swz(1) == PIPE_SWIZZLE_MAX);
      assert(instr.data_swz(2) == PIPE_SWIZZLE_Z ||
             instr.data_swz(2) == PIPE_SWIZZLE_MAX);
   }

   cf->vpm = m_bc->type == MESA_SHADER_FRAGMENT;
   cf->barrier = 1;
   cf->mark = instr.need_ack();
   cf->output.elem_size = instr.elm_size();

   m_ack_suggested |= instr.need_ack();
}

void
AssamblerVisitor::clear_states(const uint32_t& states)
{
   if (states & sf_vtx)
      vtx_fetch_results.clear();

   if (states & sf_tex)
      tex_fetch_results.clear();

   if (states & sf_alu) {
      m_last_op_was_barrier = false;
      m_last_addr = nullptr;
   }
}

void
AssamblerVisitor::visit(const Block& block)
{
   if (block.empty())
      return;

   if (block.cf_start())
      block.cf_start()->accept(*this);
   else if (block.has_instr_flag(Instr::force_cf)) {
      m_bc->force_add_cf = 1;
      m_bc->ar_loaded = 0;
      m_last_addr = nullptr;
   }
   sfn_log << SfnLog::assembly << "Translate block  size: " << block.size()
           << " new_cf:" << m_bc->force_add_cf << "\n";

   m_require_alu_extended = block.kcache_needs_extended();
   for (const auto& i : block) {
      sfn_log << SfnLog::assembly << "Translate " << *i << " ";
      i->accept(*this);
      sfn_log << SfnLog::assembly << (m_result ? "good" : "fail") << "\n";

      if (!m_result)
         break;
   }
   m_require_alu_extended = false;
}

void
AssamblerVisitor::visit(const IfInstr& instr)
{

   auto pred = instr.predicate();
   auto [addr, dummy0, dummy1] = pred->indirect_addr();
   assert(!dummy1);
   assert(!addr);

   r600_bytecode_add_cfinst(m_bc, CF_OP_JUMP);
   clear_states(sf_all);

   m_jump_tracker.push(m_bc->cf_last, jt_if);
}

void
AssamblerVisitor::visit(const ControlFlowInstr& instr)
{
   sfn_log << SfnLog::assembly << "Translate " << instr << " ";

   clear_states(sf_all);
   switch (instr.cf_type()) {
   case ControlFlowInstr::cf_else:
      emit_else();
      break;
   case ControlFlowInstr::cf_endif:
      emit_endif();
      break;
   case ControlFlowInstr::cf_loop_begin: {
      bool use_vpm = m_shader->processor_type == MESA_SHADER_FRAGMENT &&
                     instr.has_instr_flag(Instr::vpm) &&
                     !instr.has_instr_flag(Instr::helper);
      emit_loop_begin(use_vpm);
      break;
   }
   case ControlFlowInstr::cf_loop_end:
      emit_loop_end();
      break;
   case ControlFlowInstr::cf_loop_break:
      emit_loop_break();
      break;
   case ControlFlowInstr::cf_loop_continue:
      emit_loop_cont();
      break;
   case ControlFlowInstr::cf_wait_ack: {
      int r = r600_bytecode_add_cfinst(m_bc, CF_OP_WAIT_ACK);
      if (!r) {
         m_bc->cf_last->cf_addr = 0;
         m_bc->cf_last->barrier = 1;
         m_ack_suggested = false;
      } else {
         m_result = false;
      }
   } break;
   case ControlFlowInstr::cf_alu:
      r600_bytecode_add_cfinst(m_bc, CF_OP_ALU);
      break;
   case ControlFlowInstr::cf_alu_push_before:
      emit_alu_push_before();
      break;
   case ControlFlowInstr::cf_gds:
      r600_bytecode_add_cfinst(m_bc, CF_OP_GDS);
      break;
   case ControlFlowInstr::cf_tex:
      r600_bytecode_add_cfinst(m_bc, CF_OP_TEX);
      break;
   case ControlFlowInstr::cf_vtx:
      r600_bytecode_add_cfinst(m_bc, CF_OP_VTX);
      break;
   default:
      UNREACHABLE("Unknown CF instruction type");
   }
}

void
AssamblerVisitor::visit(const GDSInstr& instr)
{
   struct r600_bytecode_gds gds;

   memset(&gds, 0, sizeof(struct r600_bytecode_gds));

   gds.op = ds_opcode_map.at(instr.opcode());
   gds.uav_id = instr.resource_id();
   gds.uav_index_mode = instr.resource_index_mode();
   gds.src_gpr = instr.src().sel();

   gds.src_sel_x = instr.src()[0]->chan() < 7 ? instr.src()[0]->chan() : 4;
   gds.src_sel_y = instr.src()[1]->chan() < 7 ? instr.src()[1]->chan() : 4;
   gds.src_sel_z = instr.src()[2]->chan() < 7 ? instr.src()[2]->chan() : 4;

   gds.dst_sel_x = 7;
   gds.dst_sel_y = 7;
   gds.dst_sel_z = 7;
   gds.dst_sel_w = 7;

   if (instr.dest()) {
      gds.dst_gpr = instr.dest()->sel();
      switch (instr.dest()->chan()) {
      case 0:
         gds.dst_sel_x = 0;
         break;
      case 1:
         gds.dst_sel_y = 0;
         break;
      case 2:
         gds.dst_sel_z = 0;
         break;
      case 3:
         gds.dst_sel_w = 0;
      }
   }

   gds.src_gpr2 = 0;
   gds.alloc_consume = m_bc->gfx_level < CAYMAN ? 1 : 0; // Not Cayman

   int r = r600_bytecode_add_gds(m_bc, &gds);
   if (r) {
      m_result = false;
      return;
   }
   m_bc->cf_last->vpm = MESA_SHADER_FRAGMENT == m_bc->type;
   m_bc->cf_last->barrier = 1;
}

void
AssamblerVisitor::visit(const LDSAtomicInstr& instr)
{
   (void)instr;
   UNREACHABLE("LDSAtomicInstr must be lowered to ALUInstr");
}

void
AssamblerVisitor::visit(const LDSReadInstr& instr)
{
   (void)instr;
   UNREACHABLE("LDSReadInstr must be lowered to ALUInstr");
}

EBufferIndexMode
AssamblerVisitor::emit_index_reg(const VirtualValue& addr, unsigned idx)
{
   assert(idx < 2);

   if (!m_bc->index_loaded[idx] || m_loop_nesting ||
       m_bc->index_reg[idx] != (unsigned)addr.sel() ||
       m_bc->index_reg_chan[idx] != (unsigned)addr.chan()) {
      struct r600_bytecode_alu alu;

      // Make sure MOVA is not last instr in clause

      if (!m_bc->cf_last || (m_bc->cf_last->ndw >> 1) >= 110)
         m_bc->force_add_cf = 1;

      if (m_bc->gfx_level != CAYMAN) {

         EAluOp idxop = idx ? op1_set_cf_idx1 : op1_set_cf_idx0;

         memset(&alu, 0, sizeof(alu));
         alu.op = opcode_map.at(op1_mova_int);
         alu.dst.chan = 0;
         alu.src[0].sel = addr.sel();
         alu.src[0].chan = addr.chan();
         alu.last = 1;
         sfn_log << SfnLog::assembly << "   mova_int, ";
         int r = r600_bytecode_add_alu(m_bc, &alu);
         if (r)
            return bim_invalid;

         alu.op = opcode_map.at(idxop);
         alu.dst.chan = 0;
         alu.src[0].sel = 0;
         alu.src[0].chan = 0;
         alu.last = 1;
         sfn_log << SfnLog::assembly << "op1_set_cf_idx" << idx;
         r = r600_bytecode_add_alu(m_bc, &alu);
         if (r)
            return bim_invalid;
      } else {
         memset(&alu, 0, sizeof(alu));
         alu.op = opcode_map.at(op1_mova_int);
         alu.dst.sel = idx == 0 ? CM_V_SQ_MOVA_DST_CF_IDX0 : CM_V_SQ_MOVA_DST_CF_IDX1;
         alu.dst.chan = 0;
         alu.src[0].sel = addr.sel();
         alu.src[0].chan = addr.chan();
         alu.last = 1;
         sfn_log << SfnLog::assembly << "   mova_int, ";
         int r = r600_bytecode_add_alu(m_bc, &alu);
         if (r)
            return bim_invalid;
      }

      m_bc->ar_loaded = 0;
      m_bc->index_reg[idx] = addr.sel();
      m_bc->index_reg_chan[idx] = addr.chan();
      m_bc->index_loaded[idx] = true;
      m_bc->force_add_cf = 1;
      sfn_log << SfnLog::assembly << "\n";
   }
   return idx == 0 ? bim_zero : bim_one;
}

void
AssamblerVisitor::emit_else()
{
   r600_bytecode_add_cfinst(m_bc, CF_OP_ELSE);
   m_bc->cf_last->pop_count = 1;
   m_result &= m_jump_tracker.add_mid(m_bc->cf_last, jt_if);
}

void
AssamblerVisitor::emit_alu_push_before()
{
   int elems = m_callstack.push(FC_PUSH_VPM);
   bool needs_workaround = false;

   if (m_bc->gfx_level == CAYMAN && m_bc->stack.loop > 1)
      needs_workaround = true;

   if (m_bc->gfx_level == EVERGREEN && m_bc->family != CHIP_HEMLOCK &&
       m_bc->family != CHIP_CYPRESS && m_bc->family != CHIP_JUNIPER) {
      unsigned dmod1 = (elems - 1) % m_bc->stack.entry_size;
      unsigned dmod2 = (elems) % m_bc->stack.entry_size;

      if (elems && (!dmod1 || !dmod2))
         needs_workaround = true;
   }

   if (needs_workaround || m_require_alu_extended) {
      r600_bytecode_add_cfinst(m_bc, CF_OP_PUSH);
      m_bc->cf_last->cf_addr = m_bc->cf_last->id + 2;
      r600_bytecode_add_cfinst(m_bc, CF_OP_ALU);
      m_bc->cf_last->eg_alu_extended = m_require_alu_extended;
   } else {
      r600_bytecode_add_cfinst(m_bc, CF_OP_ALU_PUSH_BEFORE);
   }

   clear_states(sf_tex | sf_vtx);
}

void
AssamblerVisitor::emit_endif()
{
   m_callstack.pop(FC_PUSH_VPM);

   unsigned force_pop = m_bc->force_add_cf;
   if (!force_pop) {
      int alu_pop = 3;
      if (m_bc->cf_last) {
         if (m_bc->cf_last->op == CF_OP_ALU)
            alu_pop = 0;
         else if (m_bc->cf_last->op == CF_OP_ALU_POP_AFTER)
            alu_pop = 1;
      }
      alu_pop += 1;
      if (alu_pop == 1) {
         m_bc->cf_last->op = CF_OP_ALU_POP_AFTER;
         m_bc->force_add_cf = 1;
      } else {
         force_pop = 1;
      }
   }

   if (force_pop) {
      r600_bytecode_add_cfinst(m_bc, CF_OP_POP);
      m_bc->cf_last->pop_count = 1;
      m_bc->cf_last->cf_addr = m_bc->cf_last->id + 2;
   }

   m_result &= m_jump_tracker.pop(m_bc->cf_last, jt_if);
}

void
AssamblerVisitor::emit_loop_begin(bool vpm)
{
   r600_bytecode_add_cfinst(m_bc, CF_OP_LOOP_START_DX10);
   m_bc->cf_last->vpm = vpm && m_bc->type == MESA_SHADER_FRAGMENT;
   m_jump_tracker.push(m_bc->cf_last, jt_loop);
   m_callstack.push(FC_LOOP);
   ++m_loop_nesting;
}

void
AssamblerVisitor::emit_loop_end()
{
   if (m_ack_suggested) {
      emit_wait_ack();
      m_ack_suggested = false;
   }

   r600_bytecode_add_cfinst(m_bc, CF_OP_LOOP_END);
   m_callstack.pop(FC_LOOP);
   assert(m_loop_nesting);
   --m_loop_nesting;
   m_result |= m_jump_tracker.pop(m_bc->cf_last, jt_loop);
}

void
AssamblerVisitor::emit_loop_break()
{
   r600_bytecode_add_cfinst(m_bc, CF_OP_LOOP_BREAK);
   m_result |= m_jump_tracker.add_mid(m_bc->cf_last, jt_loop);
}

void
AssamblerVisitor::emit_loop_cont()
{
   r600_bytecode_add_cfinst(m_bc, CF_OP_LOOP_CONTINUE);
   m_result |= m_jump_tracker.add_mid(m_bc->cf_last, jt_loop);
}

bool
AssamblerVisitor::copy_dst(r600_bytecode_alu_dst& dst, const Register& d, bool write)
{
   if (write && d.sel() > g_clause_local_end && d.sel() != g_registers_unused) {
      R600_ASM_ERR("shader_from_nir: Don't support more then 123 GPRs + 4 clause "
                   "local, but try using %d\n",
                   d.sel());
      m_result = false;
      return false;
   }

   dst.sel = d.sel() != g_registers_unused ? d.sel() : g_registers_end;
   dst.chan = d.chan();

   if (m_last_addr && m_last_addr->equal_to(d))
      m_last_addr = nullptr;

   return true;
}

void
AssamblerVisitor::emit_wait_ack()
{
   int r = r600_bytecode_add_cfinst(m_bc, CF_OP_WAIT_ACK);
   if (!r) {
      m_bc->cf_last->cf_addr = 0;
      m_bc->cf_last->barrier = 1;
      m_ack_suggested = false;
   } else
      m_result = false;
}

class EncodeSourceVisitor : public ConstRegisterVisitor {
public:
   EncodeSourceVisitor(r600_bytecode_alu_src& s, r600_bytecode *bc);
   void visit(const Register& value) override;
   void visit(const LocalArray& value) override;
   void visit(const LocalArrayValue& value) override;
   void visit(const UniformValue& value) override;
   void visit(const LiteralConstant& value) override;
   void visit(const InlineConstant& value) override;

   r600_bytecode_alu_src& src;
   r600_bytecode *m_bc;
   PVirtualValue m_buffer_offset{nullptr};
};

PVirtualValue
AssamblerVisitor::copy_src(r600_bytecode_alu_src& src, const VirtualValue& s)
{

   EncodeSourceVisitor visitor(src, m_bc);
   src.sel = s.sel();
   src.chan = s.chan();

   if (s.sel() >= g_clause_local_start && s.sel() < g_clause_local_end ) {
      assert(m_bc->cf_last);
      int clidx = 4 * (s.sel() - g_clause_local_start) + s.chan();
      /* Ensure that the clause local register was already written */
      assert(m_bc->cf_last->clause_local_written & (1 << clidx));
   }

   s.accept(visitor);
   return visitor.m_buffer_offset;
}

EncodeSourceVisitor::EncodeSourceVisitor(r600_bytecode_alu_src& s, r600_bytecode *bc):
    src(s),
    m_bc(bc)
{
}

void
EncodeSourceVisitor::visit(const Register& value)
{
   assert(value.sel() < g_clause_local_end && "Only have 123 reisters + 4 clause local");
}

void
EncodeSourceVisitor::visit(const LocalArray& value)
{
   (void)value;
   UNREACHABLE("An array can't be a source register");
}

void
EncodeSourceVisitor::visit(const LocalArrayValue& value)
{
   src.rel = value.addr() ? 1 : 0;
}

void
EncodeSourceVisitor::visit(const UniformValue& value)
{
   assert(value.sel() >= 512 && "Uniform values must have a sel >= 512");
   m_buffer_offset = value.buf_addr();
   src.kc_bank = value.kcache_bank();
}

void
EncodeSourceVisitor::visit(const LiteralConstant& value)
{
   src.value = value.value();
}

void
EncodeSourceVisitor::visit(const InlineConstant& value)
{
   (void)value;
}

const std::map<EAluOp, int> opcode_map = {

   {op2_add,                       ALU_OP2_ADD                      },
   {op2_mul,                       ALU_OP2_MUL                      },
   {op2_mul_ieee,                  ALU_OP2_MUL_IEEE                 },
   {op2_max,                       ALU_OP2_MAX                      },
   {op2_min,                       ALU_OP2_MIN                      },
   {op2_max_dx10,                  ALU_OP2_MAX_DX10                 },
   {op2_min_dx10,                  ALU_OP2_MIN_DX10                 },
   {op2_sete,                      ALU_OP2_SETE                     },
   {op2_setgt,                     ALU_OP2_SETGT                    },
   {op2_setge,                     ALU_OP2_SETGE                    },
   {op2_setne,                     ALU_OP2_SETNE                    },
   {op2_sete_dx10,                 ALU_OP2_SETE_DX10                },
   {op2_setgt_dx10,                ALU_OP2_SETGT_DX10               },
   {op2_setge_dx10,                ALU_OP2_SETGE_DX10               },
   {op2_setne_dx10,                ALU_OP2_SETNE_DX10               },
   {op1_fract,                     ALU_OP1_FRACT                    },
   {op1_trunc,                     ALU_OP1_TRUNC                    },
   {op1_ceil,                      ALU_OP1_CEIL                     },
   {op1_rndne,                     ALU_OP1_RNDNE                    },
   {op1_floor,                     ALU_OP1_FLOOR                    },
   {op2_ashr_int,                  ALU_OP2_ASHR_INT                 },
   {op2_lshr_int,                  ALU_OP2_LSHR_INT                 },
   {op2_lshl_int,                  ALU_OP2_LSHL_INT                 },
   {op1_mov,                       ALU_OP1_MOV                      },
   {op0_nop,                       ALU_OP0_NOP                      },
   {op2_mul_64,                    ALU_OP2_MUL_64                   },
   {op1v_flt64_to_flt32,           ALU_OP1_FLT64_TO_FLT32           },
   {op1v_flt32_to_flt64,           ALU_OP1_FLT32_TO_FLT64           },
   {op2_prede_int,                 ALU_OP2_PRED_SETE_INT            },
   {op2_pred_setne_int,            ALU_OP2_PRED_SETNE_INT           },
   {op2_pred_setge_int,            ALU_OP2_PRED_SETGE_INT           },
   {op2_pred_setgt_int,            ALU_OP2_PRED_SETGT_INT           },
   {op2_pred_setgt_uint,           ALU_OP2_PRED_SETGT_UINT          },
   {op2_pred_setge_uint,           ALU_OP2_PRED_SETGE_UINT          },
   {op2_pred_sete,                 ALU_OP2_PRED_SETE                },
   {op2_pred_setgt,                ALU_OP2_PRED_SETGT               },
   {op2_pred_setge,                ALU_OP2_PRED_SETGE               },
   {op2_pred_setne,                ALU_OP2_PRED_SETNE               },
   {op0_pred_set_clr,              ALU_OP0_PRED_SET_CLR             },
   {op1_pred_set_restore,          ALU_OP1_PRED_SET_RESTORE         },
   {op2_pred_sete_push,            ALU_OP2_PRED_SETE_PUSH           },
   {op2_pred_setgt_push,           ALU_OP2_PRED_SETGT_PUSH          },
   {op2_pred_setge_push,           ALU_OP2_PRED_SETGE_PUSH          },
   {op2_pred_setne_push,           ALU_OP2_PRED_SETNE_PUSH          },
   {op2_kille,                     ALU_OP2_KILLE                    },
   {op2_killgt,                    ALU_OP2_KILLGT                   },
   {op2_killge,                    ALU_OP2_KILLGE                   },
   {op2_killne,                    ALU_OP2_KILLNE                   },
   {op2_and_int,                   ALU_OP2_AND_INT                  },
   {op2_or_int,                    ALU_OP2_OR_INT                   },
   {op2_xor_int,                   ALU_OP2_XOR_INT                  },
   {op1_not_int,                   ALU_OP1_NOT_INT                  },
   {op2_add_int,                   ALU_OP2_ADD_INT                  },
   {op2_sub_int,                   ALU_OP2_SUB_INT                  },
   {op2_max_int,                   ALU_OP2_MAX_INT                  },
   {op2_min_int,                   ALU_OP2_MIN_INT                  },
   {op2_max_uint,                  ALU_OP2_MAX_UINT                 },
   {op2_min_uint,                  ALU_OP2_MIN_UINT                 },
   {op2_sete_int,                  ALU_OP2_SETE_INT                 },
   {op2_setgt_int,                 ALU_OP2_SETGT_INT                },
   {op2_setge_int,                 ALU_OP2_SETGE_INT                },
   {op2_setne_int,                 ALU_OP2_SETNE_INT                },
   {op2_setgt_uint,                ALU_OP2_SETGT_UINT               },
   {op2_setge_uint,                ALU_OP2_SETGE_UINT               },
   {op2_killgt_uint,               ALU_OP2_KILLGT_UINT              },
   {op2_killge_uint,               ALU_OP2_KILLGE_UINT              },
   {op2_pred_setgt_int,            ALU_OP2_PRED_SETGT_INT           },
   {op2_pred_setge_int,            ALU_OP2_PRED_SETGE_INT           },
   {op2_pred_setne_int,            ALU_OP2_PRED_SETNE_INT           },
   {op2_kille_int,                 ALU_OP2_KILLE_INT                },
   {op2_killgt_int,                ALU_OP2_KILLGT_INT               },
   {op2_killge_int,                ALU_OP2_KILLGE_INT               },
   {op2_killne_int,                ALU_OP2_KILLNE_INT               },
   {op2_pred_sete_push_int,        ALU_OP2_PRED_SETE_PUSH_INT       },
   {op2_pred_setgt_push_int,       ALU_OP2_PRED_SETGT_PUSH_INT      },
   {op2_pred_setge_push_int,       ALU_OP2_PRED_SETGE_PUSH_INT      },
   {op2_pred_setne_push_int,       ALU_OP2_PRED_SETNE_PUSH_INT      },
   {op2_pred_setlt_push_int,       ALU_OP2_PRED_SETLT_PUSH_INT      },
   {op2_pred_setle_push_int,       ALU_OP2_PRED_SETLE_PUSH_INT      },
   {op1_flt_to_int,                ALU_OP1_FLT_TO_INT               },
   {op1_bfrev_int,                 ALU_OP1_BFREV_INT                },
   {op2_addc_uint,                 ALU_OP2_ADDC_UINT                },
   {op2_subb_uint,                 ALU_OP2_SUBB_UINT                },
   {op0_group_barrier,             ALU_OP0_GROUP_BARRIER            },
   {op0_group_seq_begin,           ALU_OP0_GROUP_SEQ_BEGIN          },
   {op0_group_seq_end,             ALU_OP0_GROUP_SEQ_END            },
   {op2_set_mode,                  ALU_OP2_SET_MODE                 },
   {op1_set_cf_idx0,               ALU_OP0_SET_CF_IDX0              },
   {op1_set_cf_idx1,               ALU_OP0_SET_CF_IDX1              },
   {op2_set_lds_size,              ALU_OP2_SET_LDS_SIZE             },
   {op1_exp_ieee,                  ALU_OP1_EXP_IEEE                 },
   {op1_log_clamped,               ALU_OP1_LOG_CLAMPED              },
   {op1_log_ieee,                  ALU_OP1_LOG_IEEE                 },
   {op1_recip_clamped,             ALU_OP1_RECIP_CLAMPED            },
   {op1_recip_ff,                  ALU_OP1_RECIP_FF                 },
   {op1_recip_ieee,                ALU_OP1_RECIP_IEEE               },
   {op1_recipsqrt_clamped,         ALU_OP1_RECIPSQRT_CLAMPED        },
   {op1_recipsqrt_ff,              ALU_OP1_RECIPSQRT_FF             },
   {op1_recipsqrt_ieee1,           ALU_OP1_RECIPSQRT_IEEE           },
   {op1_sqrt_ieee,                 ALU_OP1_SQRT_IEEE                },
   {op1_sin,                       ALU_OP1_SIN                      },
   {op1_cos,                       ALU_OP1_COS                      },
   {op2_mullo_int,                 ALU_OP2_MULLO_INT                },
   {op2_mulhi_int,                 ALU_OP2_MULHI_INT                },
   {op2_mullo_uint,                ALU_OP2_MULLO_UINT               },
   {op2_mulhi_uint,                ALU_OP2_MULHI_UINT               },
   {op1_recip_int,                 ALU_OP1_RECIP_INT                },
   {op1_recip_uint,                ALU_OP1_RECIP_UINT               },
   {op1_recip_64,                  ALU_OP2_RECIP_64                 },
   {op1_recip_clamped_64,          ALU_OP2_RECIP_CLAMPED_64         },
   {op1_recipsqrt_64,              ALU_OP2_RECIPSQRT_64             },
   {op1_recipsqrt_clamped_64,      ALU_OP2_RECIPSQRT_CLAMPED_64     },
   {op1_sqrt_64,                   ALU_OP2_SQRT_64                  },
   {op1_flt_to_uint,               ALU_OP1_FLT_TO_UINT              },
   {op1_int_to_flt,                ALU_OP1_INT_TO_FLT               },
   {op1_uint_to_flt,               ALU_OP1_UINT_TO_FLT              },
   {op2_bfm_int,                   ALU_OP2_BFM_INT                  },
   {op1_flt32_to_flt16,            ALU_OP1_FLT32_TO_FLT16           },
   {op1_flt16_to_flt32,            ALU_OP1_FLT16_TO_FLT32           },
   {op1_ubyte0_flt,                ALU_OP1_UBYTE0_FLT               },
   {op1_ubyte1_flt,                ALU_OP1_UBYTE1_FLT               },
   {op1_ubyte2_flt,                ALU_OP1_UBYTE2_FLT               },
   {op1_ubyte3_flt,                ALU_OP1_UBYTE3_FLT               },
   {op1_bcnt_int,                  ALU_OP1_BCNT_INT                 },
   {op1_ffbh_uint,                 ALU_OP1_FFBH_UINT                },
   {op1_ffbl_int,                  ALU_OP1_FFBL_INT                 },
   {op1_ffbh_int,                  ALU_OP1_FFBH_INT                 },
   {op1_flt_to_uint4,              ALU_OP1_FLT_TO_UINT4             },
   {op2_dot_ieee,                  ALU_OP2_DOT_IEEE                 },
   {op1_flt_to_int_rpi,            ALU_OP1_FLT_TO_INT_RPI           },
   {op1_flt_to_int_floor,          ALU_OP1_FLT_TO_INT_FLOOR         },
   {op2_mulhi_uint24,              ALU_OP2_MULHI_UINT24             },
   {op1_mbcnt_32hi_int,            ALU_OP1_MBCNT_32HI_INT           },
   {op1_offset_to_flt,             ALU_OP1_OFFSET_TO_FLT            },
   {op2_mul_uint24,                ALU_OP2_MUL_UINT24               },
   {op1_bcnt_accum_prev_int,       ALU_OP1_BCNT_ACCUM_PREV_INT      },
   {op1_mbcnt_32lo_accum_prev_int, ALU_OP1_MBCNT_32LO_ACCUM_PREV_INT},
   {op2_sete_64,                   ALU_OP2_SETE_64                  },
   {op2_setne_64,                  ALU_OP2_SETNE_64                 },
   {op2_setgt_64,                  ALU_OP2_SETGT_64                 },
   {op2_setge_64,                  ALU_OP2_SETGE_64                 },
   {op2_min_64,                    ALU_OP2_MIN_64                   },
   {op2_max_64,                    ALU_OP2_MAX_64                   },
   {op2_dot4,                      ALU_OP2_DOT4                     },
   {op2_dot4_ieee,                 ALU_OP2_DOT4_IEEE                },
   {op2_cube,                      ALU_OP2_CUBE                     },
   {op1_max4,                      ALU_OP1_MAX4                     },
   {op1_frexp_64,                  ALU_OP1_FREXP_64                 },
   {op1_ldexp_64,                  ALU_OP2_LDEXP_64                 },
   {op1_fract_64,                  ALU_OP1_FRACT_64                 },
   {op2_pred_setgt_64,             ALU_OP2_PRED_SETGT_64            },
   {op2_pred_sete_64,              ALU_OP2_PRED_SETE_64             },
   {op2_pred_setge_64,             ALU_OP2_PRED_SETGE_64            },
   {op2_add_64,                    ALU_OP2_ADD_64                   },
   {op1_mova_int,                  ALU_OP1_MOVA_INT                 },
   {op1v_flt64_to_flt32,           ALU_OP1_FLT64_TO_FLT32           },
   {op1_flt32_to_flt64,            ALU_OP1_FLT32_TO_FLT64           },
   {op2_sad_accum_prev_uint,       ALU_OP2_SAD_ACCUM_PREV_UINT      },
   {op2_dot,                       ALU_OP2_DOT                      },
   {op1_mul_prev,                  ALU_OP1_MUL_PREV                 },
   {op1_mul_ieee_prev,             ALU_OP1_MUL_IEEE_PREV            },
   {op1_add_prev,                  ALU_OP1_ADD_PREV                 },
   {op2_muladd_prev,               ALU_OP2_MULADD_PREV              },
   {op2_muladd_ieee_prev,          ALU_OP2_MULADD_IEEE_PREV         },
   {op2_interp_xy,                 ALU_OP2_INTERP_XY                },
   {op2_interp_zw,                 ALU_OP2_INTERP_ZW                },
   {op2_interp_x,                  ALU_OP2_INTERP_X                 },
   {op2_interp_z,                  ALU_OP2_INTERP_Z                 },
   {op0_store_flags,               ALU_OP1_STORE_FLAGS              },
   {op1_load_store_flags,          ALU_OP1_LOAD_STORE_FLAGS         },
   {op0_lds_1a,                    ALU_OP2_LDS_1A                   },
   {op0_lds_1a1d,                  ALU_OP2_LDS_1A1D                 },
   {op0_lds_2a,                    ALU_OP2_LDS_2A                   },
   {op1_interp_load_p0,            ALU_OP1_INTERP_LOAD_P0           },
   {op1_interp_load_p10,           ALU_OP1_INTERP_LOAD_P10          },
   {op1_interp_load_p20,           ALU_OP1_INTERP_LOAD_P20          },
   {op3_bfe_uint,                  ALU_OP3_BFE_UINT                 },
   {op3_bfe_int,                   ALU_OP3_BFE_INT                  },
   {op3_bfi_int,                   ALU_OP3_BFI_INT                  },
   {op3_fma,                       ALU_OP3_FMA                      },
   {op3_cndne_64,                  ALU_OP3_CNDNE_64                 },
   {op3_fma_64,                    ALU_OP3_FMA_64                   },
   {op3_lerp_uint,                 ALU_OP3_LERP_UINT                },
   {op3_bit_align_int,             ALU_OP3_BIT_ALIGN_INT            },
   {op3_byte_align_int,            ALU_OP3_BYTE_ALIGN_INT           },
   {op3_sad_accum_uint,            ALU_OP3_SAD_ACCUM_UINT           },
   {op3_sad_accum_hi_uint,         ALU_OP3_SAD_ACCUM_HI_UINT        },
   {op3_muladd_uint24,             ALU_OP3_MULADD_UINT24            },
   {op3_lds_idx_op,                ALU_OP3_LDS_IDX_OP               },
   {op3_muladd,                    ALU_OP3_MULADD                   },
   {op3_muladd_m2,                 ALU_OP3_MULADD_M2                },
   {op3_muladd_m4,                 ALU_OP3_MULADD_M4                },
   {op3_muladd_d2,                 ALU_OP3_MULADD_D2                },
   {op3_muladd_ieee,               ALU_OP3_MULADD_IEEE              },
   {op3_cnde,                      ALU_OP3_CNDE                     },
   {op3_cndgt,                     ALU_OP3_CNDGT                    },
   {op3_cndge,                     ALU_OP3_CNDGE                    },
   {op3_cnde_int,                  ALU_OP3_CNDE_INT                 },
   {op3_cndgt_int,                 ALU_OP3_CNDGT_INT                },
   {op3_cndge_int,                 ALU_OP3_CNDGE_INT                },
   {op3_mul_lit,                   ALU_OP3_MUL_LIT                  },
};

const std::map<ESDOp, int> ds_opcode_map = {
   {DS_OP_ADD,                      FETCH_OP_GDS_ADD                 },
   {DS_OP_SUB,                      FETCH_OP_GDS_SUB                 },
   {DS_OP_RSUB,                     FETCH_OP_GDS_RSUB                },
   {DS_OP_INC,                      FETCH_OP_GDS_INC                 },
   {DS_OP_DEC,                      FETCH_OP_GDS_DEC                 },
   {DS_OP_MIN_INT,                  FETCH_OP_GDS_MIN_INT             },
   {DS_OP_MAX_INT,                  FETCH_OP_GDS_MAX_INT             },
   {DS_OP_MIN_UINT,                 FETCH_OP_GDS_MIN_UINT            },
   {DS_OP_MAX_UINT,                 FETCH_OP_GDS_MAX_UINT            },
   {DS_OP_AND,                      FETCH_OP_GDS_AND                 },
   {DS_OP_OR,                       FETCH_OP_GDS_OR                  },
   {DS_OP_XOR,                      FETCH_OP_GDS_XOR                 },
   {DS_OP_MSKOR,                    FETCH_OP_GDS_MSKOR               },
   {DS_OP_WRITE,                    FETCH_OP_GDS_WRITE               },
   {DS_OP_WRITE_REL,                FETCH_OP_GDS_WRITE_REL           },
   {DS_OP_WRITE2,                   FETCH_OP_GDS_WRITE2              },
   {DS_OP_CMP_STORE,                FETCH_OP_GDS_CMP_STORE           },
   {DS_OP_CMP_STORE_SPF,            FETCH_OP_GDS_CMP_STORE_SPF       },
   {DS_OP_BYTE_WRITE,               FETCH_OP_GDS_BYTE_WRITE          },
   {DS_OP_SHORT_WRITE,              FETCH_OP_GDS_SHORT_WRITE         },
   {DS_OP_ADD_RET,                  FETCH_OP_GDS_ADD_RET             },
   {DS_OP_SUB_RET,                  FETCH_OP_GDS_SUB_RET             },
   {DS_OP_RSUB_RET,                 FETCH_OP_GDS_RSUB_RET            },
   {DS_OP_INC_RET,                  FETCH_OP_GDS_INC_RET             },
   {DS_OP_DEC_RET,                  FETCH_OP_GDS_DEC_RET             },
   {DS_OP_MIN_INT_RET,              FETCH_OP_GDS_MIN_INT_RET         },
   {DS_OP_MAX_INT_RET,              FETCH_OP_GDS_MAX_INT_RET         },
   {DS_OP_MIN_UINT_RET,             FETCH_OP_GDS_MIN_UINT_RET        },
   {DS_OP_MAX_UINT_RET,             FETCH_OP_GDS_MAX_UINT_RET        },
   {DS_OP_AND_RET,                  FETCH_OP_GDS_AND_RET             },
   {DS_OP_OR_RET,                   FETCH_OP_GDS_OR_RET              },
   {DS_OP_XOR_RET,                  FETCH_OP_GDS_XOR_RET             },
   {DS_OP_MSKOR_RET,                FETCH_OP_GDS_MSKOR_RET           },
   {DS_OP_XCHG_RET,                 FETCH_OP_GDS_XCHG_RET            },
   {DS_OP_XCHG_REL_RET,             FETCH_OP_GDS_XCHG_REL_RET        },
   {DS_OP_XCHG2_RET,                FETCH_OP_GDS_XCHG2_RET           },
   {DS_OP_CMP_XCHG_RET,             FETCH_OP_GDS_CMP_XCHG_RET        },
   {DS_OP_CMP_XCHG_SPF_RET,         FETCH_OP_GDS_CMP_XCHG_SPF_RET    },
   {DS_OP_READ_RET,                 FETCH_OP_GDS_READ_RET            },
   {DS_OP_READ_REL_RET,             FETCH_OP_GDS_READ_REL_RET        },
   {DS_OP_READ2_RET,                FETCH_OP_GDS_READ2_RET           },
   {DS_OP_READWRITE_RET,            FETCH_OP_GDS_READWRITE_RET       },
   {DS_OP_BYTE_READ_RET,            FETCH_OP_GDS_BYTE_READ_RET       },
   {DS_OP_UBYTE_READ_RET,           FETCH_OP_GDS_UBYTE_READ_RET      },
   {DS_OP_SHORT_READ_RET,           FETCH_OP_GDS_SHORT_READ_RET      },
   {DS_OP_USHORT_READ_RET,          FETCH_OP_GDS_USHORT_READ_RET     },
   {DS_OP_ATOMIC_ORDERED_ALLOC_RET, FETCH_OP_GDS_ATOMIC_ORDERED_ALLOC},
   {DS_OP_INVALID,                  0                                },
};

} // namespace r600
