zig/src/codegen/wasm/CodeGen.zig

const std = @import("std");
const builtin = @import("builtin");
const Allocator = std.mem.Allocator;
const assert = std.debug.assert;
const testing = std.testing;
const math = std.math;
const mem = std.mem;
const log = std.log.scoped(.codegen);

const CodeGen = @This();
const codegen = @import("../../codegen.zig");
const Zcu = @import("../../Zcu.zig");
const InternPool = @import("../../InternPool.zig");
const Decl = Zcu.Decl;
const Type = @import("../../Type.zig");
const Value = @import("../../Value.zig");
const Compilation = @import("../../Compilation.zig");
const link = @import("../../link.zig");
const Air = @import("../../Air.zig");
const Mir = @import("Mir.zig");
const assembly = @import("assembly.zig");
const abi = @import("../../codegen/wasm/abi.zig");
const Alignment = InternPool.Alignment;
const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
const errUnionErrorOffset = codegen.errUnionErrorOffset;

const target_util = @import("../../target.zig");
const libcFloatPrefix = target_util.libcFloatPrefix;
const libcFloatSuffix = target_util.libcFloatSuffix;
const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev;
const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;

pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
    return comptime &.initMany(&.{
        .expand_intcast_safe,
        .expand_int_from_float_safe,
        .expand_int_from_float_optimized_safe,
        .expand_add_safe,
        .expand_sub_safe,
        .expand_mul_safe,

        .expand_packed_load,
        .expand_packed_store,
        .expand_packed_struct_field_val,
        .expand_packed_aggregate_init,
    });
}

/// Reference to the function declaration the code
/// section belongs to
owner_nav: InternPool.Nav.Index,
/// Current block depth. Used to calculate the relative difference between a break
/// and block
block_depth: u32 = 0,
air: Air,
liveness: Air.Liveness,
gpa: mem.Allocator,
func_index: InternPool.Index,
/// Contains a list of current branches.
/// When we return from a branch, the branch will be popped from this list,
/// which means branches can only contain references from within its own branch,
/// or a branch higher (lower index) in the tree.
branches: std.ArrayList(Branch) = .empty,
/// Table to save `WValue`'s generated by an `Air.Inst`
// values: ValueTable,
/// Mapping from Air.Inst.Index to block ids
blocks: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, struct {
    label: u32,
    value: WValue,
}) = .{},
/// Maps `loop` instructions to their label. `br` to here repeats the loop.
loops: std.AutoHashMapUnmanaged(Air.Inst.Index, u32) = .empty,
/// The index the next local generated will have
/// NOTE: arguments share the index with locals therefore the first variable
/// will have the index that comes after the last argument's index
local_index: u32,
/// The index of the current argument.
/// Used to track which argument is being referenced in `airArg`.
arg_index: u32 = 0,
/// List of simd128 immediates. Each value is stored as an array of bytes.
/// This list will only be populated for 128bit-simd values when the target features
/// are enabled also.
simd_immediates: std.ArrayList([16]u8) = .empty,
/// The Target we're emitting (used to call intInfo)
target: *const std.Target,
ptr_size: enum { wasm32, wasm64 },
pt: Zcu.PerThread,
/// List of MIR Instructions
mir_instructions: std.MultiArrayList(Mir.Inst),
/// Contains extra data for MIR
mir_extra: std.ArrayList(u32),
/// List of all locals' types generated throughout this declaration
/// used to emit locals count at start of 'code' section.
mir_locals: std.ArrayList(std.wasm.Valtype),
/// Set of all UAVs referenced by this function. Key is the UAV value, value is the alignment.
/// `.none` means naturally aligned. An explicit alignment is never less than the natural alignment.
mir_uavs: std.AutoArrayHashMapUnmanaged(InternPool.Index, Alignment),
/// Set of all functions whose address this function has taken and which therefore might be called
/// via a `call_indirect` function.
mir_indirect_function_set: std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, void),
/// Set of all function types used by this function. These must be interned by the linker.
mir_func_tys: std.AutoArrayHashMapUnmanaged(InternPool.Index, void),
/// The number of `error_name_table_ref` instructions emitted.
error_name_table_ref_count: u32,
/// When a function is executing, we store the the current stack pointer's value within this local.
/// This value is then used to restore the stack pointer to the original value at the return of the function.
initial_stack_value: WValue = .none,
/// The current stack pointer subtracted with the stack size. From this value, we will calculate
/// all offsets of the stack values.
bottom_stack_value: WValue = .none,
/// Arguments of this function declaration
/// This will be set after `resolveCallingConventionValues`
args: []WValue,
/// This will only be `.none` if the function returns void, or returns an immediate.
/// When it returns a pointer to the stack, the `.local` tag will be active and must be populated
/// before this function returns its execution to the caller.
return_value: WValue,
/// The size of the stack this function occupies. In the function prologue
/// we will move the stack pointer by this number, forward aligned with the `stack_alignment`.
stack_size: u32 = 0,
/// The stack alignment, which is 16 bytes by default. This is specified by the
/// tool-conventions: https://github.com/WebAssembly/tool-conventions/blob/main/BasicCABI.md
/// and also what the llvm backend will emit.
/// However, local variables or the usage of `incoming_stack_alignment` in a `CallingConvention` can overwrite this default.
stack_alignment: Alignment = .@"16",

// For each individual Wasm valtype we store a seperate free list which
// allows us to re-use locals that are no longer used. e.g. a temporary local.
/// A list of indexes which represents a local of valtype `i32`.
/// It is illegal to store a non-i32 valtype in this list.
free_locals_i32: std.ArrayList(u32) = .empty,
/// A list of indexes which represents a local of valtype `i64`.
/// It is illegal to store a non-i64 valtype in this list.
free_locals_i64: std.ArrayList(u32) = .empty,
/// A list of indexes which represents a local of valtype `f32`.
/// It is illegal to store a non-f32 valtype in this list.
free_locals_f32: std.ArrayList(u32) = .empty,
/// A list of indexes which represents a local of valtype `f64`.
/// It is illegal to store a non-f64 valtype in this list.
free_locals_f64: std.ArrayList(u32) = .empty,
/// A list of indexes which represents a local of valtype `v127`.
/// It is illegal to store a non-v128 valtype in this list.
free_locals_v128: std.ArrayList(u32) = .empty,

/// When in debug mode, this tracks if no `finishAir` was missed.
/// Forgetting to call `finishAir` will cause the result to not be
/// stored in our `values` map and therefore cause bugs.
air_bookkeeping: @TypeOf(bookkeeping_init) = bookkeeping_init,

/// Wasm Value, created when generating an instruction
const WValue = union(enum) {
    /// `WValue` which has been freed and may no longer hold
    /// any references.
    dead: void,
    /// May be referenced but is unused
    none: void,
    /// The value lives on top of the stack
    stack: void,
    /// Index of the local
    local: struct {
        /// Contains the index to the local
        value: u32,
        /// The amount of instructions referencing this `WValue`
        references: u32,
    },
    /// An immediate 32bit value
    imm32: u32,
    /// An immediate 64bit value
    imm64: u64,
    /// Index into the list of simd128 immediates. This `WValue` is
    /// only possible in very rare cases, therefore it would be
    /// a waste of memory to store the value in a 128 bit integer.
    imm128: u32,
    /// A constant 32bit float value
    float32: f32,
    /// A constant 64bit float value
    float64: f64,
    nav_ref: struct {
        nav_index: InternPool.Nav.Index,
        offset: i32 = 0,
    },
    uav_ref: struct {
        ip_index: InternPool.Index,
        offset: i32 = 0,
        orig_ptr_ty: InternPool.Index = .none,
    },
    /// Offset from the bottom of the virtual stack, with the offset
    /// pointing to where the value lives.
    stack_offset: struct {
        /// Contains the actual value of the offset
        value: u32,
        /// The amount of instructions referencing this `WValue`
        references: u32,
    },

    /// Returns the offset from the bottom of the stack. This is useful when
    /// we use the load or store instruction to ensure we retrieve the value
    /// from the correct position, rather than the value that lives at the
    /// bottom of the stack. For instances where `WValue` is not `stack_value`
    /// this will return 0, which allows us to simply call this function for all
    /// loads and stores without requiring checks everywhere.
    fn offset(value: WValue) u32 {
        switch (value) {
            .stack_offset => |stack_offset| return stack_offset.value,
            .dead => unreachable,
            else => return 0,
        }
    }

    /// Promotes a `WValue` to a local when given value is on top of the stack.
    /// When encountering a `local` or `stack_offset` this is essentially a no-op.
    /// All other tags are illegal.
    fn toLocal(value: WValue, gen: *CodeGen, ty: Type) InnerError!WValue {
        switch (value) {
            .stack => {
                const new_local = try gen.allocLocal(ty);
                try gen.addLocal(.local_set, new_local.local.value);
                return new_local;
            },
            .local, .stack_offset => return value,
            else => unreachable,
        }
    }

    /// Marks a local as no longer being referenced and essentially allows
    /// us to re-use it somewhere else within the function.
    /// The valtype of the local is deducted by using the index of the given `WValue`.
    fn free(value: *WValue, gen: *CodeGen) void {
        if (value.* != .local) return;
        const local_value = value.local.value;
        const reserved = gen.args.len + @intFromBool(gen.return_value != .none);
        if (local_value < reserved + 2) return; // reserved locals may never be re-used. Also accounts for 2 stack locals.

        const index = local_value - reserved;
        const valtype = gen.mir_locals.items[index];
        switch (valtype) {
            .i32 => gen.free_locals_i32.append(gen.gpa, local_value) catch return, // It's ok to fail any of those, a new local can be allocated instead
            .i64 => gen.free_locals_i64.append(gen.gpa, local_value) catch return,
            .f32 => gen.free_locals_f32.append(gen.gpa, local_value) catch return,
            .f64 => gen.free_locals_f64.append(gen.gpa, local_value) catch return,
            .v128 => gen.free_locals_v128.append(gen.gpa, local_value) catch return,
        }
        log.debug("freed local ({d}) of type {}", .{ local_value, valtype });
        value.* = .dead;
    }
};

/// Hashmap to store generated `WValue` for each `Air.Inst.Ref`
const ValueTable = std.AutoArrayHashMapUnmanaged(Air.Inst.Ref, WValue);

const bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};

const InnerError = error{
    OutOfMemory,
    /// An error occurred when trying to lower AIR to MIR.
    CodegenFail,
    /// Compiler implementation could not handle a large integer.
    Overflow,
} || link.File.UpdateDebugInfoError;

pub fn deinit(cg: *CodeGen) void {
    const gpa = cg.gpa;
    for (cg.branches.items) |*branch| branch.deinit(gpa);
    cg.branches.deinit(gpa);
    cg.blocks.deinit(gpa);
    cg.loops.deinit(gpa);
    cg.simd_immediates.deinit(gpa);
    cg.free_locals_i32.deinit(gpa);
    cg.free_locals_i64.deinit(gpa);
    cg.free_locals_f32.deinit(gpa);
    cg.free_locals_f64.deinit(gpa);
    cg.free_locals_v128.deinit(gpa);
    cg.mir_instructions.deinit(gpa);
    cg.mir_extra.deinit(gpa);
    cg.mir_locals.deinit(gpa);
    cg.mir_uavs.deinit(gpa);
    cg.mir_indirect_function_set.deinit(gpa);
    cg.mir_func_tys.deinit(gpa);
    cg.* = undefined;
}

pub fn fail(cg: *CodeGen, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
    const zcu = cg.pt.zcu;
    const func = zcu.funcInfo(cg.func_index);
    return zcu.codegenFail(func.owner_nav, fmt, args);
}

/// Resolves the `WValue` for the given instruction `inst`
/// When the given instruction has a `Value`, it returns a constant instead
fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue {
    var branch_index = cg.branches.items.len;
    while (branch_index > 0) : (branch_index -= 1) {
        const branch = cg.branches.items[branch_index - 1];
        if (branch.values.get(ref)) |value| {
            return value;
        }
    }

    // when we did not find an existing instruction, it
    // means we must generate it from a constant.
    // We always store constants in the most outer branch as they must never
    // be removed. The most outer branch is always at index 0.
    const gop = try cg.branches.items[0].values.getOrPut(cg.gpa, ref);
    assert(!gop.found_existing);

    const pt = cg.pt;
    const zcu = pt.zcu;
    const val: Value = .fromInterned(ref.toInterned().?);
    const ty = cg.typeOf(ref);
    if (!ty.hasRuntimeBits(zcu) and !ty.isInt(zcu) and !ty.isError(zcu)) {
        gop.value_ptr.* = .none;
        return .none;
    }

    // When we need to pass the value by reference (such as a struct), we will
    // leverage `generateSymbol` to lower the constant to bytes and emit it
    // to the 'rodata' section. We then return the index into the section as `WValue`.
    //
    // In the other cases, we will simply lower the constant to a value that fits
    // into a single local (such as a pointer, integer, bool, etc).
    const result: WValue = if (isByRef(ty, zcu, cg.target))
        .{ .uav_ref = .{ .ip_index = val.toIntern() } }
    else
        try cg.lowerConstant(val);

    gop.value_ptr.* = result;
    return result;
}

fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue {
    const zcu = cg.pt.zcu;
    const ty = val.typeOf(zcu);

    return if (isByRef(ty, zcu, cg.target))
        .{ .uav_ref = .{ .ip_index = val.toIntern() } }
    else
        try cg.lowerConstant(val);
}

/// NOTE: if result == .stack, it will be stored in .local
fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void {
    assert(operands.len <= Air.Liveness.bpi - 1);
    var tomb_bits = cg.liveness.getTombBits(inst);
    for (operands) |operand| {
        const dies = @as(u1, @truncate(tomb_bits)) != 0;
        tomb_bits >>= 1;
        if (!dies) continue;
        processDeath(cg, operand);
    }
    try cg.finishAirResult(inst, result);
}

fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerError!void {
    // results of `none` can never be referenced.
    if (result != .none) {
        const trackable_result = if (result != .stack)
            result
        else
            try result.toLocal(cg, cg.typeOfIndex(inst));
        const branch = cg.currentBranch();
        branch.values.putAssumeCapacityNoClobber(inst.toRef(), trackable_result);
    }

    if (std.debug.runtime_safety) {
        cg.air_bookkeeping += 1;
    }
}

const Branch = struct {
    values: ValueTable = .{},

    fn deinit(branch: *Branch, gpa: Allocator) void {
        branch.values.deinit(gpa);
        branch.* = undefined;
    }
};

inline fn currentBranch(cg: *CodeGen) *Branch {
    return &cg.branches.items[cg.branches.items.len - 1];
}

fn feed(cg: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) void {
    if (bt.feed()) {
        cg.processDeath(operand);
    }
}

fn processDeath(cg: *CodeGen, ref: Air.Inst.Ref) void {
    if (ref.toIndex() == null) return;
    // Branches are currently only allowed to free locals allocated
    // within their own branch.
    // TODO: Upon branch consolidation free any locals if needed.
    const value = cg.currentBranch().values.getPtr(ref) orelse return;
    if (value.* != .local) return;
    const reserved_indexes = cg.args.len + @intFromBool(cg.return_value != .none);
    if (value.local.value < reserved_indexes) {
        return; // function arguments can never be re-used
    }
    log.debug("Decreasing reference for ref: %{d}, using local '{d}'", .{ @intFromEnum(ref.toIndex().?), value.local.value });
    value.local.references -= 1; // if this panics, a call to `reuseOperand` was forgotten by the developer
    if (value.local.references == 0) {
        value.free(cg);
    }
}

pub fn addInst(cg: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!void {
    try cg.mir_instructions.append(cg.gpa, inst);
}

pub fn addTag(cg: *CodeGen, tag: Mir.Inst.Tag) error{OutOfMemory}!void {
    try cg.addInst(.{ .tag = tag, .data = .{ .tag = {} } });
}

pub fn addExtended(cg: *CodeGen, opcode: std.wasm.MiscOpcode) error{OutOfMemory}!void {
    const extra_index: u32 = @intCast(cg.mir_extra.items.len);
    try cg.mir_extra.append(cg.gpa, @intFromEnum(opcode));
    try cg.addInst(.{ .tag = .misc_prefix, .data = .{ .payload = extra_index } });
}

pub fn addLabel(cg: *CodeGen, tag: Mir.Inst.Tag, label: u32) error{OutOfMemory}!void {
    try cg.addInst(.{ .tag = tag, .data = .{ .label = label } });
}

pub fn addLocal(cg: *CodeGen, tag: Mir.Inst.Tag, local: u32) error{OutOfMemory}!void {
    try cg.addInst(.{ .tag = tag, .data = .{ .local = local } });
}

/// Accepts an unsigned 32bit integer rather than a signed integer to
/// prevent us from having to bitcast multiple times as most values
/// within codegen are represented as unsigned rather than signed.
pub fn addImm32(cg: *CodeGen, imm: u32) error{OutOfMemory}!void {
    try cg.addInst(.{ .tag = .i32_const, .data = .{ .imm32 = @bitCast(imm) } });
}

/// Accepts an unsigned 64bit integer rather than a signed integer to
/// prevent us from having to bitcast multiple times as most values
/// within codegen are represented as unsigned rather than signed.
pub fn addImm64(cg: *CodeGen, imm: u64) error{OutOfMemory}!void {
    const extra_index = try cg.addExtra(Mir.Imm64.init(imm));
    try cg.addInst(.{ .tag = .i64_const, .data = .{ .payload = extra_index } });
}

/// Accepts the index into the list of 128bit-immediates
pub fn addImm128(cg: *CodeGen, index: u32) error{OutOfMemory}!void {
    const simd_values = cg.simd_immediates.items[index];
    const extra_index: u32 = @intCast(cg.mir_extra.items.len);
    // tag + 128bit value
    try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 5);
    cg.mir_extra.appendAssumeCapacity(@intFromEnum(std.wasm.SimdOpcode.v128_const));
    cg.mir_extra.appendSliceAssumeCapacity(@alignCast(mem.bytesAsSlice(u32, &simd_values)));
    try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
}

pub fn addFloat32(cg: *CodeGen, float: f32) error{OutOfMemory}!void {
    try cg.addInst(.{ .tag = .f32_const, .data = .{ .float32 = float } });
}

pub fn addFloat64(cg: *CodeGen, float: f64) error{OutOfMemory}!void {
    const extra_index = try cg.addExtra(Mir.Float64.init(float));
    try cg.addInst(.{ .tag = .f64_const, .data = .{ .payload = extra_index } });
}

/// Inserts an instruction to load/store from/to wasm's linear memory dependent on the given `tag`.
pub fn addMemArg(cg: *CodeGen, tag: Mir.Inst.Tag, mem_arg: Mir.MemArg) error{OutOfMemory}!void {
    const extra_index = try cg.addExtra(mem_arg);
    try cg.addInst(.{ .tag = tag, .data = .{ .payload = extra_index } });
}

/// Inserts an instruction from the 'atomics' feature which accesses wasm's linear memory dependent on the
/// given `tag`.
pub fn addAtomicMemArg(cg: *CodeGen, tag: std.wasm.AtomicsOpcode, mem_arg: Mir.MemArg) error{OutOfMemory}!void {
    const extra_index = try cg.addExtra(@as(struct { val: u32 }, .{ .val = @intFromEnum(tag) }));
    _ = try cg.addExtra(mem_arg);
    try cg.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } });
}

/// Helper function to emit atomic mir opcodes.
pub fn addAtomicTag(cg: *CodeGen, tag: std.wasm.AtomicsOpcode) error{OutOfMemory}!void {
    const extra_index = try cg.addExtra(@as(struct { val: u32 }, .{ .val = @intFromEnum(tag) }));
    try cg.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } });
}

fn addCallIntrinsic(cg: *CodeGen, intrinsic: Mir.Intrinsic) error{OutOfMemory}!void {
    try cg.addInst(.{ .tag = .call_intrinsic, .data = .{ .intrinsic = intrinsic } });
}

/// Appends entries to `mir_extra` based on the type of `extra`.
/// Returns the index into `mir_extra`
fn addExtra(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
    const fields = std.meta.fields(@TypeOf(extra));
    try cg.mir_extra.ensureUnusedCapacity(cg.gpa, fields.len);
    return cg.addExtraAssumeCapacity(extra);
}

/// Appends entries to `mir_extra` based on the type of `extra`.
/// Returns the index into `mir_extra`
fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
    const fields = std.meta.fields(@TypeOf(extra));
    const result: u32 = @intCast(cg.mir_extra.items.len);
    inline for (fields) |field| {
        cg.mir_extra.appendAssumeCapacity(switch (field.type) {
            u32 => @field(extra, field.name),
            i32 => @bitCast(@field(extra, field.name)),
            InternPool.Index,
            InternPool.Nav.Index,
            => @intFromEnum(@field(extra, field.name)),
            else => |field_type| @compileError("Unsupported field type " ++ @typeName(field_type)),
        });
    }
    return result;
}

/// For `std.builtin.CallingConvention.auto`.
pub fn typeToValtype(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.Valtype {
    return switch (ty.zigTypeTag(zcu)) {
        .float => switch (ty.floatBits(target)) {
            16 => .i32, // stored/loaded as u16
            32 => .f32,
            64 => .f64,
            80, 128 => .i32,
            else => unreachable,
        },
        .int, .@"enum" => switch (ty.intInfo(zcu).bits) {
            0...32 => .i32,
            33...64 => .i64,
            else => .i32,
        },
        .vector => switch (CodeGen.determineSimdStoreStrategy(ty, zcu, target)) {
            .direct => .v128,
            .unrolled => .i32,
        },
        .@"union", .@"struct" => switch (ty.containerLayout(zcu)) {
            .@"packed" => typeToValtype(ty.bitpackBackingInt(zcu), zcu, target),
            .auto, .@"extern" => .i32,
        },
        else => .i32, // all represented as reference/immediate
    };
}

/// Using a given `Type`, returns the corresponding wasm value type
/// Differently from `typeToValtype` this also allows `void` to create a block
/// with no return type
fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.BlockType {
    return switch (ty.ip_index) {
        .void_type, .noreturn_type => .empty,
        else => .fromValtype(typeToValtype(ty, zcu, target)),
    };
}

/// Writes the bytecode depending on the given `WValue` in `val`
fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
    switch (value) {
        .dead => unreachable, // reference to free'd `WValue` (missing reuseOperand?)
        .none, .stack => {}, // no-op
        .local => |idx| try cg.addLocal(.local_get, idx.value),
        .imm32 => |val| try cg.addImm32(val),
        .imm64 => |val| try cg.addImm64(val),
        .imm128 => |val| try cg.addImm128(val),
        .float32 => |val| try cg.addFloat32(val),
        .float64 => |val| try cg.addFloat64(val),
        .nav_ref => |nav_ref| {
            const zcu = cg.pt.zcu;
            const ip = &zcu.intern_pool;
            if (ip.zigTypeTag(ip.getNav(nav_ref.nav_index).resolved.?.type) == .@"fn") {
                assert(nav_ref.offset == 0);
                try cg.mir_indirect_function_set.put(cg.gpa, nav_ref.nav_index, {});
                try cg.addInst(.{ .tag = .func_ref, .data = .{ .nav_index = nav_ref.nav_index } });
            } else if (nav_ref.offset == 0) {
                try cg.addInst(.{ .tag = .nav_ref, .data = .{ .nav_index = nav_ref.nav_index } });
            } else {
                try cg.addInst(.{
                    .tag = .nav_ref_off,
                    .data = .{
                        .payload = try cg.addExtra(Mir.NavRefOff{
                            .nav_index = nav_ref.nav_index,
                            .offset = nav_ref.offset,
                        }),
                    },
                });
            }
        },
        .uav_ref => |uav| {
            const zcu = cg.pt.zcu;
            const ip = &zcu.intern_pool;
            assert(!ip.isFunctionType(ip.typeOf(uav.ip_index)));
            const gop = try cg.mir_uavs.getOrPut(cg.gpa, uav.ip_index);
            const this_align: Alignment = a: {
                if (uav.orig_ptr_ty == .none) break :a .none;
                const ptr_type = ip.indexToKey(uav.orig_ptr_ty).ptr_type;
                const this_align = ptr_type.flags.alignment;
                if (this_align == .none) break :a .none;
                const abi_align = Type.fromInterned(ptr_type.child).abiAlignment(zcu);
                if (this_align.compare(.lte, abi_align)) break :a .none;
                break :a this_align;
            };
            if (!gop.found_existing or
                gop.value_ptr.* == .none or
                (this_align != .none and this_align.compare(.gt, gop.value_ptr.*)))
            {
                gop.value_ptr.* = this_align;
            }
            if (uav.offset == 0) {
                try cg.addInst(.{
                    .tag = .uav_ref,
                    .data = .{ .ip_index = uav.ip_index },
                });
            } else {
                try cg.addInst(.{
                    .tag = .uav_ref_off,
                    .data = .{ .payload = try cg.addExtra(@as(Mir.UavRefOff, .{
                        .value = uav.ip_index,
                        .offset = uav.offset,
                    })) },
                });
            }
        },
        .stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), // caller must ensure to address the offset
    }
}

/// If given a local or stack-offset, increases the reference count by 1.
/// The old `WValue` found at instruction `ref` is then replaced by the
/// modified `WValue` and returned. When given a non-local or non-stack-offset,
/// returns the given `operand` itfunc instead.
fn reuseOperand(cg: *CodeGen, ref: Air.Inst.Ref, operand: WValue) WValue {
    if (operand != .local and operand != .stack_offset) return operand;
    var new_value = operand;
    switch (new_value) {
        .local => |*local| local.references += 1,
        .stack_offset => |*stack_offset| stack_offset.references += 1,
        else => unreachable,
    }
    const old_value = cg.getResolvedInst(ref);
    old_value.* = new_value;
    return new_value;
}

/// From a reference, returns its resolved `WValue`.
/// It's illegal to provide a `Air.Inst.Ref` that hasn't been resolved yet.
fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue {
    var index = cg.branches.items.len;
    while (index > 0) : (index -= 1) {
        const branch = cg.branches.items[index - 1];
        if (branch.values.getPtr(ref)) |value| {
            return value;
        }
    }
    unreachable; // developer-error: This can only be called on resolved instructions. Use `resolveInst` instead.
}

/// Creates one locals for a given `Type`.
/// Returns a corresponding `Wvalue` with `local` as active tag
fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {
    const zcu = cg.pt.zcu;
    const valtype = typeToValtype(ty, zcu, cg.target);
    const index_or_null = switch (valtype) {
        .i32 => cg.free_locals_i32.pop(),
        .i64 => cg.free_locals_i64.pop(),
        .f32 => cg.free_locals_f32.pop(),
        .f64 => cg.free_locals_f64.pop(),
        .v128 => cg.free_locals_v128.pop(),
    };
    if (index_or_null) |index| {
        log.debug("reusing local ({d}) of type {}", .{ index, valtype });
        return .{ .local = .{ .value = index, .references = 1 } };
    }
    log.debug("new local of type {}", .{valtype});
    return cg.ensureAllocLocal(ty);
}

/// Ensures a new local will be created. This is useful when it's useful
/// to use a zero-initialized local.
fn ensureAllocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {
    const zcu = cg.pt.zcu;
    try cg.mir_locals.append(cg.gpa, typeToValtype(ty, zcu, cg.target));
    const initial_index = cg.local_index;
    cg.local_index += 1;
    return .{ .local = .{ .value = initial_index, .references = 1 } };
}

pub const Error = error{
    OutOfMemory,
    /// Compiler was asked to operate on a number larger than supported.
    Overflow,
    /// Indicates the error is already stored in Zcu `failed_codegen`.
    CodegenFail,
};

pub fn generate(
    bin_file: *link.File,
    pt: Zcu.PerThread,
    src_loc: Zcu.LazySrcLoc,
    func_index: InternPool.Index,
    air: *const Air,
    liveness: *const ?Air.Liveness,
) Error!Mir {
    _ = src_loc;
    _ = bin_file;
    const zcu = pt.zcu;
    const gpa = zcu.gpa;
    const cg = zcu.funcInfo(func_index);
    const file_scope = zcu.navFileScope(cg.owner_nav);
    const target = &file_scope.mod.?.resolved_target.result;
    const fn_ty = zcu.navValue(cg.owner_nav).typeOf(zcu);
    const fn_info = zcu.typeToFunc(fn_ty).?;
    const ret_ty: Type = .fromInterned(fn_info.return_type);
    const any_returns = !firstParamSRet(fn_info.cc, ret_ty, zcu, target) and ret_ty.hasRuntimeBits(zcu);

    var cc_result = try resolveCallingConventionValues(zcu, fn_ty, target);
    defer cc_result.deinit(gpa);

    var code_gen: CodeGen = .{
        .gpa = gpa,
        .pt = pt,
        .air = air.*,
        .liveness = liveness.*.?,
        .owner_nav = cg.owner_nav,
        .target = target,
        .ptr_size = switch (target.cpu.arch) {
            .wasm32 => .wasm32,
            .wasm64 => .wasm64,
            else => unreachable,
        },
        .func_index = func_index,
        .args = cc_result.args,
        .return_value = cc_result.return_value,
        .local_index = cc_result.local_index,
        .mir_instructions = .empty,
        .mir_extra = .empty,
        .mir_locals = .empty,
        .mir_uavs = .empty,
        .mir_indirect_function_set = .empty,
        .mir_func_tys = .empty,
        .error_name_table_ref_count = 0,
    };
    defer code_gen.deinit();

    try code_gen.mir_func_tys.putNoClobber(gpa, fn_ty.toIntern(), {});

    return generateInner(&code_gen, any_returns) catch |err| switch (err) {
        error.CodegenFail,
        error.OutOfMemory,
        error.Overflow,
        => |e| return e,
        else => |e| return code_gen.fail("failed to generate function: {s}", .{@errorName(e)}),
    };
}

fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir {
    const zcu = cg.pt.zcu;
    // branch used for const values
    try cg.branches.append(cg.gpa, .{});
    // func scope branch
    try cg.branches.append(cg.gpa, .{});
    defer {
        var func_branch = cg.branches.pop().?;
        func_branch.deinit(cg.gpa);
        var const_branch = cg.branches.pop().?;
        const_branch.deinit(cg.gpa);
        assert(cg.branches.items.len == 0); // missing branch merge
    }
    // Generate MIR for function body
    try cg.genBody(cg.air.getMainBody());

    // In case we have a return value, but the last instruction is a noreturn (such as a while loop)
    // we emit an unreachable instruction to tell the stack validator that part will never be reached.
    if (any_returns and cg.air.instructions.len > 0) {
        const inst: Air.Inst.Index = @enumFromInt(cg.air.instructions.len - 1);
        const last_inst_ty = cg.typeOfIndex(inst);
        if (!last_inst_ty.hasRuntimeBits(zcu)) {
            try cg.addTag(.@"unreachable");
        }
    }
    // End of function body
    try cg.addTag(.end);
    try cg.addTag(.dbg_epilogue_begin);

    var mir: Mir = .{
        .instructions = cg.mir_instructions.toOwnedSlice(),
        .extra = &.{}, // fallible so assigned after errdefer
        .locals = &.{}, // fallible so assigned after errdefer
        .prologue = if (cg.initial_stack_value == .none) .none else .{
            .sp_local = cg.initial_stack_value.local.value,
            .flags = .{ .stack_alignment = cg.stack_alignment },
            .stack_size = cg.stack_size,
            .bottom_stack_local = cg.bottom_stack_value.local.value,
        },
        .uavs = cg.mir_uavs.move(),
        .indirect_function_set = cg.mir_indirect_function_set.move(),
        .func_tys = cg.mir_func_tys.move(),
        .error_name_table_ref_count = cg.error_name_table_ref_count,
    };
    errdefer mir.deinit(cg.gpa);
    mir.extra = try cg.mir_extra.toOwnedSlice(cg.gpa);
    mir.locals = try cg.mir_locals.toOwnedSlice(cg.gpa);
    return mir;
}

const CallWValues = struct {
    args: []WValue,
    return_value: WValue,
    local_index: u32,

    fn deinit(values: *CallWValues, gpa: Allocator) void {
        gpa.free(values.args);
        values.* = undefined;
    }
};

fn resolveCallingConventionValues(
    zcu: *const Zcu,
    fn_ty: Type,
    target: *const std.Target,
) Allocator.Error!CallWValues {
    const gpa = zcu.gpa;
    const ip = &zcu.intern_pool;
    const fn_info = zcu.typeToFunc(fn_ty).?;
    const cc = fn_info.cc;

    var result: CallWValues = .{
        .args = &.{},
        .return_value = .none,
        .local_index = 0,
    };
    if (cc == .naked) return result;

    var args = std.array_list.Managed(WValue).init(gpa);
    defer args.deinit();

    // Check if we store the result as a pointer to the stack rather than
    // by value
    if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, target)) {
        // the sret arg will be passed as first argument, therefore we
        // set the `return_value` before allocating locals for regular args.
        result.return_value = .{ .local = .{ .value = result.local_index, .references = 1 } };
        result.local_index += 1;
    }

    switch (cc) {
        .auto => {
            for (fn_info.param_types.get(ip)) |ty| {
                if (!Type.fromInterned(ty).hasRuntimeBits(zcu)) {
                    continue;
                }

                try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
                result.local_index += 1;
            }
        },
        .wasm_mvp => {
            for (fn_info.param_types.get(ip)) |ty| {
                if (!Type.fromInterned(ty).hasRuntimeBits(zcu)) {
                    continue;
                }
                switch (abi.classifyType(.fromInterned(ty), zcu)) {
                    .direct => |scalar_ty| if (!abi.lowerAsDoubleI64(scalar_ty, zcu)) {
                        try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
                        result.local_index += 1;
                    } else {
                        try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
                        try args.append(.{ .local = .{ .value = result.local_index + 1, .references = 1 } });
                        result.local_index += 2;
                    },
                    .indirect => {
                        try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
                        result.local_index += 1;
                    },
                }
            }
        },
        else => unreachable, // Frontend is responsible for emitting an error earlier.
    }
    result.args = try args.toOwnedSlice();
    return result;
}

pub fn firstParamSRet(
    cc: std.builtin.CallingConvention,
    return_type: Type,
    zcu: *const Zcu,
    target: *const std.Target,
) bool {
    if (!return_type.hasRuntimeBits(zcu)) return false;
    switch (cc) {
        .@"inline" => unreachable,
        .auto => return isByRef(return_type, zcu, target),
        .wasm_mvp => switch (abi.classifyType(return_type, zcu)) {
            .direct => |scalar_ty| return abi.lowerAsDoubleI64(scalar_ty, zcu),
            .indirect => return true,
        },
        else => return false,
    }
}

/// Lowers a Zig type and its value based on a given calling convention to ensure
/// it matches the ABI.
fn lowerArg(cg: *CodeGen, cc: std.builtin.CallingConvention, ty: Type, value: WValue) !void {
    if (cc != .wasm_mvp) {
        return cg.lowerToStack(value);
    }

    const zcu = cg.pt.zcu;

    switch (abi.classifyType(ty, zcu)) {
        .direct => |scalar_type| if (!abi.lowerAsDoubleI64(scalar_type, zcu)) {
            if (!isByRef(ty, zcu, cg.target)) {
                return cg.lowerToStack(value);
            } else {
                switch (value) {
                    .nav_ref, .stack_offset => _ = try cg.load(value, scalar_type, 0),
                    .dead => unreachable,
                    else => try cg.emitWValue(value),
                }
            }
        } else {
            assert(ty.abiSize(zcu) == 16);
            // in this case we have an integer or float that must be lowered as 2 i64's.
            try cg.emitWValue(value);
            try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = 8 });
            try cg.emitWValue(value);
            try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = 8 });
        },
        .indirect => return cg.lowerToStack(value),
    }
}

/// Lowers a `WValue` to the stack. This means when the `value` results in
/// `.stack_offset` we calculate the pointer of this offset and use that.
/// The value is left on the stack, and not stored in any temporary.
fn lowerToStack(cg: *CodeGen, value: WValue) !void {
    switch (value) {
        .stack_offset => |offset| {
            try cg.emitWValue(value);
            if (offset.value > 0) {
                switch (cg.ptr_size) {
                    .wasm32 => {
                        try cg.addImm32(offset.value);
                        try cg.addTag(.i32_add);
                    },
                    .wasm64 => {
                        try cg.addImm64(offset.value);
                        try cg.addTag(.i64_add);
                    },
                }
            }
        },
        else => try cg.emitWValue(value),
    }
}

/// Creates a local for the initial stack value
/// Asserts `initial_stack_value` is `.none`
fn initializeStack(cg: *CodeGen) !void {
    assert(cg.initial_stack_value == .none);
    // Reserve a local to store the current stack pointer
    // We can later use this local to set the stack pointer back to the value
    // we have stored here.
    cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize);
    // Also reserve a local to store the bottom stack value
    cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize);
}

/// Reads the stack pointer from `Context.initial_stack_value` and writes it
/// to the global stack pointer variable
fn restoreStackPointer(cg: *CodeGen) !void {
    // only restore the pointer if it was initialized
    if (cg.initial_stack_value == .none) return;
    // Get the original stack pointer's value
    try cg.emitWValue(cg.initial_stack_value);

    try cg.addTag(.global_set_sp);
}

/// From a given type, will create space on the virtual stack to store the value of such type.
/// This returns a `WValue` with its active tag set to `local`, containing the index to the local
/// that points to the position on the virtual stack. This function should be used instead of
/// moveStack unless a local was already created to store the pointer.
///
/// Asserts Type has codegenbits
fn allocStack(cg: *CodeGen, ty: Type) !WValue {
    const pt = cg.pt;
    const zcu = pt.zcu;
    assert(ty.hasRuntimeBits(zcu));
    if (cg.initial_stack_value == .none) {
        try cg.initializeStack();
    }

    const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse {
        return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{
            ty.fmt(pt), ty.abiSize(zcu),
        });
    };
    const abi_align = ty.abiAlignment(zcu);

    cg.stack_alignment = cg.stack_alignment.max(abi_align);

    const offset: u32 = @intCast(abi_align.forward(cg.stack_size));
    defer cg.stack_size = offset + abi_size;

    return .{ .stack_offset = .{ .value = offset, .references = 1 } };
}

fn allocInt(cg: *CodeGen, int_ty: IntType) !WValue {
    if (cg.initial_stack_value == .none) {
        try cg.initializeStack();
    }

    const abi_size = std.math.cast(u32, std.zig.target.intByteSize(cg.target, int_ty.bits)) orelse {
        return cg.fail("Integer ABI size exceeds max stack size", .{});
    };
    const abi_align: Alignment = .fromByteUnits(std.zig.target.intAlignment(cg.target, int_ty.bits));

    cg.stack_alignment = cg.stack_alignment.max(abi_align);

    const offset: u32 = @intCast(abi_align.forward(cg.stack_size));
    defer cg.stack_size = offset + abi_size;

    return .{ .stack_offset = .{ .value = offset, .references = 1 } };
}

/// From a given AIR instruction generates a pointer to the stack where
/// the value of its type will live.
/// This is different from allocStack where this will use the pointer's alignment
/// if it is set, to ensure the stack alignment will be set correctly.
fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const ptr_ty = cg.typeOfIndex(inst);
    const pointee_ty = ptr_ty.childType(zcu);

    if (cg.initial_stack_value == .none) {
        try cg.initializeStack();
    }

    if (!pointee_ty.hasRuntimeBits(zcu)) {
        return cg.allocStack(Type.usize); // create a value containing just the stack pointer.
    }

    const abi_alignment = ptr_ty.ptrAlignment(zcu);
    const abi_size = std.math.cast(u32, pointee_ty.abiSize(zcu)) orelse {
        return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{
            pointee_ty.fmt(pt), pointee_ty.abiSize(zcu),
        });
    };
    cg.stack_alignment = cg.stack_alignment.max(abi_alignment);

    const offset: u32 = @intCast(abi_alignment.forward(cg.stack_size));
    defer cg.stack_size = offset + abi_size;

    return .{ .stack_offset = .{ .value = offset, .references = 1 } };
}

/// Performs a copy of bytes for a given type. Copying all bytes
/// from rhs to lhs.
fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
    const len_known_neq_0 = switch (len) {
        .imm32 => |val| if (val != 0) true else return,
        .imm64 => |val| if (val != 0) true else return,
        else => false,
    };
    // When bulk_memory is enabled, we lower it to wasm's memcpy instruction.
    // If not, we lower it ourselves manually
    if (cg.target.cpu.has(.wasm, .bulk_memory)) {
        const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);
        const emit_check = !(len0_ok or len_known_neq_0);

        if (emit_check) {
            try cg.startBlock(.block, .empty);

            // Even if `len` is zero, the spec requires an implementation to trap if `src + len` or
            // `dst + len` are out of memory bounds. This can easily happen in Zig in a case such
            // as:
            //
            // const dst: [*]u8 = undefined;
            // const src: [*]u8 = undefined;
            // var len: usize = runtime_zero();
            // @memcpy(dst[0..len], src[0..len]);
            //
            // So explicitly avoid using `memory.copy` in the `len == 0` case. Lovely design.
            try cg.emitWValue(len);
            try cg.addTag(.i32_eqz);
            try cg.addLabel(.br_if, 0);
        }

        try cg.lowerToStack(dst);
        try cg.lowerToStack(src);
        try cg.emitWValue(len);
        try cg.addExtended(.memory_copy);

        if (emit_check) {
            try cg.endBlock();
        }

        return;
    }

    // when the length is comptime-known, rather than a runtime value, we can optimize the generated code by having
    // the loop during codegen, rather than inserting a runtime loop into the binary.
    switch (len) {
        .imm32, .imm64 => blk: {
            const length = switch (len) {
                .imm32 => |val| val,
                .imm64 => |val| val,
                else => unreachable,
            };
            // if the size (length) is more than 32 bytes, we use a runtime loop instead to prevent
            // binary size bloat.
            if (length > 32) break :blk;
            var offset: u32 = 0;
            const lhs_base = dst.offset();
            const rhs_base = src.offset();
            while (offset < length) : (offset += 1) {
                // get dst's address to store the result
                try cg.emitWValue(dst);
                // load byte from src's address
                try cg.emitWValue(src);
                switch (cg.ptr_size) {
                    .wasm32 => {
                        try cg.addMemArg(.i32_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 });
                        try cg.addMemArg(.i32_store8, .{ .offset = lhs_base + offset, .alignment = 1 });
                    },
                    .wasm64 => {
                        try cg.addMemArg(.i64_load8_u, .{ .offset = rhs_base + offset, .alignment = 1 });
                        try cg.addMemArg(.i64_store8, .{ .offset = lhs_base + offset, .alignment = 1 });
                    },
                }
            }
            return;
        },
        else => {},
    }

    // allocate a local for the offset, and set it to 0.
    // This to ensure that inside loops we correctly re-set the counter.
    var offset = try cg.allocLocal(Type.usize); // local for counter
    defer offset.free(cg);
    switch (cg.ptr_size) {
        .wasm32 => try cg.addImm32(0),
        .wasm64 => try cg.addImm64(0),
    }
    try cg.addLocal(.local_set, offset.local.value);

    // outer block to jump to when loop is done
    try cg.startBlock(.block, .empty);
    try cg.startBlock(.loop, .empty);

    // loop condition (offset == length -> break)
    {
        try cg.emitWValue(offset);
        try cg.emitWValue(len);
        switch (cg.ptr_size) {
            .wasm32 => try cg.addTag(.i32_eq),
            .wasm64 => try cg.addTag(.i64_eq),
        }
        try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
    }

    // get dst ptr
    {
        try cg.emitWValue(dst);
        try cg.emitWValue(offset);
        switch (cg.ptr_size) {
            .wasm32 => try cg.addTag(.i32_add),
            .wasm64 => try cg.addTag(.i64_add),
        }
    }

    // get src value and also store in dst
    {
        try cg.emitWValue(src);
        try cg.emitWValue(offset);
        switch (cg.ptr_size) {
            .wasm32 => {
                try cg.addTag(.i32_add);
                try cg.addMemArg(.i32_load8_u, .{ .offset = src.offset(), .alignment = 1 });
                try cg.addMemArg(.i32_store8, .{ .offset = dst.offset(), .alignment = 1 });
            },
            .wasm64 => {
                try cg.addTag(.i64_add);
                try cg.addMemArg(.i64_load8_u, .{ .offset = src.offset(), .alignment = 1 });
                try cg.addMemArg(.i64_store8, .{ .offset = dst.offset(), .alignment = 1 });
            },
        }
    }

    // increment loop counter
    {
        try cg.emitWValue(offset);
        switch (cg.ptr_size) {
            .wasm32 => {
                try cg.addImm32(1);
                try cg.addTag(.i32_add);
            },
            .wasm64 => {
                try cg.addImm64(1);
                try cg.addTag(.i64_add);
            },
        }
        try cg.addLocal(.local_set, offset.local.value);
        try cg.addLabel(.br, 0); // jump to start of loop
    }
    try cg.endBlock(); // close off loop block
    try cg.endBlock(); // close off outer block
}

fn ptrSize(cg: *const CodeGen) u16 {
    return @divExact(cg.target.ptrBitWidth(), 8);
}

/// For a given `Type`, will return true when the type will be passed
/// by reference, rather than by value
fn isByRef(ty: Type, zcu: *const Zcu, target: *const std.Target) bool {
    switch (ty.zigTypeTag(zcu)) {
        .type,
        .comptime_int,
        .comptime_float,
        .enum_literal,
        .undefined,
        .null,
        .@"opaque",
        => unreachable,

        .noreturn,
        .void,
        .bool,
        .error_set,
        .@"fn",
        .@"anyframe",
        => return false,

        .array,
        .frame,
        => return ty.hasRuntimeBits(zcu),
        .@"struct", .@"union" => switch (ty.containerLayout(zcu)) {
            .@"packed" => return isByRef(ty.bitpackBackingInt(zcu), zcu, target),
            .@"extern", .auto => return ty.hasRuntimeBits(zcu),
        },
        .vector => return determineSimdStoreStrategy(ty, zcu, target) == .unrolled,
        .int => return ty.intInfo(zcu).bits > 64,
        .@"enum" => return ty.intInfo(zcu).bits > 64,
        .float => return ty.floatBits(target) > 64,
        .error_union => {
            const pl_ty = ty.errorUnionPayload(zcu);
            if (!pl_ty.hasRuntimeBits(zcu)) {
                return false;
            }
            return true;
        },
        .optional => {
            if (ty.isPtrLikeOptional(zcu)) return false;
            const pl_type = ty.optionalChild(zcu);
            if (pl_type.zigTypeTag(zcu) == .error_set) return false;
            return pl_type.hasRuntimeBits(zcu);
        },
        .pointer => {
            // Slices act like struct and will be passed by reference
            if (ty.isSlice(zcu)) return true;
            return false;
        },
    }
}

const SimdStoreStrategy = enum {
    direct,
    unrolled,
};

/// For a given vector type, returns the `SimdStoreStrategy`.
/// This means when a given type is 128 bits and either the simd128 or relaxed-simd
/// features are enabled, the function will return `.direct`. This would allow to store
/// it using a instruction, rather than an unrolled version.
pub fn determineSimdStoreStrategy(ty: Type, zcu: *const Zcu, target: *const std.Target) SimdStoreStrategy {
    assert(ty.zigTypeTag(zcu) == .vector);
    if (ty.bitSize(zcu) != 128) return .unrolled;
    if (target.cpu.has(.wasm, .relaxed_simd) or target.cpu.has(.wasm, .simd128)) {
        return .direct;
    }
    return .unrolled;
}

/// Creates a new local for a pointer that points to memory with given offset.
/// This can be used to get a pointer to a struct field, error payload, etc.
/// By providing `modify` as action, it will modify the given `ptr_value` instead of making a new
/// local value to store the pointer. This allows for local re-use and improves binary size.
fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum { modify, new }) InnerError!WValue {
    // do not perform arithmetic when offset is 0.
    if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value;
    const result_ptr: WValue = switch (action) {
        .new => try cg.ensureAllocLocal(Type.usize),
        .modify => ptr_value,
    };
    try cg.emitWValue(ptr_value);
    if (offset + ptr_value.offset() > 0) {
        switch (cg.ptr_size) {
            .wasm32 => {
                try cg.addImm32(@intCast(offset + ptr_value.offset()));
                try cg.addTag(.i32_add);
            },
            .wasm64 => {
                try cg.addImm64(offset + ptr_value.offset());
                try cg.addTag(.i64_add);
            },
        }
    }
    try cg.addLocal(.local_set, result_ptr.local.value);
    return result_ptr;
}

fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const air_tags = cg.air.instructions.items(.tag);
    return switch (air_tags[@intFromEnum(inst)]) {
        // No "scalarize" legalizations are enabled, so these instructions never appear.
        .legalize_vec_elem_val => unreachable,
        .legalize_vec_store_elem => unreachable,
        // No soft float legalizations are enabled.
        .legalize_compiler_rt_call => unreachable,

        .inferred_alloc, .inferred_alloc_comptime => unreachable,

        .add,
        .sub,
        .mul,
        .rem,
        .mod,
        .max,
        .min,
        .div_exact,
        .div_trunc,
        .div_floor,
        => |tag| {
            const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
            const lhs = try cg.resolveInst(bin_op.lhs);
            const rhs = try cg.resolveInst(bin_op.rhs);

            const ty = cg.typeOfIndex(inst);
            const type_tag = ty.zigTypeTag(zcu);

            if (type_tag == .vector) {
                return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
            }

            if (type_tag == .float) {
                const float_ty: FloatType = .fromType(cg, ty);

                const result = switch (tag) {
                    .add => try cg.floatAdd(float_ty, lhs, rhs),
                    .sub => try cg.floatSub(float_ty, lhs, rhs),
                    .mul => try cg.floatMul(float_ty, lhs, rhs),
                    .rem => try cg.floatRem(float_ty, lhs, rhs),
                    .mod => try cg.floatMod(float_ty, lhs, rhs),
                    .max => try cg.floatMax(float_ty, lhs, rhs),
                    .min => try cg.floatMin(float_ty, lhs, rhs),
                    .div_exact => try cg.floatDiv(float_ty, lhs, rhs),
                    .div_trunc => try cg.floatDivTrunc(float_ty, lhs, rhs),
                    .div_floor => try cg.floatDivFloor(float_ty, lhs, rhs),
                    else => unreachable,
                };

                try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
            } else if (type_tag == .int) {
                const int_ty: IntType = .fromType(cg, ty);

                const result = switch (tag) {
                    .add => try cg.intAdd(int_ty, lhs, rhs),
                    .sub => try cg.intSub(int_ty, lhs, rhs),
                    .mul => try cg.intMul(int_ty, lhs, rhs),
                    .rem => try cg.intRem(int_ty, lhs, rhs),
                    .mod => try cg.intMod(int_ty, lhs, rhs),
                    .max => try cg.intMax(int_ty, lhs, rhs),
                    .min => try cg.intMin(int_ty, lhs, rhs),
                    .div_exact => try cg.intDiv(int_ty, lhs, rhs),
                    .div_trunc => try cg.intDiv(int_ty, lhs, rhs),
                    .div_floor => try cg.intDivFloor(int_ty, lhs, rhs),
                    else => unreachable,
                };

                try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
            } else {
                unreachable;
            }
        },
        .div_float => {
            const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
            const lhs = try cg.resolveInst(bin_op.lhs);
            const rhs = try cg.resolveInst(bin_op.rhs);
            const ty = cg.typeOfIndex(inst);

            if (ty.zigTypeTag(zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: div_float for vectors", .{});
            }

            const result = try cg.floatDiv(.fromType(cg, ty), lhs, rhs);
            try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
        },
        .abs => {
            const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
            const operand = try cg.resolveInst(ty_op.operand);

            const ty = cg.typeOf(ty_op.operand);
            const type_tag = ty.zigTypeTag(zcu);

            if (type_tag == .vector) {
                return cg.fail("TODO: implement AIR op: abs for vectors", .{});
            }

            if (type_tag == .float) {
                const result = try cg.floatAbs(.fromType(cg, ty), operand);
                return cg.finishAir(inst, result, &.{ty_op.operand});
            } else if (type_tag == .int) {
                const result = try cg.intAbs(.fromType(cg, ty), operand);
                return cg.finishAir(inst, result, &.{ty_op.operand});
            } else {
                unreachable;
            }
        },
        .mul_add => {
            const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
            const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data;
            const addend = try cg.resolveInst(pl_op.operand);
            const lhs = try cg.resolveInst(bin_op.lhs);
            const rhs = try cg.resolveInst(bin_op.rhs);
            const ty = cg.typeOfIndex(inst);

            if (ty.zigTypeTag(cg.pt.zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: mul_add for vectors", .{});
            }

            const result = try cg.floatMulAdd(.fromType(cg, ty), lhs, rhs, addend);
            return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs, pl_op.operand });
        },

        .add_sat,
        .sub_sat,
        .mul_sat,
        .shl_sat,
        => |tag| {
            const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
            const lhs = try cg.resolveInst(bin_op.lhs);
            const rhs = try cg.resolveInst(bin_op.rhs);
            const ty = cg.typeOfIndex(inst);

            if (ty.zigTypeTag(cg.pt.zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
            }

            const int_ty: IntType = .fromType(cg, ty);
            const result = switch (tag) {
                .add_sat => try cg.intAddSat(int_ty, lhs, rhs),
                .sub_sat => try cg.intSubSat(int_ty, lhs, rhs),
                .mul_sat => try cg.intMulSat(int_ty, lhs, rhs),
                .shl_sat => try cg.intShlSat(int_ty, lhs, rhs),
                else => unreachable,
            };

            try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
        },

        .add_with_overflow,
        .sub_with_overflow,
        .mul_with_overflow,
        .shl_with_overflow,
        => |tag| {
            const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
            const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;

            const lhs = try cg.resolveInst(extra.lhs);
            const rhs = try cg.resolveInst(extra.rhs);

            const ty = cg.typeOf(extra.lhs);
            const int_ty: IntType = .fromType(cg, ty);

            const out = switch (tag) {
                .add_with_overflow => try cg.intAddOverflow(int_ty, lhs, rhs),
                .sub_with_overflow => try cg.intSubOverflow(int_ty, lhs, rhs),
                .mul_with_overflow => try cg.intMulOverflow(int_ty, lhs, rhs),
                .shl_with_overflow => try cg.intShlOverflow(int_ty, lhs, rhs),
                else => unreachable,
            };

            var ov_tmp = try out.ov.toLocal(cg, Type.u1);
            defer ov_tmp.free(cg);

            var res_tmp = try out.result.toLocal(cg, ty);
            defer res_tmp.free(cg);

            const result = try cg.allocStack(cg.typeOfIndex(inst));
            const offset: u32 = @intCast(ty.abiSize(cg.pt.zcu));

            try cg.store(result, res_tmp, ty, 0);
            try cg.store(result, ov_tmp, Type.u1, offset);

            try cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });
        },

        .add_wrap, .sub_wrap, .mul_wrap, .shl => |tag| {
            const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
            const lhs = try cg.resolveInst(bin_op.lhs);
            const rhs = try cg.resolveInst(bin_op.rhs);
            const ty = cg.typeOfIndex(inst);

            if (ty.zigTypeTag(zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
            }

            const int_ty: IntType = .fromType(cg, ty);
            const raw_result = switch (tag) {
                .add_wrap => try cg.intAdd(int_ty, lhs, rhs),
                .sub_wrap => try cg.intSub(int_ty, lhs, rhs),
                .mul_wrap => try cg.intMul(int_ty, lhs, rhs),
                .shl => try cg.intShl(int_ty, lhs, rhs),
                else => unreachable,
            };
            const result = try cg.intWrap(int_ty, raw_result);

            try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
        },

        .bit_and, .bit_or, .xor, .shl_exact, .shr, .shr_exact => |tag| {
            const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
            const lhs = try cg.resolveInst(bin_op.lhs);
            const rhs = try cg.resolveInst(bin_op.rhs);
            const ty = cg.typeOfIndex(inst);

            if (ty.zigTypeTag(zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
            }

            const int_ty: IntType = .fromType(cg, ty);
            const result = switch (tag) {
                .bit_and => try cg.intAnd(int_ty, lhs, rhs),
                .bit_or => try cg.intOr(int_ty, lhs, rhs),
                .xor => try cg.intXor(int_ty, lhs, rhs),
                .shl_exact => try cg.intShl(int_ty, lhs, rhs),
                .shr, .shr_exact => try cg.intShr(int_ty, lhs, rhs),
                else => unreachable,
            };

            try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
        },

        .not => {
            const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
            const operand = try cg.resolveInst(ty_op.operand);
            const ty = cg.typeOf(ty_op.operand);

            if (ty.zigTypeTag(zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: not for vectors", .{});
            }

            const result = try cg.intNot(.fromType(cg, ty), operand);
            try cg.finishAir(inst, result, &.{ty_op.operand});
        },

        .bitcast => cg.airBitcast(inst),

        .intcast => {
            const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;

            const dest_ty = ty_op.ty.toType();
            const operand = try cg.resolveInst(ty_op.operand);
            const src_ty = cg.typeOf(ty_op.operand);

            if (dest_ty.zigTypeTag(zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: intcast for vectors", .{});
            }

            const src_int_ty: IntType = .fromType(cg, src_ty);
            const dest_int_ty: IntType = .fromType(cg, dest_ty);

            const src_bits = src_int_ty.bits;
            const dest_bits = dest_int_ty.bits;

            const same_class: bool = (src_bits <= 32 and dest_bits <= 32) or
                (src_bits >= 33 and src_bits <= 64 and dest_bits >= 33 and dest_bits <= 64) or
                (src_bits >= 65 and src_bits <= 128 and dest_bits >= 65 and dest_bits <= 128);

            const result = if (same_class)
                cg.reuseOperand(ty_op.operand, operand)
            else
                try cg.intCast(dest_int_ty, src_int_ty, operand);

            try cg.finishAir(inst, result, &.{ty_op.operand});
        },
        .trunc => {
            const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;

            const operand = try cg.resolveInst(ty_op.operand);
            const dest_ty = ty_op.ty.toType();
            const src_ty = cg.typeOf(ty_op.operand);

            if (dest_ty.zigTypeTag(zcu) == .vector or src_ty.zigTypeTag(zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: trunc for vectors", .{});
            }

            const src_int_ty: IntType = .fromType(cg, src_ty);
            const dest_int_ty: IntType = .fromType(cg, dest_ty);

            const result = if (src_int_ty.bits == dest_int_ty.bits)
                cg.reuseOperand(ty_op.operand, operand)
            else blk: {
                break :blk try cg.intTrunc(dest_int_ty, src_int_ty, operand);
            };

            try cg.finishAir(inst, result, &.{ty_op.operand});
        },

        .fptrunc, .fpext => |tag| {
            const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;

            const operand = try cg.resolveInst(ty_op.operand);
            const src_ty = cg.typeOf(ty_op.operand);
            const dest_ty = cg.typeOfIndex(inst);

            if (dest_ty.zigTypeTag(cg.pt.zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
            }

            const src_float_ty: FloatType = .fromType(cg, src_ty);
            const dest_float_ty: FloatType = .fromType(cg, dest_ty);

            const result = switch (tag) {
                .fptrunc => try cg.floatTruncCast(dest_float_ty, src_float_ty, operand),
                .fpext => try cg.floatExtendCast(dest_float_ty, src_float_ty, operand),
                else => unreachable,
            };

            try cg.finishAir(inst, result, &.{ty_op.operand});
        },

        .int_from_float => {
            const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
            const operand = try cg.resolveInst(ty_op.operand);
            const src_ty = cg.typeOf(ty_op.operand);
            const dest_ty = cg.typeOfIndex(inst);

            if (src_ty.zigTypeTag(zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: int_from_float for vectors", .{});
            }

            const result = try cg.intFromFloat(.fromType(cg, dest_ty), .fromType(cg, src_ty), operand);
            try cg.finishAir(inst, result, &.{ty_op.operand});
        },
        .float_from_int => {
            const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
            const operand = try cg.resolveInst(ty_op.operand);
            const src_ty = cg.typeOf(ty_op.operand);
            const dest_ty = cg.typeOfIndex(inst);

            if (src_ty.zigTypeTag(zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: float_from_int for vectors", .{});
            }

            const result = try cg.floatFromInt(.fromType(cg, dest_ty), .fromType(cg, src_ty), operand);
            try cg.finishAir(inst, result, &.{ty_op.operand});
        },

        .clz, .ctz, .popcount, .byte_swap, .bit_reverse => |tag| {
            const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
            const operand = try cg.resolveInst(ty_op.operand);

            const ty = cg.typeOf(ty_op.operand);

            if (ty.zigTypeTag(zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
            }

            const int_ty: IntType = .fromType(cg, ty);
            const result = switch (tag) {
                .clz => try cg.intClz(int_ty, operand),
                .ctz => try cg.intCtz(int_ty, operand),
                .popcount => try cg.intPopCount(int_ty, operand),
                .byte_swap => try cg.intByteSwap(int_ty, operand),
                .bit_reverse => try cg.intBitReverse(int_ty, operand),
                else => unreachable,
            };
            try cg.finishAir(inst, result, &.{ty_op.operand});
        },

        .sqrt, .sin, .cos, .tan, .exp, .exp2, .log, .log2, .log10, .floor, .ceil, .round, .trunc_float, .neg => |tag| {
            const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
            const operand = try cg.resolveInst(un_op);
            const ty = cg.typeOfIndex(inst);

            if (ty.zigTypeTag(zcu) == .vector) {
                return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
            }

            const float_ty: FloatType = .fromType(cg, ty);
            const result = switch (tag) {
                .sqrt => try cg.floatSqrt(float_ty, operand),
                .sin => try cg.floatSin(float_ty, operand),
                .cos => try cg.floatCos(float_ty, operand),
                .tan => try cg.floatTan(float_ty, operand),
                .exp => try cg.floatExp(float_ty, operand),
                .exp2 => try cg.floatExp2(float_ty, operand),
                .log => try cg.floatLog(float_ty, operand),
                .log2 => try cg.floatLog2(float_ty, operand),
                .log10 => try cg.floatLog10(float_ty, operand),
                .floor => try cg.floatFloor(float_ty, operand),
                .ceil => try cg.floatCeil(float_ty, operand),
                .round => try cg.floatRound(float_ty, operand),
                .trunc_float => try cg.floatTrunc(float_ty, operand),
                .neg => try cg.floatNeg(float_ty, operand),
                else => unreachable,
            };

            try cg.finishAir(inst, result, &.{un_op});
        },

        .cmp_eq => cg.airCmp(inst, .eq),
        .cmp_gte => cg.airCmp(inst, .gte),
        .cmp_gt => cg.airCmp(inst, .gt),
        .cmp_lte => cg.airCmp(inst, .lte),
        .cmp_lt => cg.airCmp(inst, .lt),
        .cmp_neq => cg.airCmp(inst, .neq),

        .cmp_vector => cg.airCmpVector(inst),
        .cmp_lte_errors_len => cg.airCmpLteErrorsLen(inst),

        .array_elem_val => cg.airArrayElemVal(inst),
        .array_to_slice => cg.airArrayToSlice(inst),
        .alloc => cg.airAlloc(inst),
        .arg => cg.airArg(inst),
        .block => cg.airBlock(inst),
        .trap => cg.airTrap(inst),
        .unreach => cg.airUnreachable(inst),
        .breakpoint => cg.airBreakpoint(inst),
        .br => cg.airBr(inst),
        .repeat => cg.airRepeat(inst),
        .switch_dispatch => cg.airSwitchDispatch(inst),
        .cond_br => cg.airCondBr(inst),

        .@"try" => cg.airTry(inst),
        .try_cold => cg.airTry(inst),
        .try_ptr => cg.airTryPtr(inst),
        .try_ptr_cold => cg.airTryPtr(inst),

        .dbg_stmt => cg.airDbgStmt(inst),
        .dbg_empty_stmt => try cg.finishAir(inst, .none, &.{}),
        .dbg_inline_block => cg.airDbgInlineBlock(inst),
        .dbg_var_ptr => cg.airDbgVar(inst, .local_var, true),
        .dbg_var_val => cg.airDbgVar(inst, .local_var, false),
        .dbg_arg_inline => cg.airDbgVar(inst, .arg, false),

        .call => cg.airCall(inst, .auto),
        .call_always_tail => cg.airCall(inst, .always_tail),
        .call_never_tail => cg.airCall(inst, .never_tail),
        .call_never_inline => cg.airCall(inst, .never_inline),

        .is_err => cg.airIsErr(inst, .i32_ne, .value),
        .is_non_err => cg.airIsErr(inst, .i32_eq, .value),
        .is_err_ptr => cg.airIsErr(inst, .i32_ne, .ptr),
        .is_non_err_ptr => cg.airIsErr(inst, .i32_eq, .ptr),

        .is_null => cg.airIsNull(inst, .i32_eq, .value),
        .is_non_null => cg.airIsNull(inst, .i32_ne, .value),
        .is_null_ptr => cg.airIsNull(inst, .i32_eq, .ptr),
        .is_non_null_ptr => cg.airIsNull(inst, .i32_ne, .ptr),

        .load => cg.airLoad(inst),
        .loop => cg.airLoop(inst),
        .memset => cg.airMemset(inst, false),
        .memset_safe => cg.airMemset(inst, true),
        .optional_payload => cg.airOptionalPayload(inst),
        .optional_payload_ptr => cg.airOptionalPayloadPtr(inst),
        .optional_payload_ptr_set => cg.airOptionalPayloadPtrSet(inst),
        .ptr_add => cg.airPtrBinOp(inst, .add),
        .ptr_sub => cg.airPtrBinOp(inst, .sub),
        .ptr_elem_ptr => cg.airPtrElemPtr(inst),
        .ptr_elem_val => cg.airPtrElemVal(inst),
        .ret => cg.airRet(inst),
        .ret_safe => cg.airRet(inst), // TODO
        .ret_ptr => cg.airRetPtr(inst),
        .ret_load => cg.airRetLoad(inst),
        .splat => cg.airSplat(inst),
        .select => cg.airSelect(inst),
        .shuffle_one => cg.airShuffleOne(inst),
        .shuffle_two => cg.airShuffleTwo(inst),
        .reduce => cg.airReduce(inst),
        .aggregate_init => cg.airAggregateInit(inst),
        .union_init => cg.airUnionInit(inst),
        .prefetch => cg.airPrefetch(inst),

        .slice => cg.airSlice(inst),
        .slice_len => cg.airSliceLen(inst),
        .slice_elem_val => cg.airSliceElemVal(inst),
        .slice_elem_ptr => cg.airSliceElemPtr(inst),
        .slice_ptr => cg.airSlicePtr(inst),
        .ptr_slice_len_ptr => cg.airPtrSliceFieldPtr(inst, cg.ptrSize()),
        .ptr_slice_ptr_ptr => cg.airPtrSliceFieldPtr(inst, 0),
        .store => cg.airStore(inst, false),
        .store_safe => cg.airStore(inst, true),

        .set_union_tag => cg.airSetUnionTag(inst),
        .get_union_tag => cg.airGetUnionTag(inst),
        .struct_field_ptr => cg.airStructFieldPtr(inst),
        .struct_field_ptr_index_0 => cg.airStructFieldPtrIndex(inst, 0),
        .struct_field_ptr_index_1 => cg.airStructFieldPtrIndex(inst, 1),
        .struct_field_ptr_index_2 => cg.airStructFieldPtrIndex(inst, 2),
        .struct_field_ptr_index_3 => cg.airStructFieldPtrIndex(inst, 3),
        .struct_field_val => cg.airStructFieldVal(inst),
        .field_parent_ptr => cg.airFieldParentPtr(inst),

        .switch_br => cg.airSwitchBr(inst, false),
        .loop_switch_br => cg.airSwitchBr(inst, true),

        .wrap_optional => cg.airWrapOptional(inst),
        .unwrap_errunion_payload => cg.airUnwrapErrUnionPayload(inst, false),
        .unwrap_errunion_payload_ptr => cg.airUnwrapErrUnionPayload(inst, true),
        .unwrap_errunion_err => cg.airUnwrapErrUnionError(inst, false),
        .unwrap_errunion_err_ptr => cg.airUnwrapErrUnionError(inst, true),
        .wrap_errunion_payload => cg.airWrapErrUnionPayload(inst),
        .wrap_errunion_err => cg.airWrapErrUnionErr(inst),
        .errunion_payload_ptr_set => cg.airErrUnionPayloadPtrSet(inst),
        .error_name => cg.airErrorName(inst),

        .unwrap_restricted => cg.airUnwrapRestricted(inst, false),
        .unwrap_restricted_safe => cg.airUnwrapRestricted(inst, true),

        .wasm_memory_size => cg.airWasmMemorySize(inst),
        .wasm_memory_grow => cg.airWasmMemoryGrow(inst),

        .memcpy, .memmove => cg.airMemcpy(inst),

        .ret_addr => cg.airRetAddr(inst),
        .tag_name => cg.airTagName(inst),

        .error_set_has_value => cg.airErrorSetHasValue(inst),
        .frame_addr => cg.airFrameAddress(inst),

        .runtime_nav_ptr => cg.airRuntimeNavPtr(inst),

        .assembly => cg.airAsm(inst),

        .err_return_trace,
        .set_err_return_trace,
        .save_err_return_trace_index,
        .is_named_enum_value,
        .addrspace_cast,
        .c_va_arg,
        .c_va_copy,
        .c_va_end,
        .c_va_start,
        => |tag| return cg.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}),

        .atomic_load => cg.airAtomicLoad(inst),
        .atomic_store_unordered,
        .atomic_store_monotonic,
        .atomic_store_release,
        .atomic_store_seq_cst,
        // in WebAssembly, all atomic instructions are sequentially ordered.
        => cg.airAtomicStore(inst),
        .atomic_rmw => cg.airAtomicRmw(inst),
        .cmpxchg_weak => cg.airCmpxchg(inst),
        .cmpxchg_strong => cg.airCmpxchg(inst),

        .add_optimized,
        .sub_optimized,
        .mul_optimized,
        .div_float_optimized,
        .div_trunc_optimized,
        .div_floor_optimized,
        .div_exact_optimized,
        .rem_optimized,
        .mod_optimized,
        .neg_optimized,
        .cmp_lt_optimized,
        .cmp_lte_optimized,
        .cmp_eq_optimized,
        .cmp_gte_optimized,
        .cmp_gt_optimized,
        .cmp_neq_optimized,
        .cmp_vector_optimized,
        .reduce_optimized,
        .int_from_float_optimized,
        => return cg.fail("TODO implement optimized float mode", .{}),

        .add_safe,
        .sub_safe,
        .mul_safe,
        .intcast_safe,
        .int_from_float_safe,
        .int_from_float_optimized_safe,
        => return cg.fail("TODO implement safety_checked_instructions", .{}),

        .work_item_id,
        .work_group_size,
        .work_group_id,
        => unreachable,
    };
}

fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ip = &zcu.intern_pool;

    for (body) |inst| {
        if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip)) {
            continue;
        }
        const old_bookkeeping_value = cg.air_bookkeeping;
        try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, 1);
        try cg.genInst(inst);

        if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) {
            std.debug.panic("Missing call to `finishAir` in AIR instruction %{d} ('{t}')", .{
                inst,
                cg.air.instructions.items(.tag)[@intFromEnum(inst)],
            });
        }
    }
}

fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
    const operand = try cg.resolveInst(un_op);
    const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
    const ret_ty = Type.fromInterned(fn_info.return_type);

    // result must be stored in the stack and we return a pointer
    // to the stack instead
    if (cg.return_value != .none) {
        try cg.store(cg.return_value, operand, ret_ty, 0);
    } else if (fn_info.cc == .wasm_mvp and ret_ty.hasRuntimeBits(zcu)) {
        switch (abi.classifyType(ret_ty, zcu)) {
            .direct => |scalar_type| {
                assert(!abi.lowerAsDoubleI64(scalar_type, zcu));
                if (!isByRef(ret_ty, zcu, cg.target)) {
                    try cg.emitWValue(operand);
                } else {
                    _ = try cg.load(operand, scalar_type, 0);
                }
            },
            .indirect => unreachable,
        }
    } else {
        if (!ret_ty.hasRuntimeBits(zcu) and ret_ty.isError(zcu)) {
            try cg.addImm32(0);
        } else {
            try cg.emitWValue(operand);
        }
    }
    try cg.restoreStackPointer();
    try cg.addTag(.@"return");

    return cg.finishAir(inst, .none, &.{un_op});
}

fn airRetPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const child_type = cg.typeOfIndex(inst).childType(zcu);

    const result = result: {
        if (!child_type.hasRuntimeBits(zcu)) {
            break :result try cg.allocStack(Type.usize); // create pointer to void
        }

        const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
        if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) {
            break :result cg.return_value;
        }

        break :result try cg.allocStackPtr(inst);
    };

    return cg.finishAir(inst, result, &.{});
}

fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
    const operand = try cg.resolveInst(un_op);
    const ret_ty = cg.typeOf(un_op).childType(zcu);

    const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
    if (!ret_ty.hasRuntimeBits(zcu)) {
        if (ret_ty.isError(zcu)) {
            try cg.addImm32(0);
        }
    } else if (!firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) {
        // leave on the stack
        _ = try cg.load(operand, ret_ty, 0);
    }

    try cg.restoreStackPointer();
    try cg.addTag(.@"return");
    return cg.finishAir(inst, .none, &.{un_op});
}

fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) InnerError!void {
    if (modifier == .always_tail) return cg.fail("TODO implement tail calls for wasm", .{});
    const call = cg.air.unwrapCall(inst);
    const args = call.args;
    const ty = cg.typeOf(call.callee);

    const pt = cg.pt;
    const zcu = pt.zcu;
    const ip = &zcu.intern_pool;
    const fn_ty = switch (ty.zigTypeTag(zcu)) {
        .@"fn" => ty,
        .pointer => ty.childType(zcu),
        else => unreachable,
    };
    const ret_ty = fn_ty.fnReturnType(zcu);
    const fn_info = zcu.typeToFunc(fn_ty).?;
    const first_param_sret = firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target);

    const callee: ?InternPool.Nav.Index = blk: {
        const func_val: Value = .fromInterned(call.callee.toInterned() orelse break :blk null);

        switch (ip.indexToKey(func_val.toIntern())) {
            inline .func, .@"extern" => |x| break :blk x.owner_nav,
            .ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) {
                .nav => |nav| break :blk nav,
                else => {},
            },
            else => {},
        }
        return cg.fail("unable to lower callee to a function index", .{});
    };

    const sret: WValue = if (first_param_sret) blk: {
        const sret_local = try cg.allocStack(ret_ty);
        try cg.lowerToStack(sret_local);
        break :blk sret_local;
    } else .none;

    for (args) |arg| {
        const arg_val = try cg.resolveInst(arg);

        const arg_ty = cg.typeOf(arg);
        if (!arg_ty.hasRuntimeBits(zcu)) continue;

        try cg.lowerArg(zcu.typeToFunc(fn_ty).?.cc, arg_ty, arg_val);
    }

    if (callee) |nav_index| {
        try cg.addInst(.{ .tag = .call_nav, .data = .{ .nav_index = nav_index } });
    } else {
        // in this case we call a function pointer
        // so load its value onto the stack
        assert(ty.zigTypeTag(zcu) == .pointer);
        const operand = try cg.resolveInst(call.callee);
        try cg.emitWValue(operand);

        try cg.mir_func_tys.put(cg.gpa, fn_ty.toIntern(), {});
        try cg.addInst(.{
            .tag = .call_indirect,
            .data = .{ .ip_index = fn_ty.toIntern() },
        });
    }

    const result_value = result_value: {
        if (!ret_ty.hasRuntimeBits(zcu) and !ret_ty.isError(zcu)) {
            break :result_value .none;
        } else if (first_param_sret) {
            break :result_value sret;
        } else if (zcu.typeToFunc(fn_ty).?.cc == .wasm_mvp) {
            switch (abi.classifyType(ret_ty, zcu)) {
                .direct => |scalar_type| {
                    assert(!abi.lowerAsDoubleI64(scalar_type, zcu));
                    if (!isByRef(ret_ty, zcu, cg.target)) {
                        const result_local = try cg.allocLocal(ret_ty);
                        try cg.addLocal(.local_set, result_local.local.value);
                        break :result_value result_local;
                    } else {
                        const result_local = try cg.allocLocal(ret_ty);
                        try cg.addLocal(.local_set, result_local.local.value);
                        const result = try cg.allocStack(ret_ty);
                        try cg.store(result, result_local, scalar_type, 0);
                        break :result_value result;
                    }
                },
                .indirect => unreachable,
            }
        } else {
            const result_local = try cg.allocLocal(ret_ty);
            try cg.addLocal(.local_set, result_local.local.value);
            break :result_value result_local;
        }
    };

    var bt = cg.liveness.iterateBigTomb(inst);
    cg.feed(&bt, call.callee);
    for (args) |arg| cg.feed(&bt, arg);
    return cg.finishAirResult(inst, result_value);
}

fn airAlloc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const value = try cg.allocStackPtr(inst);
    return cg.finishAir(inst, value, &.{});
}

fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;

    const lhs = try cg.resolveInst(bin_op.lhs);
    const rhs = try cg.resolveInst(bin_op.rhs);
    const ptr_ty = cg.typeOf(bin_op.lhs);
    const ptr_info = ptr_ty.ptrInfo(zcu);
    const ty = ptr_ty.childType(zcu);

    if (!safety and bin_op.rhs == .undef) {
        return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
    }

    assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store

    try cg.store(lhs, rhs, ty, 0);

    return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
}

fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void {
    assert(!(lhs != .stack and rhs == .stack));
    const pt = cg.pt;
    const zcu = pt.zcu;
    const abi_size = ty.abiSize(zcu);

    if (!ty.hasRuntimeBits(zcu)) return;

    if (isByRef(ty, zcu, cg.target)) {
        return cg.memcpy(lhs, rhs, .{ .imm32 = @intCast(abi_size) });
    }

    if (ty.zigTypeTag(zcu) == .vector) {
        try cg.emitWValue(lhs);
        try cg.lowerToStack(rhs);
        // TODO: Add helper functions for simd opcodes
        const extra_index: u32 = @intCast(cg.mir_extra.items.len);
        // stores as := opcode, offset, alignment (opcode::memarg)
        try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
            @intFromEnum(std.wasm.SimdOpcode.v128_store),
            offset + lhs.offset(),
            @intCast(ty.abiAlignment(zcu).toByteUnits() orelse 0),
        });
        return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
    }

    const store_opcode: Mir.Inst.Tag = opcode: {
        if (ty.isAnyFloat()) {
            break :opcode switch (abi_size) {
                2 => .i32_store16,
                4 => .f32_store,
                8 => .f64_store,
                else => unreachable,
            };
        } else {
            break :opcode switch (abi_size) {
                1 => .i32_store8,
                2 => .i32_store16,
                4 => .i32_store,
                8 => .i64_store,
                else => unreachable,
            };
        }
    };

    try cg.emitWValue(lhs);
    try cg.lowerToStack(rhs);

    try cg.addMemArg(
        store_opcode,
        .{
            .offset = offset + lhs.offset(),
            .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
        },
    );
}

fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
    const operand = try cg.resolveInst(ty_op.operand);
    const ty = ty_op.ty.toType();
    const ptr_ty = cg.typeOf(ty_op.operand);
    const ptr_info = ptr_ty.ptrInfo(zcu);

    if (!ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{ty_op.operand});

    assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load

    const result = result: {
        if (isByRef(ty, zcu, cg.target)) {
            const new_local = try cg.allocStack(ty);
            try cg.store(new_local, operand, ty, 0);
            break :result new_local;
        }

        const loaded = try cg.load(operand, ty, 0);
        const ty_size = ty.abiSize(zcu);
        if (ty.isAbiInt(zcu) and ty_size * 8 > ty.bitSize(zcu)) {
            const int_info = ty.intInfo(zcu);
            const loaded_int_ty: IntType = .{
                .is_signed = int_info.signedness == .signed,
                .bits = @intCast(ty_size * 8),
            };
            break :result try cg.intTrunc(.fromType(cg, ty), loaded_int_ty, loaded);
        } else {
            break :result loaded;
        }
    };
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

/// Loads an operand from the linear memory section.
/// NOTE: Leaves the value on the stack.
fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
    const zcu = cg.pt.zcu;
    // load local's value from memory by its stack position
    try cg.emitWValue(operand);

    if (ty.zigTypeTag(zcu) == .vector) {
        // TODO: Add helper functions for simd opcodes
        const extra_index: u32 = @intCast(cg.mir_extra.items.len);
        // stores as := opcode, offset, alignment (opcode::memarg)
        try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
            @intFromEnum(std.wasm.SimdOpcode.v128_load),
            offset + operand.offset(),
            @intCast(ty.abiAlignment(zcu).toByteUnits().?),
        });
        try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
        return .stack;
    }

    const abi_size = ty.abiSize(zcu);
    const load_opcode: Mir.Inst.Tag = opcode: {
        if (ty.isAnyFloat()) {
            break :opcode switch (abi_size) {
                2 => .i32_load16_u,
                4 => .f32_load,
                8 => .f64_load,
                else => unreachable,
            };
        } else {
            const is_signed = if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness == .signed else false;
            break :opcode switch (abi_size) {
                1 => if (is_signed) .i32_load8_s else .i32_load8_u,
                2 => if (is_signed) .i32_load16_s else .i32_load16_u,
                4 => .i32_load,
                8 => .i64_load,
                else => unreachable,
            };
        }
    };

    try cg.addMemArg(
        load_opcode,
        .{
            .offset = offset + operand.offset(),
            .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
        },
    );

    return .stack;
}

fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const arg_index = cg.arg_index;
    const arg = cg.args[arg_index];
    const cc = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?.cc;
    const arg_ty = cg.typeOfIndex(inst);
    if (cc == .wasm_mvp) {
        switch (abi.classifyType(arg_ty, zcu)) {
            .direct => |scalar_ty| if (!abi.lowerAsDoubleI64(scalar_ty, zcu)) {
                cg.arg_index += 1;
            } else {
                cg.arg_index += 2;
                const result = try cg.allocStack(arg_ty);
                try cg.store(result, arg, Type.u64, 0);
                try cg.store(result, cg.args[arg_index + 1], Type.u64, 8);
                return cg.finishAir(inst, result, &.{});
            },
            .indirect => cg.arg_index += 1,
        }
    } else {
        cg.arg_index += 1;
    }

    return cg.finishAir(inst, arg, &.{});
}

const IntType = struct {
    is_signed: bool,
    bits: u16,

    const @"i32": IntType = .{ .is_signed = true, .bits = 32 };
    const @"i64": IntType = .{ .is_signed = true, .bits = 64 };
    const @"u32": IntType = .{ .is_signed = false, .bits = 32 };
    const @"u64": IntType = .{ .is_signed = false, .bits = 64 };

    // Adapted from x86_64 backend
    // Differ from Type.intInfo as it treats pointers/booleans/packed/enums/errors as integer
    fn fromType(cg: *CodeGen, ty: Type) IntType {
        const zcu = cg.pt.zcu;
        const ip = &zcu.intern_pool;
        var ty_index = ty.ip_index;
        while (true) switch (ip.indexToKey(ty_index)) {
            .int_type => |int_type| return .{ .is_signed = int_type.signedness == .signed, .bits = int_type.bits },
            .ptr_type => |ptr_type| return switch (ptr_type.flags.size) {
                .one, .many, .c => .{ .is_signed = false, .bits = cg.target.ptrBitWidth() },
                .slice => unreachable,
            },
            .opt_type => |opt_child| return if (!Type.fromInterned(opt_child).hasRuntimeBits(zcu))
                .{ .is_signed = false, .bits = 1 }
            else switch (ip.indexToKey(opt_child)) {
                .ptr_type => |ptr_type| switch (ptr_type.flags.size) {
                    .one, .many => switch (ptr_type.flags.is_allowzero) {
                        false => .{ .is_signed = false, .bits = cg.target.ptrBitWidth() },
                        true => unreachable,
                    },
                    .slice, .c => unreachable,
                },
                else => unreachable,
            },
            .error_union_type => |error_union_type| return if (!Type.fromInterned(error_union_type.payload_type)
                .hasRuntimeBits(zcu)) .{ .is_signed = false, .bits = zcu.errorSetBits() } else unreachable,
            .simple_type => |simple_type| return switch (simple_type) {
                .bool => .{ .is_signed = false, .bits = 1 },
                .anyerror => .{ .is_signed = false, .bits = zcu.errorSetBits() },
                .isize => .{ .is_signed = true, .bits = cg.target.ptrBitWidth() },
                .usize => .{ .is_signed = false, .bits = cg.target.ptrBitWidth() },
                .c_char => .{ .is_signed = cg.target.cCharSignedness() == .signed, .bits = cg.target.cTypeBitSize(.char) },
                .c_short => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.short) },
                .c_ushort => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.short) },
                .c_int => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.int) },
                .c_uint => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.int) },
                .c_long => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.long) },
                .c_ulong => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.long) },
                .c_longlong => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.longlong) },
                .c_ulonglong => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.longlong) },
                .f16, .f32, .f64, .f80, .f128, .c_longdouble => unreachable,
                .anyopaque, .void, .type, .comptime_int, .comptime_float, .noreturn, .null, .undefined, .enum_literal, .adhoc_inferred_error_set, .generic_poison => unreachable,
            },
            .struct_type => {
                const loaded_struct = ip.loadStructType(ty_index);
                switch (loaded_struct.layout) {
                    .auto, .@"extern" => unreachable,
                    .@"packed" => ty_index = loaded_struct.packed_backing_int_type,
                }
            },
            .union_type => return switch (ip.loadUnionType(ty_index).layout) {
                .auto, .@"extern" => unreachable,
                .@"packed" => .{ .is_signed = false, .bits = @intCast(ty.bitSize(zcu)) },
            },
            .enum_type => ty_index = ip.loadEnumType(ty_index).int_tag_type,
            .error_set_type, .inferred_error_set_type => return .{ .is_signed = false, .bits = zcu.errorSetBits() },
            else => unreachable,
        };
    }
};

fn intBackingBits(cg: *CodeGen, bits: u16) u16 {
    return switch (bits) {
        0 => unreachable,
        1...32 => 32,
        33...64 => 64,
        else => std.zig.target.intByteSize(cg.target, bits) * 8,
    };
}

fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i32_add);
            return .stack;
        },
        33...64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i64_add);
            return .stack;
        },
        65...128 => {
            const result = try cg.allocStack(Type.u128);

            var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
            defer lhs_lsb.free(cg);
            var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
            defer rhs_lsb.free(cg);
            var op_lsb = try (try cg.intAdd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
            defer op_lsb.free(cg);

            const lhs_msb = try cg.load(lhs, Type.u64, 8);
            const rhs_msb = try cg.load(rhs, Type.u64, 8);
            const op_msb = try cg.intAdd(.u64, lhs_msb, rhs_msb);

            const lt = try cg.intCmp(.u64, .lt, op_lsb, rhs_lsb);
            const tmp = try cg.intCast(.u64, .u32, lt);
            var tmp_op = try (try cg.intAdd(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
            defer tmp_op.free(cg);

            try cg.store(result, op_lsb, Type.u64, 0);
            try cg.store(result, tmp_op, Type.u64, 8);
            return result;
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            try cg.addImm32(@intFromBool(ty.is_signed));
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__addo_limb64);
            try cg.addTag(.drop);
            return result;
        },
    }
}

fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i32_sub);
            return .stack;
        },
        33...64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i64_sub);
            return .stack;
        },
        65...128 => {
            const result = try cg.allocStack(Type.u128);

            var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
            defer lhs_lsb.free(cg);
            var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
            defer rhs_lsb.free(cg);
            var op_lsb = try (try cg.intSub(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
            defer op_lsb.free(cg);

            const lhs_msb = try cg.load(lhs, Type.u64, 8);
            const rhs_msb = try cg.load(rhs, Type.u64, 8);
            const op_msb = try cg.intSub(.u64, lhs_msb, rhs_msb);

            const lt = try cg.intCmp(.u64, .lt, lhs_lsb, rhs_lsb);
            const tmp = try cg.intCast(.u64, .u32, lt);
            var tmp_op = try (try cg.intSub(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
            defer tmp_op.free(cg);

            try cg.store(result, op_lsb, Type.u64, 0);
            try cg.store(result, tmp_op, Type.u64, 8);
            return result;
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            try cg.addImm32(@intFromBool(ty.is_signed));
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__subo_limb64);
            try cg.addTag(.drop);

            return result;
        },
    }
}

fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i32_mul);
            return .stack;
        },
        33...64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i64_mul);
            return .stack;
        },
        65...128 => return cg.callIntrinsic(.__multi3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs }),
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            try cg.addImm32(@intFromBool(ty.is_signed));
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__mulo_limb64);
            try cg.addTag(.drop);

            return result;
        },
    }
}

fn intDiv(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(if (ty.is_signed) .i32_div_s else .i32_div_u);
            return .stack;
        },
        33...64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(if (ty.is_signed) .i64_div_s else .i64_div_u);
            return .stack;
        },
        65...128 => {
            if (ty.is_signed) {
                return cg.callIntrinsic(.__divti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
            } else {
                return cg.callIntrinsic(.__udivti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
            }
        },
        else => {
            const result = try cg.allocInt(ty);
            const bits = cg.intBackingBits(ty.bits);
            var tmp = try cg.allocInt(.{ .is_signed = false, .bits = bits * 2 });
            if (ty.is_signed) {
                _ = try cg.callIntrinsic(
                    .__divei5,
                    &.{ .usize_type, .usize_type, .usize_type, .usize_type, .usize_type },
                    .void,
                    &.{ result, lhs, rhs, tmp, .{ .imm32 = ty.bits } },
                );
            } else {
                _ = try cg.callIntrinsic(
                    .__udivei5,
                    &.{ .usize_type, .usize_type, .usize_type, .usize_type, .usize_type },
                    .void,
                    &.{ result, lhs, rhs, tmp, .{ .imm32 = ty.bits } },
                );
            }
            tmp.free(cg);
            return result;
        },
    }
}

fn intDivFloor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    if (!ty.is_signed) {
        return cg.intDiv(ty, lhs, rhs);
    }

    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.i32);
            defer q.free(cg);

            const zero: WValue = .{ .imm32 = 0 };

            const r = try cg.intRem(ty, lhs, rhs);
            var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.i32);
            defer r_nonzero.free(cg);

            const sign_xor = try cg.intXor(ty, lhs, rhs);
            var sign_diff = try (try cg.intCmp(ty, .lt, sign_xor, zero)).toLocal(cg, Type.i32);
            defer sign_diff.free(cg);

            try cg.emitWValue(q);
            const need_adjust = try cg.intAnd(.u32, r_nonzero, sign_diff);
            try cg.emitWValue(need_adjust);
            try cg.addTag(.i32_sub);
            return .stack;
        },
        33...64 => {
            var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.i64);
            defer q.free(cg);

            const zero: WValue = .{ .imm64 = 0 };

            const r = try cg.intRem(ty, lhs, rhs);
            var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.i32);
            defer r_nonzero.free(cg);

            const sign_xor = try cg.intXor(ty, lhs, rhs);
            var sign_diff = try (try cg.intCmp(ty, .lt, sign_xor, zero)).toLocal(cg, Type.i32);
            defer sign_diff.free(cg);

            try cg.emitWValue(q);
            const need_adjust = try cg.intAnd(.u32, r_nonzero, sign_diff);
            try cg.emitWValue(need_adjust);
            try cg.addTag(.i64_extend_i32_u);
            try cg.addTag(.i64_sub);
            return .stack;
        },
        else => {
            const q = try cg.intDiv(ty, lhs, rhs);

            const zero = try cg.intZeroValue(ty);

            const r = try cg.intRem(ty, lhs, rhs);
            _ = try cg.intCmp(ty, .neq, r, zero);

            const sign_xor = try cg.intXor(ty, lhs, rhs);
            _ = try cg.intCmp(ty, .lt, sign_xor, zero);
            var adjust = try (try cg.intAnd(.u32, .stack, .stack)).toLocal(cg, Type.u32);

            const adjust_bigint = try cg.intCast(ty, .u32, adjust);
            adjust.free(cg);
            return try cg.intSub(ty, q, adjust_bigint);
        },
    }
}

fn intRem(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(if (ty.is_signed) .i32_rem_s else .i32_rem_u);
            return .stack;
        },
        33...64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(if (ty.is_signed) .i64_rem_s else .i64_rem_u);
            return .stack;
        },
        65...128 => {
            if (ty.is_signed) {
                return cg.callIntrinsic(.__modti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
            } else {
                return cg.callIntrinsic(.__umodti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
            }
        },
        else => {
            const result = try cg.allocInt(ty);
            const bits = cg.intBackingBits(ty.bits);
            var tmp = try cg.allocInt(.{ .is_signed = false, .bits = bits * 2 });
            if (ty.is_signed) {
                _ = try cg.callIntrinsic(
                    .__modei5,
                    &.{ .usize_type, .usize_type, .usize_type, .usize_type, .usize_type },
                    .void,
                    &.{ result, lhs, rhs, tmp, .{ .imm32 = ty.bits } },
                );
            } else {
                _ = try cg.callIntrinsic(
                    .__umodei5,
                    &.{ .usize_type, .usize_type, .usize_type, .usize_type, .usize_type },
                    .void,
                    &.{ result, lhs, rhs, tmp, .{ .imm32 = ty.bits } },
                );
            }
            tmp.free(cg);
            return result;
        },
    }
}

fn intMod(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    if (!ty.is_signed) {
        return cg.intRem(ty, lhs, rhs);
    }

    // mod_s(a, b) = rem_s(rem_s(a, b) + b, b)
    const rem = try cg.intRem(ty, lhs, rhs);
    const sum = try cg.intAdd(ty, rem, rhs);
    return cg.intRem(ty, sum, rhs);
}

fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i32_and);
            return .stack;
        },
        33...64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i64_and);
            return .stack;
        },
        65...128 => {
            const result = try cg.allocStack(Type.u128);

            const lhs_lsb = try cg.load(lhs, Type.u64, 0);
            const rhs_lsb = try cg.load(rhs, Type.u64, 0);
            const and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
            try cg.store(result, and_lsb, Type.u64, 0);

            const lhs_msb = try cg.load(lhs, Type.u64, 8);
            const rhs_msb = try cg.load(rhs, Type.u64, 8);
            const and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
            try cg.store(result, and_msb, Type.u64, 8);

            return result;
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__and_limb64);

            return result;
        },
    }
}

fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i32_or);
            return .stack;
        },
        33...64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i64_or);
            return .stack;
        },
        65...128 => {
            const result = try cg.allocStack(Type.u128);

            const lhs_lsb = try cg.load(lhs, Type.u64, 0);
            const rhs_lsb = try cg.load(rhs, Type.u64, 0);
            const or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
            try cg.store(result, or_lsb, Type.u64, 0);

            const lhs_msb = try cg.load(lhs, Type.u64, 8);
            const rhs_msb = try cg.load(rhs, Type.u64, 8);
            const or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
            try cg.store(result, or_msb, Type.u64, 8);

            return result;
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__or_limb64);

            return result;
        },
    }
}

fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i32_xor);
            return .stack;
        },
        33...64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i64_xor);
            return .stack;
        },
        65...128 => {
            const result = try cg.allocStack(Type.u128);

            const lhs_lsb = try cg.load(lhs, Type.u64, 0);
            const rhs_lsb = try cg.load(rhs, Type.u64, 0);
            const xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
            try cg.store(result, xor_lsb, Type.u64, 0);

            const lhs_msb = try cg.load(lhs, Type.u64, 8);
            const rhs_msb = try cg.load(rhs, Type.u64, 8);
            const xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
            try cg.store(result, xor_msb, Type.u64, 8);

            return result;
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__xor_limb64);

            return result;
        },
    }
}

fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1 => {
            try cg.emitWValue(operand);
            if (ty.is_signed) {
                try cg.addImm32(~@as(u32, 0));
                try cg.addTag(.i32_xor);
            } else {
                try cg.addTag(.i32_eqz);
            }
            return .stack;
        },
        2...32 => {
            const mask: u32 = if (ty.is_signed)
                ~@as(u32, 0)
            else
                ~@as(u32, 0) >> @intCast(32 - ty.bits);
            try cg.emitWValue(operand);
            try cg.addImm32(mask);
            try cg.addTag(.i32_xor);
            return .stack;
        },
        33...64 => {
            const mask: u64 = if (ty.is_signed)
                ~@as(u64, 0)
            else
                ~@as(u64, 0) >> @intCast(64 - ty.bits);
            try cg.emitWValue(operand);
            try cg.addImm64(mask);
            try cg.addTag(.i64_xor);
            return .stack;
        },
        65...128 => {
            const result = try cg.allocStack(Type.u128);

            try cg.emitWValue(result);
            _ = try cg.load(operand, Type.u64, 0);
            try cg.addImm64(~@as(u64, 0));
            try cg.addTag(.i64_xor);
            try cg.store(.stack, .stack, Type.u64, result.offset());

            try cg.emitWValue(result);
            _ = try cg.load(operand, Type.u64, 8);
            const high_mask: u64 = if (ty.is_signed)
                ~@as(u64, 0)
            else
                ~@as(u64, 0) >> @intCast(128 - ty.bits);
            try cg.addImm64(high_mask);
            try cg.addTag(.i64_xor);
            try cg.store(.stack, .stack, Type.u64, result.offset() + 8);

            return result;
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(operand);
            try cg.addImm32(@intFromBool(ty.is_signed));
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__not_limb64);

            return result;
        },
    }
}

// rhs is a shift count, pointing to i32 value
// does not perform wrapping, padding bits does not satisfy invariant
fn intShl(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i32_shl);
            return .stack;
        },
        33...64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i64_extend_i32_u);
            try cg.addTag(.i64_shl);
            return .stack;
        },
        65...128 => return cg.callIntrinsic(.__ashlti3, &.{ .i128_type, .i32_type }, Type.i128, &.{ lhs, rhs }),
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            try cg.addImm32(@intFromBool(ty.is_signed));
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__shlo_limb64);
            try cg.addTag(.drop);

            return result;
        },
    }
}

// rhs is a shift count, pointing to i32 value
fn intShr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(if (ty.is_signed) .i32_shr_s else .i32_shr_u);
            return .stack;
        },
        33...64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.i64_extend_i32_u);
            try cg.addTag(if (ty.is_signed) .i64_shr_s else .i64_shr_u);
            return .stack;
        },
        65...128 => {
            if (ty.is_signed) {
                return cg.callIntrinsic(.__ashrti3, &.{ .i128_type, .i32_type }, Type.i128, &.{ lhs, rhs });
            } else {
                return cg.callIntrinsic(.__lshrti3, &.{ .i128_type, .i32_type }, Type.i128, &.{ lhs, rhs });
            }
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            try cg.addImm32(@intFromBool(ty.is_signed));
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__shr_limb64);

            return result;
        },
    }
}

fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
    if (!ty.is_signed) return operand;
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            try cg.emitWValue(operand);
            try cg.addImm32(31);
            try cg.addTag(.i32_shr_s);

            var mask = try cg.allocLocal(Type.i32);
            defer mask.free(cg);
            try cg.addLocal(.local_tee, mask.local.value);

            try cg.emitWValue(operand);
            try cg.addTag(.i32_xor);
            try cg.emitWValue(mask);
            try cg.addTag(.i32_sub);
            return .stack;
        },
        33...64 => {
            try cg.emitWValue(operand);
            try cg.addImm64(63);
            try cg.addTag(.i64_shr_s);

            var mask = try cg.allocLocal(Type.i64);
            defer mask.free(cg);
            try cg.addLocal(.local_tee, mask.local.value);

            try cg.emitWValue(operand);
            try cg.addTag(.i64_xor);
            try cg.emitWValue(mask);
            try cg.addTag(.i64_sub);
            return .stack;
        },
        65...128 => {
            const u128_ty: IntType = .{ .is_signed = false, .bits = 128 };

            const mask = try cg.allocStack(Type.u128);
            try cg.emitWValue(mask);
            try cg.emitWValue(mask);

            _ = try cg.load(operand, Type.u64, 8);
            try cg.addImm64(63);
            try cg.addTag(.i64_shr_s);

            var tmp = try cg.allocLocal(Type.u64);
            defer tmp.free(cg);
            try cg.addLocal(.local_tee, tmp.local.value);
            try cg.store(.stack, .stack, Type.u64, mask.offset() + 0);
            try cg.emitWValue(tmp);
            try cg.store(.stack, .stack, Type.u64, mask.offset() + 8);

            const a = try cg.intXor(u128_ty, operand, mask);
            const b = try cg.intSub(u128_ty, a, mask);
            return b;
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(operand);
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__abs_limb64);

            return result;
        },
    }
}

fn intMax(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    try cg.lowerToStack(lhs);
    try cg.lowerToStack(rhs);
    _ = try cg.intCmp(ty, .gt, lhs, rhs);
    try cg.addTag(.select);
    return .stack;
}

fn intMin(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    try cg.lowerToStack(lhs);
    try cg.lowerToStack(rhs);
    _ = try cg.intCmp(ty, .lt, lhs, rhs);
    try cg.addTag(.select);
    return .stack;
}

fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            if (ty.is_signed and ty.bits < 32) {
                const mask: u32 = ~@as(u32, 0) >> @intCast(32 - ty.bits);
                _ = try cg.intAnd(.u32, operand, .{ .imm32 = mask });
            } else {
                try cg.emitWValue(operand);
            }
            try cg.addTag(.i32_clz);
            if (ty.bits < 32) {
                try cg.addImm32(32 - ty.bits);
                try cg.addTag(.i32_sub);
            }
            return .stack;
        },
        33...64 => {
            if (ty.is_signed and ty.bits < 64) {
                const mask: u64 = ~@as(u64, 0) >> @intCast(64 - ty.bits);
                _ = try cg.intAnd(.u64, operand, .{ .imm64 = mask });
            } else {
                try cg.emitWValue(operand);
            }
            try cg.addTag(.i64_clz);
            try cg.addTag(.i32_wrap_i64);
            if (ty.bits < 64) {
                try cg.addImm32(64 - ty.bits);
                try cg.addTag(.i32_sub);
            }
            return .stack;
        },
        65...128 => {
            var msb = try (try cg.load(operand, Type.u64, 8)).toLocal(cg, Type.u64);
            defer msb.free(cg);

            try cg.emitWValue(msb);
            try cg.addTag(.i64_clz);
            _ = try cg.load(operand, Type.u64, 0);
            try cg.addTag(.i64_clz);
            try cg.emitWValue(.{ .imm64 = 64 });
            try cg.addTag(.i64_add);
            _ = try cg.intCmp(.u64, .neq, msb, .{ .imm64 = 0 });
            try cg.addTag(.select);
            try cg.addTag(.i32_wrap_i64);
            return .stack;
        },
        else => {
            try cg.lowerToStack(operand);
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__clz_limb64);

            return .stack;
        },
    }
}

fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            if (ty.bits < 32) {
                _ = try cg.intOr(.u32, operand, .{ .imm32 = @as(u32, 1) << @intCast(ty.bits) });
            } else {
                try cg.emitWValue(operand);
            }
            try cg.addTag(.i32_ctz);
            return .stack;
        },
        33...64 => {
            if (ty.bits < 64) {
                _ = try cg.intOr(.u64, operand, .{ .imm64 = @as(u64, 1) << @intCast(ty.bits) });
            } else {
                try cg.emitWValue(operand);
            }
            try cg.addTag(.i64_ctz);
            try cg.addTag(.i32_wrap_i64);
            return .stack;
        },
        65...128 => {
            var lsb = try (try cg.load(operand, Type.u64, 0)).toLocal(cg, Type.u64);
            defer lsb.free(cg);

            try cg.emitWValue(lsb);
            try cg.addTag(.i64_ctz);

            _ = try cg.load(operand, Type.u64, 8);
            if (ty.bits < 128) {
                try cg.addImm64(@as(u64, 1) << @intCast(ty.bits - 64));
                try cg.addTag(.i64_or);
            }
            try cg.addTag(.i64_ctz);
            try cg.addImm64(64);
            try cg.addTag(.i64_add);
            _ = try cg.intCmp(.u64, .neq, lsb, .{ .imm64 = 0 });
            try cg.addTag(.select);
            try cg.addTag(.i32_wrap_i64);
            return .stack;
        },
        else => {
            try cg.lowerToStack(operand);
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__ctz_limb64);

            return .stack;
        },
    }
}

fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            try cg.emitWValue(operand);
            if (ty.is_signed and ty.bits < 32) {
                try cg.addImm32(32 - ty.bits);
                try cg.addTag(.i32_shl);
            }
            try cg.addTag(.i32_popcnt);
            return .stack;
        },
        33...64 => {
            try cg.emitWValue(operand);
            if (ty.is_signed and ty.bits < 64) {
                try cg.addImm64(64 - ty.bits);
                try cg.addTag(.i64_shl);
            }
            try cg.addTag(.i64_popcnt);
            try cg.addTag(.i32_wrap_i64);
            return .stack;
        },
        65...128 => {
            _ = try cg.load(operand, Type.u64, 0);
            try cg.addTag(.i64_popcnt);
            _ = try cg.load(operand, Type.u64, 8);
            if (ty.is_signed and ty.bits < 128) {
                try cg.addImm64(128 - ty.bits);
                try cg.addTag(.i64_shl);
            }
            try cg.addTag(.i64_popcnt);

            try cg.addTag(.i64_add);
            try cg.addTag(.i32_wrap_i64);
            return .stack;
        },
        else => {
            try cg.lowerToStack(operand);
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__popcount_limb64);

            return .stack;
        },
    }
}

fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            const intrin_ret = try cg.callIntrinsic(
                .__bitreversesi2,
                &.{.u32_type},
                Type.u32,
                &.{operand},
            );
            if (ty.bits == 32) return intrin_ret;
            return cg.intShr(ty, intrin_ret, .{ .imm32 = 32 - ty.bits });
        },
        33...64 => {
            const intrin_ret = try cg.callIntrinsic(
                .__bitreversedi2,
                &.{.u64_type},
                Type.u64,
                &.{operand},
            );
            if (ty.bits == 64) return intrin_ret;
            return cg.intShr(ty, intrin_ret, .{ .imm32 = 64 - ty.bits });
        },
        65...128 => {
            const tmp = try cg.allocStack(Type.u128);

            try cg.emitWValue(tmp);
            const hi = try cg.load(operand, Type.u64, 8);
            const hi_rev = try cg.callIntrinsic(
                .__bitreversedi2,
                &.{.u64_type},
                Type.u64,
                &.{hi},
            );
            try cg.emitWValue(hi_rev);
            try cg.store(.stack, .stack, Type.u64, tmp.offset());

            try cg.emitWValue(tmp);
            const lo = try cg.load(operand, Type.u64, 0);
            const lo_rev = try cg.callIntrinsic(
                .__bitreversedi2,
                &.{.u64_type},
                Type.u64,
                &.{lo},
            );
            try cg.emitWValue(lo_rev);
            try cg.store(.stack, .stack, Type.u64, tmp.offset() + 8);

            if (ty.bits < 128) {
                const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
                return cg.intShr(shift_ty, tmp, .{ .imm32 = 128 - ty.bits });
            } else {
                return tmp;
            }
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(operand);
            try cg.addImm32(@intFromBool(ty.is_signed));
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__bitreverse_limb64);

            return result;
        },
    }
}

fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            const intrin_ret = try cg.callIntrinsic(
                .__bswapsi2,
                &.{.u32_type},
                Type.u32,
                &.{operand},
            );
            if (ty.bits == 32) return intrin_ret;
            return cg.intShr(ty, intrin_ret, .{ .imm32 = 32 - ty.bits });
        },
        33...64 => {
            const intrin_ret = try cg.callIntrinsic(
                .__bswapdi2,
                &.{.u64_type},
                Type.u64,
                &.{operand},
            );
            if (ty.bits == 64) return intrin_ret;
            return cg.intShr(ty, intrin_ret, .{ .imm32 = 64 - ty.bits });
        },
        65...128 => {
            const result = try cg.allocStack(Type.u128);

            try cg.emitWValue(result);

            const low = try cg.load(operand, Type.u64, 0);
            const swap_low = try cg.callIntrinsic(
                .__bswapdi2,
                &.{.u64_type},
                Type.u64,
                &.{low},
            );
            try cg.store(.stack, swap_low, Type.u64, result.offset() + 8);

            try cg.emitWValue(result);

            const high = try cg.load(operand, Type.u64, 8);
            const swap_high = try cg.callIntrinsic(
                .__bswapdi2,
                &.{.u64_type},
                Type.u64,
                &.{high},
            );
            try cg.store(.stack, swap_high, Type.u64, result.offset());

            if (ty.bits < 128) {
                const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
                return cg.intShr(shift_ty, result, .{ .imm32 = 128 - ty.bits });
            } else {
                return result;
            }
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(operand);
            try cg.addImm32(@intFromBool(ty.is_signed));
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__byteswap_limb64);

            return result;
        },
    }
}

fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...31 => {
            try cg.emitWValue(operand);
            if (ty.is_signed) {
                try cg.addImm32(32 - ty.bits);
                try cg.addTag(.i32_shl);
                try cg.addImm32(32 - ty.bits);
                try cg.addTag(.i32_shr_s);
            } else {
                try cg.addImm32(~@as(u32, 0) >> @intCast(32 - ty.bits));
                try cg.addTag(.i32_and);
            }
            return .stack;
        },
        32 => return operand,
        33...63 => {
            try cg.emitWValue(operand);
            if (ty.is_signed) {
                try cg.addImm64(64 - ty.bits);
                try cg.addTag(.i64_shl);
                try cg.addImm64(64 - ty.bits);
                try cg.addTag(.i64_shr_s);
            } else {
                try cg.addImm64(~@as(u64, 0) >> @intCast(64 - ty.bits));
                try cg.addTag(.i64_and);
            }
            return .stack;
        },
        64 => return operand,
        65...127 => {
            const result = try cg.allocStack(Type.u128);

            try cg.emitWValue(result);
            _ = try cg.load(operand, Type.u64, 0);
            try cg.store(.stack, .stack, Type.u64, result.offset());

            try cg.emitWValue(result);
            _ = try cg.load(operand, Type.u64, 8);
            if (ty.is_signed) {
                try cg.addImm64(128 - ty.bits);
                try cg.addTag(.i64_shl);
                try cg.addImm64(128 - ty.bits);
                try cg.addTag(.i64_shr_s);
            } else {
                try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));
                try cg.addTag(.i64_and);
            }
            try cg.store(.stack, .stack, Type.u64, result.offset() + 8);

            return result;
        },
        128 => return operand,
        else => {
            const bits = cg.intBackingBits(ty.bits);
            if (ty.bits == bits) return operand;

            const result = try cg.allocInt(ty);

            const used_len = (math.divCeil(u16, ty.bits, 64) catch unreachable) * 8;

            if (ty.bits % 64 != 0) {
                try cg.memcpy(result, operand, .{ .imm32 = used_len - 8 });
                const pad = 64 - ty.bits % 64;

                try cg.emitWValue(result);
                _ = try cg.load(operand, Type.u64, used_len - 8);
                if (ty.is_signed) {
                    try cg.addImm64(pad);
                    try cg.addTag(.i64_shl);
                    try cg.addImm64(pad);
                    try cg.addTag(.i64_shr_s);
                } else {
                    try cg.addImm64(~@as(u64, 0) >> @intCast(pad));
                    try cg.addTag(.i64_and);
                }
                try cg.store(.stack, .stack, Type.u64, result.offset() + used_len - 8);
            } else {
                try cg.memcpy(result, operand, .{ .imm32 = used_len });
            }

            const full_len = @divExact(bits, 8);
            if (used_len + 8 == full_len) { // last limb needs sign extended
                try cg.emitWValue(result);
                if (ty.is_signed) {
                    _ = try cg.load(result, Type.u64, used_len - 8);
                    try cg.addImm64(63);
                    try cg.addTag(.i64_shr_s);
                } else {
                    try cg.addImm64(0);
                }
                try cg.store(.stack, .stack, Type.u64, result.offset() + used_len);
            }

            return result;
        },
    }
}

fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
    if (int_ty.bits <= 32) {
        if (int_ty.is_signed) {
            return .{ .imm32 = (~@as(u32, 0) >> @intCast(32 - int_ty.bits)) >> 1 };
        } else {
            return .{ .imm32 = ~@as(u32, 0) >> @intCast(32 - int_ty.bits) };
        }
    } else if (int_ty.bits <= 64) {
        if (int_ty.is_signed) {
            return .{ .imm64 = (~@as(u64, 0) >> @intCast(64 - int_ty.bits)) >> 1 };
        } else {
            return .{ .imm64 = ~@as(u64, 0) >> @intCast(64 - int_ty.bits) };
        }
    } else if (int_ty.bits <= 128) {
        const result = try cg.allocInt(int_ty);
        try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, 0);

        if (int_ty.is_signed) {
            try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, Type.u64, 8);
        } else {
            try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, Type.u64, 8);
        }
        return result;
    } else {
        const result = try cg.allocInt(int_ty);
        const full_len = @divExact(cg.intBackingBits(int_ty.bits), 8);
        const used_len = (math.divCeil(u16, int_ty.bits, 64) catch unreachable) * 8;

        try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0xFF });

        if (int_ty.is_signed) {
            try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, Type.u64, used_len - 8);
        } else {
            try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, Type.u64, used_len - 8);
        }

        if (used_len + 8 == full_len) {
            try cg.store(result, .{ .imm64 = 0 }, Type.u64, full_len - 8);
        }

        return result;
    }
}

fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
    if (!int_ty.is_signed) {
        return cg.intZeroValue(int_ty);
    }
    if (int_ty.bits <= 32) {
        return .{ .imm32 = ~@as(u32, 0) << @intCast(int_ty.bits - 1) };
    } else if (int_ty.bits <= 64) {
        return .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 1) };
    } else if (int_ty.bits <= 128) {
        const result = try cg.allocInt(int_ty);
        try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
        try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, Type.u64, 8);
        return result;
    } else {
        const result = try cg.allocInt(int_ty);
        const full_len = @divExact(cg.intBackingBits(int_ty.bits), 8);
        const used_len = (math.divCeil(u16, int_ty.bits, 64) catch unreachable) * 8;

        try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0 });
        try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, Type.u64, used_len - 8);

        if (used_len + 8 == full_len) {
            try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, full_len - 8);
        }

        return result;
    }
}

fn intAddSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    const raw_val = try cg.intAdd(int_ty, lhs, rhs);
    var op_val = try cg.toLocalInt(try cg.intWrap(int_ty, raw_val), int_ty);
    defer op_val.free(cg);

    const max_val = try cg.intMaxValue(int_ty);

    if (int_ty.is_signed) {
        const zero = try cg.intZeroValue(int_ty);
        var rhs_is_neg = try cg.toLocalInt(try cg.intCmp(int_ty, .lt, rhs, zero), .u32);
        defer rhs_is_neg.free(cg);
        const min_val = try cg.intMinValue(int_ty);

        try cg.lowerToStack(min_val);
        try cg.lowerToStack(max_val);
        try cg.emitWValue(rhs_is_neg);
        try cg.addTag(.select);

        try cg.lowerToStack(op_val);
        const overflow_cmp = try cg.intCmp(int_ty, .lt, op_val, lhs);
        const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
        try cg.emitWValue(is_overflow);
        try cg.addTag(.select);
        return .stack;
    } else {
        try cg.lowerToStack(max_val);
        try cg.lowerToStack(op_val);

        const is_overflow = try cg.intCmp(int_ty, .lt, op_val, lhs);
        try cg.emitWValue(is_overflow);
        try cg.addTag(.select);
        return .stack;
    }
}

fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    const raw_val = try cg.intSub(int_ty, lhs, rhs);
    var op_val = try cg.toLocalInt(try cg.intWrap(int_ty, raw_val), int_ty);
    defer op_val.free(cg);

    if (int_ty.is_signed) {
        const zero = try cg.intZeroValue(int_ty);
        var rhs_is_neg = try cg.toLocalInt(try cg.intCmp(int_ty, .lt, rhs, zero), .u32);
        defer rhs_is_neg.free(cg);
        const max_val = try cg.intMaxValue(int_ty);
        const min_val = try cg.intMinValue(int_ty);

        try cg.lowerToStack(max_val);
        try cg.lowerToStack(min_val);
        try cg.emitWValue(rhs_is_neg);
        try cg.addTag(.select);

        try cg.lowerToStack(op_val);
        const overflow_cmp = try cg.intCmp(int_ty, .gt, op_val, lhs);
        const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
        try cg.emitWValue(is_overflow);
        try cg.addTag(.select);
        return .stack;
    } else {
        const zero = try cg.intZeroValue(int_ty);

        try cg.lowerToStack(zero);
        try cg.lowerToStack(op_val);
        const is_overflow = try cg.intCmp(int_ty, .lt, lhs, rhs);
        try cg.emitWValue(is_overflow);
        try cg.addTag(.select);
        return .stack;
    }
}

fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    const ext_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = int_ty.bits * 2 };

    const lhs_ext = try cg.intCast(ext_ty, int_ty, lhs);
    const rhs_ext = try cg.intCast(ext_ty, int_ty, rhs);

    var mul_ext = try cg.toLocalInt(try cg.intMul(ext_ty, lhs_ext, rhs_ext), ext_ty);
    defer mul_ext.free(cg);

    var op_val = try cg.toLocalInt(try cg.intTrunc(int_ty, ext_ty, mul_ext), int_ty);
    defer op_val.free(cg);
    const max_val = try cg.intMaxValue(int_ty);

    if (int_ty.is_signed) {
        const min_val = try cg.intMinValue(int_ty);

        try cg.lowerToStack(min_val);

        try cg.lowerToStack(max_val);
        try cg.lowerToStack(op_val);
        const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
        const ov_pos = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
        try cg.emitWValue(ov_pos);
        try cg.addTag(.select);

        const min_ext = try cg.intCast(ext_ty, int_ty, min_val);
        const ov_neg = try cg.intCmp(ext_ty, .gt, min_ext, mul_ext);
        try cg.lowerToStack(ov_neg);
        try cg.addTag(.select);
        return .stack;
    } else {
        try cg.lowerToStack(max_val);
        try cg.lowerToStack(op_val);
        const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
        const is_overflow = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
        try cg.emitWValue(is_overflow);
        try cg.addTag(.select);
        return .stack;
    }
}

fn intShlSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
    const raw_val = try cg.intShl(int_ty, lhs, rhs);
    var op_val = try cg.toLocalInt(try cg.intWrap(int_ty, raw_val), int_ty);
    defer op_val.free(cg);

    var check_val = try cg.toLocalInt(try cg.intShr(int_ty, op_val, rhs), int_ty);
    defer check_val.free(cg);

    const max_val = try cg.intMaxValue(int_ty);

    if (int_ty.is_signed) {
        const zero = try cg.intZeroValue(int_ty);
        const min_val = try cg.intMinValue(int_ty);

        try cg.lowerToStack(min_val);
        try cg.lowerToStack(max_val);
        const lhs_is_neg = try cg.intCmp(int_ty, .lt, lhs, zero);
        try cg.emitWValue(lhs_is_neg);
        try cg.addTag(.select);

        try cg.lowerToStack(op_val);
        const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
        try cg.emitWValue(is_overflow);
        try cg.addTag(.select);
        return .stack;
    } else {
        try cg.lowerToStack(max_val);
        try cg.lowerToStack(op_val);
        const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
        try cg.emitWValue(is_overflow);
        try cg.addTag(.select);
        return .stack;
    }
}

fn intZeroValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
    switch (int_ty.bits) {
        0 => unreachable,
        1...32 => return .{ .imm32 = 0 },
        33...64 => return .{ .imm64 = 0 },
        65...128 => {
            const result = try cg.allocInt(int_ty);
            try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
            try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);
            return result;
        },
        else => {
            const result = try cg.allocInt(int_ty);
            const full_len = @divExact(cg.intBackingBits(int_ty.bits), 8);
            try cg.memset(Type.u8, result, .{ .imm32 = full_len }, .{ .imm32 = 0 });
            return result;
        },
    }
}

fn toLocalInt(cg: *CodeGen, value: WValue, int_ty: IntType) InnerError!WValue {
    switch (value) {
        .stack => {
            const ty: Type = switch (int_ty.bits) {
                0 => unreachable,
                1...32 => .u32,
                33...64 => .u64,
                65...128 => .u128,
                else => return cg.fail("TODO: Support toLocalInt for integer bitsize: {d}", .{int_ty.bits}),
            };
            const new_local = try cg.allocLocal(ty);
            try cg.addLocal(.local_set, new_local.local.value);
            return new_local;
        },
        .local, .stack_offset => return value,
        else => unreachable,
    }
}

const OverflowResult = struct {
    result: WValue,
    ov: WValue,
};

fn intAddOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
    switch (ty.bits) {
        0 => unreachable,
        1...128 => {
            const raw_result = try cg.intAdd(ty, lhs, rhs);
            const op_result = try cg.intWrap(ty, raw_result);
            const op_tmp = try cg.toLocalInt(op_result, ty);

            const overflow_bit = if (ty.is_signed) blk: {
                const zero = try cg.intZeroValue(ty);
                const rhs_is_neg = try cg.intCmp(ty, .lt, rhs, zero);
                const overflow_cmp = try cg.intCmp(ty, .lt, op_tmp, lhs);
                break :blk try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
            } else try cg.intCmp(ty, .lt, op_tmp, lhs);

            return .{ .result = op_tmp, .ov = overflow_bit };
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            try cg.addImm32(@intFromBool(ty.is_signed));
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__addo_limb64);

            return .{ .result = result, .ov = .stack };
        },
    }
}

fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
    switch (ty.bits) {
        0 => unreachable,
        1...128 => {
            const raw_result = try cg.intSub(ty, lhs, rhs);
            const op_result = try cg.intWrap(ty, raw_result);
            const op_tmp = try cg.toLocalInt(op_result, ty);

            const overflow_bit = if (ty.is_signed) blk: {
                const zero = try cg.intZeroValue(ty);
                const rhs_is_neg = try cg.intCmp(ty, .lt, rhs, zero);
                const overflow_cmp = try cg.intCmp(ty, .gt, op_tmp, lhs);
                break :blk try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
            } else try cg.intCmp(ty, .gt, op_tmp, lhs);

            return .{ .result = op_tmp, .ov = overflow_bit };
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            try cg.addImm32(@intFromBool(ty.is_signed));
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__subo_limb64);
            return .{ .result = result, .ov = .stack };
        },
    }
}

fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
    const overflow_bit = try cg.allocLocal(Type.u32);
    try cg.addImm32(0);
    try cg.addLocal(.local_set, overflow_bit.local.value);

    const result_val = if (int_ty.bits <= 32) blk: {
        const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 64 };
        const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);
        const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);
        const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);
        const bin_op = try cg.toLocalInt(mul_raw, new_ty);

        const res = try cg.intTrunc(int_ty, new_ty, bin_op);
        const res_tmp = try cg.toLocalInt(res, int_ty);

        const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);
        _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
        try cg.addLocal(.local_set, overflow_bit.local.value);
        break :blk res_tmp;
    } else if (int_ty.bits <= 64) blk: {
        const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 128 };
        const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);
        const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);
        const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);
        const bin_op = try cg.toLocalInt(mul_raw, new_ty);

        const res = try cg.intTrunc(int_ty, new_ty, bin_op);
        const res_tmp = try cg.toLocalInt(res, int_ty);

        const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);
        _ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
        try cg.addLocal(.local_set, overflow_bit.local.value);
        break :blk res_tmp;
    } else if (int_ty.bits == 128 and int_ty.is_signed) blk: {
        const overflow_ret = try cg.allocStack(Type.i32);
        const res = try cg.callIntrinsic(
            .__muloti4,
            &[_]InternPool.Index{ .i128_type, .i128_type, .usize_type },
            Type.i128,
            &.{ lhs, rhs, overflow_ret },
        );
        _ = try cg.load(overflow_ret, Type.i32, 0);
        try cg.addLocal(.local_set, overflow_bit.local.value);
        break :blk res;
    } else {
        const result = try cg.allocInt(int_ty);

        try cg.lowerToStack(result);
        try cg.lowerToStack(lhs);
        try cg.lowerToStack(rhs);
        try cg.addImm32(@intFromBool(int_ty.is_signed));
        try cg.addImm32(int_ty.bits);
        try cg.addCallIntrinsic(.__mulo_limb64);

        return .{ .result = result, .ov = .stack };
    };

    return .{ .result = result_val, .ov = .{ .local = overflow_bit.local } };
}

fn intShlOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
    switch (ty.bits) {
        0 => unreachable,
        1...128 => {
            const raw_shl = try cg.intShl(ty, lhs, rhs);
            const wrapped_shl = try cg.intWrap(ty, raw_shl);
            const shl_tmp = try cg.toLocalInt(wrapped_shl, ty);

            const shr = try cg.intShr(ty, shl_tmp, rhs);
            const overflow_bit = try cg.intCmp(ty, .neq, shr, lhs);

            return .{ .result = shl_tmp, .ov = overflow_bit };
        },
        else => {
            const result = try cg.allocInt(ty);

            try cg.lowerToStack(result);
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            try cg.addImm32(@intFromBool(ty.is_signed));
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__shlo_limb64);

            return .{ .result = result, .ov = .stack };
        },
    }
}

fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) InnerError!WValue {
    const src_bits: u16 = cg.intBackingBits(src_ty.bits);
    const dest_bits: u16 = cg.intBackingBits(dest_ty.bits);

    if (src_bits == dest_bits) {
        return operand;
    }

    if (src_bits == 64 and dest_bits == 32) {
        try cg.emitWValue(operand);
        try cg.addTag(.i32_wrap_i64);
        return .stack;
    } else if (src_bits == 32 and dest_bits == 64) {
        try cg.emitWValue(operand);
        try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
        return .stack;
    } else if (dest_bits >= 128) {
        const result = try cg.allocInt(dest_ty);

        const dest_len = dest_bits / 8;

        if (dest_bits <= src_bits) {
            assert(src_bits >= 128);
            try cg.memcpy(result, operand, .{ .imm32 = dest_len });
        } else {
            var src_len: u32 = undefined;
            if (src_bits == 32) {
                try cg.emitWValue(result);
                try cg.emitWValue(operand);
                try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
                try cg.store(.stack, .stack, Type.u64, result.offset());
                src_len = 8;
            } else if (src_bits == 64) {
                try cg.emitWValue(result);
                try cg.emitWValue(operand);
                try cg.store(.stack, .stack, Type.u64, result.offset());
                src_len = 8;
            } else {
                src_len = src_bits / 8;
                try cg.memcpy(result, operand, .{ .imm32 = src_len });
            }

            if (dest_bits == 128) {
                if (src_ty.is_signed) {
                    try cg.emitWValue(result);
                    if (src_bits == 32) {
                        try cg.emitWValue(operand);
                        try cg.addTag(if (dest_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
                    } else if (src_bits == 64) {
                        try cg.emitWValue(operand);
                    } else unreachable;
                    const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
                    try cg.store(.stack, shr, Type.u64, 8 + result.offset());
                } else {
                    try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);
                }
            } else {
                var pad = result;
                pad.stack_offset.value += src_len;
                const memset_len = dest_len - src_len;
                if (src_ty.is_signed) {
                    if (src_bits == 32) {
                        try cg.emitWValue(operand);
                        _ = try cg.intShr(IntType.i32, .stack, .{ .imm32 = 31 });
                    } else if (src_bits == 64) {
                        try cg.emitWValue(operand);
                        _ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
                        try cg.addTag(.i32_wrap_i64);
                    } else {
                        _ = try cg.load(operand, Type.u64, src_len - 8);
                        _ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
                        try cg.addTag(.i32_wrap_i64);
                    }
                    var sign_byte = try @as(WValue, .stack).toLocal(cg, Type.u32);
                    try cg.memset(Type.u8, pad, .{ .imm32 = memset_len }, sign_byte);
                    sign_byte.free(cg);
                } else {
                    try cg.memset(Type.u8, pad, .{ .imm32 = memset_len }, .{ .imm32 = 0 });
                }
            }
        }

        return result;
    } else {
        assert(dest_bits <= 64);
        assert(src_bits >= 128);
        const load_ty = if (dest_bits == 32) Type.u32 else Type.u64;
        return cg.load(operand, load_ty, 0);
    }
}

fn intTrunc(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) InnerError!WValue {
    var result = try cg.intCast(dest_ty, src_ty, operand);

    const dest_wasm_bits = cg.intBackingBits(dest_ty.bits);

    if (dest_wasm_bits != dest_ty.bits) {
        result = try cg.intWrap(dest_ty, result);
    }

    return result;
}

const FloatType = enum {
    f16,
    f32,
    f64,
    f80,
    f128,

    fn fromType(cg: *CodeGen, ty: Type) FloatType {
        assert(ty.isRuntimeFloat());
        return switch (ty.floatBits(cg.target)) {
            16 => .f16,
            32 => .f32,
            64 => .f64,
            80 => .f80,
            128 => .f128,
            else => unreachable,
        };
    }
};

fn floatAdd(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__addhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
        .f32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.f32_add);
            return .stack;
        },
        .f64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.f64_add);
            return .stack;
        },
        .f80 => return cg.callIntrinsic(.__addxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
        .f128 => return cg.callIntrinsic(.__addtf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
    }
}

fn floatSub(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__subhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
        .f32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.f32_sub);
            return .stack;
        },
        .f64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.f64_sub);
            return .stack;
        },
        .f80 => return cg.callIntrinsic(.__subxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
        .f128 => return cg.callIntrinsic(.__subtf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
    }
}

fn floatMul(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__mulhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
        .f32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.f32_mul);
            return .stack;
        },
        .f64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.f64_mul);
            return .stack;
        },
        .f80 => return cg.callIntrinsic(.__mulxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
        .f128 => return cg.callIntrinsic(.__multf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
    }
}

fn floatMulAdd(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue, addend: WValue) InnerError!WValue {
    const mul_result = try cg.floatMul(ty, lhs, rhs);
    return cg.floatAdd(ty, mul_result, addend);
}

fn floatDiv(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__divhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
        .f32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.f32_div);
            return .stack;
        },
        .f64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(.f64_div);
            return .stack;
        },
        .f80 => return cg.callIntrinsic(.__divxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
        .f128 => return cg.callIntrinsic(.__divtf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
    }
}

fn floatRem(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__fmodh, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
        .f32 => return cg.callIntrinsic(.fmodf, &.{ .f32_type, .f32_type }, Type.f32, &.{ lhs, rhs }),
        .f64 => return cg.callIntrinsic(.fmod, &.{ .f64_type, .f64_type }, Type.f64, &.{ lhs, rhs }),
        .f80 => return cg.callIntrinsic(.__fmodx, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
        .f128 => return cg.callIntrinsic(.fmodq, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
    }
}

// div_trunc(a, b) = trunc(a / b)
fn floatDivTrunc(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
    const div_result = try cg.floatDiv(ty, lhs, rhs);
    return cg.floatTrunc(ty, div_result);
}

// div_floor(a, b) = floor(a / b)
fn floatDivFloor(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
    const div_result = try cg.floatDiv(ty, lhs, rhs);
    return cg.floatFloor(ty, div_result);
}

// mod(a, b) = fmod(fmod(a, b) + b, b)
fn floatMod(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
    const r = try cg.floatRem(ty, lhs, rhs);
    const s = try cg.floatAdd(ty, r, rhs);
    return cg.floatRem(ty, s, rhs);
}

// wasm fN_max NaN semantics differ with Zig
fn floatMax(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__fmaxh, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
        .f32 => return cg.callIntrinsic(.fmaxf, &.{ .f32_type, .f32_type }, Type.f32, &.{ lhs, rhs }),
        .f64 => return cg.callIntrinsic(.fmax, &.{ .f64_type, .f64_type }, Type.f64, &.{ lhs, rhs }),
        .f80 => return cg.callIntrinsic(.__fmaxx, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
        .f128 => return cg.callIntrinsic(.fmaxq, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
    }
}

// wasm fN_min NaN semantics differ with Zig
fn floatMin(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__fminh, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
        .f32 => return cg.callIntrinsic(.fminf, &.{ .f32_type, .f32_type }, Type.f32, &.{ lhs, rhs }),
        .f64 => return cg.callIntrinsic(.fmin, &.{ .f64_type, .f64_type }, Type.f64, &.{ lhs, rhs }),
        .f80 => return cg.callIntrinsic(.__fminx, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
        .f128 => return cg.callIntrinsic(.fminq, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
    }
}

fn floatSqrt(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__sqrth, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f32_sqrt);
            return .stack;
        },
        .f64 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f64_sqrt);
            return .stack;
        },
        .f80 => return cg.callIntrinsic(.__sqrtx, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.sqrtq, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatSin(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__sinh, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => return cg.callIntrinsic(.sinf, &.{.f32_type}, Type.f32, &.{arg}),
        .f64 => return cg.callIntrinsic(.sin, &.{.f64_type}, Type.f64, &.{arg}),
        .f80 => return cg.callIntrinsic(.__sinx, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.sinq, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatCos(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__cosh, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => return cg.callIntrinsic(.cosf, &.{.f32_type}, Type.f32, &.{arg}),
        .f64 => return cg.callIntrinsic(.cos, &.{.f64_type}, Type.f64, &.{arg}),
        .f80 => return cg.callIntrinsic(.__cosx, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.cosq, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatTan(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__tanh, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => return cg.callIntrinsic(.tanf, &.{.f32_type}, Type.f32, &.{arg}),
        .f64 => return cg.callIntrinsic(.tan, &.{.f64_type}, Type.f64, &.{arg}),
        .f80 => return cg.callIntrinsic(.__tanx, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.tanq, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatExp(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__exph, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => return cg.callIntrinsic(.expf, &.{.f32_type}, Type.f32, &.{arg}),
        .f64 => return cg.callIntrinsic(.exp, &.{.f64_type}, Type.f64, &.{arg}),
        .f80 => return cg.callIntrinsic(.__expx, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.expq, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatExp2(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__exp2h, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => return cg.callIntrinsic(.exp2f, &.{.f32_type}, Type.f32, &.{arg}),
        .f64 => return cg.callIntrinsic(.exp2, &.{.f64_type}, Type.f64, &.{arg}),
        .f80 => return cg.callIntrinsic(.__exp2x, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.exp2q, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatLog(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__logh, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => return cg.callIntrinsic(.logf, &.{.f32_type}, Type.f32, &.{arg}),
        .f64 => return cg.callIntrinsic(.log, &.{.f64_type}, Type.f64, &.{arg}),
        .f80 => return cg.callIntrinsic(.__logx, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.logq, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatLog2(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__log2h, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => return cg.callIntrinsic(.log2f, &.{.f32_type}, Type.f32, &.{arg}),
        .f64 => return cg.callIntrinsic(.log2, &.{.f64_type}, Type.f64, &.{arg}),
        .f80 => return cg.callIntrinsic(.__log2x, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.log2q, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatLog10(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__log10h, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => return cg.callIntrinsic(.log10f, &.{.f32_type}, Type.f32, &.{arg}),
        .f64 => return cg.callIntrinsic(.log10, &.{.f64_type}, Type.f64, &.{arg}),
        .f80 => return cg.callIntrinsic(.__log10x, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.log10q, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatFloor(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__floorh, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f32_floor);
            return .stack;
        },
        .f64 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f64_floor);
            return .stack;
        },
        .f80 => return cg.callIntrinsic(.__floorx, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.floorq, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatCeil(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__ceilh, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f32_ceil);
            return .stack;
        },
        .f64 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f64_ceil);
            return .stack;
        },
        .f80 => return cg.callIntrinsic(.__ceilx, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.ceilq, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatRound(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__roundh, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f32_nearest);
            return .stack;
        },
        .f64 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f64_nearest);
            return .stack;
        },
        .f80 => return cg.callIntrinsic(.__roundx, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.roundq, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatTrunc(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__trunch, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f32_trunc);
            return .stack;
        },
        .f64 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f64_trunc);
            return .stack;
        },
        .f80 => return cg.callIntrinsic(.__truncx, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.truncq, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => {
            try cg.emitWValue(arg);
            try cg.addImm32(0x8000);
            try cg.addTag(.i32_xor);
            return .stack;
        },
        .f32 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f32_neg);
            return .stack;
        },
        .f64 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f64_neg);
            return .stack;
        },
        .f80 => {
            const result = try cg.allocStack(Type.f80);
            try cg.emitWValue(result);
            try cg.emitWValue(arg);
            try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
            try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
            try cg.emitWValue(result);
            try cg.emitWValue(arg);
            try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });
            try cg.addImm64(0x8000);
            try cg.addTag(.i64_xor);
            try cg.addMemArg(.i64_store16, .{ .offset = 8 + result.offset(), .alignment = 2 });
            return result;
        },
        .f128 => {
            const result = try cg.allocStack(Type.f128);
            try cg.emitWValue(result);
            try cg.emitWValue(arg);
            try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
            try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
            try cg.emitWValue(result);
            try cg.emitWValue(arg);
            try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });
            try cg.addImm64(0x8000000000000000);
            try cg.addTag(.i64_xor);
            try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });
            return result;
        },
    }
}

fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => return cg.callIntrinsic(.__fabsh, &.{.f16_type}, Type.f16, &.{arg}),
        .f32 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f32_abs);
            return .stack;
        },
        .f64 => {
            try cg.emitWValue(arg);
            try cg.addTag(.f64_abs);
            return .stack;
        },
        .f80 => return cg.callIntrinsic(.__fabsx, &.{.f80_type}, Type.f80, &.{arg}),
        .f128 => return cg.callIntrinsic(.fabsq, &.{.f128_type}, Type.f128, &.{arg}),
    }
}

fn floatExtendCast(cg: *CodeGen, dest_ty: FloatType, src_ty: FloatType, operand: WValue) InnerError!WValue {
    switch (dest_ty) {
        .f16 => unreachable,
        .f32 => switch (src_ty) {
            .f16 => {
                _ = try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand});
                return .stack;
            },
            else => unreachable,
        },
        .f64 => switch (src_ty) {
            .f16 => {
                _ = try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand});
                try cg.addTag(.f64_promote_f32);
                return .stack;
            },
            .f32 => {
                try cg.emitWValue(operand);
                try cg.addTag(.f64_promote_f32);
                return .stack;
            },
            else => unreachable,
        },
        .f80 => switch (src_ty) {
            .f16 => return cg.callIntrinsic(.__extendhfxf2, &.{.f16_type}, Type.f80, &.{operand}),
            .f32 => return cg.callIntrinsic(.__extendsfxf2, &.{.f32_type}, Type.f80, &.{operand}),
            .f64 => return cg.callIntrinsic(.__extenddfxf2, &.{.f64_type}, Type.f80, &.{operand}),
            else => unreachable,
        },
        .f128 => switch (src_ty) {
            .f16 => return cg.callIntrinsic(.__extendhftf2, &.{.f16_type}, Type.f128, &.{operand}),
            .f32 => return cg.callIntrinsic(.__extendsftf2, &.{.f32_type}, Type.f128, &.{operand}),
            .f64 => return cg.callIntrinsic(.__extenddftf2, &.{.f64_type}, Type.f128, &.{operand}),
            .f80 => return cg.callIntrinsic(.__extendxftf2, &.{.f80_type}, Type.f128, &.{operand}),
            else => unreachable,
        },
    }
}

fn floatTruncCast(cg: *CodeGen, dest_ty: FloatType, src_ty: FloatType, operand: WValue) InnerError!WValue {
    switch (dest_ty) {
        .f16 => switch (src_ty) {
            .f32 => return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{operand}),
            .f64 => {
                try cg.emitWValue(operand);
                try cg.addTag(.f32_demote_f64);
                return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{.stack});
            },
            .f80 => return cg.callIntrinsic(.__truncxfhf2, &.{.f80_type}, Type.f16, &.{operand}),
            .f128 => return cg.callIntrinsic(.__trunctfhf2, &.{.f128_type}, Type.f16, &.{operand}),
            else => unreachable,
        },
        .f32 => switch (src_ty) {
            .f64 => {
                try cg.emitWValue(operand);
                try cg.addTag(.f32_demote_f64);
                return .stack;
            },
            .f80 => return cg.callIntrinsic(.__truncxfsf2, &.{.f80_type}, Type.f32, &.{operand}),
            .f128 => return cg.callIntrinsic(.__trunctfsf2, &.{.f128_type}, Type.f32, &.{operand}),
            else => unreachable,
        },
        .f64 => switch (src_ty) {
            .f80 => return cg.callIntrinsic(.__truncxfdf2, &.{.f80_type}, Type.f64, &.{operand}),
            .f128 => return cg.callIntrinsic(.__trunctfdf2, &.{.f128_type}, Type.f64, &.{operand}),
            else => unreachable,
        },
        .f80 => switch (src_ty) {
            .f128 => return cg.callIntrinsic(.__trunctfxf2, &.{.f128_type}, Type.f80, &.{operand}),
            else => unreachable,
        },
        .f128 => unreachable,
    }
}

fn intFromFloat(cg: *CodeGen, dest_ty: IntType, src_ty: FloatType, operand: WValue) InnerError!WValue {
    switch (dest_ty.bits) {
        0 => unreachable,
        1...32 => switch (src_ty) {
            .f16 => {
                const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfsi else .__fixunshfsi;
                return cg.callIntrinsic(intrinsic, &.{.f16_type}, Type.u32, &.{operand});
            },
            .f32 => {
                try cg.emitWValue(operand);
                try cg.addTag(if (dest_ty.is_signed) .i32_trunc_f32_s else .i32_trunc_f32_u);
                return .stack;
            },
            .f64 => {
                try cg.emitWValue(operand);
                try cg.addTag(if (dest_ty.is_signed) .i32_trunc_f64_s else .i32_trunc_f64_u);
                return .stack;
            },
            .f80 => {
                const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfsi else .__fixunsxfsi;
                return cg.callIntrinsic(intrinsic, &.{.f80_type}, Type.u32, &.{operand});
            },
            .f128 => {
                const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfsi else .__fixunstfsi;
                return cg.callIntrinsic(intrinsic, &.{.f128_type}, Type.u32, &.{operand});
            },
        },
        33...64 => switch (src_ty) {
            .f16 => {
                const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfdi else .__fixunshfdi;
                return cg.callIntrinsic(intrinsic, &.{.f16_type}, Type.u64, &.{operand});
            },
            .f32 => {
                try cg.emitWValue(operand);
                try cg.addTag(if (dest_ty.is_signed) .i64_trunc_f32_s else .i64_trunc_f32_u);
                return .stack;
            },
            .f64 => {
                try cg.emitWValue(operand);
                try cg.addTag(if (dest_ty.is_signed) .i64_trunc_f64_s else .i64_trunc_f64_u);
                return .stack;
            },
            .f80 => {
                const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfdi else .__fixunsxfdi;
                return cg.callIntrinsic(intrinsic, &.{.f80_type}, Type.u64, &.{operand});
            },
            .f128 => {
                const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfdi else .__fixunstfdi;
                return cg.callIntrinsic(intrinsic, &.{.f128_type}, Type.u64, &.{operand});
            },
        },
        65...128 => switch (src_ty) {
            .f16 => {
                const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfti else .__fixunshfti;
                return cg.callIntrinsic(intrinsic, &.{.f16_type}, Type.u128, &.{operand});
            },
            .f32 => {
                const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixsfti else .__fixunssfti;
                return cg.callIntrinsic(intrinsic, &.{.f32_type}, Type.u128, &.{operand});
            },
            .f64 => {
                const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixdfti else .__fixunsdfti;
                return cg.callIntrinsic(intrinsic, &.{.f64_type}, Type.u128, &.{operand});
            },
            .f80 => {
                const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfti else .__fixunsxfti;
                return cg.callIntrinsic(intrinsic, &.{.f80_type}, Type.u128, &.{operand});
            },
            .f128 => {
                const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfti else .__fixunstfti;
                return cg.callIntrinsic(intrinsic, &.{.f128_type}, Type.u128, &.{operand});
            },
        },
        else => {
            const result = try cg.allocInt(dest_ty);

            switch (src_ty) {
                .f16 => {
                    const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfei else .__fixunshfei;
                    _ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f16_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
                },
                .f32 => {
                    const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixsfei else .__fixunssfei;
                    _ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f32_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
                },
                .f64 => {
                    const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixdfei else .__fixunsdfei;
                    _ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f64_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
                },
                .f80 => {
                    const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfei else .__fixunsxfei;
                    _ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f80_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
                },
                .f128 => {
                    const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfei else .__fixunstfei;
                    _ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f128_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
                },
            }

            return result;
        },
    }
}

fn floatFromInt(cg: *CodeGen, dest_ty: FloatType, src_ty: IntType, operand: WValue) InnerError!WValue {
    switch (dest_ty) {
        .f16 => switch (src_ty.bits) {
            0 => unreachable,
            1...32 => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatsihf else .__floatunsihf;
                return cg.callIntrinsic(intrinsic, &.{.i32_type}, Type.f16, &.{operand});
            },
            33...64 => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatdihf else .__floatundihf;
                return cg.callIntrinsic(intrinsic, &.{.i64_type}, Type.f16, &.{operand});
            },
            65...128 => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattihf else .__floatuntihf;
                return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f16, &.{operand});
            },
            else => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateihf else .__floatuneihf;
                return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f16, &.{ operand, .{ .imm32 = src_ty.bits } });
            },
        },
        .f32 => switch (src_ty.bits) {
            0 => unreachable,
            1...32 => {
                try cg.emitWValue(operand);
                try cg.addTag(if (src_ty.is_signed) .f32_convert_i32_s else .f32_convert_i32_u);
                return .stack;
            },
            33...64 => {
                try cg.emitWValue(operand);
                try cg.addTag(if (src_ty.is_signed) .f32_convert_i64_s else .f32_convert_i64_u);
                return .stack;
            },
            65...128 => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattisf else .__floatuntisf;
                return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f32, &.{operand});
            },
            else => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateisf else .__floatuneisf;
                return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f32, &.{ operand, .{ .imm32 = src_ty.bits } });
            },
        },
        .f64 => switch (src_ty.bits) {
            0 => unreachable,
            1...32 => {
                try cg.emitWValue(operand);
                try cg.addTag(if (src_ty.is_signed) .f64_convert_i32_s else .f64_convert_i32_u);
                return .stack;
            },
            33...64 => {
                try cg.emitWValue(operand);
                try cg.addTag(if (src_ty.is_signed) .f64_convert_i64_s else .f64_convert_i64_u);
                return .stack;
            },
            65...128 => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattidf else .__floatuntidf;
                return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f64, &.{operand});
            },
            else => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateidf else .__floatuneidf;
                return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f64, &.{ operand, .{ .imm32 = src_ty.bits } });
            },
        },
        .f80 => switch (src_ty.bits) {
            0 => unreachable,
            1...32 => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatsixf else .__floatunsixf;
                return cg.callIntrinsic(intrinsic, &.{.i32_type}, Type.f80, &.{operand});
            },
            33...64 => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatdixf else .__floatundixf;
                return cg.callIntrinsic(intrinsic, &.{.i64_type}, Type.f80, &.{operand});
            },
            65...128 => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattixf else .__floatuntixf;
                return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f80, &.{operand});
            },
            else => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateixf else .__floatuneixf;
                return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f80, &.{ operand, .{ .imm32 = src_ty.bits } });
            },
        },
        .f128 => switch (src_ty.bits) {
            0 => unreachable,
            1...32 => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatsitf else .__floatunsitf;
                return cg.callIntrinsic(intrinsic, &.{.i32_type}, Type.f128, &.{operand});
            },
            33...64 => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatditf else .__floatunditf;
                return cg.callIntrinsic(intrinsic, &.{.i64_type}, Type.f128, &.{operand});
            },
            65...128 => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattitf else .__floatuntitf;
                return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f128, &.{operand});
            },
            else => {
                const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateitf else .__floatuneitf;
                return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f128, &.{ operand, .{ .imm32 = src_ty.bits } });
            },
        },
    }
}

fn lowerPtr(cg: *CodeGen, ptr_val: InternPool.Index, prev_offset: u64) InnerError!WValue {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr;
    const offset: u64 = prev_offset + ptr.byte_offset;
    return switch (ptr.base_addr) {
        .nav => |nav| return .{ .nav_ref = .{ .nav_index = nav, .offset = @intCast(offset) } },
        .uav => |uav| return .{ .uav_ref = .{ .ip_index = uav.val, .offset = @intCast(offset), .orig_ptr_ty = uav.orig_ty } },
        .int => return cg.lowerConstant(try pt.intValue(.usize, offset)),
        .eu_payload => |eu_ptr| try cg.lowerPtr(
            eu_ptr,
            offset + codegen.errUnionPayloadOffset(
                Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu),
                zcu,
            ),
        ),
        .opt_payload => |opt_ptr| return cg.lowerPtr(opt_ptr, offset),
        .field => |field| {
            const base_ptr = Value.fromInterned(field.base);
            const base_ty = base_ptr.typeOf(zcu).childType(zcu);
            const field_off: u64 = switch (base_ty.zigTypeTag(zcu)) {
                .pointer => off: {
                    assert(base_ty.isSlice(zcu));
                    break :off switch (field.index) {
                        Value.slice_ptr_index => 0,
                        Value.slice_len_index => @divExact(cg.target.ptrBitWidth(), 8),
                        else => unreachable,
                    };
                },
                .@"struct" => switch (base_ty.containerLayout(zcu)) {
                    .auto => base_ty.structFieldOffset(@intCast(field.index), zcu),
                    .@"extern", .@"packed" => unreachable,
                },
                .@"union" => switch (base_ty.containerLayout(zcu)) {
                    .auto => base_ty.structFieldOffset(@intCast(field.index), zcu),
                    .@"extern", .@"packed" => unreachable,
                },
                else => unreachable,
            };
            return cg.lowerPtr(field.base, offset + field_off);
        },
        .arr_elem, .comptime_field, .comptime_alloc => unreachable,
    };
}

/// Asserts that `isByRef` returns `false` for `val.typeOf(zcu)`.
fn lowerConstant(cg: *CodeGen, val: Value) InnerError!WValue {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const ty = val.typeOf(zcu);
    assert(!isByRef(ty, zcu, cg.target));
    const ip = &zcu.intern_pool;
    if (val.isUndef(zcu)) return cg.emitUndefined(ty);

    switch (ip.indexToKey(val.ip_index)) {
        .int_type,
        .ptr_type,
        .restricted_ptr_type,
        .array_type,
        .vector_type,
        .opt_type,
        .anyframe_type,
        .error_union_type,
        .simple_type,
        .struct_type,
        .tuple_type,
        .union_type,
        .opaque_type,
        .enum_type,
        .func_type,
        .error_set_type,
        .inferred_error_set_type,
        => unreachable, // types, not values

        .undef => unreachable, // handled above
        .simple_value => |simple_value| switch (simple_value) {
            .void,
            .null,
            .@"unreachable",
            => unreachable, // non-runtime values
            .false, .true => return .{ .imm32 = switch (simple_value) {
                .false => 0,
                .true => 1,
                else => unreachable,
            } },
        },
        .@"extern",
        .func,
        .enum_literal,
        => unreachable, // non-runtime values
        .int => {
            const int_info = ty.intInfo(zcu);
            switch (int_info.signedness) {
                .signed => switch (int_info.bits) {
                    0...32 => return .{ .imm32 = @bitCast(@as(i32, @intCast(val.toSignedInt(zcu)))) },
                    33...64 => return .{ .imm64 = @bitCast(val.toSignedInt(zcu)) },
                    else => unreachable,
                },
                .unsigned => switch (int_info.bits) {
                    0...32 => return .{ .imm32 = @intCast(val.toUnsignedInt(zcu)) },
                    33...64 => return .{ .imm64 = val.toUnsignedInt(zcu) },
                    else => unreachable,
                },
            }
        },
        .err => |err| {
            const int = try pt.getErrorValue(err.name);
            return .{ .imm32 = int };
        },
        .error_union => |error_union| {
            const err_int_ty = try pt.errorIntType();
            const err_val: Value = switch (error_union.val) {
                .err_name => |err_name| .fromInterned(try pt.intern(.{ .err = .{
                    .ty = ty.errorUnionSet(zcu).toIntern(),
                    .name = err_name,
                } })),
                .payload => try pt.intValue(err_int_ty, 0),
            };
            const payload_type = ty.errorUnionPayload(zcu);
            if (!payload_type.hasRuntimeBits(zcu)) {
                // We use the error type directly as the type.
                return cg.lowerConstant(err_val);
            }

            return cg.fail("Wasm TODO: lowerConstant error union with non-zero-bit payload type", .{});
        },
        .enum_tag => |enum_tag| return cg.lowerConstant(.fromInterned(enum_tag.int)),
        .float => |float| switch (float.storage) {
            .f16 => |f16_val| return .{ .imm32 = @as(u16, @bitCast(f16_val)) },
            .f32 => |f32_val| return .{ .float32 = f32_val },
            .f64 => |f64_val| return .{ .float64 = f64_val },
            else => unreachable,
        },
        .slice => unreachable, // isByRef == true
        .ptr => return cg.lowerPtr(val.toIntern(), 0),
        .opt => if (ty.optionalReprIsPayload(zcu)) {
            if (val.optionalValue(zcu)) |payload| {
                return cg.lowerConstant(payload);
            } else {
                return .{ .imm32 = 0 };
            }
        } else {
            return .{ .imm32 = @intFromBool(!val.isNull(zcu)) };
        },
        .aggregate => switch (ip.indexToKey(ty.ip_index)) {
            .array_type => return cg.fail("Wasm TODO: LowerConstant for {f}", .{ty.fmt(pt)}),
            .vector_type => {
                assert(determineSimdStoreStrategy(ty, zcu, cg.target) == .direct);
                var buf: [16]u8 = undefined;
                val.writeToMemory(zcu, &buf) catch unreachable;
                return cg.storeSimdImmd(buf);
            },
            .struct_type => unreachable, // packed structs use `bitpack`
            else => unreachable,
        },
        .un => unreachable, // packed unions use `bitpack`
        .bitpack => |bitpack| return cg.lowerConstant(.fromInterned(bitpack.backing_int_val)),
        .memoized_call => unreachable,
    }
}

/// Stores the value as a 128bit-immediate value by storing it inside
/// the list and returning the index into this list as `WValue`.
fn storeSimdImmd(cg: *CodeGen, value: [16]u8) !WValue {
    const index = @as(u32, @intCast(cg.simd_immediates.items.len));
    try cg.simd_immediates.append(cg.gpa, value);
    return .{ .imm128 = index };
}

fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue {
    const zcu = cg.pt.zcu;
    switch (ty.zigTypeTag(zcu)) {
        .bool, .error_set => return .{ .imm32 = 0xaaaaaaaa },
        .int, .@"enum" => switch (ty.intInfo(zcu).bits) {
            0...32 => return .{ .imm32 = 0xaaaaaaaa },
            33...64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa },
            else => unreachable,
        },
        .float => switch (ty.floatBits(cg.target)) {
            16 => return .{ .imm32 = 0xaaaaaaaa },
            32 => return .{ .float32 = @as(f32, @bitCast(@as(u32, 0xaaaaaaaa))) },
            64 => return .{ .float64 = @as(f64, @bitCast(@as(u64, 0xaaaaaaaaaaaaaaaa))) },
            else => unreachable,
        },
        .pointer => switch (cg.ptr_size) {
            .wasm32 => return .{ .imm32 = 0xaaaaaaaa },
            .wasm64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa },
        },
        .optional => {
            const pl_ty = ty.optionalChild(zcu);
            if (ty.optionalReprIsPayload(zcu)) {
                return cg.emitUndefined(pl_ty);
            }
            return .{ .imm32 = 0xaaaaaaaa };
        },
        .error_union => {
            return .{ .imm32 = 0xaaaaaaaa };
        },
        .@"struct", .@"union" => {
            const backing_int_ty = ty.bitpackBackingInt(zcu);
            return cg.emitUndefined(backing_int_ty);
        },
        else => return cg.fail("Wasm TODO: emitUndefined for type: {t}\n", .{ty.zigTypeTag(zcu)}),
    }
}

fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const block = cg.air.unwrapBlock(inst);
    try cg.lowerBlock(inst, block.ty, block.body);
}

fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    // if wasm_block_ty is non-empty, we create a register to store the temporary value
    const block_result: WValue = if (block_ty.hasRuntimeBits(zcu))
        try cg.allocLocal(block_ty)
    else
        .none;

    try cg.startBlock(.block, .empty);
    // Here we set the current block idx, so breaks know the depth to jump
    // to when breaking out.
    try cg.blocks.putNoClobber(cg.gpa, inst, .{
        .label = cg.block_depth,
        .value = block_result,
    });

    {
        try cg.branches.append(cg.gpa, .{});
        defer {
            var branch = cg.branches.pop().?;
            branch.deinit(cg.gpa);
        }
        try cg.genBody(body);
        try cg.endBlock();
    }

    return cg.finishAir(inst, block_result, &.{});
}

/// appends a new wasm block to the code section and increases the `block_depth` by 1
fn startBlock(cg: *CodeGen, block_tag: std.wasm.Opcode, block_type: std.wasm.BlockType) !void {
    cg.block_depth += 1;
    try cg.addInst(.{
        .tag = Mir.Inst.Tag.fromOpcode(block_tag),
        .data = .{ .block_type = block_type },
    });
}

/// Ends the current wasm block and decreases the `block_depth` by 1
fn endBlock(cg: *CodeGen) !void {
    try cg.addTag(.end);
    cg.block_depth -= 1;
}

fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const block = cg.air.unwrapBlock(inst);

    // result type of loop is always 'noreturn', meaning we can always
    // emit the wasm type 'block_empty'.
    try cg.startBlock(.loop, .empty);

    try cg.loops.putNoClobber(cg.gpa, inst, cg.block_depth);
    defer assert(cg.loops.remove(inst));

    try cg.genBody(block.body);
    try cg.endBlock();

    return cg.finishAir(inst, .none, &.{});
}

fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const cond_br = cg.air.unwrapCondBr(inst);
    const condition = try cg.resolveInst(cond_br.condition);
    const then_body = cond_br.then_body;
    const else_body = cond_br.else_body;

    // result type is always noreturn, so use `block_empty` as type.
    try cg.startBlock(.block, .empty);
    // emit the conditional value
    try cg.emitWValue(condition);

    // we inserted the block in front of the condition
    // so now check if condition matches. If not, break outside this block
    // and continue with the then codepath
    try cg.addLabel(.br_if, 0);

    try cg.branches.ensureUnusedCapacity(cg.gpa, 2);
    {
        cg.branches.appendAssumeCapacity(.{});
        defer {
            var else_stack = cg.branches.pop().?;
            else_stack.deinit(cg.gpa);
        }
        try cg.genBody(else_body);
        try cg.endBlock();
    }

    // Outer block that matches the condition
    {
        cg.branches.appendAssumeCapacity(.{});
        defer {
            var then_stack = cg.branches.pop().?;
            then_stack.deinit(cg.gpa);
        }
        try cg.genBody(then_body);
    }

    return cg.finishAir(inst, .none, &.{});
}

fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) InnerError!void {
    const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
    const lhs = try cg.resolveInst(bin_op.lhs);
    const rhs = try cg.resolveInst(bin_op.rhs);
    const operand_ty = cg.typeOf(bin_op.lhs);
    const zcu = cg.pt.zcu;

    const type_tag = operand_ty.zigTypeTag(zcu);

    if (type_tag == .vector) {
        return cg.fail("TODO: implement AIR op: cmp for vectors", .{});
    }

    if (type_tag == .optional and !operand_ty.optionalReprIsPayload(zcu)) {
        const payload_ty = operand_ty.optionalChild(zcu);

        if (payload_ty.hasRuntimeBits(zcu)) {
            assert(op == .eq or op == .neq);
            assert(!isByRef(payload_ty, zcu, cg.target));

            var result = try cg.allocLocal(Type.i32);
            defer result.free(cg);

            var lhs_null = try cg.allocLocal(Type.i32);
            defer lhs_null.free(cg);

            try cg.startBlock(.block, .empty);

            try cg.addImm32(if (op == .eq) 0 else 1);
            try cg.addLocal(.local_set, result.local.value);

            _ = try cg.isNull(lhs, operand_ty, .i32_eq);
            try cg.addLocal(.local_tee, lhs_null.local.value);
            _ = try cg.isNull(rhs, operand_ty, .i32_eq);
            try cg.addTag(.i32_ne);
            try cg.addLabel(.br_if, 0);

            try cg.addImm32(if (op == .eq) 1 else 0);
            try cg.addLocal(.local_set, result.local.value);

            try cg.addLocal(.local_get, lhs_null.local.value);
            try cg.addLabel(.br_if, 0);

            _ = try cg.load(lhs, payload_ty, 0);
            _ = try cg.load(rhs, payload_ty, 0);

            if (payload_ty.isAnyFloat()) {
                _ = try cg.floatCmp(.fromType(cg, payload_ty), op, .stack, .stack);
            } else {
                _ = try cg.intCmp(.fromType(cg, payload_ty), op, .stack, .stack);
            }

            try cg.addLocal(.local_set, result.local.value);
            try cg.endBlock();

            try cg.addLocal(.local_get, result.local.value);
            try cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
        } else {
            const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs);
            try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
        }
    } else if (type_tag == .float) {
        const result = try cg.floatCmp(.fromType(cg, operand_ty), op, lhs, rhs);
        try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
    } else {
        const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs);
        try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
    }
}

fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty.bits) {
        0 => unreachable,
        1...32 => {
            // lhs or rhs could be stack pointers
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            const opcode: Mir.Inst.Tag = switch (op) {
                .eq => .i32_eq,
                .neq => .i32_ne,
                .lt => if (ty.is_signed) .i32_lt_s else .i32_lt_u,
                .lte => if (ty.is_signed) .i32_le_s else .i32_le_u,
                .gte => if (ty.is_signed) .i32_ge_s else .i32_ge_u,
                .gt => if (ty.is_signed) .i32_gt_s else .i32_gt_u,
            };
            try cg.addTag(opcode);
            return .stack;
        },
        33...64 => {
            // lhs or rhs could be stack pointers
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            const opcode: Mir.Inst.Tag = switch (op) {
                .eq => .i64_eq,
                .neq => .i64_ne,
                .lt => if (ty.is_signed) .i64_lt_s else .i64_lt_u,
                .lte => if (ty.is_signed) .i64_le_s else .i64_le_u,
                .gte => if (ty.is_signed) .i64_ge_s else .i64_ge_u,
                .gt => if (ty.is_signed) .i64_gt_s else .i64_gt_u,
            };
            try cg.addTag(opcode);
            return .stack;
        },
        65...128 => {
            var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64);
            defer lhs_msb.free(cg);
            var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64);
            defer rhs_msb.free(cg);

            switch (op) {
                .eq, .neq => {
                    const xor_high = try cg.intXor(.u64, lhs_msb, rhs_msb);
                    const lhs_lsb = try cg.load(lhs, Type.u64, 0);
                    const rhs_lsb = try cg.load(rhs, Type.u64, 0);
                    const xor_low = try cg.intXor(.u64, lhs_lsb, rhs_lsb);
                    const or_result = try cg.intOr(.u64, xor_high, xor_low);

                    switch (op) {
                        .eq => return cg.intCmp(.u64, .eq, or_result, .{ .imm64 = 0 }),
                        .neq => return cg.intCmp(.u64, .neq, or_result, .{ .imm64 = 0 }),
                        else => unreachable,
                    }
                },
                else => {
                    const word_int_ty: IntType = if (ty.is_signed) .i64 else .u64;

                    const lhs_lsb = try cg.load(lhs, Type.u64, 0);
                    const rhs_lsb = try cg.load(rhs, Type.u64, 0);

                    // leave values on stack for 'select'
                    _ = try cg.intCmp(.u64, op, lhs_lsb, rhs_lsb);
                    _ = try cg.intCmp(word_int_ty, op, lhs_msb, rhs_msb);
                    _ = try cg.intCmp(word_int_ty, .eq, lhs_msb, rhs_msb);
                    try cg.addTag(.select);
                },
            }

            return .stack;
        },
        else => {
            try cg.lowerToStack(lhs);
            try cg.lowerToStack(rhs);
            try cg.addImm32(@intFromBool(ty.is_signed));
            try cg.addImm32(ty.bits);
            try cg.addCallIntrinsic(.__cmp_limb64);
            try cg.addImm32(0);
            try cg.addTag(switch (op) {
                .eq => .i32_eq,
                .neq => .i32_ne,
                .lt => .i32_lt_s,
                .lte => .i32_le_s,
                .gte => .i32_ge_s,
                .gt => .i32_gt_s,
            });
            return .stack;
        },
    }
}

fn floatCmp(cg: *CodeGen, ty: FloatType, op: std.math.CompareOperator, lhs: WValue, rhs: WValue) InnerError!WValue {
    switch (ty) {
        .f16 => {
            _ = try cg.floatExtendCast(.f32, .f16, lhs);
            _ = try cg.floatExtendCast(.f32, .f16, rhs);
            try cg.addTag(switch (op) {
                .eq => .f32_eq,
                .neq => .f32_ne,
                .lt => .f32_lt,
                .lte => .f32_le,
                .gte => .f32_ge,
                .gt => .f32_gt,
            });
            return .stack;
        },
        .f32 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(switch (op) {
                .eq => .f32_eq,
                .neq => .f32_ne,
                .lt => .f32_lt,
                .lte => .f32_le,
                .gte => .f32_ge,
                .gt => .f32_gt,
            });
            return .stack;
        },
        .f64 => {
            try cg.emitWValue(lhs);
            try cg.emitWValue(rhs);
            try cg.addTag(switch (op) {
                .eq => .f64_eq,
                .neq => .f64_ne,
                .lt => .f64_lt,
                .lte => .f64_le,
                .gte => .f64_ge,
                .gt => .f64_gt,
            });
            return .stack;
        },
        .f80 => {
            const intrinsic: Mir.Intrinsic = switch (op) {
                .lt => .__ltxf2,
                .lte => .__lexf2,
                .eq => .__eqxf2,
                .neq => .__nexf2,
                .gte => .__gexf2,
                .gt => .__gtxf2,
            };
            const result = try cg.callIntrinsic(intrinsic, &.{ .f80_type, .f80_type }, Type.bool, &.{ lhs, rhs });
            return cg.intCmp(.i32, op, result, .{ .imm32 = 0 });
        },
        .f128 => {
            const intrinsic: Mir.Intrinsic = switch (op) {
                .lt => .__lttf2,
                .lte => .__letf2,
                .eq => .__eqtf2,
                .neq => .__netf2,
                .gte => .__getf2,
                .gt => .__gttf2,
            };
            const result = try cg.callIntrinsic(intrinsic, &.{ .f128_type, .f128_type }, Type.bool, &.{ lhs, rhs });
            return cg.intCmp(.i32, op, result, .{ .imm32 = 0 });
        },
    }
}

fn airCmpVector(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    _ = inst;
    return cg.fail("TODO implement airCmpVector for wasm", .{});
}

fn airCmpLteErrorsLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
    const operand = try cg.resolveInst(un_op);

    try cg.emitWValue(operand);
    const pt = cg.pt;
    const err_int_ty = try pt.errorIntType();
    try cg.addTag(.errors_len);
    const result = try cg.intCmp(.fromType(cg, err_int_ty), .lt, .stack, .stack);

    return cg.finishAir(inst, result, &.{un_op});
}

fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const br = cg.air.instructions.items(.data)[@intFromEnum(inst)].br;
    const block = cg.blocks.get(br.block_inst).?;

    // if operand has codegen bits we should break with a value
    if (block.value != .none) {
        const operand = try cg.resolveInst(br.operand);
        try cg.lowerToStack(operand);
        try cg.addLocal(.local_set, block.value.local.value);
    }

    // We map every block to its block index.
    // We then determine how far we have to jump to it by subtracting it from current block depth
    const idx: u32 = cg.block_depth - block.label;
    try cg.addLabel(.br, idx);

    return cg.finishAir(inst, .none, &.{br.operand});
}

fn airRepeat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const repeat = cg.air.instructions.items(.data)[@intFromEnum(inst)].repeat;
    const loop_label = cg.loops.get(repeat.loop_inst).?;

    const idx: u32 = cg.block_depth - loop_label;
    try cg.addLabel(.br, idx);

    return cg.finishAir(inst, .none, &.{});
}

fn airTrap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    try cg.addTag(.@"unreachable");
    return cg.finishAir(inst, .none, &.{});
}

fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    // unsupported by wasm itfunc. Can be implemented once we support DWARF
    // for wasm
    try cg.addTag(.@"unreachable");
    return cg.finishAir(inst, .none, &.{});
}

fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    try cg.addTag(.@"unreachable");
    return cg.finishAir(inst, .none, &.{});
}

fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
    const operand = try cg.resolveInst(ty_op.operand);
    const dest_ty = cg.typeOfIndex(inst);
    const src_ty = cg.typeOf(ty_op.operand);

    const result = (try cg.bitcast(dest_ty, src_ty, operand)) orelse cg.reuseOperand(ty_op.operand, operand);

    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerError!?WValue {
    const zcu = cg.pt.zcu;
    const bit_size = src_ty.bitSize(zcu);
    const needs_wrapping = (src_ty.isSignedInt(zcu) != dest_ty.isSignedInt(zcu)) and
        bit_size != 32 and bit_size != 64 and bit_size != 128;

    if (src_ty.isAnyFloat() or dest_ty.isAnyFloat()) {
        if (dest_ty.ip_index == .f16_type or src_ty.ip_index == .f16_type) return null;
        if (dest_ty.bitSize(zcu) > 64) return null;
        assert((dest_ty.isInt(zcu) and src_ty.isAnyFloat()) or (dest_ty.isAnyFloat() and src_ty.isInt(zcu)));

        const dest_valtype = typeToValtype(dest_ty, zcu, cg.target);
        const opcode: Mir.Inst.Tag = switch (dest_valtype) {
            .i32 => .i32_reinterpret_f32,
            .i64 => .i64_reinterpret_f64,
            .f32 => .f32_reinterpret_i32,
            .f64 => .f64_reinterpret_i64,
            else => unreachable,
        };

        try cg.emitWValue(operand);
        try cg.addTag(opcode);
        return .stack;
    }

    if (isByRef(src_ty, zcu, cg.target) and !isByRef(dest_ty, zcu, cg.target)) {
        const loaded_memory = try cg.load(operand, dest_ty, 0);
        if (needs_wrapping) {
            const int_ty: IntType = .fromType(cg, dest_ty);
            return try cg.intWrap(int_ty, loaded_memory);
        } else {
            return loaded_memory;
        }
    }

    if (!isByRef(src_ty, zcu, cg.target) and isByRef(dest_ty, zcu, cg.target)) {
        const stack_memory = try cg.allocStack(dest_ty);
        try cg.store(stack_memory, operand, src_ty, 0);
        if (needs_wrapping) {
            const int_ty: IntType = .fromType(cg, dest_ty);
            return try cg.intWrap(int_ty, stack_memory);
        } else {
            return stack_memory;
        }
    }

    if (needs_wrapping) {
        const int_ty: IntType = .fromType(cg, dest_ty);
        return try cg.intWrap(int_ty, operand);
    }

    return switch (operand) {
        // for stack offset, return a pointer to this offset.
        .stack_offset => try cg.buildPointerOffset(operand, 0, .new),
        else => null, // caller should use cg.reuseOperand, if returnes for AIR
    };
}

fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
    const extra = cg.air.extraData(Air.StructField, ty_pl.payload);

    const struct_ptr = try cg.resolveInst(extra.data.struct_operand);
    const struct_ptr_ty = cg.typeOf(extra.data.struct_operand);
    const struct_ty = struct_ptr_ty.childType(zcu);
    const result = try cg.structFieldPtr(inst, extra.data.struct_operand, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index);
    return cg.finishAir(inst, result, &.{extra.data.struct_operand});
}

fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, index: u32) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
    const struct_ptr = try cg.resolveInst(ty_op.operand);
    const struct_ptr_ty = cg.typeOf(ty_op.operand);
    const struct_ty = struct_ptr_ty.childType(zcu);

    const result = try cg.structFieldPtr(inst, ty_op.operand, struct_ptr, struct_ptr_ty, struct_ty, index);
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn structFieldPtr(
    cg: *CodeGen,
    inst: Air.Inst.Index,
    ref: Air.Inst.Ref,
    struct_ptr: WValue,
    struct_ptr_ty: Type,
    struct_ty: Type,
    index: u32,
) InnerError!WValue {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const result_ty = cg.typeOfIndex(inst);
    const struct_ptr_ty_info = struct_ptr_ty.ptrInfo(zcu);

    const offset = switch (struct_ty.containerLayout(zcu)) {
        .@"packed" => switch (struct_ty.zigTypeTag(zcu)) {
            .@"struct" => offset: {
                if (result_ty.ptrInfo(zcu).packed_offset.host_size != 0) {
                    break :offset @as(u32, 0);
                }
                const struct_type = zcu.typeToStruct(struct_ty).?;
                break :offset @divExact(zcu.structPackedFieldBitOffset(struct_type, index) + struct_ptr_ty_info.packed_offset.bit_offset, 8);
            },
            .@"union" => 0,
            else => unreachable,
        },
        else => struct_ty.structFieldOffset(index, zcu),
    };
    // save a load and store when we can simply reuse the operand
    if (offset == 0) {
        return cg.reuseOperand(ref, struct_ptr);
    }
    switch (struct_ptr) {
        .stack_offset => |stack_offset| {
            return .{ .stack_offset = .{ .value = stack_offset.value + @as(u32, @intCast(offset)), .references = 1 } };
        },
        else => return cg.buildPointerOffset(struct_ptr, offset, .new),
    }
}

fn airStructFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
    const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data;

    const struct_ty = cg.typeOf(struct_field.struct_operand);
    const operand = try cg.resolveInst(struct_field.struct_operand);
    const field_index = struct_field.field_index;
    const field_ty = struct_ty.fieldType(field_index, zcu);
    if (!field_ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{struct_field.struct_operand});

    const result: WValue = switch (struct_ty.containerLayout(zcu)) {
        .@"packed" => unreachable, // legalize .expand_packed_struct_field_val
        else => result: {
            const offset = std.math.cast(u32, struct_ty.structFieldOffset(field_index, zcu)) orelse {
                return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)});
            };
            if (isByRef(field_ty, zcu, cg.target)) {
                switch (operand) {
                    .stack_offset => |stack_offset| {
                        break :result .{ .stack_offset = .{ .value = stack_offset.value + offset, .references = 1 } };
                    },
                    else => break :result try cg.buildPointerOffset(operand, offset, .new),
                }
            }
            break :result try cg.load(operand, field_ty, offset);
        },
    };

    return cg.finishAir(inst, result, &.{struct_field.struct_operand});
}

fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) InnerError!void {
    const pt = cg.pt;
    const zcu = pt.zcu;

    const switch_br = cg.air.unwrapSwitch(inst);
    const target_ty = cg.typeOf(switch_br.operand);

    assert(target_ty.hasRuntimeBits(zcu));

    // swap target value with placeholder local, for dispatching
    const target = if (is_dispatch_loop) target: {
        const initial_target = try cg.resolveInst(switch_br.operand);
        const target: WValue = try cg.allocLocal(target_ty);
        try cg.lowerToStack(initial_target);
        try cg.addLocal(.local_set, target.local.value);

        try cg.startBlock(.loop, .empty); // dispatch loop start
        try cg.blocks.putNoClobber(cg.gpa, inst, .{
            .label = cg.block_depth,
            .value = target,
        });

        break :target target;
    } else try cg.resolveInst(switch_br.operand);

    const has_else_body = switch_br.else_body_len != 0;
    const branch_count = switch_br.cases_len + 1; // if else branch is missing, we trap when failing all conditions
    try cg.branches.ensureUnusedCapacity(cg.gpa, switch_br.cases_len + @intFromBool(has_else_body));

    if (switch_br.cases_len == 0) {
        assert(has_else_body);

        var it = switch_br.iterateCases();
        const else_body = it.elseBody();

        cg.branches.appendAssumeCapacity(.{});
        defer {
            var else_branch = cg.branches.pop().?;
            else_branch.deinit(cg.gpa);
        }
        try cg.genBody(else_body);

        if (is_dispatch_loop) {
            try cg.endBlock(); // dispatch loop end
        }
        return cg.finishAir(inst, .none, &.{});
    }

    var min: ?Value = null;
    var max: ?Value = null;
    var branching_size: u32 = 0; // single item +1, range +2

    {
        var cases_it = switch_br.iterateCases();
        while (cases_it.next()) |case| {
            for (case.items) |item| {
                const val = Value.fromInterned(item.toInterned().?);
                if (min == null or val.compareHetero(.lt, min.?, zcu)) min = val;
                if (max == null or val.compareHetero(.gt, max.?, zcu)) max = val;
                branching_size += 1;
            }
            for (case.ranges) |range| {
                const low = Value.fromInterned(range[0].toInterned().?);
                if (min == null or low.compareHetero(.lt, min.?, zcu)) min = low;
                const high = Value.fromInterned(range[1].toInterned().?);
                if (max == null or high.compareHetero(.gt, max.?, zcu)) max = high;
                branching_size += 2;
            }
        }
    }

    var min_space: Value.BigIntSpace = undefined;
    const min_bigint = min.?.toBigInt(&min_space, zcu);
    var max_space: Value.BigIntSpace = undefined;
    const max_bigint = max.?.toBigInt(&max_space, zcu);
    const limbs = try cg.gpa.alloc(
        std.math.big.Limb,
        @max(min_bigint.limbs.len, max_bigint.limbs.len) + 1,
    );
    defer cg.gpa.free(limbs);

    const width_maybe: ?u32 = width: {
        var width_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
        width_bigint.sub(max_bigint, min_bigint);
        width_bigint.addScalar(width_bigint.toConst(), 1);
        break :width width_bigint.toConst().toInt(u32) catch null;
    };

    try cg.startBlock(.block, .empty); // whole switch block start

    for (0..branch_count) |_| {
        try cg.startBlock(.block, .empty);
    }

    // Heuristic on deciding when to use .br_table instead of .br_if jump table
    // 1. Differences between lowest and highest values should fit into u32
    // 2. .br_table should be applied for "dense" switch, we test it by checking .br_if jumps will need more instructions
    // 3. Do not use .br_table for tiny switches
    const use_br_table = cond: {
        const width = width_maybe orelse break :cond false;
        if (width > 2 * branching_size) break :cond false;
        if (width < 2 or branch_count < 2) break :cond false;
        break :cond true;
    };

    const int_ty: IntType = .fromType(cg, target_ty);

    if (use_br_table) {
        const width = width_maybe.?;

        const br_value_original = try cg.intSub(int_ty, target, try cg.resolveValue(min.?));
        _ = try cg.intCast(.u32, int_ty, br_value_original);

        const jump_table: Mir.JumpTable = .{ .length = width + 1 };
        const table_extra_index = try cg.addExtra(jump_table);
        try cg.addInst(.{ .tag = .br_table, .data = .{ .payload = table_extra_index } });

        const branch_list = try cg.mir_extra.addManyAsSlice(cg.gpa, width + 1);
        @memset(branch_list, branch_count - 1);

        var cases_it = switch_br.iterateCases();
        while (cases_it.next()) |case| {
            for (case.items) |item| {
                const val = Value.fromInterned(item.toInterned().?);
                var val_space: Value.BigIntSpace = undefined;
                const val_bigint = val.toBigInt(&val_space, zcu);
                var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
                index_bigint.sub(val_bigint, min_bigint);
                branch_list[index_bigint.toConst().toInt(u32) catch unreachable] = case.idx;
            }
            for (case.ranges) |range| {
                var low_space: Value.BigIntSpace = undefined;
                const low_bigint = Value.fromInterned(range[0].toInterned().?).toBigInt(&low_space, zcu);
                var high_space: Value.BigIntSpace = undefined;
                const high_bigint = Value.fromInterned(range[1].toInterned().?).toBigInt(&high_space, zcu);
                var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
                index_bigint.sub(low_bigint, min_bigint);
                const start = index_bigint.toConst().toInt(u32) catch unreachable;
                index_bigint.sub(high_bigint, min_bigint);
                const end = (index_bigint.toConst().toInt(u32) catch unreachable) + 1;
                @memset(branch_list[start..end], case.idx);
            }
        }
    } else {
        var cases_it = switch_br.iterateCases();
        while (cases_it.next()) |case| {
            for (case.items) |ref| {
                const val = try cg.resolveInst(ref);
                _ = try cg.intCmp(int_ty, .eq, target, val);
                try cg.addLabel(.br_if, case.idx); // item match found
            }
            for (case.ranges) |range| {
                const low = try cg.resolveInst(range[0]);
                const high = try cg.resolveInst(range[1]);

                const gte = try cg.intCmp(int_ty, .gte, target, low);
                const lte = try cg.intCmp(int_ty, .lte, target, high);
                _ = try cg.intAnd(.u32, gte, lte);
                try cg.addLabel(.br_if, case.idx); // range match found
            }
        }
        try cg.addLabel(.br, branch_count - 1);
    }

    var cases_it = switch_br.iterateCases();
    while (cases_it.next()) |case| {
        try cg.endBlock();

        cg.branches.appendAssumeCapacity(.{});
        defer {
            var case_branch = cg.branches.pop().?;
            case_branch.deinit(cg.gpa);
        }
        try cg.genBody(case.body);

        try cg.addLabel(.br, branch_count - case.idx - 1); // matching case found and executed => exit switch
    }

    try cg.endBlock();
    if (has_else_body) {
        const else_body = cases_it.elseBody();

        cg.branches.appendAssumeCapacity(.{});
        defer {
            var else_branch = cg.branches.pop().?;
            else_branch.deinit(cg.gpa);
        }
        try cg.genBody(else_body);
    } else {
        try cg.addTag(.@"unreachable");
    }

    try cg.endBlock(); // whole switch block end

    if (is_dispatch_loop) {
        try cg.endBlock(); // dispatch loop end
    }

    return cg.finishAir(inst, .none, &.{});
}

fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const br = cg.air.instructions.items(.data)[@intFromEnum(inst)].br;
    const switch_loop = cg.blocks.get(br.block_inst).?;

    const operand = try cg.resolveInst(br.operand);
    try cg.lowerToStack(operand);
    try cg.addLocal(.local_set, switch_loop.value.local.value);

    const idx: u32 = cg.block_depth - switch_loop.label;
    try cg.addLabel(.br, idx);

    return cg.finishAir(inst, .none, &.{br.operand});
}

fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: enum { value, ptr }) InnerError!void {
    const zcu = cg.pt.zcu;
    const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
    const operand = try cg.resolveInst(un_op);
    const err_union_ty = switch (op_kind) {
        .value => cg.typeOf(un_op),
        .ptr => cg.typeOf(un_op).childType(zcu),
    };
    const pl_ty = err_union_ty.errorUnionPayload(zcu);

    const result: WValue = result: {
        if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
            switch (opcode) {
                .i32_ne => break :result .{ .imm32 = 0 },
                .i32_eq => break :result .{ .imm32 = 1 },
                else => unreachable,
            }
        }

        try cg.emitWValue(operand);
        if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {
            try cg.addMemArg(.i32_load16_u, .{
                .offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))),
                .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
            });
        }

        // Compare the error value with '0'
        try cg.addImm32(0);
        try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
        break :result .stack;
    };
    return cg.finishAir(inst, result, &.{un_op});
}

/// E!T -> T op_is_ptr == false
/// *(E!T) -> *T op_is_prt == true
fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;

    const operand = try cg.resolveInst(ty_op.operand);
    const op_ty = cg.typeOf(ty_op.operand);
    const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty;
    const payload_ty = eu_ty.errorUnionPayload(zcu);

    const result: WValue = result: {
        if (!payload_ty.hasRuntimeBits(zcu)) {
            if (op_is_ptr) {
                break :result cg.reuseOperand(ty_op.operand, operand);
            } else {
                break :result .none;
            }
        }

        const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
        if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) {
            break :result try cg.buildPointerOffset(operand, pl_offset, .new);
        } else {
            assert(isByRef(eu_ty, zcu, cg.target));
            break :result try cg.load(operand, payload_ty, pl_offset);
        }
    };
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

/// E!T -> E op_is_ptr == false
/// *(E!T) -> E op_is_ptr == true
/// NOTE: op_is_ptr will not change return type
fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;

    const operand = try cg.resolveInst(ty_op.operand);
    const op_ty = cg.typeOf(ty_op.operand);
    const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty;
    const payload_ty = eu_ty.errorUnionPayload(zcu);

    const result: WValue = result: {
        if (eu_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
            break :result .{ .imm32 = 0 };
        }

        const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
        if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) {
            break :result try cg.load(operand, Type.anyerror, err_offset);
        } else {
            assert(!payload_ty.hasRuntimeBits(zcu));
            break :result cg.reuseOperand(ty_op.operand, operand);
        }
    };
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;

    const operand = try cg.resolveInst(ty_op.operand);
    const err_ty = cg.typeOfIndex(inst);

    const pl_ty = cg.typeOf(ty_op.operand);
    const result = result: {
        if (!pl_ty.hasRuntimeBits(zcu)) {
            break :result cg.reuseOperand(ty_op.operand, operand);
        }

        const err_union = try cg.allocStack(err_ty);
        const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
        try cg.store(payload_ptr, operand, pl_ty, 0);

        // ensure we also write '0' to the error part, so any present stack value gets overwritten by it.
        try cg.emitWValue(err_union);
        try cg.addImm32(0);
        const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
        try cg.addMemArg(.i32_store16, .{
            .offset = err_union.offset() + err_val_offset,
            .alignment = 2,
        });
        break :result err_union;
    };
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;

    const operand = try cg.resolveInst(ty_op.operand);
    const err_ty = ty_op.ty.toType();
    const pl_ty = err_ty.errorUnionPayload(zcu);

    const result = result: {
        if (!pl_ty.hasRuntimeBits(zcu)) {
            break :result cg.reuseOperand(ty_op.operand, operand);
        }

        const err_union = try cg.allocStack(err_ty);
        // store error value
        try cg.store(err_union, operand, Type.anyerror, @intCast(errUnionErrorOffset(pl_ty, zcu)));

        // write 'undefined' to the payload
        const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
        const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu)));
        try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });

        break :result err_union;
    };
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: enum { value, ptr }) InnerError!void {
    const zcu = cg.pt.zcu;
    const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
    const operand = try cg.resolveInst(un_op);

    const op_ty = cg.typeOf(un_op);
    const optional_ty = if (op_kind == .ptr) op_ty.childType(zcu) else op_ty;
    const result = try cg.isNull(operand, optional_ty, opcode);
    return cg.finishAir(inst, result, &.{un_op});
}

/// For a given type and operand, checks if it's considered `null`.
/// NOTE: Leaves the result on the stack
fn isNull(cg: *CodeGen, operand: WValue, optional_ty: Type, opcode: std.wasm.Opcode) InnerError!WValue {
    const pt = cg.pt;
    const zcu = pt.zcu;
    try cg.emitWValue(operand);
    const payload_ty = optional_ty.optionalChild(zcu);
    if (!optional_ty.optionalReprIsPayload(zcu)) {
        // When payload is zero-bits, we can treat operand as a value, rather than
        // a pointer to the stack value
        if (payload_ty.hasRuntimeBits(zcu)) {
            const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
                return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});
            };
            try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 });
        }
    } else if (payload_ty.isSlice(zcu)) {
        switch (cg.ptr_size) {
            .wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = 4 }),
            .wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = 8 }),
        }
    }

    // Compare the null value with '0'
    try cg.addImm32(0);
    try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));

    return .stack;
}

fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
    const opt_ty = cg.typeOf(ty_op.operand);
    const payload_ty = cg.typeOfIndex(inst);
    if (!payload_ty.hasRuntimeBits(zcu)) {
        return cg.finishAir(inst, .none, &.{ty_op.operand});
    }

    const result = result: {
        const operand = try cg.resolveInst(ty_op.operand);
        if (opt_ty.optionalReprIsPayload(zcu)) break :result cg.reuseOperand(ty_op.operand, operand);

        if (isByRef(payload_ty, zcu, cg.target)) {
            break :result try cg.buildPointerOffset(operand, 0, .new);
        }

        break :result try cg.load(operand, payload_ty, 0);
    };
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
    const operand = try cg.resolveInst(ty_op.operand);
    const opt_ty = cg.typeOf(ty_op.operand).childType(zcu);

    const result = result: {
        const payload_ty = opt_ty.optionalChild(zcu);
        if (!payload_ty.hasRuntimeBits(zcu) or opt_ty.optionalReprIsPayload(zcu)) {
            break :result cg.reuseOperand(ty_op.operand, operand);
        }

        break :result try cg.buildPointerOffset(operand, 0, .new);
    };
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
    const operand = try cg.resolveInst(ty_op.operand);
    const opt_ty = cg.typeOf(ty_op.operand).childType(zcu);
    const payload_ty = opt_ty.optionalChild(zcu);

    if (opt_ty.optionalReprIsPayload(zcu)) {
        return cg.finishAir(inst, operand, &.{ty_op.operand});
    }

    const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
        return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)});
    };

    try cg.emitWValue(operand);
    try cg.addImm32(1);
    try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 });

    const result = try cg.buildPointerOffset(operand, 0, .new);
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
    const payload_ty = cg.typeOf(ty_op.operand);
    const pt = cg.pt;
    const zcu = pt.zcu;

    const result = result: {
        if (!payload_ty.hasRuntimeBits(zcu)) {
            const non_null_bit = try cg.allocStack(Type.u1);
            try cg.emitWValue(non_null_bit);
            try cg.addImm32(1);
            try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = 1 });
            break :result non_null_bit;
        }

        const operand = try cg.resolveInst(ty_op.operand);
        const op_ty = cg.typeOfIndex(inst);
        if (op_ty.optionalReprIsPayload(zcu)) {
            break :result cg.reuseOperand(ty_op.operand, operand);
        }
        const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
            return cg.fail("Optional type {f} too big to fit into stack frame", .{op_ty.fmt(pt)});
        };

        // Create optional type, set the non-null bit, and store the operand inside the optional type
        const result_ptr = try cg.allocStack(op_ty);
        try cg.emitWValue(result_ptr);
        try cg.addImm32(1);
        try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 });

        const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new);
        try cg.store(payload_ptr, operand, payload_ty, 0);
        break :result result_ptr;
    };

    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
    const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;

    const lhs = try cg.resolveInst(bin_op.lhs);
    const rhs = try cg.resolveInst(bin_op.rhs);
    const slice_ty = cg.typeOfIndex(inst);

    const slice = try cg.allocStack(slice_ty);
    try cg.store(slice, lhs, Type.usize, 0);
    try cg.store(slice, rhs, Type.usize, cg.ptrSize());

    return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs });
}

fn airSliceLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;

    const operand = try cg.resolveInst(ty_op.operand);
    return cg.finishAir(inst, try cg.sliceLen(operand), &.{ty_op.operand});
}

fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;

    const slice_ty = cg.typeOf(bin_op.lhs);
    const slice = try cg.resolveInst(bin_op.lhs);
    const index = try cg.resolveInst(bin_op.rhs);
    const elem_ty = slice_ty.childType(zcu);
    const elem_size = elem_ty.abiSize(zcu);

    // load pointer onto stack
    _ = try cg.load(slice, Type.usize, 0);

    // calculate index into slice
    try cg.emitWValue(index);
    try cg.addImm32(@intCast(elem_size));
    try cg.addTag(.i32_mul);
    try cg.addTag(.i32_add);

    const elem_result = if (isByRef(elem_ty, zcu, cg.target))
        .stack
    else
        try cg.load(.stack, elem_ty, 0);

    return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
}

fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
    const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;

    const elem_ty = ty_pl.ty.toType().childType(zcu);
    const elem_size = elem_ty.abiSize(zcu);

    const slice = try cg.resolveInst(bin_op.lhs);
    const index = try cg.resolveInst(bin_op.rhs);

    _ = try cg.load(slice, Type.usize, 0);

    // calculate index into slice
    try cg.emitWValue(index);
    try cg.addImm32(@intCast(elem_size));
    try cg.addTag(.i32_mul);
    try cg.addTag(.i32_add);

    return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
}

fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
    const operand = try cg.resolveInst(ty_op.operand);
    return cg.finishAir(inst, try cg.slicePtr(operand), &.{ty_op.operand});
}

fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue {
    const ptr = try cg.load(operand, Type.usize, 0);
    return ptr.toLocal(cg, Type.usize);
}

fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue {
    const len = try cg.load(operand, Type.usize, cg.ptrSize());
    return len.toLocal(cg, Type.usize);
}

fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;

    const operand = try cg.resolveInst(ty_op.operand);
    const array_ty = cg.typeOf(ty_op.operand).childType(zcu);
    const slice_ty = ty_op.ty.toType();

    // create a slice on the stack
    const slice_local = try cg.allocStack(slice_ty);

    // store the array ptr in the slice
    if (array_ty.hasRuntimeBits(zcu)) {
        try cg.store(slice_local, operand, Type.usize, 0);
    }

    // store the length of the array in the slice
    const array_len: u32 = @intCast(array_ty.arrayLen(zcu));
    try cg.store(slice_local, .{ .imm32 = array_len }, Type.usize, cg.ptrSize());

    return cg.finishAir(inst, slice_local, &.{ty_op.operand});
}

fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;

    const ptr_ty = cg.typeOf(bin_op.lhs);
    const ptr = try cg.resolveInst(bin_op.lhs);
    const index = try cg.resolveInst(bin_op.rhs);
    const elem_ty = ptr_ty.childType(zcu);
    const elem_size = elem_ty.abiSize(zcu);

    // load pointer onto the stack
    if (ptr_ty.isSlice(zcu)) {
        _ = try cg.load(ptr, Type.usize, 0);
    } else {
        try cg.lowerToStack(ptr);
    }

    // calculate index into slice
    try cg.emitWValue(index);
    try cg.addImm32(@intCast(elem_size));
    try cg.addTag(.i32_mul);
    try cg.addTag(.i32_add);

    const elem_result = if (isByRef(elem_ty, zcu, cg.target))
        .stack
    else
        try cg.load(.stack, elem_ty, 0);

    return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
}

fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
    const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;

    const ptr_ty = cg.typeOf(bin_op.lhs);
    const elem_ty = ty_pl.ty.toType().childType(zcu);
    const elem_size = elem_ty.abiSize(zcu);

    const ptr = try cg.resolveInst(bin_op.lhs);
    const index = try cg.resolveInst(bin_op.rhs);

    // load pointer onto the stack
    if (ptr_ty.isSlice(zcu)) {
        _ = try cg.load(ptr, Type.usize, 0);
    } else {
        try cg.lowerToStack(ptr);
    }

    // calculate index into ptr
    try cg.emitWValue(index);
    try cg.addImm32(@intCast(elem_size));
    try cg.addTag(.i32_mul);
    try cg.addTag(.i32_add);

    return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
}

fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
    const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;

    const ptr = try cg.resolveInst(bin_op.lhs);
    const offset = try cg.resolveInst(bin_op.rhs);
    const ptr_ty = cg.typeOf(bin_op.lhs);
    const pointee_ty = switch (ptr_ty.ptrSize(zcu)) {
        .one => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type
        else => ptr_ty.childType(zcu),
    };

    try cg.lowerToStack(ptr);
    try cg.emitWValue(offset);

    switch (cg.ptr_size) {
        .wasm32 => {
            try cg.addImm32(@intCast(pointee_ty.abiSize(zcu)));
            try cg.addTag(.i32_mul);
            try cg.addTag(switch (op) {
                .add => .i32_add,
                .sub => .i32_sub,
            });
        },
        .wasm64 => {
            try cg.addImm64(pointee_ty.abiSize(zcu));
            try cg.addTag(.i64_mul);
            try cg.addTag(switch (op) {
                .add => .i64_add,
                .sub => .i64_sub,
            });
        },
    }

    return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
}

fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
    const zcu = cg.pt.zcu;
    const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;

    const ptr = try cg.resolveInst(bin_op.lhs);
    const ptr_ty = cg.typeOf(bin_op.lhs);
    const value = try cg.resolveInst(bin_op.rhs);
    const len = switch (ptr_ty.ptrSize(zcu)) {
        .slice => try cg.sliceLen(ptr),
        .one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }),
        .c, .many => unreachable,
    };

    const elem_ty = if (ptr_ty.ptrSize(zcu) == .one)
        ptr_ty.childType(zcu).childType(zcu)
    else
        ptr_ty.childType(zcu);

    if (!safety and bin_op.rhs == .undef) {
        return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
    }

    const dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty);
    try cg.memset(elem_ty, dst_ptr, len, value);

    return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
}

/// Sets a region of memory at `ptr` to the value of `value`
/// When the user has enabled the bulk_memory feature, we lower
/// this to wasm's memset instruction. When the feature is not present,
/// we implement it manually.
fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {
    const zcu = cg.pt.zcu;
    const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));

    // When bulk_memory is enabled, we lower it to wasm's memset instruction.
    // If not, we lower it ourselves.
    if (cg.target.cpu.has(.wasm, .bulk_memory) and abi_size == 1) {
        const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);

        if (!len0_ok) {
            try cg.startBlock(.block, .empty);

            // Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is
            // out of memory bounds. This can easily happen in Zig in a case such as:
            //
            // const ptr: [*]u8 = undefined;
            // var len: usize = runtime_zero();
            // @memset(ptr[0..len], 42);
            //
            // So explicitly avoid using `memory.fill` in the `len == 0` case. Lovely design.
            try cg.emitWValue(len);
            try cg.addTag(.i32_eqz);
            try cg.addLabel(.br_if, 0);
        }

        try cg.lowerToStack(ptr);
        try cg.emitWValue(value);
        try cg.emitWValue(len);
        try cg.addExtended(.memory_fill);

        if (!len0_ok) {
            try cg.endBlock();
        }

        return;
    }

    const final_len: WValue = switch (len) {
        .imm32 => |val| .{ .imm32 = val * abi_size },
        .imm64 => |val| .{ .imm64 = val * abi_size },
        else => if (abi_size != 1) blk: {
            const new_len = try cg.ensureAllocLocal(Type.usize);
            try cg.emitWValue(len);
            switch (cg.ptr_size) {
                .wasm32 => {
                    try cg.emitWValue(.{ .imm32 = abi_size });
                    try cg.addTag(.i32_mul);
                },
                .wasm64 => {
                    try cg.emitWValue(.{ .imm64 = abi_size });
                    try cg.addTag(.i64_mul);
                },
            }
            try cg.addLocal(.local_set, new_len.local.value);
            break :blk new_len;
        } else len,
    };

    var end_ptr = try cg.allocLocal(Type.usize);
    defer end_ptr.free(cg);
    var new_ptr = try cg.buildPointerOffset(ptr, 0, .new);
    defer new_ptr.free(cg);

    // get the loop conditional: if current pointer address equals final pointer's address
    try cg.lowerToStack(ptr);
    try cg.emitWValue(final_len);
    switch (cg.ptr_size) {
        .wasm32 => try cg.addTag(.i32_add),
        .wasm64 => try cg.addTag(.i64_add),
    }
    try cg.addLocal(.local_set, end_ptr.local.value);

    // outer block to jump to when loop is done
    try cg.startBlock(.block, .empty);
    try cg.startBlock(.loop, .empty);

    // check for condition for loop end
    try cg.emitWValue(new_ptr);
    try cg.emitWValue(end_ptr);
    switch (cg.ptr_size) {
        .wasm32 => try cg.addTag(.i32_eq),
        .wasm64 => try cg.addTag(.i64_eq),
    }
    try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished)

    // store the value at the current position of the pointer
    try cg.store(new_ptr, value, elem_ty, 0);

    // move the pointer to the next element
    try cg.emitWValue(new_ptr);
    switch (cg.ptr_size) {
        .wasm32 => {
            try cg.emitWValue(.{ .imm32 = abi_size });
            try cg.addTag(.i32_add);
        },
        .wasm64 => {
            try cg.emitWValue(.{ .imm64 = abi_size });
            try cg.addTag(.i64_add);
        },
    }
    try cg.addLocal(.local_set, new_ptr.local.value);

    // end of loop
    try cg.addLabel(.br, 0); // jump to start of loop
    try cg.endBlock();
    try cg.endBlock();
}

fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;

    const array_ty = cg.typeOf(bin_op.lhs);
    const array = try cg.resolveInst(bin_op.lhs);
    const index = try cg.resolveInst(bin_op.rhs);
    const elem_ty = array_ty.childType(zcu);
    const elem_size = elem_ty.abiSize(zcu);

    if (isByRef(array_ty, zcu, cg.target)) {
        try cg.lowerToStack(array);
        try cg.emitWValue(index);
        try cg.addImm32(@intCast(elem_size));
        try cg.addTag(.i32_mul);
        try cg.addTag(.i32_add);
    } else {
        assert(array_ty.zigTypeTag(zcu) == .vector);

        switch (index) {
            inline .imm32, .imm64 => |lane| {
                const opcode: std.wasm.SimdOpcode = switch (elem_ty.bitSize(zcu)) {
                    8 => if (elem_ty.isSignedInt(zcu)) .i8x16_extract_lane_s else .i8x16_extract_lane_u,
                    16 => if (elem_ty.isSignedInt(zcu)) .i16x8_extract_lane_s else .i16x8_extract_lane_u,
                    32 => if (elem_ty.isInt(zcu)) .i32x4_extract_lane else .f32x4_extract_lane,
                    64 => if (elem_ty.isInt(zcu)) .i64x2_extract_lane else .f64x2_extract_lane,
                    else => unreachable,
                };

                var operands = [_]u32{ @intFromEnum(opcode), @as(u8, @intCast(lane)) };

                try cg.emitWValue(array);

                const extra_index: u32 = @intCast(cg.mir_extra.items.len);
                try cg.mir_extra.appendSlice(cg.gpa, &operands);
                try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });

                return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
            },
            else => {
                const stack_vec = try cg.allocStack(array_ty);
                try cg.store(stack_vec, array, array_ty, 0);

                // Is a non-unrolled vector (v128)
                try cg.lowerToStack(stack_vec);
                try cg.emitWValue(index);
                try cg.addImm32(@intCast(elem_size));
                try cg.addTag(.i32_mul);
                try cg.addTag(.i32_add);
            },
        }
    }

    const elem_result = if (isByRef(elem_ty, zcu, cg.target))
        .stack
    else
        try cg.load(.stack, elem_ty, 0);

    return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
}

fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
    const operand = try cg.resolveInst(ty_op.operand);
    const ty = cg.typeOfIndex(inst);
    const elem_ty = ty.childType(zcu);

    if (determineSimdStoreStrategy(ty, zcu, cg.target) == .direct) blk: {
        switch (operand) {
            // when the operand lives in the linear memory section, we can directly
            // load and splat the value at once. Meaning we do not first have to load
            // the scalar value onto the stack.
            .stack_offset, .nav_ref, .uav_ref => {
                const opcode = switch (elem_ty.bitSize(zcu)) {
                    8 => @intFromEnum(std.wasm.SimdOpcode.v128_load8_splat),
                    16 => @intFromEnum(std.wasm.SimdOpcode.v128_load16_splat),
                    32 => @intFromEnum(std.wasm.SimdOpcode.v128_load32_splat),
                    64 => @intFromEnum(std.wasm.SimdOpcode.v128_load64_splat),
                    else => break :blk, // Cannot make use of simd-instructions
                };
                try cg.emitWValue(operand);
                const extra_index: u32 = @intCast(cg.mir_extra.items.len);
                // stores as := opcode, offset, alignment (opcode::memarg)
                try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
                    opcode,
                    operand.offset(),
                    @intCast(elem_ty.abiAlignment(zcu).toByteUnits().?),
                });
                try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
                return cg.finishAir(inst, .stack, &.{ty_op.operand});
            },
            .local => {
                const opcode = switch (elem_ty.bitSize(zcu)) {
                    8 => @intFromEnum(std.wasm.SimdOpcode.i8x16_splat),
                    16 => @intFromEnum(std.wasm.SimdOpcode.i16x8_splat),
                    32 => if (elem_ty.isInt(zcu)) @intFromEnum(std.wasm.SimdOpcode.i32x4_splat) else @intFromEnum(std.wasm.SimdOpcode.f32x4_splat),
                    64 => if (elem_ty.isInt(zcu)) @intFromEnum(std.wasm.SimdOpcode.i64x2_splat) else @intFromEnum(std.wasm.SimdOpcode.f64x2_splat),
                    else => break :blk, // Cannot make use of simd-instructions
                };
                try cg.emitWValue(operand);
                const extra_index: u32 = @intCast(cg.mir_extra.items.len);
                try cg.mir_extra.append(cg.gpa, opcode);
                try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
                return cg.finishAir(inst, .stack, &.{ty_op.operand});
            },
            else => unreachable,
        }
    }
    const elem_size = elem_ty.bitSize(zcu);
    const vector_len = @as(usize, @intCast(ty.vectorLen(zcu)));
    if ((!std.math.isPowerOfTwo(elem_size) or elem_size % 8 != 0) and vector_len > 1) {
        return cg.fail("TODO: WebAssembly `@splat` for arbitrary element bitsize {d}", .{elem_size});
    }

    const result = try cg.allocStack(ty);
    const elem_byte_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
    var index: usize = 0;
    var offset: u32 = 0;
    while (index < vector_len) : (index += 1) {
        try cg.store(result, operand, elem_ty, offset);
        offset += elem_byte_size;
    }

    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn airSelect(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
    const operand = try cg.resolveInst(pl_op.operand);

    _ = operand;
    return cg.fail("TODO: Implement wasm airSelect", .{});
}

fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const pt = cg.pt;
    const zcu = pt.zcu;

    const unwrapped = cg.air.unwrapShuffleOne(zcu, inst);
    const result_ty = unwrapped.result_ty;
    const mask = unwrapped.mask;
    const operand = try cg.resolveInst(unwrapped.operand);

    const elem_ty = result_ty.childType(zcu);
    const elem_size = elem_ty.abiSize(zcu);

    // TODO: this function could have an `i8x16_shuffle` fast path like `airShuffleTwo` if we were
    // to lower the comptime-known operands to a non-by-ref vector value.

    // TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible.
    // I tried to fix it, but I couldn't make much sense of how this backend handles memory.
    if (!isByRef(result_ty, zcu, cg.target) or
        !isByRef(cg.typeOf(unwrapped.operand), zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{});

    const dest_alloc = try cg.allocStack(result_ty);
    for (mask, 0..) |mask_elem, out_idx| {
        try cg.emitWValue(dest_alloc);
        const elem_val = switch (mask_elem.unwrap()) {
            .elem => |idx| try cg.load(operand, elem_ty, @intCast(elem_size * idx)),
            .value => |val| try cg.lowerConstant(.fromInterned(val)),
        };
        try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
    }
    return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});
}

fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const pt = cg.pt;
    const zcu = pt.zcu;

    const unwrapped = cg.air.unwrapShuffleTwo(zcu, inst);
    const result_ty = unwrapped.result_ty;
    const mask = unwrapped.mask;
    const operand_a = try cg.resolveInst(unwrapped.operand_a);
    const operand_b = try cg.resolveInst(unwrapped.operand_b);

    const a_ty = cg.typeOf(unwrapped.operand_a);
    const b_ty = cg.typeOf(unwrapped.operand_b);
    const elem_ty = result_ty.childType(zcu);
    const elem_size = elem_ty.abiSize(zcu);

    // WASM has `i8x16_shuffle`, which we can apply if the element type bit size is a multiple of 8
    // and the input and output vectors have a bit size of 128 (and are hence not by-ref). Otherwise,
    // we fall back to a naive loop lowering.
    if (!isByRef(a_ty, zcu, cg.target) and
        !isByRef(b_ty, zcu, cg.target) and
        !isByRef(result_ty, zcu, cg.target) and
        elem_ty.bitSize(zcu) % 8 == 0)
    {
        var lane_map: [16]u8 align(4) = undefined;
        const lanes_per_elem: usize = @intCast(elem_ty.bitSize(zcu) / 8);
        for (mask, 0..) |mask_elem, out_idx| {
            const out_first_lane = out_idx * lanes_per_elem;
            const in_first_lane = switch (mask_elem.unwrap()) {
                .a_elem => |i| i * lanes_per_elem,
                .b_elem => |i| i * lanes_per_elem + 16,
                .undef => 0, // doesn't matter
            };
            for (lane_map[out_first_lane..][0..lanes_per_elem], in_first_lane..) |*out, in| {
                out.* = @intCast(in);
            }
        }
        try cg.emitWValue(operand_a);
        try cg.emitWValue(operand_b);
        const extra_index: u32 = @intCast(cg.mir_extra.items.len);
        try cg.mir_extra.appendSlice(cg.gpa, &.{
            @intFromEnum(std.wasm.SimdOpcode.i8x16_shuffle),
            @bitCast(lane_map[0..4].*),
            @bitCast(lane_map[4..8].*),
            @bitCast(lane_map[8..12].*),
            @bitCast(lane_map[12..].*),
        });
        try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
        return cg.finishAir(inst, .stack, &.{ unwrapped.operand_a, unwrapped.operand_b });
    }

    // TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible.
    // I tried to fix it, but I couldn't make much sense of how this backend handles memory.
    if (!isByRef(result_ty, zcu, cg.target) or
        !isByRef(a_ty, zcu, cg.target) or
        !isByRef(b_ty, zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{});

    const dest_alloc = try cg.allocStack(result_ty);
    for (mask, 0..) |mask_elem, out_idx| {
        try cg.emitWValue(dest_alloc);
        const elem_val = switch (mask_elem.unwrap()) {
            .a_elem => |idx| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)),
            .b_elem => |idx| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)),
            .undef => try cg.emitUndefined(elem_ty),
        };
        try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
    }
    return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });
}

fn airReduce(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const reduce = cg.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
    const operand = try cg.resolveInst(reduce.operand);

    _ = operand;
    return cg.fail("TODO: Implement wasm airReduce", .{});
}

fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
    const result_ty = cg.typeOfIndex(inst);
    const len = @as(usize, @intCast(result_ty.arrayLen(zcu)));
    const elements: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..len]);

    const result: WValue = result_value: {
        switch (result_ty.zigTypeTag(zcu)) {
            .array => {
                const result = try cg.allocStack(result_ty);
                const elem_ty = result_ty.childType(zcu);
                const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
                const sentinel = result_ty.sentinel(zcu);

                // When the element type is by reference, we must copy the entire
                // value. It is therefore safer to move the offset pointer and store
                // each value individually, instead of using store offsets.
                if (isByRef(elem_ty, zcu, cg.target)) {
                    // copy stack pointer into a temporary local, which is
                    // moved for each element to store each value in the right position.
                    const offset = try cg.buildPointerOffset(result, 0, .new);
                    for (elements, 0..) |elem, elem_index| {
                        const elem_val = try cg.resolveInst(elem);
                        try cg.store(offset, elem_val, elem_ty, 0);

                        if (elem_index < elements.len - 1 or sentinel != null) {
                            _ = try cg.buildPointerOffset(offset, elem_size, .modify);
                        }
                    }
                    if (sentinel) |s| {
                        const val = try cg.resolveValue(s);
                        try cg.store(offset, val, elem_ty, 0);
                    }
                } else {
                    var offset: u32 = 0;
                    for (elements) |elem| {
                        const elem_val = try cg.resolveInst(elem);
                        try cg.store(result, elem_val, elem_ty, offset);
                        offset += elem_size;
                    }
                    if (sentinel) |s| {
                        const val = try cg.resolveValue(s);
                        try cg.store(result, val, elem_ty, offset);
                    }
                }
                break :result_value result;
            },
            .@"struct" => switch (result_ty.containerLayout(zcu)) {
                .@"packed" => unreachable, // legalize .expand_packed_aggregate_init
                else => {
                    const result = try cg.allocStack(result_ty);
                    const offset = try cg.buildPointerOffset(result, 0, .new); // pointer to offset
                    var prev_field_offset: u64 = 0;
                    for (elements, 0..) |elem, elem_index| {
                        if (try result_ty.structFieldValueComptime(pt, elem_index) != null) continue;

                        const elem_ty = result_ty.fieldType(elem_index, zcu);
                        const field_offset = result_ty.structFieldOffset(elem_index, zcu);
                        _ = try cg.buildPointerOffset(offset, @intCast(field_offset - prev_field_offset), .modify);
                        prev_field_offset = field_offset;

                        const value = try cg.resolveInst(elem);
                        try cg.store(offset, value, elem_ty, 0);
                    }

                    break :result_value result;
                },
            },
            .vector => return cg.fail("TODO: Wasm backend: implement airAggregateInit for vectors", .{}),
            else => unreachable,
        }
    };

    var bt = cg.liveness.iterateBigTomb(inst);
    for (elements) |arg| cg.feed(&bt, arg);
    return cg.finishAirResult(inst, result);
}

fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const ip = &zcu.intern_pool;
    const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
    const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data;

    const result = result: {
        const union_ty = cg.typeOfIndex(inst);
        const layout = union_ty.unionGetLayout(zcu);
        const union_obj = zcu.typeToUnion(union_ty).?;
        const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
        const field_name = ip.loadEnumType(union_obj.enum_tag_type).field_names.get(ip)[extra.field_index];

        const tag_int = blk: {
            const tag_ty = union_ty.unionTagTypeHypothetical(zcu);
            const enum_field_index = tag_ty.enumFieldIndex(field_name, zcu).?;
            const tag_val = try pt.enumValueFieldIndex(tag_ty, enum_field_index);
            break :blk try cg.lowerConstant(tag_val);
        };
        if (layout.payload_size == 0) {
            if (layout.tag_size == 0) {
                break :result .none;
            }
            assert(!isByRef(union_ty, zcu, cg.target));
            break :result tag_int;
        }

        if (isByRef(union_ty, zcu, cg.target)) {
            const result_ptr = try cg.allocStack(union_ty);
            const payload = try cg.resolveInst(extra.init);
            if (layout.tag_align.compare(.gte, layout.payload_align)) {
                if (isByRef(field_ty, zcu, cg.target)) {
                    const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new);
                    try cg.store(payload_ptr, payload, field_ty, 0);
                } else {
                    try cg.store(result_ptr, payload, field_ty, @intCast(layout.tag_size));
                }

                if (layout.tag_size > 0) {
                    try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), 0);
                }
            } else {
                try cg.store(result_ptr, payload, field_ty, 0);
                if (layout.tag_size > 0) {
                    try cg.store(
                        result_ptr,
                        tag_int,
                        .fromInterned(union_obj.enum_tag_type),
                        @intCast(layout.payload_size),
                    );
                }
            }
            break :result result_ptr;
        } else {
            const operand = try cg.resolveInst(extra.init);
            break :result (try cg.bitcast(union_ty, field_ty, operand)) orelse cg.reuseOperand(extra.init, operand);
        }
    };

    return cg.finishAir(inst, result, &.{extra.init});
}

fn airPrefetch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const prefetch = cg.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
    return cg.finishAir(inst, .none, &.{prefetch.ptr});
}

fn airWasmMemorySize(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;

    try cg.addLabel(.memory_size, pl_op.payload);
    return cg.finishAir(inst, .stack, &.{pl_op.operand});
}

fn airWasmMemoryGrow(cg: *CodeGen, inst: Air.Inst.Index) !void {
    const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;

    const operand = try cg.resolveInst(pl_op.operand);
    try cg.emitWValue(operand);
    try cg.addLabel(.memory_grow, pl_op.payload);
    return cg.finishAir(inst, .stack, &.{pl_op.operand});
}

fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
    const un_ty = cg.typeOf(bin_op.lhs).childType(zcu);
    const tag_ty = cg.typeOf(bin_op.rhs);
    const layout = un_ty.unionGetLayout(zcu);
    if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });

    const union_ptr = try cg.resolveInst(bin_op.lhs);
    const new_tag = try cg.resolveInst(bin_op.rhs);
    if (layout.payload_size == 0) {
        try cg.store(union_ptr, new_tag, tag_ty, 0);
        return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
    }

    // when the tag alignment is smaller than the payload, the field will be stored
    // after the payload.
    const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: {
        break :blk @intCast(layout.payload_size);
    } else 0;
    try cg.store(union_ptr, new_tag, tag_ty, offset);
    return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
}

fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;

    const un_ty = cg.typeOf(ty_op.operand);
    const tag_ty = cg.typeOfIndex(inst);
    const layout = un_ty.unionGetLayout(zcu);
    if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ty_op.operand});

    const operand = try cg.resolveInst(ty_op.operand);
    // when the tag alignment is smaller than the payload, the field will be stored
    // after the payload.
    const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align))
        @intCast(layout.payload_size)
    else
        0;
    const result = try cg.load(operand, tag_ty, offset);
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;

    const err_set_ty = cg.typeOf(ty_op.operand).childType(zcu);
    const payload_ty = err_set_ty.errorUnionPayload(zcu);
    const operand = try cg.resolveInst(ty_op.operand);

    // set error-tag to '0' to annotate error union is non-error
    try cg.store(
        operand,
        .{ .imm32 = 0 },
        Type.anyerror,
        @intCast(errUnionErrorOffset(payload_ty, zcu)),
    );

    const result = result: {
        if (!payload_ty.hasRuntimeBits(zcu)) {
            break :result cg.reuseOperand(ty_op.operand, operand);
        }

        break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new);
    };
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn airUnwrapRestricted(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
    const operand = try cg.resolveInst(ty_op.operand);
    const unrestricted_ty = ty_op.ty.toType();
    const restricted_ty = cg.typeOf(ty_op.operand);
    const result = result: switch (restricted_ty.restrictedRepr(zcu)) {
        .indirect => {
            _ = safety; // TODO
            break :result try cg.load(operand, unrestricted_ty, 0);
        },
        .direct => cg.reuseOperand(ty_op.operand, operand),
    };
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const pt = cg.pt;
    const zcu = pt.zcu;
    const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
    const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;

    const field_ptr = try cg.resolveInst(extra.field_ptr);
    const parent_ptr_ty = cg.typeOfIndex(inst);
    const parent_ty = parent_ptr_ty.childType(zcu);
    const field_ptr_ty = cg.typeOf(extra.field_ptr);
    const field_index = extra.field_index;
    const field_offset = switch (parent_ty.containerLayout(zcu)) {
        .auto, .@"extern" => parent_ty.structFieldOffset(field_index, zcu),
        .@"packed" => offset: {
            const parent_ptr_offset = parent_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset;
            const field_offset = if (zcu.typeToStruct(parent_ty)) |loaded_struct| zcu.structPackedFieldBitOffset(loaded_struct, field_index) else 0;
            const field_ptr_offset = field_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset;
            break :offset @divExact(parent_ptr_offset + field_offset - field_ptr_offset, 8);
        },
    };

    const result = if (field_offset != 0) result: {
        const base = try cg.buildPointerOffset(field_ptr, 0, .new);
        try cg.addLocal(.local_get, base.local.value);
        try cg.addImm32(@intCast(field_offset));
        try cg.addTag(.i32_sub);
        try cg.addLocal(.local_set, base.local.value);
        break :result base;
    } else cg.reuseOperand(extra.field_ptr, field_ptr);

    return cg.finishAir(inst, result, &.{extra.field_ptr});
}

fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue {
    const zcu = cg.pt.zcu;
    if (ptr_ty.isSlice(zcu)) {
        return cg.slicePtr(ptr);
    } else {
        return ptr;
    }
}

fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
    const dst = try cg.resolveInst(bin_op.lhs);
    const dst_ty = cg.typeOf(bin_op.lhs);
    const ptr_elem_ty = dst_ty.childType(zcu);
    const src = try cg.resolveInst(bin_op.rhs);
    const src_ty = cg.typeOf(bin_op.rhs);
    const len = switch (dst_ty.ptrSize(zcu)) {
        .slice => blk: {
            const slice_len = try cg.sliceLen(dst);
            if (ptr_elem_ty.abiSize(zcu) != 1) {
                try cg.emitWValue(slice_len);
                try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });
                try cg.addTag(.i32_mul);
                try cg.addLocal(.local_set, slice_len.local.value);
            }
            break :blk slice_len;
        },
        .one => @as(WValue, .{
            .imm32 = @as(u32, @intCast(ptr_elem_ty.arrayLen(zcu) * ptr_elem_ty.childType(zcu).abiSize(zcu))),
        }),
        .c, .many => unreachable,
    };
    const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
    const src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
    try cg.memcpy(dst_ptr, src_ptr, len);

    return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
}

fn airRetAddr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    // TODO: Implement this properly once stack serialization is solved
    return cg.finishAir(inst, switch (cg.ptr_size) {
        .wasm32 => .{ .imm32 = 0 },
        .wasm64 => .{ .imm64 = 0 },
    }, &.{});
}

fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
    const operand = try cg.resolveInst(un_op);
    // Each entry to this table is a slice (ptr+len).
    // The operand in this instruction represents the index within this table.
    // This means to get the final name, we emit the base pointer and then perform
    // pointer arithmetic to find the pointer to this slice and return that.
    //
    // As the names are global and the slice elements are constant, we do not have
    // to make a copy of the ptr+value but can point towards them directly.
    const pt = cg.pt;
    const name_ty = Type.slice_const_u8_sentinel_0;
    const abi_size = name_ty.abiSize(pt.zcu);

    // Lowers to a i32.const or i64.const with the error table memory address.
    cg.error_name_table_ref_count += 1;
    try cg.addTag(.error_name_table_ref);
    try cg.emitWValue(operand);
    switch (cg.ptr_size) {
        .wasm32 => {
            try cg.addImm32(@intCast(abi_size));
            try cg.addTag(.i32_mul);
            try cg.addTag(.i32_add);
        },
        .wasm64 => {
            try cg.addImm64(abi_size);
            try cg.addTag(.i64_mul);
            try cg.addTag(.i64_add);
        },
    }

    return cg.finishAir(inst, .stack, &.{un_op});
}

fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void {
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
    const slice_ptr = try cg.resolveInst(ty_op.operand);
    const result = try cg.buildPointerOffset(slice_ptr, offset, .new);
    return cg.finishAir(inst, result, &.{ty_op.operand});
}

fn airDbgStmt(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const dbg_stmt = cg.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
    try cg.addInst(.{ .tag = .dbg_line, .data = .{
        .payload = try cg.addExtra(Mir.DbgLineColumn{
            .line = dbg_stmt.line,
            .column = dbg_stmt.column,
        }),
    } });
    return cg.finishAir(inst, .none, &.{});
}

fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const block = cg.air.unwrapDbgBlock(inst);
    // TODO
    try cg.lowerBlock(inst, block.ty, block.body);
}

fn airDbgVar(
    cg: *CodeGen,
    inst: Air.Inst.Index,
    local_tag: link.File.Dwarf.WipNav.LocalVarTag,
    is_ptr: bool,
) InnerError!void {
    _ = is_ptr;
    _ = local_tag;
    return cg.finishAir(inst, .none, &.{});
}

fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const unwrapped_try = cg.air.unwrapTry(inst);
    const body = unwrapped_try.else_body;
    const err_union = try cg.resolveInst(unwrapped_try.error_union);
    const err_union_ty = cg.typeOf(unwrapped_try.error_union);
    const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false);
    return cg.finishAir(inst, result, &.{unwrapped_try.error_union});
}

fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const unwrapped_try = cg.air.unwrapTryPtr(inst);
    const err_union_ptr = try cg.resolveInst(unwrapped_try.error_union_ptr);
    const body = unwrapped_try.else_body;
    const err_union_ty = cg.typeOf(unwrapped_try.error_union_ptr).childType(zcu);
    const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true);
    return cg.finishAir(inst, result, &.{unwrapped_try.error_union_ptr});
}

fn lowerTry(
    cg: *CodeGen,
    inst: Air.Inst.Index,
    err_union: WValue,
    body: []const Air.Inst.Index,
    err_union_ty: Type,
    operand_is_ptr: bool,
) InnerError!WValue {
    _ = inst;
    const zcu = cg.pt.zcu;

    const pl_ty = err_union_ty.errorUnionPayload(zcu);
    const pl_has_bits = pl_ty.hasRuntimeBits(zcu);

    if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
        // Block we can jump out of when error is not set
        try cg.startBlock(.block, .empty);

        // check if the error tag is set for the error union.
        try cg.emitWValue(err_union);
        if (pl_has_bits or operand_is_ptr) {
            const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
            try cg.addMemArg(.i32_load16_u, .{
                .offset = err_union.offset() + err_offset,
                .alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
            });
        }
        try cg.addTag(.i32_eqz);
        try cg.addLabel(.br_if, 0); // jump out of block when error is '0'

        try cg.branches.append(cg.gpa, .{});
        defer {
            var branch = cg.branches.pop().?;
            branch.deinit(cg.gpa);
        }
        try cg.genBody(body);
        try cg.endBlock();
    }

    // if we reach here it means error was not set, and we want the payload
    if (!pl_has_bits and !operand_is_ptr) {
        return .none;
    }

    const pl_offset: u32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
    if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {
        return buildPointerOffset(cg, err_union, pl_offset, .new);
    }
    const payload = try cg.load(err_union, pl_ty, pl_offset);
    return payload.toLocal(cg, pl_ty);
}

/// Calls a compiler-rt intrinsic by creating an undefined symbol,
/// then lowering the arguments and calling the symbol as a function call.
/// This function call assumes the C-ABI.
/// Asserts arguments are not stack values when the return value is
/// passed as the first parameter.
/// May leave the return value on the stack.
fn callIntrinsic(
    cg: *CodeGen,
    intrinsic: Mir.Intrinsic,
    param_types: []const InternPool.Index,
    return_type: Type,
    args: []const WValue,
) InnerError!WValue {
    assert(param_types.len == args.len);
    const zcu = cg.pt.zcu;

    // Always pass over C-ABI

    const want_sret_param = firstParamSRet(.{ .wasm_mvp = .{} }, return_type, zcu, cg.target);
    // if we want return as first param, we allocate a pointer to stack,
    // and emit it as our first argument
    const sret = if (want_sret_param) blk: {
        const sret_local = try cg.allocStack(return_type);
        try cg.lowerToStack(sret_local);
        break :blk sret_local;
    } else .none;

    // Lower all arguments to the stack before we call our function
    for (args, 0..) |arg, arg_i| {
        assert(!(want_sret_param and arg == .stack));
        assert(Type.fromInterned(param_types[arg_i]).hasRuntimeBits(zcu));
        try cg.lowerArg(.{ .wasm_mvp = .{} }, Type.fromInterned(param_types[arg_i]), arg);
    }

    try cg.addInst(.{ .tag = .call_intrinsic, .data = .{ .intrinsic = intrinsic } });

    if (!return_type.hasRuntimeBits(zcu)) {
        return .none;
    } else if (want_sret_param) {
        return sret;
    } else {
        return .stack;
    }
}

fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
    const operand = try cg.resolveInst(un_op);
    const enum_ty = cg.typeOf(un_op);

    const result_ptr = try cg.allocStack(cg.typeOfIndex(inst));
    try cg.lowerToStack(result_ptr);
    try cg.emitWValue(operand);
    try cg.addInst(.{ .tag = .call_tag_name, .data = .{ .ip_index = enum_ty.toIntern() } });

    return cg.finishAir(inst, result_ptr, &.{un_op});
}

fn airErrorSetHasValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ip = &zcu.intern_pool;
    const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;

    const operand = try cg.resolveInst(ty_op.operand);
    const error_set_ty = ty_op.ty.toType();
    const result = try cg.allocLocal(Type.bool);

    const names = error_set_ty.errorSetNames(zcu);
    var values = try std.array_list.Managed(u32).initCapacity(cg.gpa, names.len);
    defer values.deinit();

    var lowest: ?u32 = null;
    var highest: ?u32 = null;
    for (0..names.len) |name_index| {
        const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
        if (lowest) |*l| {
            if (err_int < l.*) {
                l.* = err_int;
            }
        } else {
            lowest = err_int;
        }
        if (highest) |*h| {
            if (err_int > h.*) {
                highest = err_int;
            }
        } else {
            highest = err_int;
        }

        values.appendAssumeCapacity(err_int);
    }

    // start block for 'true' branch
    try cg.startBlock(.block, .empty);
    // start block for 'false' branch
    try cg.startBlock(.block, .empty);
    // block for the jump table itself
    try cg.startBlock(.block, .empty);

    // lower operand to determine jump table target
    try cg.emitWValue(operand);
    try cg.addImm32(lowest.?);
    try cg.addTag(.i32_sub);

    // Account for default branch so always add '1'
    const depth = @as(u32, @intCast(highest.? - lowest.? + 1));
    const jump_table: Mir.JumpTable = .{ .length = depth };
    const table_extra_index = try cg.addExtra(jump_table);
    try cg.addInst(.{ .tag = .br_table, .data = .{ .payload = table_extra_index } });
    try cg.mir_extra.ensureUnusedCapacity(cg.gpa, depth);

    var value: u32 = lowest.?;
    while (value <= highest.?) : (value += 1) {
        const idx: u32 = blk: {
            for (values.items) |val| {
                if (val == value) break :blk 1;
            }
            break :blk 0;
        };
        cg.mir_extra.appendAssumeCapacity(idx);
    }
    try cg.endBlock();

    // 'false' branch (i.e. error set does not have value
    // ensure we set local to 0 in case the local was re-used.
    try cg.addImm32(0);
    try cg.addLocal(.local_set, result.local.value);
    try cg.addLabel(.br, 1);
    try cg.endBlock();

    // 'true' branch
    try cg.addImm32(1);
    try cg.addLocal(.local_set, result.local.value);
    try cg.addLabel(.br, 0);
    try cg.endBlock();

    return cg.finishAir(inst, result, &.{ty_op.operand});
}

inline fn useAtomicFeature(cg: *const CodeGen) bool {
    return cg.target.cpu.has(.wasm, .atomics);
}

fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
    const extra = cg.air.extraData(Air.Cmpxchg, ty_pl.payload).data;

    const ptr_ty = cg.typeOf(extra.ptr);
    const ty = ptr_ty.childType(zcu);
    const result_ty = cg.typeOfIndex(inst);

    const int_ty: IntType = .fromType(cg, ty);

    const ptr_operand = try cg.resolveInst(extra.ptr);
    const expected_val = try cg.resolveInst(extra.expected_value);
    const new_val = try cg.resolveInst(extra.new_value);

    const cmp_result = try cg.allocLocal(Type.bool);

    const ptr_val = if (cg.useAtomicFeature()) val: {
        const val_local = try cg.allocLocal(ty);
        try cg.emitWValue(ptr_operand);
        try cg.lowerToStack(expected_val);
        try cg.lowerToStack(new_val);
        try cg.addAtomicMemArg(switch (ty.abiSize(zcu)) {
            1 => .i32_atomic_rmw8_cmpxchg_u,
            2 => .i32_atomic_rmw16_cmpxchg_u,
            4 => .i32_atomic_rmw_cmpxchg,
            8 => .i32_atomic_rmw_cmpxchg,
            else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}),
        }, .{
            .offset = ptr_operand.offset(),
            .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
        });
        try cg.addLocal(.local_tee, val_local.local.value);
        _ = try cg.intCmp(int_ty, .eq, .stack, expected_val);
        try cg.addLocal(.local_set, cmp_result.local.value);
        break :val val_local;
    } else val: {
        if (ty.abiSize(zcu) > 8) {
            return cg.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{});
        }
        const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, 0), cg, ty);

        try cg.lowerToStack(ptr_operand);
        try cg.lowerToStack(new_val);
        try cg.emitWValue(ptr_val);
        _ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);
        try cg.addLocal(.local_tee, cmp_result.local.value);
        try cg.addTag(.select);
        try cg.store(.stack, .stack, ty, 0);

        break :val ptr_val;
    };

    const result = if (isByRef(result_ty, zcu, cg.target)) val: {
        try cg.emitWValue(cmp_result);
        try cg.addImm32(~@as(u32, 0));
        try cg.addTag(.i32_xor);
        try cg.addImm32(1);
        try cg.addTag(.i32_and);
        const and_result = try WValue.toLocal(.stack, cg, Type.bool);
        const result_ptr = try cg.allocStack(result_ty);
        try cg.store(result_ptr, and_result, Type.bool, @as(u32, @intCast(ty.abiSize(zcu))));
        try cg.store(result_ptr, ptr_val, ty, 0);
        break :val result_ptr;
    } else val: {
        try cg.addImm32(0);
        try cg.emitWValue(ptr_val);
        try cg.emitWValue(cmp_result);
        try cg.addTag(.select);
        break :val .stack;
    };

    return cg.finishAir(inst, result, &.{ extra.ptr, extra.expected_value, extra.new_value });
}

fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const atomic_load = cg.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
    const ptr = try cg.resolveInst(atomic_load.ptr);
    const ty = cg.typeOfIndex(inst);

    if (cg.useAtomicFeature()) {
        const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
            1 => .i32_atomic_load8_u,
            2 => .i32_atomic_load16_u,
            4 => .i32_atomic_load,
            8 => .i64_atomic_load,
            else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
        };
        try cg.emitWValue(ptr);
        try cg.addAtomicMemArg(tag, .{
            .offset = ptr.offset(),
            .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
        });
    } else {
        _ = try cg.load(ptr, ty, 0);
    }

    return cg.finishAir(inst, .stack, &.{atomic_load.ptr});
}

fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
    const extra = cg.air.extraData(Air.AtomicRmw, pl_op.payload).data;

    const ptr = try cg.resolveInst(pl_op.operand);
    const operand = try cg.resolveInst(extra.operand);
    const ty = cg.typeOfIndex(inst);
    const op: std.builtin.AtomicRmwOp = extra.op();

    const int_ty: IntType = .fromType(cg, ty);

    if (cg.useAtomicFeature()) {
        switch (op) {
            .Max,
            .Min,
            .Nand,
            => {
                const tmp = try cg.load(ptr, ty, 0);
                const value = try tmp.toLocal(cg, ty);

                // create a loop to cmpxchg the new value
                try cg.startBlock(.loop, .empty);

                try cg.emitWValue(ptr);
                try cg.emitWValue(value);
                if (op == .Nand) {
                    const and_res = try cg.intAnd(int_ty, value, operand);
                    if (int_ty.bits <= 32) {
                        try cg.addImm32(~@as(u32, 0));
                    } else if (int_ty.bits <= 64) {
                        try cg.addImm64(~@as(u64, 0));
                    } else {
                        return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
                    }
                    _ = try cg.intXor(int_ty, and_res, .stack);
                } else {
                    try cg.emitWValue(value);
                    try cg.emitWValue(operand);
                    _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, value, operand);
                    try cg.addTag(.select);
                }
                try cg.addAtomicMemArg(
                    switch (ty.abiSize(zcu)) {
                        1 => .i32_atomic_rmw8_cmpxchg_u,
                        2 => .i32_atomic_rmw16_cmpxchg_u,
                        4 => .i32_atomic_rmw_cmpxchg,
                        8 => .i64_atomic_rmw_cmpxchg,
                        else => return cg.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}),
                    },
                    .{
                        .offset = ptr.offset(),
                        .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
                    },
                );
                const select_res = try cg.allocLocal(ty);
                try cg.addLocal(.local_tee, select_res.local.value);
                _ = try cg.intCmp(int_ty, .neq, .stack, value); // leave on stack so we can use it for br_if

                try cg.emitWValue(select_res);
                try cg.addLocal(.local_set, value.local.value);

                try cg.addLabel(.br_if, 0);
                try cg.endBlock();
                return cg.finishAir(inst, value, &.{ pl_op.operand, extra.operand });
            },

            // the other operations have their own instructions for Wasm.
            else => {
                try cg.emitWValue(ptr);
                try cg.emitWValue(operand);
                const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
                    1 => switch (op) {
                        .Xchg => .i32_atomic_rmw8_xchg_u,
                        .Add => .i32_atomic_rmw8_add_u,
                        .Sub => .i32_atomic_rmw8_sub_u,
                        .And => .i32_atomic_rmw8_and_u,
                        .Or => .i32_atomic_rmw8_or_u,
                        .Xor => .i32_atomic_rmw8_xor_u,
                        else => unreachable,
                    },
                    2 => switch (op) {
                        .Xchg => .i32_atomic_rmw16_xchg_u,
                        .Add => .i32_atomic_rmw16_add_u,
                        .Sub => .i32_atomic_rmw16_sub_u,
                        .And => .i32_atomic_rmw16_and_u,
                        .Or => .i32_atomic_rmw16_or_u,
                        .Xor => .i32_atomic_rmw16_xor_u,
                        else => unreachable,
                    },
                    4 => switch (op) {
                        .Xchg => .i32_atomic_rmw_xchg,
                        .Add => .i32_atomic_rmw_add,
                        .Sub => .i32_atomic_rmw_sub,
                        .And => .i32_atomic_rmw_and,
                        .Or => .i32_atomic_rmw_or,
                        .Xor => .i32_atomic_rmw_xor,
                        else => unreachable,
                    },
                    8 => switch (op) {
                        .Xchg => .i64_atomic_rmw_xchg,
                        .Add => .i64_atomic_rmw_add,
                        .Sub => .i64_atomic_rmw_sub,
                        .And => .i64_atomic_rmw_and,
                        .Or => .i64_atomic_rmw_or,
                        .Xor => .i64_atomic_rmw_xor,
                        else => unreachable,
                    },
                    else => |size| return cg.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}),
                };
                try cg.addAtomicMemArg(tag, .{
                    .offset = ptr.offset(),
                    .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
                });
                return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand });
            },
        }
    } else {
        const loaded = try cg.load(ptr, ty, 0);
        const result = try loaded.toLocal(cg, ty);

        switch (op) {
            .Xchg => {
                try cg.store(ptr, operand, ty, 0);
            },
            .Add,
            .Sub,
            .And,
            .Or,
            .Xor,
            => {
                try cg.emitWValue(ptr);
                _ = switch (op) {
                    .Add => try cg.intAdd(int_ty, result, operand),
                    .Sub => try cg.intSub(int_ty, result, operand),
                    .And => try cg.intAnd(int_ty, result, operand),
                    .Or => try cg.intOr(int_ty, result, operand),
                    .Xor => try cg.intXor(int_ty, result, operand),
                    else => unreachable,
                };
                if (ty.isInt(zcu) and (op == .Add or op == .Sub)) {
                    _ = try cg.intWrap(int_ty, .stack);
                }
                try cg.store(.stack, .stack, ty, ptr.offset());
            },
            .Max,
            .Min,
            => {
                try cg.emitWValue(ptr);
                try cg.emitWValue(result);
                try cg.emitWValue(operand);
                _ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);
                try cg.addTag(.select);
                try cg.store(.stack, .stack, ty, ptr.offset());
            },
            .Nand => {
                try cg.emitWValue(ptr);
                const and_res = try cg.intAnd(int_ty, result, operand);
                if (int_ty.bits <= 32) {
                    try cg.addImm32(~@as(u32, 0));
                } else if (int_ty.bits <= 64) {
                    try cg.addImm64(~@as(u64, 0));
                } else {
                    return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
                }
                _ = try cg.intXor(int_ty, and_res, .stack);
                try cg.store(.stack, .stack, ty, ptr.offset());
            },
        }

        return cg.finishAir(inst, result, &.{ pl_op.operand, extra.operand });
    }
}

fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const zcu = cg.pt.zcu;
    const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;

    const ptr = try cg.resolveInst(bin_op.lhs);
    const operand = try cg.resolveInst(bin_op.rhs);
    const ptr_ty = cg.typeOf(bin_op.lhs);
    const ty = ptr_ty.childType(zcu);

    if (cg.useAtomicFeature()) {
        const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
            1 => .i32_atomic_store8,
            2 => .i32_atomic_store16,
            4 => .i32_atomic_store,
            8 => .i64_atomic_store,
            else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
        };
        try cg.emitWValue(ptr);
        try cg.lowerToStack(operand);
        try cg.addAtomicMemArg(tag, .{
            .offset = ptr.offset(),
            .alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
        });
    } else {
        try cg.store(ptr, operand, ty, 0);
    }

    return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
}

fn airFrameAddress(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    if (cg.initial_stack_value == .none) {
        try cg.initializeStack();
    }
    try cg.emitWValue(cg.bottom_stack_value);
    return cg.finishAir(inst, .stack, &.{});
}

fn airRuntimeNavPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const ty_nav = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav;
    const mod = cg.pt.zcu.navFileScope(cg.owner_nav).mod.?;
    if (mod.single_threaded) {
        const result: WValue = .{ .nav_ref = .{
            .nav_index = ty_nav.nav,
            .offset = 0,
        } };
        return cg.finishAir(inst, result, &.{});
    }
    return cg.fail("TODO: thread-local variables", .{});
}

fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
    const unwrapped_asm = cg.air.unwrapAsm(inst);
    const outputs = unwrapped_asm.outputs;
    const inputs = unwrapped_asm.inputs;

    const zcu = cg.pt.zcu;
    const output_ty = cg.typeOfIndex(inst);

    const result: WValue = if (output_ty.hasRuntimeBits(zcu))
        try cg.allocLocal(output_ty)
    else
        .none;

    var local_map: assembly.LocalMap = .empty;
    defer local_map.deinit(cg.gpa);

    {
        var it = unwrapped_asm.iterateOutputs();
        if (it.next()) |output| {
            const constraint = output.constraint;
            assert(output.operand == .none);
            const name = output.name;

            if (!mem.eql(u8, constraint, "=r")) {
                return cg.fail("Self-hosted wasm backend requires output constraint to be equal \"=r\"", .{});
            }

            const gop = try local_map.getOrPutValue(cg.gpa, name, result.local.value);
            assert(!gop.found_existing); // first value

            assert(it.next() == null);
        }
    }

    {
        var it = unwrapped_asm.iterateInputs();
        while (it.next()) |input| {
            const constraint = input.constraint;
            const operand = try cg.resolveInst(input.operand);
            const name = input.name;

            if (!mem.eql(u8, constraint, "r")) {
                return cg.fail("Self-hosted wasm backend requires input constraint to be equal \"r\"", .{});
            }

            try cg.lowerToStack(operand);
            const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand));

            const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local.value);
            if (gop.found_existing) {
                return cg.fail("Duplicate asm variable name \"{s}\"", .{name});
            }
        }
    }

    try assembly.assemble(cg, unwrapped_asm.source, &local_map);

    var bt = cg.liveness.iterateBigTomb(inst);
    for (outputs) |output| if (output != .none) cg.feed(&bt, output);
    for (inputs) |input| cg.feed(&bt, input);
    return cg.finishAirResult(inst, result);
}

fn typeOf(cg: *CodeGen, inst: Air.Inst.Ref) Type {
    const zcu = cg.pt.zcu;
    return cg.air.typeOf(inst, &zcu.intern_pool);
}

fn typeOfIndex(cg: *CodeGen, inst: Air.Inst.Index) Type {
    const zcu = cg.pt.zcu;
    return cg.air.typeOfIndex(inst, &zcu.intern_pool);
}