Merge pull request #14993 from jacobly0/x86_64

x86_64: implement more operations

src/arch/aarch64/CodeGen.zig

@@ -3011,41 +3011,16 @@ fn optionalPayload(self: *Self, inst: Air.Inst.Index, mcv: MCValue, optional_ty:
         return MCValue{ .register = reg };
     }
 
-    const offset = @intCast(u32, optional_ty.abiSize(self.target.*) - payload_ty.abiSize(self.target.*));
     switch (mcv) {
-        .register => |source_reg| {
+        .register => {
             // TODO should we reuse the operand here?
             const raw_reg = try self.register_manager.allocReg(inst, gp);
             const dest_reg = raw_reg.toX();
 
-            const shift = @intCast(u6, offset * 8);
-            if (shift == 0) {
-                try self.genSetReg(payload_ty, dest_reg, mcv);
-            } else {
-                _ = try self.addInst(.{
-                    .tag = if (payload_ty.isSignedInt())
-                        Mir.Inst.Tag.asr_immediate
-                    else
-                        Mir.Inst.Tag.lsr_immediate,
-                    .data = .{ .rr_shift = .{
-                        .rd = dest_reg,
-                        .rn = source_reg.toX(),
-                        .shift = shift,
-                    } },
-                });
-            }
-
+            try self.genSetReg(payload_ty, dest_reg, mcv);
             return MCValue{ .register = self.registerAlias(dest_reg, payload_ty) };
         },
-        .stack_argument_offset => |off| {
-            return MCValue{ .stack_argument_offset = off + offset };
-        },
-        .stack_offset => |off| {
-            return MCValue{ .stack_offset = off - offset };
-        },
-        .memory => |addr| {
-            return MCValue{ .memory = addr + offset };
-        },
+        .stack_argument_offset, .stack_offset, .memory => return mcv,
         else => unreachable, // invalid MCValue for an error union
     }
 }
@@ -3289,12 +3264,11 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
 
         const optional_abi_size = @intCast(u32, optional_ty.abiSize(self.target.*));
         const optional_abi_align = optional_ty.abiAlignment(self.target.*);
-        const payload_abi_size = @intCast(u32, payload_ty.abiSize(self.target.*));
-        const offset = optional_abi_size - payload_abi_size;
+        const offset = @intCast(u32, payload_ty.abiSize(self.target.*));
 
         const stack_offset = try self.allocMem(optional_abi_size, optional_abi_align, inst);
-        try self.genSetStack(Type.bool, stack_offset, .{ .immediate = 1 });
-        try self.genSetStack(payload_ty, stack_offset - @intCast(u32, offset), operand);
+        try self.genSetStack(payload_ty, stack_offset, operand);
+        try self.genSetStack(Type.bool, stack_offset - offset, .{ .immediate = 1 });
 
         break :result MCValue{ .stack_offset = stack_offset };
     };
@@ -4834,13 +4808,49 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
 }
 
 fn isNull(self: *Self, operand_bind: ReadArg.Bind, operand_ty: Type) !MCValue {
-    const sentinel_ty: Type = if (!operand_ty.isPtrLikeOptional()) blk: {
+    const sentinel: struct { ty: Type, bind: ReadArg.Bind } = if (!operand_ty.isPtrLikeOptional()) blk: {
         var buf: Type.Payload.ElemType = undefined;
         const payload_ty = operand_ty.optionalChild(&buf);
-        break :blk if (payload_ty.hasRuntimeBitsIgnoreComptime()) Type.bool else operand_ty;
-    } else operand_ty;
+        if (!payload_ty.hasRuntimeBitsIgnoreComptime())
+            break :blk .{ .ty = operand_ty, .bind = operand_bind };
+
+        const offset = @intCast(u32, payload_ty.abiSize(self.target.*));
+        const operand_mcv = try operand_bind.resolveToMcv(self);
+        const new_mcv: MCValue = switch (operand_mcv) {
+            .register => |source_reg| new: {
+                // TODO should we reuse the operand here?
+                const raw_reg = try self.register_manager.allocReg(null, gp);
+                const dest_reg = raw_reg.toX();
+
+                const shift = @intCast(u6, offset * 8);
+                if (shift == 0) {
+                    try self.genSetReg(payload_ty, dest_reg, operand_mcv);
+                } else {
+                    _ = try self.addInst(.{
+                        .tag = if (payload_ty.isSignedInt())
+                            Mir.Inst.Tag.asr_immediate
+                        else
+                            Mir.Inst.Tag.lsr_immediate,
+                        .data = .{ .rr_shift = .{
+                            .rd = dest_reg,
+                            .rn = source_reg.toX(),
+                            .shift = shift,
+                        } },
+                    });
+                }
+
+                break :new .{ .register = self.registerAlias(dest_reg, payload_ty) };
+            },
+            .stack_argument_offset => |off| .{ .stack_argument_offset = off + offset },
+            .stack_offset => |off| .{ .stack_offset = off - offset },
+            .memory => |addr| .{ .memory = addr + offset },
+            else => unreachable, // invalid MCValue for an optional
+        };
+
+        break :blk .{ .ty = Type.bool, .bind = .{ .mcv = new_mcv } };
+    } else .{ .ty = operand_ty, .bind = operand_bind };
     const imm_bind: ReadArg.Bind = .{ .mcv = .{ .immediate = 0 } };
-    return self.cmp(operand_bind, imm_bind, sentinel_ty, .eq);
+    return self.cmp(sentinel.bind, imm_bind, sentinel.ty, .eq);
 }
 
 fn isNonNull(self: *Self, operand_bind: ReadArg.Bind, operand_ty: Type) !MCValue {

src/arch/wasm/CodeGen.zig

@@ -2715,20 +2715,7 @@ fn lowerParentPtr(func: *CodeGen, ptr_val: Value, ptr_child_ty: Type) InnerError
         },
         .opt_payload_ptr => {
             const payload_ptr = ptr_val.castTag(.opt_payload_ptr).?.data;
-            const parent_ptr = try func.lowerParentPtr(payload_ptr.container_ptr, payload_ptr.container_ty);
-            var buf: Type.Payload.ElemType = undefined;
-            const payload_ty = payload_ptr.container_ty.optionalChild(&buf);
-            if (!payload_ty.hasRuntimeBitsIgnoreComptime() or payload_ty.optionalReprIsPayload()) {
-                return parent_ptr;
-            }
-
-            const abi_size = payload_ptr.container_ty.abiSize(func.target);
-            const offset = abi_size - payload_ty.abiSize(func.target);
-
-            return WValue{ .memory_offset = .{
-                .pointer = parent_ptr.memory,
-                .offset = @intCast(u32, offset),
-            } };
+            return func.lowerParentPtr(payload_ptr.container_ptr, payload_ptr.container_ty);
         },
         else => |tag| return func.fail("TODO: Implement lowerParentPtr for tag: {}", .{tag}),
     }
@@ -2889,7 +2876,7 @@ fn lowerConstant(func: *CodeGen, arg_val: Value, ty: Type) InnerError!WValue {
             }
         } else {
             const is_pl = val.tag() == .opt_payload;
-            return WValue{ .imm32 = if (is_pl) @as(u32, 1) else 0 };
+            return WValue{ .imm32 = @boolToInt(is_pl) };
         },
         .Struct => {
             const struct_obj = ty.castTag(.@"struct").?.data;
@@ -3882,7 +3869,11 @@ fn isNull(func: *CodeGen, operand: WValue, optional_ty: Type, opcode: wasm.Opcod
         // When payload is zero-bits, we can treat operand as a value, rather than
         // a pointer to the stack value
         if (payload_ty.hasRuntimeBitsIgnoreComptime()) {
-            try func.addMemArg(.i32_load8_u, .{ .offset = operand.offset(), .alignment = 1 });
+            const offset = std.math.cast(u32, payload_ty.abiSize(func.target)) orelse {
+                const module = func.bin_file.base.options.module.?;
+                return func.fail("Optional type {} too big to fit into stack frame", .{optional_ty.fmt(module)});
+            };
+            try func.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 });
         }
     } else if (payload_ty.isSlice()) {
         switch (func.arch()) {
@@ -3911,13 +3902,11 @@ fn airOptionalPayload(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
         const operand = try func.resolveInst(ty_op.operand);
         if (opt_ty.optionalReprIsPayload()) break :result func.reuseOperand(ty_op.operand, operand);
 
-        const offset = opt_ty.abiSize(func.target) - payload_ty.abiSize(func.target);
-
         if (isByRef(payload_ty, func.target)) {
-            break :result try func.buildPointerOffset(operand, offset, .new);
+            break :result try func.buildPointerOffset(operand, 0, .new);
         }
 
-        const payload = try func.load(operand, payload_ty, @intCast(u32, offset));
+        const payload = try func.load(operand, payload_ty, 0);
         break :result try payload.toLocal(func, payload_ty);
     };
     func.finishAir(inst, result, &.{ty_op.operand});
@@ -3936,8 +3925,7 @@ fn airOptionalPayloadPtr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
             break :result func.reuseOperand(ty_op.operand, operand);
         }
 
-        const offset = opt_ty.abiSize(func.target) - payload_ty.abiSize(func.target);
-        break :result try func.buildPointerOffset(operand, offset, .new);
+        break :result try func.buildPointerOffset(operand, 0, .new);
     };
     func.finishAir(inst, result, &.{ty_op.operand});
 }
@@ -3956,16 +3944,16 @@ fn airOptionalPayloadPtrSet(func: *CodeGen, inst: Air.Inst.Index) InnerError!voi
         return func.finishAir(inst, operand, &.{ty_op.operand});
     }
 
-    const offset = std.math.cast(u32, opt_ty.abiSize(func.target) - payload_ty.abiSize(func.target)) orelse {
+    const offset = std.math.cast(u32, payload_ty.abiSize(func.target)) orelse {
         const module = func.bin_file.base.options.module.?;
         return func.fail("Optional type {} too big to fit into stack frame", .{opt_ty.fmt(module)});
     };
 
     try func.emitWValue(operand);
     try func.addImm32(1);
-    try func.addMemArg(.i32_store8, .{ .offset = operand.offset(), .alignment = 1 });
+    try func.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 });
 
-    const result = try func.buildPointerOffset(operand, offset, .new);
+    const result = try func.buildPointerOffset(operand, 0, .new);
     return func.finishAir(inst, result, &.{ty_op.operand});
 }
 
@@ -3988,7 +3976,7 @@ fn airWrapOptional(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
         if (op_ty.optionalReprIsPayload()) {
             break :result func.reuseOperand(ty_op.operand, operand);
         }
-        const offset = std.math.cast(u32, op_ty.abiSize(func.target) - payload_ty.abiSize(func.target)) orelse {
+        const offset = std.math.cast(u32, payload_ty.abiSize(func.target)) orelse {
             const module = func.bin_file.base.options.module.?;
             return func.fail("Optional type {} too big to fit into stack frame", .{op_ty.fmt(module)});
         };
@@ -3997,9 +3985,9 @@ fn airWrapOptional(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
         const result_ptr = try func.allocStack(op_ty);
         try func.emitWValue(result_ptr);
         try func.addImm32(1);
-        try func.addMemArg(.i32_store8, .{ .offset = result_ptr.offset(), .alignment = 1 });
+        try func.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 });
 
-        const payload_ptr = try func.buildPointerOffset(result_ptr, offset, .new);
+        const payload_ptr = try func.buildPointerOffset(result_ptr, 0, .new);
         try func.store(payload_ptr, operand, payload_ty, 0);
         break :result result_ptr;
     };
@@ -4719,7 +4707,6 @@ fn cmpOptionals(func: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op:
     assert(op == .eq or op == .neq);
     var buf: Type.Payload.ElemType = undefined;
     const payload_ty = operand_ty.optionalChild(&buf);
-    const offset = @intCast(u32, operand_ty.abiSize(func.target) - payload_ty.abiSize(func.target));
 
     // We store the final result in here that will be validated
     // if the optional is truly equal.
@@ -4732,8 +4719,8 @@ fn cmpOptionals(func: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op:
     try func.addTag(.i32_ne); // inverse so we can exit early
     try func.addLabel(.br_if, 0);
 
-    _ = try func.load(lhs, payload_ty, offset);
-    _ = try func.load(rhs, payload_ty, offset);
+    _ = try func.load(lhs, payload_ty, 0);
+    _ = try func.load(rhs, payload_ty, 0);
     const opcode = buildOpcode(.{ .op = .ne, .valtype1 = typeToValtype(payload_ty, func.target) });
     try func.addTag(Mir.Inst.Tag.fromOpcode(opcode));
     try func.addLabel(.br_if, 0);

src/arch/x86_64/CodeGen.zig

@@ -402,19 +402,38 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
 fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
     _ = try self.addInst(.{
         .tag = .setcc,
-        .ops = .r_c,
-        .data = .{ .r_c = .{
+        .ops = .r_cc,
+        .data = .{ .r_cc = .{
             .r1 = reg,
             .cc = cc,
         } },
     });
 }
 
+fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
+    _ = try self.addInst(.{
+        .tag = .setcc,
+        .ops = switch (m) {
+            .sib => .m_sib_cc,
+            .rip => .m_rip_cc,
+            else => unreachable,
+        },
+        .data = .{ .x_cc = .{
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
+            .cc = cc,
+        } },
+    });
+}
+
 fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
     _ = try self.addInst(.{
         .tag = .cmovcc,
-        .ops = .rr_c,
-        .data = .{ .rr_c = .{
+        .ops = .rr_cc,
+        .data = .{ .rr_cc = .{
             .r1 = reg1,
             .r2 = reg2,
             .cc = cc,
@@ -422,6 +441,26 @@ fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bi
     });
 }
 
+fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
+    _ = try self.addInst(.{
+        .tag = .cmovcc,
+        .ops = switch (m) {
+            .sib => .rm_sib_cc,
+            .rip => .rm_rip_cc,
+            else => unreachable,
+        },
+        .data = .{ .rx_cc = .{
+            .r1 = reg,
+            .cc = cc,
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
+        } },
+    });
+}
+
 fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
     return self.addInst(.{
         .tag = .jmp_reloc,
@@ -793,18 +832,23 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
 
         switch (air_tags[inst]) {
             // zig fmt: off
-            .add             => try self.airBinOp(inst, .add),
-            .addwrap         => try self.airBinOp(inst, .addwrap),
-            .sub             => try self.airBinOp(inst, .sub),
-            .subwrap         => try self.airBinOp(inst, .subwrap),
-            .bool_and        => try self.airBinOp(inst, .bool_and),
-            .bool_or         => try self.airBinOp(inst, .bool_or),
-            .bit_and         => try self.airBinOp(inst, .bit_and),
-            .bit_or          => try self.airBinOp(inst, .bit_or),
-            .xor             => try self.airBinOp(inst, .xor),
+            .not,
+            => |tag| try self.airUnOp(inst, tag),
 
-            .ptr_add         => try self.airPtrArithmetic(inst, .ptr_add),
-            .ptr_sub         => try self.airPtrArithmetic(inst, .ptr_sub),
+            .add,
+            .addwrap,
+            .sub,
+            .subwrap,
+            .bool_and,
+            .bool_or,
+            .bit_and,
+            .bit_or,
+            .xor,
+            .min,
+            .max,
+            => |tag| try self.airBinOp(inst, tag),
+
+            .ptr_add, .ptr_sub => |tag| try self.airPtrArithmetic(inst, tag),
 
             .shr, .shr_exact => try self.airShlShrBinOp(inst),
             .shl, .shl_exact => try self.airShlShrBinOp(inst),
@@ -818,8 +862,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .sub_sat         => try self.airSubSat(inst),
             .mul_sat         => try self.airMulSat(inst),
             .shl_sat         => try self.airShlSat(inst),
-            .min             => try self.airMin(inst),
-            .max             => try self.airMax(inst),
             .slice           => try self.airSlice(inst),
 
             .sqrt,
@@ -867,7 +909,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .breakpoint      => try self.airBreakpoint(),
             .ret_addr        => try self.airRetAddr(inst),
             .frame_addr      => try self.airFrameAddress(inst),
-            .fence           => try self.airFence(),
+            .fence           => try self.airFence(inst),
             .cond_br         => try self.airCondBr(inst),
             .dbg_stmt        => try self.airDbgStmt(inst),
             .fptrunc         => try self.airFptrunc(inst),
@@ -885,7 +927,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .is_err_ptr      => try self.airIsErrPtr(inst),
             .load            => try self.airLoad(inst),
             .loop            => try self.airLoop(inst),
-            .not             => try self.airNot(inst),
             .ptrtoint        => try self.airPtrToInt(inst),
             .ret             => try self.airRet(inst),
             .ret_load        => try self.airRetLoad(inst),
@@ -972,10 +1013,10 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .optional_payload           => try self.airOptionalPayload(inst),
             .optional_payload_ptr       => try self.airOptionalPayloadPtr(inst),
             .optional_payload_ptr_set   => try self.airOptionalPayloadPtrSet(inst),
-            .unwrap_errunion_err        => try self.airUnwrapErrErr(inst),
-            .unwrap_errunion_payload    => try self.airUnwrapErrPayload(inst),
-            .unwrap_errunion_err_ptr    => try self.airUnwrapErrErrPtr(inst),
-            .unwrap_errunion_payload_ptr=> try self.airUnwrapErrPayloadPtr(inst),
+            .unwrap_errunion_err        => try self.airUnwrapErrUnionErr(inst),
+            .unwrap_errunion_payload    => try self.airUnwrapErrUnionPayload(inst),
+            .unwrap_errunion_err_ptr    => try self.airUnwrapErrUnionErrPtr(inst),
+            .unwrap_errunion_payload_ptr=> try self.airUnwrapErrUnionPayloadPtr(inst),
             .errunion_payload_ptr_set   => try self.airErrUnionPayloadPtrSet(inst),
             .err_return_trace           => try self.airErrReturnTrace(inst),
             .set_err_return_trace       => try self.airSetErrReturnTrace(inst),
@@ -1266,7 +1307,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
     }
 }
 
-pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [count]Register) !void {
+pub fn spillRegisters(self: *Self, registers: []const Register) !void {
     for (registers) |reg| {
         try self.register_manager.getReg(reg, null);
     }
@@ -1399,8 +1440,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
 
     // when truncating a `u16` to `u5`, for example, those top 3 bits in the result
     // have to be removed. this only happens if the dst if not a power-of-two size.
-    const dst_bit_size = dst_ty.bitSize(self.target.*);
-    if (!math.isPowerOfTwo(dst_bit_size) or dst_bit_size < 8) {
+    if (self.regExtraBits(dst_ty) > 0) {
         try self.truncateRegister(dst_ty, reg);
     }
 
@@ -1414,113 +1454,6 @@ fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAir(inst, result, .{ un_op, .none, .none });
 }
 
-fn airNot(self: *Self, inst: Air.Inst.Index) !void {
-    const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-
-    if (self.liveness.isUnused(inst)) {
-        return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
-    }
-
-    const operand_ty = self.air.typeOf(ty_op.operand);
-    const operand = try self.resolveInst(ty_op.operand);
-
-    const result: MCValue = result: {
-        switch (operand) {
-            .dead => unreachable,
-            .unreach => unreachable,
-            .eflags => |cc| {
-                break :result MCValue{ .eflags = cc.negate() };
-            },
-            else => {},
-        }
-
-        const operand_lock: ?RegisterLock = switch (operand) {
-            .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
-            else => null,
-        };
-        defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
-
-        const dst_mcv: MCValue = blk: {
-            if (self.reuseOperand(inst, ty_op.operand, 0, operand) and operand.isRegister()) {
-                break :blk operand;
-            }
-            break :blk try self.copyToRegisterWithInstTracking(inst, operand_ty, operand);
-        };
-        const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
-            .register => |reg| self.register_manager.lockReg(reg),
-            else => null,
-        };
-        defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);
-
-        const mask = switch (operand_ty.abiSize(self.target.*)) {
-            1 => ~@as(u8, 0),
-            2 => ~@as(u16, 0),
-            4 => ~@as(u32, 0),
-            8 => ~@as(u64, 0),
-            else => unreachable,
-        };
-        try self.genBinOpMir(.xor, operand_ty, dst_mcv, .{ .immediate = mask });
-
-        break :result dst_mcv;
-    };
-    return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
-}
-
-fn airMin(self: *Self, inst: Air.Inst.Index) !void {
-    const bin_op = self.air.instructions.items(.data)[inst].bin_op;
-    if (self.liveness.isUnused(inst)) {
-        return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
-    }
-
-    const ty = self.air.typeOfIndex(inst);
-    if (ty.zigTypeTag() != .Int) {
-        return self.fail("TODO implement min for type {}", .{ty.fmtDebug()});
-    }
-    const signedness = ty.intInfo(self.target.*).signedness;
-    const result: MCValue = result: {
-        // TODO improve by checking if any operand can be reused.
-        // TODO audit register allocation
-        const lhs = try self.resolveInst(bin_op.lhs);
-        const lhs_lock: ?RegisterLock = switch (lhs) {
-            .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
-            else => null,
-        };
-        defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
-
-        const lhs_reg = try self.copyToTmpRegister(ty, lhs);
-        const lhs_reg_lock = self.register_manager.lockRegAssumeUnused(lhs_reg);
-        defer self.register_manager.unlockReg(lhs_reg_lock);
-
-        const rhs_mcv = try self.limitImmediateType(bin_op.rhs, i32);
-        const rhs_lock: ?RegisterLock = switch (rhs_mcv) {
-            .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
-            else => null,
-        };
-        defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
-
-        try self.genBinOpMir(.cmp, ty, .{ .register = lhs_reg }, rhs_mcv);
-
-        const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, rhs_mcv);
-        const cc: Condition = switch (signedness) {
-            .unsigned => .b,
-            .signed => .l,
-        };
-        try self.asmCmovccRegisterRegister(dst_mcv.register, lhs_reg, cc);
-
-        break :result dst_mcv;
-    };
-    return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
-}
-
-fn airMax(self: *Self, inst: Air.Inst.Index) !void {
-    const bin_op = self.air.instructions.items(.data)[inst].bin_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement max for {}", .{self.target.cpu.arch});
-    return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
-}
-
 fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
     const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
     const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
@@ -1542,14 +1475,23 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
 }
 
+fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
+    const ty_op = self.air.instructions.items(.data)[inst].ty_op;
+
+    const result = if (self.liveness.isUnused(inst))
+        .dead
+    else
+        try self.genUnOp(inst, tag, ty_op.operand);
+    return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
+}
+
 fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
     const bin_op = self.air.instructions.items(.data)[inst].bin_op;
 
-    if (self.liveness.isUnused(inst)) {
-        return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
-    }
-
-    const result = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
+    const result = if (self.liveness.isUnused(inst))
+        .dead
+    else
+        try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
     return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
 }
 
@@ -1557,31 +1499,32 @@ fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void
     const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
     const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
 
-    if (self.liveness.isUnused(inst)) {
-        return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
-    }
-
-    const result = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
+    const result = if (self.liveness.isUnused(inst))
+        .dead
+    else
+        try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
     return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
 }
 
 fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
     const bin_op = self.air.instructions.items(.data)[inst].bin_op;
+    const result = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
 
-    if (self.liveness.isUnused(inst)) {
-        return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
-    }
+        const tag = self.air.instructions.items(.tag)[inst];
+        const ty = self.air.typeOfIndex(inst);
 
-    const tag = self.air.instructions.items(.tag)[inst];
-    const ty = self.air.typeOfIndex(inst);
+        if (ty.zigTypeTag() == .Float) {
+            break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
+        }
 
-    try self.spillRegisters(2, .{ .rax, .rdx });
+        try self.spillRegisters(&.{ .rax, .rdx });
 
-    const lhs = try self.resolveInst(bin_op.lhs);
-    const rhs = try self.resolveInst(bin_op.rhs);
-
-    const result = try self.genMulDivBinOp(tag, inst, ty, lhs, rhs);
+        const lhs = try self.resolveInst(bin_op.lhs);
+        const rhs = try self.resolveInst(bin_op.rhs);
 
+        break :result try self.genMulDivBinOp(tag, inst, ty, lhs, rhs);
+    };
     return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
 }
 
@@ -1629,7 +1572,7 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
                 try self.spillEflagsIfOccupied();
 
                 if (tag == .shl_with_overflow) {
-                    try self.spillRegisters(1, .{.rcx});
+                    try self.spillRegisters(&.{.rcx});
                 }
 
                 const partial: MCValue = switch (tag) {
@@ -1756,7 +1699,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
                     try self.spillEflagsIfOccupied();
                     self.eflags_inst = inst;
 
-                    try self.spillRegisters(2, .{ .rax, .rdx });
+                    try self.spillRegisters(&.{ .rax, .rdx });
 
                     const lhs = try self.resolveInst(bin_op.lhs);
                     const rhs = try self.resolveInst(bin_op.rhs);
@@ -1809,7 +1752,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
                             break :dst_reg dst_reg;
                         },
                         .unsigned => {
-                            try self.spillRegisters(2, .{ .rax, .rdx });
+                            try self.spillRegisters(&.{ .rax, .rdx });
 
                             const lhs = try self.resolveInst(bin_op.lhs);
                             const rhs = try self.resolveInst(bin_op.rhs);
@@ -1923,7 +1866,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
         return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
     }
 
-    try self.spillRegisters(1, .{.rcx});
+    try self.spillRegisters(&.{.rcx});
 
     const tag = self.air.instructions.items(.tag)[inst];
     const lhs = try self.resolveInst(bin_op.lhs);
@@ -1947,59 +1890,82 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
 
 fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    if (self.liveness.isUnused(inst)) {
-        return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
-    }
-
-    const payload_ty = self.air.typeOfIndex(inst);
-    const optional_ty = self.air.typeOf(ty_op.operand);
-    const operand = try self.resolveInst(ty_op.operand);
     const result: MCValue = result: {
-        if (!payload_ty.hasRuntimeBits()) break :result MCValue.none;
-        if (optional_ty.isPtrLikeOptional()) {
-            if (self.reuseOperand(inst, ty_op.operand, 0, operand)) {
-                break :result operand;
+        if (self.liveness.isUnused(inst)) break :result .none;
+
+        const pl_ty = self.air.typeOfIndex(inst);
+        const opt_mcv = try self.resolveInst(ty_op.operand);
+
+        if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) {
+            switch (opt_mcv) {
+                .register => |reg| try self.truncateRegister(pl_ty, reg),
+                .register_overflow => |ro| try self.truncateRegister(pl_ty, ro.reg),
+                else => {},
             }
-            break :result try self.copyToRegisterWithInstTracking(inst, payload_ty, operand);
+            break :result opt_mcv;
         }
 
-        const offset = optional_ty.abiSize(self.target.*) - payload_ty.abiSize(self.target.*);
-        switch (operand) {
-            .stack_offset => |off| {
-                break :result MCValue{ .stack_offset = off - @intCast(i32, offset) };
-            },
-            .register => {
-                // TODO reuse the operand
-                const result = try self.copyToRegisterWithInstTracking(inst, optional_ty, operand);
-                const shift = @intCast(u8, offset * @sizeOf(usize));
-                try self.genShiftBinOpMir(.shr, optional_ty, result.register, .{ .immediate = @intCast(u8, shift) });
-                break :result result;
-            },
-            else => return self.fail("TODO implement optional_payload when operand is {}", .{operand}),
-        }
+        const pl_mcv = try self.allocRegOrMem(inst, true);
+        try self.setRegOrMem(pl_ty, pl_mcv, switch (opt_mcv) {
+            else => opt_mcv,
+            .register_overflow => |ro| .{ .register = ro.reg },
+        });
+        break :result pl_mcv;
     };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
 fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch});
+    const result: MCValue = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        const dst_ty = self.air.typeOfIndex(inst);
+        const opt_mcv = try self.resolveInst(ty_op.operand);
+
+        break :result if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv))
+            opt_mcv
+        else
+            try self.copyToRegisterWithInstTracking(inst, dst_ty, opt_mcv);
+    };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
 fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement .optional_payload_ptr_set for {}", .{self.target.cpu.arch});
+    const result = result: {
+        const dst_ty = self.air.typeOfIndex(inst);
+        const src_ty = self.air.typeOf(ty_op.operand);
+        const opt_ty = src_ty.childType();
+        const src_mcv = try self.resolveInst(ty_op.operand);
+
+        if (opt_ty.optionalReprIsPayload()) {
+            break :result if (self.liveness.isUnused(inst))
+                .dead
+            else if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+                src_mcv
+            else
+                try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
+        }
+
+        const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+            src_mcv
+        else
+            try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
+
+        const pl_ty = dst_ty.childType();
+        const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
+        try self.asmMemoryImmediate(
+            .mov,
+            Memory.sib(.byte, .{ .base = dst_mcv.register, .disp = pl_abi_size }),
+            Immediate.u(1),
+        );
+        break :result if (self.liveness.isUnused(inst)) .dead else dst_mcv;
+    };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
-fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
+fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
     if (self.liveness.isUnused(inst)) {
         return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
@@ -2026,8 +1992,14 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
             },
             .register => |reg| {
                 // TODO reuse operand
-                const lock = self.register_manager.lockRegAssumeUnused(reg);
-                defer self.register_manager.unlockReg(lock);
+                self.register_manager.getRegAssumeFree(.rcx, null);
+                const rcx_lock =
+                    if (err_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null;
+                defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);
+
+                const eu_lock = self.register_manager.lockReg(reg);
+                defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
+
                 const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
                 if (err_off > 0) {
                     const shift = @intCast(u6, err_off * 8);
@@ -2043,7 +2015,7 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
-fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
+fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
     if (self.liveness.isUnused(inst)) {
         return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
@@ -2075,8 +2047,14 @@ fn genUnwrapErrorUnionPayloadMir(
             },
             .register => |reg| {
                 // TODO reuse operand
-                const lock = self.register_manager.lockRegAssumeUnused(reg);
-                defer self.register_manager.unlockReg(lock);
+                self.register_manager.getRegAssumeFree(.rcx, null);
+                const rcx_lock =
+                    if (payload_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null;
+                defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);
+
+                const eu_lock = self.register_manager.lockReg(reg);
+                defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
+
                 const result_reg: Register = if (maybe_inst) |inst|
                     (try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)).register
                 else
@@ -2097,31 +2075,118 @@ fn genUnwrapErrorUnionPayloadMir(
 }
 
 // *(E!T) -> E
-fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void {
+fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch});
+    const result: MCValue = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        const src_ty = self.air.typeOf(ty_op.operand);
+        const src_mcv = try self.resolveInst(ty_op.operand);
+        const src_reg = switch (src_mcv) {
+            .register => |reg| reg,
+            else => try self.copyToTmpRegister(src_ty, src_mcv),
+        };
+        const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
+        defer self.register_manager.unlockReg(src_lock);
+
+        const dst_reg = try self.register_manager.allocReg(inst, gp);
+        const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
+        defer self.register_manager.unlockReg(dst_lock);
+
+        const eu_ty = src_ty.childType();
+        const pl_ty = eu_ty.errorUnionPayload();
+        const err_ty = eu_ty.errorUnionSet();
+        const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
+        const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
+        try self.asmRegisterMemory(
+            .mov,
+            registerAlias(dst_reg, err_abi_size),
+            Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }),
+        );
+        break :result .{ .register = dst_reg };
+    };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
 // *(E!T) -> *T
-fn airUnwrapErrPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
+fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch});
+    const result: MCValue = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        const src_ty = self.air.typeOf(ty_op.operand);
+        const src_mcv = try self.resolveInst(ty_op.operand);
+        const src_reg = switch (src_mcv) {
+            .register => |reg| reg,
+            else => try self.copyToTmpRegister(src_ty, src_mcv),
+        };
+        const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
+        defer self.register_manager.unlockReg(src_lock);
+
+        const dst_ty = self.air.typeOfIndex(inst);
+        const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+            src_reg
+        else
+            try self.register_manager.allocReg(inst, gp);
+        const dst_lock = self.register_manager.lockReg(dst_reg);
+        defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+
+        const eu_ty = src_ty.childType();
+        const pl_ty = eu_ty.errorUnionPayload();
+        const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
+        const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
+        try self.asmRegisterMemory(
+            .lea,
+            registerAlias(dst_reg, dst_abi_size),
+            Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }),
+        );
+        break :result .{ .register = dst_reg };
+    };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
 fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement .errunion_payload_ptr_set for {}", .{self.target.cpu.arch});
+    const result: MCValue = result: {
+        const src_ty = self.air.typeOf(ty_op.operand);
+        const src_mcv = try self.resolveInst(ty_op.operand);
+        const src_reg = switch (src_mcv) {
+            .register => |reg| reg,
+            else => try self.copyToTmpRegister(src_ty, src_mcv),
+        };
+        const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
+        defer self.register_manager.unlockReg(src_lock);
+
+        const eu_ty = src_ty.childType();
+        const pl_ty = eu_ty.errorUnionPayload();
+        const err_ty = eu_ty.errorUnionSet();
+        const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
+        const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
+        try self.asmMemoryImmediate(
+            .mov,
+            Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }),
+            Immediate.u(0),
+        );
+
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        const dst_ty = self.air.typeOfIndex(inst);
+        const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+            src_reg
+        else
+            try self.register_manager.allocReg(inst, gp);
+        const dst_lock = self.register_manager.lockReg(dst_reg);
+        defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+
+        const pl_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
+        const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
+        try self.asmRegisterMemory(
+            .lea,
+            registerAlias(dst_reg, dst_abi_size),
+            Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }),
+        );
+        break :result .{ .register = dst_reg };
+    };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
@@ -2145,41 +2210,45 @@ fn airSaveErrReturnTraceIndex(self: *Self, inst: Air.Inst.Index) !void {
 
 fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    if (self.liveness.isUnused(inst)) {
-        return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
-    }
-
-    const payload_ty = self.air.typeOf(ty_op.operand);
     const result: MCValue = result: {
-        if (!payload_ty.hasRuntimeBits()) {
-            break :result MCValue{ .immediate = 1 };
-        }
+        if (self.liveness.isUnused(inst)) break :result .dead;
 
-        const optional_ty = self.air.typeOfIndex(inst);
-        const operand = try self.resolveInst(ty_op.operand);
-        const operand_lock: ?RegisterLock = switch (operand) {
+        const pl_ty = self.air.typeOf(ty_op.operand);
+        if (!pl_ty.hasRuntimeBits()) break :result .{ .immediate = 1 };
+
+        const opt_ty = self.air.typeOfIndex(inst);
+        const pl_mcv = try self.resolveInst(ty_op.operand);
+        const same_repr = opt_ty.optionalReprIsPayload();
+        if (same_repr and self.reuseOperand(inst, ty_op.operand, 0, pl_mcv)) break :result pl_mcv;
+
+        const pl_lock: ?RegisterLock = switch (pl_mcv) {
             .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
             else => null,
         };
-        defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
+        defer if (pl_lock) |lock| self.register_manager.unlockReg(lock);
 
-        if (optional_ty.isPtrLikeOptional()) {
-            // TODO should we check if we can reuse the operand?
-            if (self.reuseOperand(inst, ty_op.operand, 0, operand)) {
-                break :result operand;
+        const opt_mcv = try self.allocRegOrMem(inst, true);
+        try self.setRegOrMem(pl_ty, opt_mcv, pl_mcv);
+
+        if (!same_repr) {
+            const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
+            switch (opt_mcv) {
+                else => unreachable,
+
+                .register => |opt_reg| try self.asmRegisterImmediate(
+                    .bts,
+                    opt_reg,
+                    Immediate.u(@intCast(u6, pl_abi_size * 8)),
+                ),
+
+                .stack_offset => |off| try self.asmMemoryImmediate(
+                    .mov,
+                    Memory.sib(.byte, .{ .base = .rsp, .disp = pl_abi_size - off }),
+                    Immediate.u(0),
+                ),
             }
-            break :result try self.copyToRegisterWithInstTracking(inst, payload_ty, operand);
         }
-
-        const optional_abi_size = @intCast(u32, optional_ty.abiSize(self.target.*));
-        const optional_abi_align = optional_ty.abiAlignment(self.target.*);
-        const payload_abi_size = @intCast(u32, payload_ty.abiSize(self.target.*));
-        const offset = optional_abi_size - payload_abi_size;
-
-        const stack_offset = @intCast(i32, try self.allocMem(inst, optional_abi_size, optional_abi_align));
-        try self.genSetStack(Type.bool, stack_offset, .{ .immediate = 1 }, .{});
-        try self.genSetStack(payload_ty, stack_offset - @intCast(i32, offset), operand, .{});
-        break :result MCValue{ .stack_offset = stack_offset };
+        break :result opt_mcv;
     };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
@@ -2281,19 +2350,53 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
 
 fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement ptr_slice_len_ptr for {}", .{self.target.cpu.arch});
+    const result: MCValue = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        const src_ty = self.air.typeOf(ty_op.operand);
+        const src_mcv = try self.resolveInst(ty_op.operand);
+        const src_reg = switch (src_mcv) {
+            .register => |reg| reg,
+            else => try self.copyToTmpRegister(src_ty, src_mcv),
+        };
+        const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
+        defer self.register_manager.unlockReg(src_lock);
+
+        const dst_ty = self.air.typeOfIndex(inst);
+        const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+            src_reg
+        else
+            try self.register_manager.allocReg(inst, gp);
+        const dst_lock = self.register_manager.lockReg(dst_reg);
+        defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+
+        const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
+        try self.asmRegisterMemory(
+            .lea,
+            registerAlias(dst_reg, dst_abi_size),
+            Memory.sib(.qword, .{
+                .base = src_reg,
+                .disp = @divExact(self.target.cpu.arch.ptrBitWidth(), 8),
+            }),
+        );
+        break :result .{ .register = dst_reg };
+    };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
 fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement ptr_slice_ptr_ptr for {}", .{self.target.cpu.arch});
+    const result: MCValue = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        const dst_ty = self.air.typeOfIndex(inst);
+        const opt_mcv = try self.resolveInst(ty_op.operand);
+
+        break :result if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv))
+            opt_mcv
+        else
+            try self.copyToRegisterWithInstTracking(inst, dst_ty, opt_mcv);
+    };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
@@ -2492,10 +2595,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
             try self.asmRegisterMemory(
                 .mov,
                 registerAlias(dst_mcv.register, elem_abi_size),
-                Memory.sib(Memory.PtrSize.fromSize(elem_abi_size), .{
-                    .base = dst_mcv.register,
-                    .disp = 0,
-                }),
+                Memory.sib(Memory.PtrSize.fromSize(elem_abi_size), .{ .base = dst_mcv.register }),
             );
             break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
         }
@@ -2614,7 +2714,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
             },
             .register => {
                 const shift: u6 = if (layout.tag_align < layout.payload_align)
-                    @intCast(u6, layout.payload_size * @sizeOf(usize))
+                    @intCast(u6, layout.payload_size * 8)
                 else
                     0;
                 const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
@@ -2632,46 +2732,418 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
 
 fn airClz(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch});
+    const result = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        const dst_ty = self.air.typeOfIndex(inst);
+        const src_ty = self.air.typeOf(ty_op.operand);
+
+        const src_mcv = try self.resolveInst(ty_op.operand);
+        const mat_src_mcv = switch (src_mcv) {
+            .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
+            else => src_mcv,
+        };
+        const mat_src_lock = switch (mat_src_mcv) {
+            .register => |reg| self.register_manager.lockReg(reg),
+            else => null,
+        };
+        defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
+
+        const dst_reg = try self.register_manager.allocReg(inst, gp);
+        const dst_mcv = MCValue{ .register = dst_reg };
+        const dst_lock = self.register_manager.lockReg(dst_reg);
+        defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+
+        if (Target.x86.featureSetHas(self.target.cpu.features, .lzcnt)) {
+            try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
+            const extra_bits = self.regExtraBits(src_ty);
+            if (extra_bits > 0) {
+                try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });
+            }
+            break :result dst_mcv;
+        }
+
+        const src_bits = src_ty.bitSize(self.target.*);
+        const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
+        const width_mcv = MCValue{ .register = width_reg };
+        try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
+
+        const dst_abi_size = @intCast(u32, @max(dst_ty.abiSize(self.target.*), 2));
+        try self.asmCmovccRegisterRegister(
+            registerAlias(dst_reg, dst_abi_size),
+            registerAlias(width_reg, dst_abi_size),
+            .z,
+        );
+
+        try self.genBinOpMir(.sub, dst_ty, width_mcv, dst_mcv);
+        break :result width_mcv;
+    };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
 fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch});
+    const result = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        const dst_ty = self.air.typeOfIndex(inst);
+        const src_ty = self.air.typeOf(ty_op.operand);
+        const src_bits = src_ty.bitSize(self.target.*);
+
+        const src_mcv = try self.resolveInst(ty_op.operand);
+        const mat_src_mcv = switch (src_mcv) {
+            .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
+            else => src_mcv,
+        };
+        const mat_src_lock = switch (mat_src_mcv) {
+            .register => |reg| self.register_manager.lockReg(reg),
+            else => null,
+        };
+        defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
+
+        const dst_reg = try self.register_manager.allocReg(inst, gp);
+        const dst_mcv = MCValue{ .register = dst_reg };
+        const dst_lock = self.register_manager.lockReg(dst_reg);
+        defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+
+        if (Target.x86.featureSetHas(self.target.cpu.features, .bmi)) {
+            const extra_bits = self.regExtraBits(src_ty);
+            const masked_mcv = if (extra_bits > 0) masked: {
+                const mask_mcv = MCValue{
+                    .immediate = ((@as(u64, 1) << @intCast(u6, extra_bits)) - 1) << @intCast(u6, src_bits),
+                };
+                const tmp_mcv = tmp: {
+                    if (src_mcv.isImmediate() or self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
+                        break :tmp src_mcv;
+                    }
+                    try self.genSetReg(src_ty, dst_reg, src_mcv);
+                    break :tmp dst_mcv;
+                };
+                try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);
+                break :masked tmp_mcv;
+            } else mat_src_mcv;
+            try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);
+            break :result dst_mcv;
+        }
+
+        const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
+        try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
+
+        const abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
+        try self.asmCmovccRegisterRegister(
+            registerAlias(dst_reg, abi_size),
+            registerAlias(width_reg, abi_size),
+            .z,
+        );
+
+        break :result dst_mcv;
+    };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
 fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch});
+    const result: MCValue = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        const src_ty = self.air.typeOf(ty_op.operand);
+        const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
+        const src_mcv = try self.resolveInst(ty_op.operand);
+
+        if (Target.x86.featureSetHas(self.target.cpu.features, .popcnt)) {
+            const mat_src_mcv = switch (src_mcv) {
+                .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
+                else => src_mcv,
+            };
+            const mat_src_lock = switch (mat_src_mcv) {
+                .register => |reg| self.register_manager.lockReg(reg),
+                else => null,
+            };
+            defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
+
+            const dst_mcv: MCValue = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+                src_mcv
+            else
+                .{ .register = try self.register_manager.allocReg(inst, gp) };
+
+            const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16;
+            try self.genBinOpMir(.popcnt, popcnt_ty, dst_mcv, mat_src_mcv);
+            break :result dst_mcv;
+        }
+
+        const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - src_abi_size * 8);
+        const imm_0_1 = Immediate.u(mask / 0b1_1);
+        const imm_00_11 = Immediate.u(mask / 0b01_01);
+        const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);
+        const imm_0000_0001 = Immediate.u(mask / 0b1111_1111);
+
+        const tmp_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+            src_mcv.register
+        else
+            try self.copyToTmpRegister(src_ty, src_mcv);
+        const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
+        defer self.register_manager.unlockReg(tmp_lock);
+
+        const dst_reg = try self.register_manager.allocReg(inst, gp);
+        const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
+        defer self.register_manager.unlockReg(dst_lock);
+
+        {
+            const dst = registerAlias(dst_reg, src_abi_size);
+            const tmp = registerAlias(tmp_reg, src_abi_size);
+            const imm = if (src_abi_size > 4)
+                try self.register_manager.allocReg(null, gp)
+            else
+                undefined;
+
+            // tmp = operand
+            try self.asmRegisterRegister(.mov, dst, tmp);
+            // dst = operand
+            try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));
+            // tmp = operand >> 1
+            if (src_abi_size > 4) {
+                try self.asmRegisterImmediate(.mov, imm, imm_0_1);
+                try self.asmRegisterRegister(.@"and", tmp, imm);
+            } else try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);
+            // tmp = (operand >> 1) & 0x55...55
+            try self.asmRegisterRegister(.sub, dst, tmp);
+            // dst = temp1 = operand - ((operand >> 1) & 0x55...55)
+            try self.asmRegisterRegister(.mov, tmp, dst);
+            // tmp = temp1
+            try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));
+            // dst = temp1 >> 2
+            if (src_abi_size > 4) {
+                try self.asmRegisterImmediate(.mov, imm, imm_00_11);
+                try self.asmRegisterRegister(.@"and", tmp, imm);
+                try self.asmRegisterRegister(.@"and", dst, imm);
+            } else {
+                try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);
+                try self.asmRegisterImmediate(.@"and", dst, imm_00_11);
+            }
+            // tmp = temp1 & 0x33...33
+            // dst = (temp1 >> 2) & 0x33...33
+            try self.asmRegisterRegister(.add, tmp, dst);
+            // tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
+            try self.asmRegisterRegister(.mov, dst, tmp);
+            // dst = temp2
+            try self.asmRegisterImmediate(.shr, tmp, Immediate.u(4));
+            // tmp = temp2 >> 4
+            try self.asmRegisterRegister(.add, dst, tmp);
+            // dst = temp2 + (temp2 >> 4)
+            if (src_abi_size > 4) {
+                try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);
+                try self.asmRegisterImmediate(.mov, tmp, imm_0000_0001);
+                try self.asmRegisterRegister(.@"and", dst, imm);
+                try self.asmRegisterRegister(.imul, dst, tmp);
+            } else {
+                try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);
+                if (src_abi_size > 1) {
+                    try self.asmRegisterRegisterImmediate(.imul, dst, dst, imm_0000_0001);
+                }
+            }
+            // dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
+            // dst = temp3 * 0x01...01
+            if (src_abi_size > 1) {
+                try self.asmRegisterImmediate(.shr, dst, Immediate.u((src_abi_size - 1) * 8));
+            }
+            // dst = (temp3 * 0x01...01) >> (bits - 8)
+        }
+        break :result .{ .register = dst_reg };
+    };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
+fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, mem_ok: bool) !MCValue {
+    const ty_op = self.air.instructions.items(.data)[inst].ty_op;
+
+    const src_bits = self.regBitSize(src_ty);
+    const src_lock = switch (src_mcv) {
+        .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
+        else => null,
+    };
+    defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
+
+    switch (src_bits) {
+        else => unreachable,
+        8 => return if ((mem_ok or src_mcv.isRegister()) and
+            self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+            src_mcv
+        else
+            try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv),
+        16 => if ((mem_ok or src_mcv.isRegister()) and
+            self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+        {
+            try self.genBinOpMir(.rol, src_ty, src_mcv, .{ .immediate = 8 });
+            return src_mcv;
+        },
+        32, 64 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
+            try self.genUnOpMir(.bswap, src_ty, src_mcv);
+            return src_mcv;
+        },
+    }
+
+    if (src_mcv.isRegister()) {
+        const dst_mcv: MCValue = if (mem_ok)
+            try self.allocRegOrMem(inst, true)
+        else
+            .{ .register = try self.register_manager.allocReg(inst, gp) };
+        if (dst_mcv.isRegister()) {
+            const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register);
+            defer self.register_manager.unlockReg(dst_lock);
+
+            try self.genSetReg(src_ty, dst_mcv.register, src_mcv);
+            switch (src_bits) {
+                else => unreachable,
+                16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),
+                32, 64 => try self.genUnOpMir(.bswap, src_ty, dst_mcv),
+            }
+        } else try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);
+        return dst_mcv;
+    }
+
+    const dst_reg = try self.register_manager.allocReg(inst, gp);
+    const dst_mcv = MCValue{ .register = dst_reg };
+    const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
+    defer self.register_manager.unlockReg(dst_lock);
+
+    try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);
+    return dst_mcv;
+}
+
 fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement airByteSwap for {}", .{self.target.cpu.arch});
+    const result = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        const src_ty = self.air.typeOf(ty_op.operand);
+        const src_mcv = try self.resolveInst(ty_op.operand);
+
+        const dst_mcv = try self.byteSwap(inst, src_ty, src_mcv, true);
+        switch (self.regExtraBits(src_ty)) {
+            0 => {},
+            else => |extra| try self.genBinOpMir(
+                if (src_ty.isSignedInt()) .sar else .shr,
+                src_ty,
+                dst_mcv,
+                .{ .immediate = extra },
+            ),
+        }
+        break :result dst_mcv;
+    };
+
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
 fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    const result: MCValue = if (self.liveness.isUnused(inst))
-        .dead
-    else
-        return self.fail("TODO implement airBitReverse for {}", .{self.target.cpu.arch});
+    const result = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        const src_ty = self.air.typeOf(ty_op.operand);
+        const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
+        const src_mcv = try self.resolveInst(ty_op.operand);
+
+        const dst_mcv = try self.byteSwap(inst, src_ty, src_mcv, false);
+        const dst_reg = dst_mcv.register;
+        const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
+        defer self.register_manager.unlockReg(dst_lock);
+
+        const tmp_reg = try self.register_manager.allocReg(null, gp);
+        const tmp_lock = self.register_manager.lockReg(tmp_reg);
+        defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
+
+        {
+            const dst = registerAlias(dst_reg, src_abi_size);
+            const tmp = registerAlias(tmp_reg, src_abi_size);
+            const imm = if (src_abi_size > 4)
+                try self.register_manager.allocReg(null, gp)
+            else
+                undefined;
+
+            const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - src_abi_size * 8);
+            const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);
+            const imm_00_11 = Immediate.u(mask / 0b01_01);
+            const imm_0_1 = Immediate.u(mask / 0b1_1);
+
+            // dst = temp1 = bswap(operand)
+            try self.asmRegisterRegister(.mov, tmp, dst);
+            // tmp = temp1
+            try self.asmRegisterImmediate(.shr, dst, Immediate.u(4));
+            // dst = temp1 >> 4
+            if (src_abi_size > 4) {
+                try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);
+                try self.asmRegisterRegister(.@"and", tmp, imm);
+                try self.asmRegisterRegister(.@"and", dst, imm);
+            } else {
+                try self.asmRegisterImmediate(.@"and", tmp, imm_0000_1111);
+                try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);
+            }
+            // tmp = temp1 & 0x0F...0F
+            // dst = (temp1 >> 4) & 0x0F...0F
+            try self.asmRegisterImmediate(.shl, tmp, Immediate.u(4));
+            // tmp = (temp1 & 0x0F...0F) << 4
+            try self.asmRegisterRegister(.@"or", dst, tmp);
+            // dst = temp2 = ((temp1 >> 4) & 0x0F...0F) | ((temp1 & 0x0F...0F) << 4)
+            try self.asmRegisterRegister(.mov, tmp, dst);
+            // tmp = temp2
+            try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));
+            // dst = temp2 >> 2
+            if (src_abi_size > 4) {
+                try self.asmRegisterImmediate(.mov, imm, imm_00_11);
+                try self.asmRegisterRegister(.@"and", tmp, imm);
+                try self.asmRegisterRegister(.@"and", dst, imm);
+            } else {
+                try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);
+                try self.asmRegisterImmediate(.@"and", dst, imm_00_11);
+            }
+            // tmp = temp2 & 0x33...33
+            // dst = (temp2 >> 2) & 0x33...33
+            try self.asmRegisterMemory(
+                .lea,
+                if (src_abi_size > 4) tmp.to64() else tmp.to32(),
+                Memory.sib(.qword, .{
+                    .base = dst.to64(),
+                    .scale_index = .{ .index = tmp.to64(), .scale = 1 << 2 },
+                }),
+            );
+            // tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
+            try self.asmRegisterRegister(.mov, dst, tmp);
+            // dst = temp3
+            try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));
+            // tmp = temp3 >> 1
+            if (src_abi_size > 4) {
+                try self.asmRegisterImmediate(.mov, imm, imm_0_1);
+                try self.asmRegisterRegister(.@"and", dst, imm);
+                try self.asmRegisterRegister(.@"and", tmp, imm);
+            } else {
+                try self.asmRegisterImmediate(.@"and", dst, imm_0_1);
+                try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);
+            }
+            // dst = temp3 & 0x55...55
+            // tmp = (temp3 >> 1) & 0x55...55
+            try self.asmRegisterMemory(
+                .lea,
+                if (src_abi_size > 4) dst.to64() else dst.to32(),
+                Memory.sib(.qword, .{
+                    .base = tmp.to64(),
+                    .scale_index = .{ .index = dst.to64(), .scale = 1 << 1 },
+                }),
+            );
+            // dst = ((temp3 >> 1) & 0x55...55) + ((temp3 & 0x55...55) << 1)
+        }
+
+        switch (self.regExtraBits(src_ty)) {
+            0 => {},
+            else => |extra| try self.genBinOpMir(
+                if (src_ty.isSignedInt()) .sar else .shr,
+                src_ty,
+                dst_mcv,
+                .{ .immediate = extra },
+            ),
+        }
+        break :result dst_mcv;
+    };
+
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
@@ -2753,7 +3225,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
                     try self.asmRegisterMemory(
                         .mov,
                         registerAlias(dst_reg, abi_size),
-                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg, .disp = 0 }),
+                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg }),
                     );
                 },
                 .stack_offset => |off| {
@@ -2865,8 +3337,14 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
                 .none => unreachable,
                 .dead => unreachable,
                 .unreach => unreachable,
-                .eflags => unreachable,
+                .eflags => |cc| {
+                    try self.asmSetccMemory(Memory.sib(
+                        Memory.PtrSize.fromSize(abi_size),
+                        .{ .base = reg.to64() },
+                    ), cc);
+                },
                 .undef => {
+                    if (!self.wantSafety()) return; // The already existing value will do just fine.
                     switch (abi_size) {
                         1 => try self.store(ptr, .{ .immediate = 0xaa }, ptr_ty, value_ty),
                         2 => try self.store(ptr, .{ .immediate = 0xaaaa }, ptr_ty, value_ty),
@@ -2882,10 +3360,10 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
                                 Immediate.s(@intCast(i32, @bitCast(i64, imm)))
                             else
                                 Immediate.u(@truncate(u32, imm));
-                            try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
-                                .base = reg.to64(),
-                                .disp = 0,
-                            }), immediate);
+                            try self.asmMemoryImmediate(.mov, Memory.sib(
+                                Memory.PtrSize.fromSize(abi_size),
+                                .{ .base = reg.to64() },
+                            ), immediate);
                         },
                         8 => {
                             // TODO: optimization: if the imm is only using the lower
@@ -2957,10 +3435,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
             try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr);
 
             // To get the actual address of the value we want to modify we have to go through the GOT
-            try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
-                .base = addr_reg.to64(),
-                .disp = 0,
-            }));
+            try self.asmRegisterMemory(
+                .mov,
+                addr_reg.to64(),
+                Memory.sib(.qword, .{ .base = addr_reg.to64() }),
+            );
 
             const new_ptr = MCValue{ .register = addr_reg.to64() };
 
@@ -2982,10 +3461,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
                             return self.fail("TODO imm64 would get incorrectly sign extended", .{});
                         }
                     }
-                    try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
-                        .base = addr_reg.to64(),
-                        .disp = 0,
-                    }), Immediate.u(@intCast(u32, imm)));
+                    try self.asmMemoryImmediate(
+                        .mov,
+                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg.to64() }),
+                        Immediate.u(@intCast(u32, imm)),
+                    );
                 },
                 .register => {
                     return self.store(new_ptr, value, ptr_ty, value_ty);
@@ -2997,10 +3477,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
                         defer self.register_manager.unlockReg(tmp_reg_lock);
 
                         try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);
-                        try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(.qword, .{
-                            .base = tmp_reg,
-                            .disp = 0,
-                        }));
+                        try self.asmRegisterMemory(
+                            .mov,
+                            tmp_reg,
+                            Memory.sib(.qword, .{ .base = tmp_reg }),
+                        );
 
                         return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
                     }
@@ -3152,7 +3633,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
                 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);
 
                 // Shift by struct_field_offset.
-                const shift = @intCast(u8, struct_field_offset * @sizeOf(usize));
+                const shift = @intCast(u8, struct_field_offset * 8);
                 try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv.register, .{ .immediate = shift });
 
                 // Mask with reg.bitSize() - struct_field_size
@@ -3212,6 +3693,107 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
 }
 
+fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
+    const src_ty = self.air.typeOf(src_air);
+    const src_mcv = try self.resolveInst(src_air);
+    if (src_ty.zigTypeTag() == .Vector) {
+        return self.fail("TODO implement genBinOp for {}", .{src_ty.fmt(self.bin_file.options.module.?)});
+    }
+    if (src_ty.abiSize(self.target.*) > 8) {
+        return self.fail("TODO implement genBinOp for {}", .{src_ty.fmt(self.bin_file.options.module.?)});
+    }
+
+    switch (src_mcv) {
+        .eflags => |cc| switch (tag) {
+            .not => return .{ .eflags = cc.negate() },
+            else => {},
+        },
+        else => {},
+    }
+
+    const src_lock = switch (src_mcv) {
+        .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
+        else => null,
+    };
+    defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
+
+    const dst_mcv: MCValue = if (maybe_inst) |inst|
+        if (self.reuseOperand(inst, src_air, 0, src_mcv))
+            src_mcv
+        else
+            try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv)
+    else
+        .{ .register = try self.copyToTmpRegister(src_ty, src_mcv) };
+    const dst_lock = switch (dst_mcv) {
+        .register => |reg| self.register_manager.lockReg(reg),
+        else => null,
+    };
+    defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+
+    switch (tag) {
+        .not => {
+            const int_info = if (src_ty.tag() == .bool)
+                std.builtin.Type.Int{ .signedness = .unsigned, .bits = 1 }
+            else
+                src_ty.intInfo(self.target.*);
+            const extra_bits = self.regExtraBits(src_ty);
+            if (int_info.signedness == .unsigned and extra_bits > 0) {
+                const mask = (@as(u64, 1) << @intCast(u6, src_ty.bitSize(self.target.*))) - 1;
+                try self.genBinOpMir(.xor, src_ty, dst_mcv, .{ .immediate = mask });
+            } else try self.genUnOpMir(.not, src_ty, dst_mcv);
+        },
+
+        .neg => try self.genUnOpMir(.neg, src_ty, dst_mcv),
+
+        else => unreachable,
+    }
+    return dst_mcv;
+}
+
+fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {
+    const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
+    switch (dst_mcv) {
+        .none => unreachable,
+        .undef => unreachable,
+        .dead, .unreach, .immediate => unreachable,
+        .eflags => unreachable,
+        .register_overflow => unreachable,
+        .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)),
+        .ptr_stack_offset, .stack_offset => |off| {
+            if (off > math.maxInt(i32)) {
+                return self.fail("stack offset too large", .{});
+            }
+            if (abi_size > 8) {
+                return self.fail("TODO implement {} for stack dst with large ABI", .{mir_tag});
+            }
+
+            try self.asmMemory(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
+                .base = .rbp,
+                .disp = -off,
+            }));
+        },
+        .memory, .linker_load => {
+            const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
+            const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
+            defer self.register_manager.unlockReg(addr_reg_lock);
+
+            try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_mcv);
+
+            // To get the actual address of the value we want to modify we have to go through the GOT
+            try self.asmRegisterMemory(
+                .mov,
+                addr_reg,
+                Memory.sib(.qword, .{ .base = addr_reg }),
+            );
+
+            try self.asmMemory(
+                mir_tag,
+                Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
+            );
+        },
+    }
+}
+
 /// Clobbers .rcx for non-immediate shift value.
 fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shift: MCValue) !void {
     assert(reg.to64() != .rcx);
@@ -3233,8 +3815,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
             },
             else => {},
         }
-        assert(self.register_manager.isRegFree(.rcx));
-        try self.register_manager.getReg(.rcx, null);
+        self.register_manager.getRegAssumeFree(.rcx, null);
         try self.genSetReg(Type.u8, .rcx, shift);
     }
 
@@ -3274,8 +3855,7 @@ fn genShiftBinOp(
     };
     defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
 
-    assert(self.register_manager.isRegFree(.rcx));
-    try self.register_manager.getReg(.rcx, null);
+    self.register_manager.getRegAssumeFree(.rcx, null);
     const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx);
     defer self.register_manager.unlockReg(rcx_lock);
 
@@ -3325,10 +3905,10 @@ fn genMulDivBinOp(
     rhs: MCValue,
 ) !MCValue {
     if (ty.zigTypeTag() == .Vector or ty.zigTypeTag() == .Float) {
-        return self.fail("TODO implement genBinOp for {}", .{ty.fmtDebug()});
+        return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()});
     }
     if (ty.abiSize(self.target.*) > 8) {
-        return self.fail("TODO implement genBinOp for {}", .{ty.fmtDebug()});
+        return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()});
     }
     if (tag == .div_float) {
         return self.fail("TODO implement genMulDivBinOp for div_float", .{});
@@ -3486,10 +4066,21 @@ fn genBinOp(
     const lhs_ty = self.air.typeOf(lhs_air);
     const rhs_ty = self.air.typeOf(rhs_air);
     if (lhs_ty.zigTypeTag() == .Vector) {
-        return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()});
+        return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmt(self.bin_file.options.module.?)});
     }
     if (lhs_ty.abiSize(self.target.*) > 8) {
-        return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()});
+        return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmt(self.bin_file.options.module.?)});
+    }
+
+    switch (lhs) {
+        .immediate => |imm| switch (imm) {
+            0 => switch (tag) {
+                .sub, .subwrap => return self.genUnOp(maybe_inst, .neg, rhs_air),
+                else => {},
+            },
+            else => {},
+        },
+        else => {},
     }
 
     const is_commutative: bool = switch (tag) {
@@ -3500,6 +4091,8 @@ fn genBinOp(
         .bool_and,
         .bit_and,
         .xor,
+        .min,
+        .max,
         => true,
 
         else => false,
@@ -3512,7 +4105,7 @@ fn genBinOp(
     defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
 
     const rhs_lock: ?RegisterLock = switch (rhs) {
-        .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
+        .register => |reg| self.register_manager.lockReg(reg),
         else => null,
     };
     defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
@@ -3520,10 +4113,10 @@ fn genBinOp(
     var flipped: bool = false;
     const dst_mcv: MCValue = blk: {
         if (maybe_inst) |inst| {
-            if (self.reuseOperand(inst, lhs_air, 0, lhs) and lhs.isRegister()) {
+            if (lhs.isRegister() and self.reuseOperand(inst, lhs_air, 0, lhs)) {
                 break :blk lhs;
             }
-            if (is_commutative and self.reuseOperand(inst, rhs_air, 1, rhs) and rhs.isRegister()) {
+            if (is_commutative and rhs.isRegister() and self.reuseOperand(inst, rhs_air, 1, rhs)) {
                 flipped = true;
                 break :blk rhs;
             }
@@ -3551,11 +4144,56 @@ fn genBinOp(
     switch (tag) {
         .add,
         .addwrap,
-        => try self.genBinOpMir(.add, lhs_ty, dst_mcv, src_mcv),
+        => try self.genBinOpMir(switch (lhs_ty.tag()) {
+            else => .add,
+            .f32 => .addss,
+            .f64 => .addsd,
+        }, lhs_ty, dst_mcv, src_mcv),
 
         .sub,
         .subwrap,
-        => try self.genBinOpMir(.sub, lhs_ty, dst_mcv, src_mcv),
+        => try self.genBinOpMir(switch (lhs_ty.tag()) {
+            else => .sub,
+            .f32 => .subss,
+            .f64 => .subsd,
+        }, lhs_ty, dst_mcv, src_mcv),
+
+        .mul => try self.genBinOpMir(switch (lhs_ty.tag()) {
+            .f32 => .mulss,
+            .f64 => .mulsd,
+            else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
+        }, lhs_ty, dst_mcv, src_mcv),
+
+        .div_float,
+        .div_exact,
+        => try self.genBinOpMir(switch (lhs_ty.tag()) {
+            .f32 => .divss,
+            .f64 => .divsd,
+            else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
+        }, lhs_ty, dst_mcv, src_mcv),
+
+        .div_trunc,
+        .div_floor,
+        => {
+            try self.genBinOpMir(switch (lhs_ty.tag()) {
+                .f32 => .divss,
+                .f64 => .divsd,
+                else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
+            }, lhs_ty, dst_mcv, src_mcv);
+            if (Target.x86.featureSetHas(self.target.cpu.features, .sse4_1)) {
+                const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
+                const dst_alias = registerAlias(dst_mcv.register, abi_size);
+                try self.asmRegisterRegisterImmediate(switch (lhs_ty.tag()) {
+                    .f32 => .roundss,
+                    .f64 => .roundsd,
+                    else => unreachable,
+                }, dst_alias, dst_alias, Immediate.u(switch (tag) {
+                    .div_trunc => 0b1_0_11,
+                    .div_floor => 0b1_0_01,
+                    else => unreachable,
+                }));
+            } else return self.fail("TODO implement round without sse4_1", .{});
+        },
 
         .ptr_add,
         .ptr_sub,
@@ -3580,6 +4218,113 @@ fn genBinOp(
 
         .xor => try self.genBinOpMir(.xor, lhs_ty, dst_mcv, src_mcv),
 
+        .min,
+        .max,
+        => switch (lhs_ty.zigTypeTag()) {
+            .Int => {
+                const mat_src_mcv = switch (src_mcv) {
+                    .immediate => MCValue{ .register = try self.copyToTmpRegister(rhs_ty, src_mcv) },
+                    else => src_mcv,
+                };
+                const mat_mcv_lock = switch (mat_src_mcv) {
+                    .register => |reg| self.register_manager.lockReg(reg),
+                    else => null,
+                };
+                defer if (mat_mcv_lock) |lock| self.register_manager.unlockReg(lock);
+
+                try self.genBinOpMir(.cmp, lhs_ty, dst_mcv, mat_src_mcv);
+
+                const int_info = lhs_ty.intInfo(self.target.*);
+                const cc: Condition = switch (int_info.signedness) {
+                    .unsigned => switch (tag) {
+                        .min => .a,
+                        .max => .b,
+                        else => unreachable,
+                    },
+                    .signed => switch (tag) {
+                        .min => .g,
+                        .max => .l,
+                        else => unreachable,
+                    },
+                };
+
+                const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
+                switch (dst_mcv) {
+                    .none,
+                    .undef,
+                    .dead,
+                    .unreach,
+                    .immediate,
+                    .eflags,
+                    .register_overflow,
+                    .stack_offset,
+                    .ptr_stack_offset,
+                    .memory,
+                    .linker_load,
+                    => unreachable,
+                    .register => |dst_reg| switch (mat_src_mcv) {
+                        .none,
+                        .undef,
+                        .dead,
+                        .unreach,
+                        .immediate,
+                        .eflags,
+                        .register_overflow,
+                        .ptr_stack_offset,
+                        => unreachable,
+                        .register => |src_reg| try self.asmCmovccRegisterRegister(
+                            registerAlias(dst_reg, abi_size),
+                            registerAlias(src_reg, abi_size),
+                            cc,
+                        ),
+                        .stack_offset => |off| try self.asmCmovccRegisterMemory(
+                            registerAlias(dst_reg, abi_size),
+                            Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
+                                .base = .rbp,
+                                .disp = -off,
+                            }),
+                            cc,
+                        ),
+                        .memory, .linker_load => {
+                            const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
+                            const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
+                            defer self.register_manager.unlockReg(addr_reg_lock);
+
+                            try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_mcv);
+
+                            // To get the actual address of the value we want to modify we
+                            // we have to go through the GOT
+                            try self.asmRegisterMemory(
+                                .mov,
+                                addr_reg,
+                                Memory.sib(.qword, .{ .base = addr_reg }),
+                            );
+
+                            try self.asmCmovccRegisterMemory(
+                                registerAlias(dst_reg, abi_size),
+                                Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
+                                cc,
+                            );
+                        },
+                    },
+                }
+            },
+            .Float => try self.genBinOpMir(switch (lhs_ty.tag()) {
+                .f32 => switch (tag) {
+                    .min => .minss,
+                    .max => .maxss,
+                    else => unreachable,
+                },
+                .f64 => switch (tag) {
+                    .min => .minsd,
+                    .max => .maxsd,
+                    else => unreachable,
+                },
+                else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
+            }, lhs_ty, dst_mcv, src_mcv),
+            else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
+        },
+
         else => unreachable,
     }
     return dst_mcv;
@@ -3609,29 +4354,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
                 .register => |src_reg| switch (dst_ty.zigTypeTag()) {
                     .Float => {
                         if (intrinsicsAllowed(self.target.*, dst_ty)) {
-                            const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) {
-                                .f32 => switch (mir_tag) {
-                                    .add => .addss,
-                                    .cmp => .ucomiss,
-                                    else => return self.fail(
-                                        "TODO genBinOpMir for f32 register-register with MIR tag {}",
-                                        .{mir_tag},
-                                    ),
-                                },
-                                .f64 => switch (mir_tag) {
-                                    .add => .addsd,
-                                    .cmp => .ucomisd,
-                                    else => return self.fail(
-                                        "TODO genBinOpMir for f64 register-register with MIR tag {}",
-                                        .{mir_tag},
-                                    ),
-                                },
-                                else => return self.fail(
-                                    "TODO genBinOpMir for float register-register and type {}",
-                                    .{dst_ty.fmtDebug()},
-                                ),
-                            };
-                            return self.asmRegisterRegister(actual_tag, dst_reg.to128(), src_reg.to128());
+                            return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
                         }
 
                         return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});
@@ -3643,16 +4366,15 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
                     ),
                 },
                 .immediate => |imm| {
-                    switch (abi_size) {
-                        0 => unreachable,
-                        1...4 => {
+                    switch (self.regBitSize(dst_ty)) {
+                        8, 16, 32 => {
                             try self.asmRegisterImmediate(
                                 mir_tag,
                                 registerAlias(dst_reg, abi_size),
                                 Immediate.u(@intCast(u32, imm)),
                             );
                         },
-                        5...8 => {
+                        64 => {
                             if (math.cast(i32, @bitCast(i64, imm))) |small| {
                                 try self.asmRegisterImmediate(
                                     mir_tag,
@@ -3660,13 +4382,10 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
                                     Immediate.s(small),
                                 );
                             } else {
-                                const tmp_reg = try self.register_manager.allocReg(null, gp);
-                                const tmp_alias = registerAlias(tmp_reg, abi_size);
-                                try self.asmRegisterImmediate(.mov, tmp_alias, Immediate.u(imm));
                                 try self.asmRegisterRegister(
                                     mir_tag,
                                     registerAlias(dst_reg, abi_size),
-                                    tmp_alias,
+                                    registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
                                 );
                             }
                         },
@@ -3716,11 +4435,43 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
                     }), registerAlias(src_reg, abi_size));
                 },
                 .immediate => |imm| {
-                    // TODO
-                    try self.asmMemoryImmediate(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
-                        .base = .rbp,
-                        .disp = -off,
-                    }), Immediate.u(@intCast(u32, imm)));
+                    switch (self.regBitSize(dst_ty)) {
+                        8, 16, 32 => {
+                            try self.asmMemoryImmediate(
+                                mir_tag,
+                                Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
+                                    .base = .rbp,
+                                    .disp = -off,
+                                }),
+                                if (math.cast(i32, @bitCast(i64, imm))) |small|
+                                    Immediate.s(small)
+                                else
+                                    Immediate.u(@intCast(u32, imm)),
+                            );
+                        },
+                        64 => {
+                            if (math.cast(i32, @bitCast(i64, imm))) |small| {
+                                try self.asmMemoryImmediate(
+                                    mir_tag,
+                                    Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
+                                        .base = .rbp,
+                                        .disp = -off,
+                                    }),
+                                    Immediate.s(small),
+                                );
+                            } else {
+                                try self.asmMemoryRegister(
+                                    mir_tag,
+                                    Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
+                                        .base = .rbp,
+                                        .disp = -off,
+                                    }),
+                                    registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
+                                );
+                            }
+                        },
+                        else => return self.fail("TODO getBinOpMir implement large immediate ABI", .{}),
+                    }
                 },
                 .memory,
                 .stack_offset,
@@ -3952,18 +4703,33 @@ fn airBreakpoint(self: *Self) !void {
 }
 
 fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
-    const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airRetAddr for x86_64", .{});
+    const result = if (self.liveness.isUnused(inst)) .dead else result: {
+        const dst_mcv = try self.allocRegOrMem(inst, true);
+        try self.setRegOrMem(Type.usize, dst_mcv, .{
+            .stack_offset = -@as(i32, @divExact(self.target.cpu.arch.ptrBitWidth(), 8)),
+        });
+        break :result dst_mcv;
+    };
     return self.finishAir(inst, result, .{ .none, .none, .none });
 }
 
 fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
-    const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFrameAddress for x86_64", .{});
+    const result = if (self.liveness.isUnused(inst)) .dead else result: {
+        const dst_mcv = try self.allocRegOrMem(inst, true);
+        try self.setRegOrMem(Type.usize, dst_mcv, .{ .register = .rbp });
+        break :result dst_mcv;
+    };
     return self.finishAir(inst, result, .{ .none, .none, .none });
 }
 
-fn airFence(self: *Self) !void {
-    return self.fail("TODO implement fence() for {}", .{self.target.cpu.arch});
-    //return self.finishAirBookkeeping();
+fn airFence(self: *Self, inst: Air.Inst.Index) !void {
+    const order = self.air.instructions.items(.data)[inst].fence;
+    switch (order) {
+        .Unordered, .Monotonic => unreachable,
+        .Acquire, .Release, .AcqRel => {},
+        .SeqCst => try self.asmOpOnly(.mfence),
+    }
+    return self.finishAirBookkeeping();
 }
 
 fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
@@ -3984,11 +4750,40 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
     defer info.deinit(self);
 
     try self.spillEflagsIfOccupied();
+    try self.spillRegisters(abi.getCallerPreservedRegs(self.target.*));
 
-    for (abi.getCallerPreservedRegs(self.target.*)) |reg| {
-        try self.register_manager.getReg(reg, null);
+    // set stack arguments first because this can clobber registers
+    // also clobber spill arguments as we go
+    if (info.return_value == .stack_offset) {
+        try self.spillRegisters(&.{abi.getCAbiIntParamRegs(self.target.*)[0]});
+    }
+    for (args, info.args) |arg, mc_arg| {
+        const arg_ty = self.air.typeOf(arg);
+        const arg_mcv = try self.resolveInst(arg);
+        // Here we do not use setRegOrMem even though the logic is similar, because
+        // the function call will move the stack pointer, so the offsets are different.
+        switch (mc_arg) {
+            .none => {},
+            .register => |reg| try self.spillRegisters(&.{reg}),
+            .stack_offset => |off| {
+                // TODO rewrite using `genSetStack`
+                try self.genSetStackArg(arg_ty, off, arg_mcv);
+            },
+            .ptr_stack_offset => {
+                return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
+            },
+            .undef => unreachable,
+            .immediate => unreachable,
+            .unreach => unreachable,
+            .dead => unreachable,
+            .memory => unreachable,
+            .linker_load => unreachable,
+            .eflags => unreachable,
+            .register_overflow => unreachable,
+        }
     }
 
+    // now we are free to set register arguments
     const ret_reg_lock: ?RegisterLock = blk: {
         if (info.return_value == .stack_offset) {
             const ret_ty = fn_ty.fnReturnType();
@@ -3998,7 +4793,6 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
             log.debug("airCall: return value on stack at offset {}", .{stack_offset});
 
             const ret_reg = abi.getCAbiIntParamRegs(self.target.*)[0];
-            try self.register_manager.getReg(ret_reg, null);
             try self.genSetReg(Type.usize, ret_reg, .{ .ptr_stack_offset = stack_offset });
             const ret_reg_lock = self.register_manager.lockRegAssumeUnused(ret_reg);
 
@@ -4010,25 +4804,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
     };
     defer if (ret_reg_lock) |lock| self.register_manager.unlockReg(lock);
 
-    for (args, info.args) |arg, info_arg| {
-        const mc_arg = info_arg;
+    for (args, info.args) |arg, mc_arg| {
         const arg_ty = self.air.typeOf(arg);
         const arg_mcv = try self.resolveInst(arg);
-        // Here we do not use setRegOrMem even though the logic is similar, because
-        // the function call will move the stack pointer, so the offsets are different.
         switch (mc_arg) {
-            .none => continue,
-            .register => |reg| {
-                try self.register_manager.getReg(reg, null);
-                try self.genSetReg(arg_ty, reg, arg_mcv);
-            },
-            .stack_offset => |off| {
-                // TODO rewrite using `genSetStack`
-                try self.genSetStackArg(arg_ty, off, arg_mcv);
-            },
-            .ptr_stack_offset => {
-                return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
-            },
+            .none, .stack_offset, .ptr_stack_offset => {},
+            .register => |reg| try self.genSetReg(arg_ty, reg, arg_mcv),
             .undef => unreachable,
             .immediate => unreachable,
             .unreach => unreachable,
@@ -4293,7 +5074,11 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
         };
         defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
 
-        try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv);
+        try self.genBinOpMir(switch (ty.tag()) {
+            else => .cmp,
+            .f32 => .ucomiss,
+            .f64 => .ucomisd,
+        }, ty, dst_mcv, src_mcv);
 
         break :result switch (signedness) {
             .signed => MCValue{ .eflags = Condition.fromCompareOperatorSigned(op) },
@@ -4510,25 +5295,113 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAir(inst, .unreach, .{ .none, .none, .none });
 }
 
-fn isNull(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue {
+fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
+    switch (opt_mcv) {
+        .register_overflow => |ro| return .{ .eflags = ro.eflags.negate() },
+        else => {},
+    }
+
     try self.spillEflagsIfOccupied();
     self.eflags_inst = inst;
 
-    const cmp_ty: Type = if (!ty.isPtrLikeOptional()) blk: {
-        var buf: Type.Payload.ElemType = undefined;
-        const payload_ty = ty.optionalChild(&buf);
-        break :blk if (payload_ty.hasRuntimeBitsIgnoreComptime()) Type.bool else ty;
-    } else ty;
+    var pl_buf: Type.Payload.ElemType = undefined;
+    const pl_ty = opt_ty.optionalChild(&pl_buf);
 
-    try self.genBinOpMir(.cmp, cmp_ty, operand, MCValue{ .immediate = 0 });
+    var ptr_buf: Type.SlicePtrFieldTypeBuffer = undefined;
+    const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload())
+        .{ .off = 0, .ty = if (pl_ty.isSlice()) pl_ty.slicePtrFieldType(&ptr_buf) else pl_ty }
+    else
+        .{ .off = @intCast(i32, pl_ty.abiSize(self.target.*)), .ty = Type.bool };
 
-    return MCValue{ .eflags = .e };
+    switch (opt_mcv) {
+        .none,
+        .unreach,
+        .dead,
+        .undef,
+        .immediate,
+        .register_overflow,
+        .ptr_stack_offset,
+        .eflags,
+        => unreachable,
+
+        .register => |opt_reg| {
+            if (some_info.off == 0) {
+                const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
+                const alias_reg = registerAlias(opt_reg, some_abi_size);
+                assert(some_abi_size * 8 == alias_reg.bitSize());
+                try self.asmRegisterRegister(.@"test", alias_reg, alias_reg);
+                return .{ .eflags = .z };
+            }
+            assert(some_info.ty.tag() == .bool);
+            const opt_abi_size = @intCast(u32, opt_ty.abiSize(self.target.*));
+            try self.asmRegisterImmediate(
+                .bt,
+                registerAlias(opt_reg, opt_abi_size),
+                Immediate.u(@intCast(u6, some_info.off * 8)),
+            );
+            return .{ .eflags = .nc };
+        },
+
+        .memory, .linker_load => {
+            const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
+            const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
+            defer self.register_manager.unlockReg(addr_reg_lock);
+
+            try self.loadMemPtrIntoRegister(addr_reg, Type.usize, opt_mcv);
+
+            // To get the actual address of the value we want to modify we have to go through the GOT
+            try self.asmRegisterMemory(
+                .mov,
+                addr_reg,
+                Memory.sib(.qword, .{ .base = addr_reg }),
+            );
+
+            const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
+            try self.asmMemoryImmediate(.cmp, Memory.sib(
+                Memory.PtrSize.fromSize(some_abi_size),
+                .{ .base = addr_reg, .disp = some_info.off },
+            ), Immediate.u(0));
+            return .{ .eflags = .e };
+        },
+
+        .stack_offset => |off| {
+            const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
+            try self.asmMemoryImmediate(.cmp, Memory.sib(
+                Memory.PtrSize.fromSize(some_abi_size),
+                .{ .base = .rbp, .disp = some_info.off - off },
+            ), Immediate.u(0));
+            return .{ .eflags = .e };
+        },
+    }
 }
 
-fn isNonNull(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue {
-    const is_null_res = try self.isNull(inst, ty, operand);
-    assert(is_null_res.eflags == .e);
-    return MCValue{ .eflags = is_null_res.eflags.negate() };
+fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
+    try self.spillEflagsIfOccupied();
+    self.eflags_inst = inst;
+
+    const opt_ty = ptr_ty.childType();
+    var pl_buf: Type.Payload.ElemType = undefined;
+    const pl_ty = opt_ty.optionalChild(&pl_buf);
+
+    var ptr_buf: Type.SlicePtrFieldTypeBuffer = undefined;
+    const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload())
+        .{ .off = 0, .ty = if (pl_ty.isSlice()) pl_ty.slicePtrFieldType(&ptr_buf) else pl_ty }
+    else
+        .{ .off = @intCast(i32, pl_ty.abiSize(self.target.*)), .ty = Type.bool };
+
+    const ptr_reg = switch (ptr_mcv) {
+        .register => |reg| reg,
+        else => try self.copyToTmpRegister(ptr_ty, ptr_mcv),
+    };
+    const ptr_lock = self.register_manager.lockReg(ptr_reg);
+    defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
+
+    const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
+    try self.asmMemoryImmediate(.cmp, Memory.sib(
+        Memory.PtrSize.fromSize(some_abi_size),
+        .{ .base = ptr_reg, .disp = some_info.off },
+    ), Immediate.u(0));
+    return .{ .eflags = .e };
 }
 
 fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !MCValue {
@@ -4550,8 +5423,13 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
             try self.genBinOpMir(.cmp, Type.anyerror, .{ .stack_offset = offset }, .{ .immediate = 0 });
         },
         .register => |reg| {
-            const maybe_lock = self.register_manager.lockReg(reg);
-            defer if (maybe_lock) |lock| self.register_manager.unlockReg(lock);
+            self.register_manager.getRegAssumeFree(.rcx, null);
+            const rcx_lock = if (err_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null;
+            defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);
+
+            const eu_lock = self.register_manager.lockReg(reg);
+            defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
+
             const tmp_reg = try self.copyToTmpRegister(ty, operand);
             if (err_off > 0) {
                 const shift = @intCast(u6, err_off * 8);
@@ -4594,29 +5472,11 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
 
 fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
-
-    if (self.liveness.isUnused(inst)) {
-        return self.finishAir(inst, .dead, .{ un_op, .none, .none });
-    }
-
-    const operand_ptr = try self.resolveInst(un_op);
-    const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
-        .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
-        else => null,
+    const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
+        const operand = try self.resolveInst(un_op);
+        const ty = self.air.typeOf(un_op);
+        break :result try self.isNullPtr(inst, ty, operand);
     };
-    defer if (operand_ptr_lock) |lock| self.register_manager.unlockReg(lock);
-
-    const ptr_ty = self.air.typeOf(un_op);
-    const elem_ty = ptr_ty.childType();
-    const operand = if (elem_ty.isPtrLikeOptional() and self.reuseOperand(inst, un_op, 0, operand_ptr))
-        // The MCValue that holds the pointer can be re-used as the value.
-        operand_ptr
-    else
-        try self.allocTempRegOrMem(elem_ty, true);
-    try self.load(operand, operand_ptr, ptr_ty);
-
-    const result = try self.isNull(inst, elem_ty, operand);
-
     return self.finishAir(inst, result, .{ un_op, .none, .none });
 }
 
@@ -4625,36 +5485,24 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
     const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
         const operand = try self.resolveInst(un_op);
         const ty = self.air.typeOf(un_op);
-        break :result try self.isNonNull(inst, ty, operand);
+        break :result switch (try self.isNull(inst, ty, operand)) {
+            .eflags => |cc| .{ .eflags = cc.negate() },
+            else => unreachable,
+        };
     };
     return self.finishAir(inst, result, .{ un_op, .none, .none });
 }
 
 fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
-
-    if (self.liveness.isUnused(inst)) {
-        return self.finishAir(inst, .dead, .{ un_op, .none, .none });
-    }
-
-    const operand_ptr = try self.resolveInst(un_op);
-    const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
-        .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
-        else => null,
+    const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
+        const operand = try self.resolveInst(un_op);
+        const ty = self.air.typeOf(un_op);
+        break :result switch (try self.isNullPtr(inst, ty, operand)) {
+            .eflags => |cc| .{ .eflags = cc.negate() },
+            else => unreachable,
+        };
     };
-    defer if (operand_ptr_lock) |lock| self.register_manager.unlockReg(lock);
-
-    const ptr_ty = self.air.typeOf(un_op);
-    const elem_ty = ptr_ty.childType();
-    const operand = if (elem_ty.isPtrLikeOptional() and self.reuseOperand(inst, un_op, 0, operand_ptr))
-        // The MCValue that holds the pointer can be re-used as the value.
-        operand_ptr
-    else
-        try self.allocTempRegOrMem(elem_ty, true);
-    try self.load(operand, operand_ptr, ptr_ty);
-
-    const result = try self.isNonNull(inst, ptr_ty.elemType(), operand);
-
     return self.finishAir(inst, result, .{ un_op, .none, .none });
 }
 
@@ -4773,69 +5621,6 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAir(inst, result, .{ .none, .none, .none });
 }
 
-fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u32 {
-    const abi_size = @intCast(u32, ty.abiSize(self.target.*));
-    switch (condition) {
-        .none => unreachable,
-        .undef => unreachable,
-        .dead, .unreach => unreachable,
-        .eflags => unreachable,
-        .register => |cond_reg| {
-            try self.spillEflagsIfOccupied();
-
-            const cond_reg_lock = self.register_manager.lockReg(cond_reg);
-            defer if (cond_reg_lock) |lock| self.register_manager.unlockReg(lock);
-
-            switch (case) {
-                .none => unreachable,
-                .undef => unreachable,
-                .dead, .unreach => unreachable,
-                .immediate => |imm| try self.asmRegisterImmediate(
-                    .xor,
-                    registerAlias(cond_reg, abi_size),
-                    Immediate.u(imm),
-                ),
-                .register => |reg| try self.asmRegisterRegister(
-                    .xor,
-                    registerAlias(cond_reg, abi_size),
-                    registerAlias(reg, abi_size),
-                ),
-                .stack_offset => {
-                    if (abi_size <= 8) {
-                        const reg = try self.copyToTmpRegister(ty, case);
-                        return self.genCondSwitchMir(ty, condition, .{ .register = reg });
-                    }
-
-                    return self.fail("TODO implement switch mir when case is stack offset with abi larger than 8 bytes", .{});
-                },
-                else => {
-                    return self.fail("TODO implement switch mir when case is {}", .{case});
-                },
-            }
-
-            const aliased_reg = registerAlias(cond_reg, abi_size);
-            try self.asmRegisterRegister(.@"test", aliased_reg, aliased_reg);
-            return self.asmJccReloc(undefined, .ne);
-        },
-        .stack_offset => {
-            try self.spillEflagsIfOccupied();
-
-            if (abi_size <= 8) {
-                const reg = try self.copyToTmpRegister(ty, condition);
-                const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
-                defer self.register_manager.unlockReg(reg_lock);
-                return self.genCondSwitchMir(ty, .{ .register = reg }, case);
-            }
-
-            return self.fail("TODO implement switch mir when condition is stack offset with abi larger than 8 bytes", .{});
-        },
-        else => {
-            return self.fail("TODO implemenent switch mir when condition is {}", .{condition});
-        },
-    }
-    return 0; // TODO
-}
-
 fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
     const pl_op = self.air.instructions.items(.data)[inst].pl_op;
     const condition = try self.resolveInst(pl_op.operand);
@@ -4880,8 +5665,10 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
         defer self.gpa.free(relocs);
 
         for (items, relocs) |item, *reloc| {
+            try self.spillEflagsIfOccupied();
             const item_mcv = try self.resolveInst(item);
-            reloc.* = try self.genCondSwitchMir(condition_ty, condition, item_mcv);
+            try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);
+            reloc.* = try self.asmJccReloc(undefined, .ne);
         }
 
         // Capture the state of register and stack allocation state so that we can revert to it.
@@ -4976,14 +5763,8 @@ fn canonicaliseBranches(self: *Self, parent_branch: *Branch, canon_branch: *Bran
             if (target_value == .dead)
                 continue;
             // The instruction is only overridden in the else branch.
-            var i: usize = self.branch_stack.items.len - 1;
-            while (true) {
-                i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
-                if (self.branch_stack.items[i].inst_table.get(target_key)) |mcv| {
-                    assert(mcv != .dead);
-                    break :blk mcv;
-                }
-            }
+            // If integer overflows occurs, the question is: why wasn't the instruction marked dead?
+            break :blk self.getResolvedInstValue(target_key).?;
         };
         log.debug("consolidating target_entry {d} {}=>{}", .{ target_key, target_value, canon_mcv });
         // TODO make sure the destination stack offset / register does not already have something
@@ -5000,16 +5781,7 @@ fn canonicaliseBranches(self: *Self, parent_branch: *Branch, canon_branch: *Bran
         log.debug("canon_value = {}", .{canon_value});
         if (canon_value == .dead)
             continue;
-        const parent_mcv = blk: {
-            var i: usize = self.branch_stack.items.len - 1;
-            while (true) {
-                i -= 1;
-                if (self.branch_stack.items[i].inst_table.get(canon_key)) |mcv| {
-                    assert(mcv != .dead);
-                    break :blk mcv;
-                }
-            }
-        };
+        const parent_mcv = self.getResolvedInstValue(canon_key).?;
         log.debug("consolidating canon_entry {d} {}=>{}", .{ canon_key, parent_mcv, canon_value });
         // TODO make sure the destination stack offset / register does not already have something
         // going on there.
@@ -5237,6 +6009,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
         .dead => unreachable,
         .unreach, .none => return,
         .undef => {
+            if (!self.wantSafety()) return; // The already existing value will do just fine.
             if (abi_size <= 8) {
                 const reg = try self.copyToTmpRegister(ty, mcv);
                 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
@@ -5344,8 +6117,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
         .dead => unreachable,
         .unreach, .none => return, // Nothing to do.
         .undef => {
-            if (!self.wantSafety())
-                return; // The already existing value will do just fine.
+            if (!self.wantSafety()) return; // The already existing value will do just fine.
             // TODO Upgrade this to a memset call when we have that available.
             switch (abi_size) {
                 1, 2, 4 => {
@@ -5567,19 +6339,14 @@ fn genInlineMemcpy(
         null;
     defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock);
 
-    const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);
-    const dst_addr_reg = regs[0];
-    const src_addr_reg = regs[1];
-    const index_reg = regs[2].to64();
-    const count_reg = regs[3].to64();
-    const tmp_reg = regs[4].to8();
+    try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
 
     switch (dst_ptr) {
         .memory, .linker_load => {
-            try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr);
+            try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);
         },
         .ptr_stack_offset, .stack_offset => |off| {
-            try self.asmRegisterMemory(.lea, dst_addr_reg.to64(), Memory.sib(.qword, .{
+            try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{
                 .base = opts.dest_stack_base orelse .rbp,
                 .disp = -off,
             }));
@@ -5587,7 +6354,7 @@ fn genInlineMemcpy(
         .register => |reg| {
             try self.asmRegisterRegister(
                 .mov,
-                registerAlias(dst_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
+                registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
                 reg,
             );
         },
@@ -5598,10 +6365,10 @@ fn genInlineMemcpy(
 
     switch (src_ptr) {
         .memory, .linker_load => {
-            try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr);
+            try self.loadMemPtrIntoRegister(.rsi, Type.usize, src_ptr);
         },
         .ptr_stack_offset, .stack_offset => |off| {
-            try self.asmRegisterMemory(.lea, src_addr_reg.to64(), Memory.sib(.qword, .{
+            try self.asmRegisterMemory(.lea, .rsi, Memory.sib(.qword, .{
                 .base = opts.source_stack_base orelse .rbp,
                 .disp = -off,
             }));
@@ -5609,7 +6376,7 @@ fn genInlineMemcpy(
         .register => |reg| {
             try self.asmRegisterRegister(
                 .mov,
-                registerAlias(src_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
+                registerAlias(.rsi, @intCast(u32, @divExact(reg.bitSize(), 8))),
                 reg,
             );
         },
@@ -5618,37 +6385,12 @@ fn genInlineMemcpy(
         },
     }
 
-    try self.genSetReg(Type.usize, count_reg, len);
-    try self.asmRegisterImmediate(.mov, index_reg, Immediate.u(0));
-    const loop_start = try self.addInst(.{
-        .tag = .cmp,
-        .ops = .ri_u,
-        .data = .{ .ri = .{
-            .r1 = count_reg,
-            .imm = 0,
-        } },
+    try self.genSetReg(Type.usize, .rcx, len);
+    _ = try self.addInst(.{
+        .tag = .movs,
+        .ops = .string,
+        .data = .{ .string = .{ .repeat = .rep, .width = .b } },
     });
-    const loop_reloc = try self.asmJccReloc(undefined, .e);
-    try self.asmRegisterMemory(.mov, tmp_reg.to8(), Memory.sib(.byte, .{
-        .base = src_addr_reg,
-        .scale_index = .{
-            .scale = 1,
-            .index = index_reg,
-        },
-        .disp = 0,
-    }));
-    try self.asmMemoryRegister(.mov, Memory.sib(.byte, .{
-        .base = dst_addr_reg,
-        .scale_index = .{
-            .scale = 1,
-            .index = index_reg,
-        },
-        .disp = 0,
-    }), tmp_reg.to8());
-    try self.asmRegisterImmediate(.add, index_reg, Immediate.u(1));
-    try self.asmRegisterImmediate(.sub, count_reg, Immediate.u(1));
-    _ = try self.asmJmpReloc(loop_start);
-    try self.performReloc(loop_reloc);
 }
 
 fn genInlineMemset(
@@ -5658,28 +6400,20 @@ fn genInlineMemset(
     len: MCValue,
     opts: InlineMemcpyOpts,
 ) InnerError!void {
-    const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) |reg|
-        self.register_manager.lockReg(reg)
-    else
-        null;
-    defer if (ssbase_lock) |reg| self.register_manager.unlockReg(reg);
-
     const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) |reg|
         self.register_manager.lockReg(reg)
     else
         null;
     defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock);
 
-    const regs = try self.register_manager.allocRegs(2, .{ null, null }, gp);
-    const addr_reg = regs[0];
-    const index_reg = regs[1].to64();
+    try self.spillRegisters(&.{ .rdi, .al, .rcx });
 
     switch (dst_ptr) {
         .memory, .linker_load => {
-            try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_ptr);
+            try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);
         },
         .ptr_stack_offset, .stack_offset => |off| {
-            try self.asmRegisterMemory(.lea, addr_reg.to64(), Memory.sib(.qword, .{
+            try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{
                 .base = opts.dest_stack_base orelse .rbp,
                 .disp = -off,
             }));
@@ -5687,48 +6421,22 @@ fn genInlineMemset(
         .register => |reg| {
             try self.asmRegisterRegister(
                 .mov,
-                registerAlias(addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
+                registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
                 reg,
             );
         },
         else => {
-            return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
+            return self.fail("TODO implement memset for setting stack when dest is {}", .{dst_ptr});
         },
     }
 
-    try self.genSetReg(Type.usize, index_reg, len);
-    try self.genBinOpMir(.sub, Type.usize, .{ .register = index_reg }, .{ .immediate = 1 });
-
-    const loop_start = try self.addInst(.{
-        .tag = .cmp,
-        .ops = .ri_s,
-        .data = .{ .ri = .{
-            .r1 = index_reg,
-            .imm = @bitCast(u32, @as(i32, -1)),
-        } },
+    try self.genSetReg(Type.u8, .al, value);
+    try self.genSetReg(Type.usize, .rcx, len);
+    _ = try self.addInst(.{
+        .tag = .stos,
+        .ops = .string,
+        .data = .{ .string = .{ .repeat = .rep, .width = .b } },
     });
-    const loop_reloc = try self.asmJccReloc(undefined, .e);
-
-    switch (value) {
-        .immediate => |x| {
-            if (x > math.maxInt(i32)) {
-                return self.fail("TODO inline memset for value immediate larger than 32bits", .{});
-            }
-            try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{
-                .base = addr_reg,
-                .scale_index = .{
-                    .scale = 1,
-                    .index = index_reg,
-                },
-                .disp = 0,
-            }), Immediate.u(@intCast(u8, x)));
-        },
-        else => return self.fail("TODO inline memset for value of type {}", .{value}),
-    }
-
-    try self.asmRegisterImmediate(.sub, index_reg, Immediate.u(1));
-    _ = try self.asmJmpReloc(loop_start);
-    try self.performReloc(loop_reloc);
 }
 
 fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
@@ -5748,10 +6456,9 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
         },
         .unreach, .none => return, // Nothing to do.
         .undef => {
-            if (!self.wantSafety())
-                return; // The already existing value will do just fine.
+            if (!self.wantSafety()) return; // The already existing value will do just fine.
             // Write the debug undefined value.
-            switch (registerAlias(reg, abi_size).bitSize()) {
+            switch (self.regBitSize(ty)) {
                 8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }),
                 16 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaa }),
                 32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }),
@@ -5762,27 +6469,27 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
         .eflags => |cc| {
             return self.asmSetccRegister(reg.to8(), cc);
         },
-        .immediate => |x| {
-            if (x == 0) {
+        .immediate => |imm| {
+            if (imm == 0) {
                 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
                 // register is the fastest way to zero a register.
-                return self.asmRegisterRegister(.xor, reg.to32(), reg.to32());
+                try self.asmRegisterRegister(.xor, reg.to32(), reg.to32());
+            } else if (abi_size > 4 and math.cast(u32, imm) != null) {
+                // 32-bit moves zero-extend to 64-bit.
+                try self.asmRegisterImmediate(.mov, reg.to32(), Immediate.u(imm));
+            } else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
+                try self.asmRegisterImmediate(
+                    .mov,
+                    registerAlias(reg, abi_size),
+                    Immediate.s(@intCast(i32, @bitCast(i64, imm))),
+                );
+            } else {
+                try self.asmRegisterImmediate(
+                    .mov,
+                    registerAlias(reg, abi_size),
+                    Immediate.u(imm),
+                );
             }
-            if (ty.isSignedInt()) {
-                const signed_x = @bitCast(i64, x);
-                if (math.minInt(i32) <= signed_x and signed_x <= math.maxInt(i32)) {
-                    return self.asmRegisterImmediate(
-                        .mov,
-                        registerAlias(reg, abi_size),
-                        Immediate.s(@intCast(i32, signed_x)),
-                    );
-                }
-            }
-            return self.asmRegisterImmediate(
-                .mov,
-                registerAlias(reg, abi_size),
-                Immediate.u(x),
-            );
         },
         .register => |src_reg| {
             // If the registers are the same, nothing to do.
@@ -5843,10 +6550,11 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
                             .f64 => .qword,
                             else => unreachable,
                         };
-                        return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{
-                            .base = base_reg.to64(),
-                            .disp = 0,
-                        }));
+                        return self.asmRegisterMemory(
+                            tag,
+                            reg.to128(),
+                            Memory.sib(ptr_size, .{ .base = base_reg.to64() }),
+                        );
                     }
 
                     return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
@@ -5856,7 +6564,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
                     try self.asmRegisterMemory(
                         .mov,
                         registerAlias(reg, abi_size),
-                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64(), .disp = 0 }),
+                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
                     );
                 },
             }
@@ -5877,10 +6585,11 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
                         .f64 => .qword,
                         else => unreachable,
                     };
-                    return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{
-                        .base = base_reg.to64(),
-                        .disp = 0,
-                    }));
+                    return self.asmRegisterMemory(
+                        tag,
+                        reg.to128(),
+                        Memory.sib(ptr_size, .{ .base = base_reg.to64() }),
+                    );
                 }
 
                 return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
@@ -5916,7 +6625,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
                         try self.asmRegisterMemory(
                             .mov,
                             registerAlias(reg, abi_size),
-                            Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64(), .disp = 0 }),
+                            Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
                         );
                     }
                 }
@@ -6088,26 +6797,178 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
 
 fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
     const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
-    const extra = self.air.extraData(Air.Block, ty_pl.payload);
-    _ = extra;
-    return self.fail("TODO implement x86 airCmpxchg", .{});
-    // return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
+    const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
+
+    const ptr_ty = self.air.typeOf(extra.ptr);
+    const ptr_mcv = try self.resolveInst(extra.ptr);
+    const val_ty = self.air.typeOf(extra.expected_value);
+
+    const exp_mcv = try self.resolveInst(extra.expected_value);
+    try self.genSetReg(val_ty, .rax, exp_mcv);
+    const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
+    defer self.register_manager.unlockReg(rax_lock);
+
+    const new_mcv = try self.resolveInst(extra.new_value);
+    const new_reg = try self.copyToTmpRegister(val_ty, new_mcv);
+    const new_lock = self.register_manager.lockRegAssumeUnused(new_reg);
+    defer self.register_manager.unlockReg(new_lock);
+
+    const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
+    const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
+    const ptr_mem: Memory = switch (ptr_mcv) {
+        .register => |reg| Memory.sib(ptr_size, .{ .base = reg }),
+        .ptr_stack_offset => |off| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),
+        else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),
+    };
+    const mem_lock = if (ptr_mem.base()) |reg| self.register_manager.lockReg(reg) else null;
+    defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
+
+    try self.spillEflagsIfOccupied();
+    _ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
+        .r1 = new_reg,
+        .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
+    } } });
+
+    const result: MCValue = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        self.eflags_inst = inst;
+        break :result .{ .register_overflow = .{ .reg = .rax, .eflags = .ne } };
+    };
+    return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
+}
+
+fn atomicOp(
+    self: *Self,
+    dst_reg: Register,
+    ptr_mcv: MCValue,
+    val_mcv: MCValue,
+    ptr_ty: Type,
+    val_ty: Type,
+    unused: bool,
+    op: ?std.builtin.AtomicRmwOp,
+    order: std.builtin.AtomicOrder,
+) InnerError!void {
+    const dst_lock = self.register_manager.lockReg(dst_reg);
+    defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+
+    const ptr_lock = switch (ptr_mcv) {
+        .register => |reg| self.register_manager.lockReg(reg),
+        else => null,
+    };
+    defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
+
+    const val_lock = switch (val_mcv) {
+        .register => |reg| self.register_manager.lockReg(reg),
+        else => null,
+    };
+    defer if (val_lock) |lock| self.register_manager.unlockReg(lock);
+
+    const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
+    const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
+    const ptr_mem: Memory = switch (ptr_mcv) {
+        .register => |reg| Memory.sib(ptr_size, .{ .base = reg }),
+        .ptr_stack_offset => |off| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),
+        else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),
+    };
+    const mem_lock = if (ptr_mem.base()) |reg| self.register_manager.lockReg(reg) else null;
+    defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
+
+    try self.genSetReg(val_ty, dst_reg, val_mcv);
+
+    const need_loop = val_ty.isRuntimeFloat() or if (op) |rmw| switch (rmw) {
+        .Xchg, .Add, .Sub => false,
+        .And, .Or, .Xor => !unused,
+        .Nand, .Max, .Min => true,
+    } else false;
+    if (!need_loop) {
+        const tag: Mir.Inst.Tag = if (op) |rmw| switch (rmw) {
+            .Xchg => if (unused) .mov else .xchg,
+            .Add => if (unused) .add else .xadd,
+            .Sub => if (unused) .sub else .xadd,
+            .And => .@"and",
+            .Or => .@"or",
+            .Xor => .xor,
+            else => unreachable,
+        } else switch (order) {
+            .Unordered, .Monotonic, .Release, .AcqRel => .mov,
+            .Acquire => unreachable,
+            .SeqCst => .xchg,
+        };
+        if (op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
+            try self.genUnOpMir(.neg, val_ty, .{ .register = dst_reg });
+        }
+        _ = try self.addInst(.{ .tag = tag, .ops = switch (tag) {
+            .mov, .xchg => .mr_sib,
+            .xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
+            else => unreachable,
+        }, .data = .{ .rx = .{
+            .r1 = registerAlias(dst_reg, val_abi_size),
+            .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
+        } } });
+        return;
+    }
+
+    return self.fail("TODO implement x86 atomic loop", .{});
 }
 
 fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
-    _ = inst;
-    return self.fail("TODO implement x86 airAtomicRaw", .{});
+    const pl_op = self.air.instructions.items(.data)[inst].pl_op;
+    const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
+
+    const dst_reg = try self.register_manager.allocReg(inst, gp);
+
+    const ptr_ty = self.air.typeOf(pl_op.operand);
+    const ptr_mcv = try self.resolveInst(pl_op.operand);
+
+    const val_ty = self.air.typeOf(extra.operand);
+    const val_mcv = try self.resolveInst(extra.operand);
+
+    const unused = self.liveness.isUnused(inst);
+    try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering());
+    const result: MCValue = if (unused) .dead else .{ .register = dst_reg };
+    return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
 }
 
 fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
-    _ = inst;
-    return self.fail("TODO implement airAtomicLoad for {}", .{self.target.cpu.arch});
+    const atomic_load = self.air.instructions.items(.data)[inst].atomic_load;
+
+    const result: MCValue = result: {
+        if (self.liveness.isUnused(inst)) break :result .dead;
+
+        const ptr_ty = self.air.typeOf(atomic_load.ptr);
+        const ptr_mcv = try self.resolveInst(atomic_load.ptr);
+        const ptr_lock = switch (ptr_mcv) {
+            .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
+            else => null,
+        };
+        defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
+
+        const dst_mcv =
+            if (self.reuseOperand(inst, atomic_load.ptr, 0, ptr_mcv))
+            ptr_mcv
+        else
+            try self.allocRegOrMem(inst, true);
+
+        try self.load(dst_mcv, ptr_mcv, ptr_ty);
+        break :result dst_mcv;
+    };
+    return self.finishAir(inst, result, .{ atomic_load.ptr, .none, .none });
 }
 
 fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
-    _ = inst;
-    _ = order;
-    return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
+    const bin_op = self.air.instructions.items(.data)[inst].bin_op;
+
+    const dst_reg = try self.register_manager.allocReg(null, gp);
+
+    const ptr_ty = self.air.typeOf(bin_op.lhs);
+    const ptr_mcv = try self.resolveInst(bin_op.lhs);
+
+    const val_ty = self.air.typeOf(bin_op.rhs);
+    const val_mcv = try self.resolveInst(bin_op.rhs);
+
+    try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, true, null, order);
+    return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
 }
 
 fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
@@ -6137,7 +6998,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
 
     try self.genInlineMemset(dst_ptr, src_val, len, .{});
 
-    return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none });
+    return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs });
 }
 
 fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
@@ -6172,10 +7033,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
             .linker_load, .memory => {
                 const reg = try self.register_manager.allocReg(null, gp);
                 try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
-                try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{
-                    .base = reg,
-                    .disp = 0,
-                }));
+                try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{ .base = reg }));
                 break :blk MCValue{ .register = reg };
             },
             else => break :blk src_ptr,
@@ -6189,7 +7047,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
 
     try self.genInlineMemcpy(dst_ptr, src, len, .{});
 
-    return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none });
+    return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs });
 }
 
 fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
@@ -6629,7 +7487,10 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
 /// Truncates the value in the register in place.
 /// Clobbers any remaining bits.
 fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
-    const int_info = ty.intInfo(self.target.*);
+    const int_info = if (ty.isAbiInt()) ty.intInfo(self.target.*) else std.builtin.Type.Int{
+        .signedness = .unsigned,
+        .bits = @intCast(u16, ty.bitSize(self.target.*)),
+    };
     const max_reg_bit_width = Register.rax.bitSize();
     switch (int_info.signedness) {
         .signed => {
@@ -6650,6 +7511,27 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
     }
 }
 
+fn regBitSize(self: *Self, ty: Type) u64 {
+    return switch (ty.zigTypeTag()) {
+        else => switch (ty.abiSize(self.target.*)) {
+            1 => 8,
+            2 => 16,
+            3...4 => 32,
+            5...8 => 64,
+            else => unreachable,
+        },
+        .Float => switch (ty.abiSize(self.target.*)) {
+            1...16 => 128,
+            17...32 => 256,
+            else => unreachable,
+        },
+    };
+}
+
+fn regExtraBits(self: *Self, ty: Type) u64 {
+    return self.regBitSize(ty) - ty.bitSize(self.target.*);
+}
+
 fn intrinsicsAllowed(target: Target, ty: Type) bool {
     return switch (ty.tag()) {
         .f32,

src/arch/x86_64/Emit.zig

@@ -73,6 +73,13 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
             .adc,
             .add,
             .@"and",
+            .bsf,
+            .bsr,
+            .bswap,
+            .bt,
+            .btc,
+            .btr,
+            .bts,
             .call,
             .cbw,
             .cwde,
@@ -81,6 +88,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
             .cdq,
             .cqo,
             .cmp,
+            .cmpxchg,
             .div,
             .fisttp,
             .fld,
@@ -89,35 +97,71 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
             .int3,
             .jmp,
             .lea,
+            .lfence,
+            .lzcnt,
+            .mfence,
             .mov,
+            .movbe,
             .movzx,
             .mul,
+            .neg,
             .nop,
+            .not,
             .@"or",
             .pop,
+            .popcnt,
             .push,
+            .rcl,
+            .rcr,
             .ret,
+            .rol,
+            .ror,
             .sal,
             .sar,
             .sbb,
+            .sfence,
             .shl,
             .shr,
             .sub,
             .syscall,
             .@"test",
+            .tzcnt,
             .ud2,
+            .xadd,
+            .xchg,
             .xor,
 
             .addss,
             .cmpss,
+            .divss,
+            .maxss,
+            .minss,
             .movss,
+            .mulss,
+            .roundss,
+            .subss,
             .ucomiss,
             .addsd,
             .cmpsd,
+            .divsd,
+            .maxsd,
+            .minsd,
             .movsd,
+            .mulsd,
+            .roundsd,
+            .subsd,
             .ucomisd,
             => try emit.mirEncodeGeneric(tag, inst),
 
+            .cmps,
+            .lods,
+            .movs,
+            .scas,
+            .stos,
+            => try emit.mirString(tag, inst),
+
+            .cmpxchgb => try emit.mirCmpxchgBytes(inst),
+
             .jmp_reloc => try emit.mirJmpReloc(inst),
 
             .call_extern => try emit.mirCallExtern(inst),
@@ -171,18 +215,8 @@ fn fixupRelocs(emit: *Emit) InnerError!void {
     }
 }
 
-fn encode(emit: *Emit, mnemonic: Instruction.Mnemonic, ops: struct {
-    op1: Instruction.Operand = .none,
-    op2: Instruction.Operand = .none,
-    op3: Instruction.Operand = .none,
-    op4: Instruction.Operand = .none,
-}) InnerError!void {
-    const inst = try Instruction.new(mnemonic, .{
-        .op1 = ops.op1,
-        .op2 = ops.op2,
-        .op3 = ops.op3,
-        .op4 = ops.op4,
-    });
+fn encode(emit: *Emit, mnemonic: Instruction.Mnemonic, ops: Instruction.Init) InnerError!void {
+    const inst = try Instruction.new(mnemonic, ops);
     return inst.encode(emit.code.writer());
 }
 
@@ -194,6 +228,20 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
     const ops = emit.mir.instructions.items(.ops)[inst];
     const data = emit.mir.instructions.items(.data)[inst];
 
+    const prefix: Instruction.Prefix = switch (ops) {
+        .lock_m_sib,
+        .lock_m_rip,
+        .lock_mi_u_sib,
+        .lock_mi_u_rip,
+        .lock_mi_s_sib,
+        .lock_mi_s_rip,
+        .lock_mr_sib,
+        .lock_mr_rip,
+        .lock_moffs_rax,
+        => .lock,
+        else => .none,
+    };
+
     var op1: Instruction.Operand = .none;
     var op2: Instruction.Operand = .none;
     var op3: Instruction.Operand = .none;
@@ -232,35 +280,35 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
             op2 = .{ .reg = data.rri.r2 };
             op3 = .{ .imm = imm };
         },
-        .m_sib => {
+        .m_sib, .lock_m_sib => {
             const msib = emit.mir.extraData(Mir.MemorySib, data.payload).data;
             op1 = .{ .mem = Mir.MemorySib.decode(msib) };
         },
-        .m_rip => {
+        .m_rip, .lock_m_rip => {
             const mrip = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
             op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
         },
-        .mi_s_sib, .mi_u_sib => {
+        .mi_s_sib, .mi_u_sib, .lock_mi_s_sib, .lock_mi_u_sib => {
             const msib = emit.mir.extraData(Mir.MemorySib, data.xi.payload).data;
             const imm = switch (ops) {
-                .mi_s_sib => Immediate.s(@bitCast(i32, data.xi.imm)),
-                .mi_u_sib => Immediate.u(data.xi.imm),
+                .mi_s_sib, .lock_mi_s_sib => Immediate.s(@bitCast(i32, data.xi.imm)),
+                .mi_u_sib, .lock_mi_u_sib => Immediate.u(data.xi.imm),
                 else => unreachable,
             };
             op1 = .{ .mem = Mir.MemorySib.decode(msib) };
             op2 = .{ .imm = imm };
         },
-        .mi_u_rip, .mi_s_rip => {
+        .mi_u_rip, .mi_s_rip, .lock_mi_u_rip, .lock_mi_s_rip => {
             const mrip = emit.mir.extraData(Mir.MemoryRip, data.xi.payload).data;
             const imm = switch (ops) {
-                .mi_s_rip => Immediate.s(@bitCast(i32, data.xi.imm)),
-                .mi_u_rip => Immediate.u(data.xi.imm),
+                .mi_s_rip, .lock_mi_s_rip => Immediate.s(@bitCast(i32, data.xi.imm)),
+                .mi_u_rip, .lock_mi_u_rip => Immediate.u(data.xi.imm),
                 else => unreachable,
             };
             op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
             op2 = .{ .imm = imm };
         },
-        .rm_sib, .mr_sib => {
+        .rm_sib, .mr_sib, .lock_mr_sib => {
             const msib = emit.mir.extraData(Mir.MemorySib, data.rx.payload).data;
             const op_r = .{ .reg = data.rx.r1 };
             const op_m = .{ .mem = Mir.MemorySib.decode(msib) };
@@ -269,23 +317,23 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
                     op1 = op_r;
                     op2 = op_m;
                 },
-                .mr_sib => {
+                .mr_sib, .lock_mr_sib => {
                     op1 = op_m;
                     op2 = op_r;
                 },
                 else => unreachable,
             }
         },
-        .rm_rip, .mr_rip => {
+        .rm_rip, .mr_rip, .lock_mr_rip => {
             const mrip = emit.mir.extraData(Mir.MemoryRip, data.rx.payload).data;
             const op_r = .{ .reg = data.rx.r1 };
             const op_m = .{ .mem = Mir.MemoryRip.decode(mrip) };
             switch (ops) {
-                .rm_sib => {
+                .rm_rip => {
                     op1 = op_r;
                     op2 = op_m;
                 },
-                .mr_sib => {
+                .mr_rip, .lock_mr_rip => {
                     op1 = op_m;
                     op2 = op_r;
                 },
@@ -299,6 +347,7 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
     }
 
     return emit.encode(mnemonic, .{
+        .prefix = prefix,
         .op1 = op1,
         .op2 = op2,
         .op3 = op3,
@@ -306,6 +355,61 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
     });
 }
 
+fn mirString(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
+    const ops = emit.mir.instructions.items(.ops)[inst];
+    switch (ops) {
+        .string => {
+            const data = emit.mir.instructions.items(.data)[inst].string;
+            const mnemonic = switch (tag) {
+                inline .cmps, .lods, .movs, .scas, .stos => |comptime_tag| switch (data.width) {
+                    inline else => |comptime_width| @field(
+                        Instruction.Mnemonic,
+                        @tagName(comptime_tag) ++ @tagName(comptime_width),
+                    ),
+                },
+                else => unreachable,
+            };
+            return emit.encode(mnemonic, .{ .prefix = switch (data.repeat) {
+                inline else => |comptime_repeat| @field(Instruction.Prefix, @tagName(comptime_repeat)),
+            } });
+        },
+        else => unreachable,
+    }
+}
+
+fn mirCmpxchgBytes(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
+    const ops = emit.mir.instructions.items(.ops)[inst];
+    const data = emit.mir.instructions.items(.data)[inst];
+
+    var op1: Instruction.Operand = .none;
+    switch (ops) {
+        .m_sib, .lock_m_sib => {
+            const sib = emit.mir.extraData(Mir.MemorySib, data.payload).data;
+            op1 = .{ .mem = Mir.MemorySib.decode(sib) };
+        },
+        .m_rip, .lock_m_rip => {
+            const rip = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
+            op1 = .{ .mem = Mir.MemoryRip.decode(rip) };
+        },
+        else => unreachable,
+    }
+
+    const mnemonic: Instruction.Mnemonic = switch (op1.mem.bitSize()) {
+        64 => .cmpxchg8b,
+        128 => .cmpxchg16b,
+        else => unreachable,
+    };
+
+    return emit.encode(mnemonic, .{
+        .prefix = switch (ops) {
+            .m_sib, .m_rip => .none,
+            .lock_m_sib, .lock_m_rip => .lock,
+            else => unreachable,
+        },
+        .op1 = op1,
+    });
+}
+
 fn mirMovMoffs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const ops = emit.mir.instructions.items(.ops)[inst];
     const payload = emit.mir.instructions.items(.data)[inst].payload;
@@ -319,8 +423,13 @@ fn mirMovMoffs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
                 .op2 = .{ .mem = Memory.moffs(seg, offset) },
             });
         },
-        .moffs_rax => {
+        .moffs_rax, .lock_moffs_rax => {
             try emit.encode(.mov, .{
+                .prefix = switch (ops) {
+                    .moffs_rax => .none,
+                    .lock_moffs_rax => .lock,
+                    else => unreachable,
+                },
                 .op1 = .{ .mem = Memory.moffs(seg, offset) },
                 .op2 = .{ .reg = .rax },
             });
@@ -365,37 +474,70 @@ fn mirMovsx(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
 }
 
 fn mnemonicFromConditionCode(comptime basename: []const u8, cc: bits.Condition) Instruction.Mnemonic {
-    inline for (@typeInfo(bits.Condition).Enum.fields) |field| {
-        if (mem.eql(u8, field.name, @tagName(cc)))
-            return @field(Instruction.Mnemonic, basename ++ field.name);
-    } else unreachable;
+    return switch (cc) {
+        inline else => |comptime_cc| @field(Instruction.Mnemonic, basename ++ @tagName(comptime_cc)),
+    };
 }
 
 fn mirCmovcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const ops = emit.mir.instructions.items(.ops)[inst];
     switch (ops) {
-        .rr_c => {
-            const data = emit.mir.instructions.items(.data)[inst].rr_c;
+        .rr_cc => {
+            const data = emit.mir.instructions.items(.data)[inst].rr_cc;
             const mnemonic = mnemonicFromConditionCode("cmov", data.cc);
             return emit.encode(mnemonic, .{
                 .op1 = .{ .reg = data.r1 },
                 .op2 = .{ .reg = data.r2 },
             });
         },
-        else => unreachable, // TODO
+        .rm_sib_cc => {
+            const data = emit.mir.instructions.items(.data)[inst].rx_cc;
+            const extra = emit.mir.extraData(Mir.MemorySib, data.payload).data;
+            const mnemonic = mnemonicFromConditionCode("cmov", data.cc);
+            return emit.encode(mnemonic, .{
+                .op1 = .{ .reg = data.r1 },
+                .op2 = .{ .mem = Mir.MemorySib.decode(extra) },
+            });
+        },
+        .rm_rip_cc => {
+            const data = emit.mir.instructions.items(.data)[inst].rx_cc;
+            const extra = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
+            const mnemonic = mnemonicFromConditionCode("cmov", data.cc);
+            return emit.encode(mnemonic, .{
+                .op1 = .{ .reg = data.r1 },
+                .op2 = .{ .mem = Mir.MemoryRip.decode(extra) },
+            });
+        },
+        else => unreachable,
     }
 }
 
 fn mirSetcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
     const ops = emit.mir.instructions.items(.ops)[inst];
     switch (ops) {
-        .r_c => {
-            const data = emit.mir.instructions.items(.data)[inst].r_c;
+        .r_cc => {
+            const data = emit.mir.instructions.items(.data)[inst].r_cc;
             const mnemonic = mnemonicFromConditionCode("set", data.cc);
             return emit.encode(mnemonic, .{
                 .op1 = .{ .reg = data.r1 },
             });
         },
+        .m_sib_cc => {
+            const data = emit.mir.instructions.items(.data)[inst].x_cc;
+            const extra = emit.mir.extraData(Mir.MemorySib, data.payload).data;
+            const mnemonic = mnemonicFromConditionCode("set", data.cc);
+            return emit.encode(mnemonic, .{
+                .op1 = .{ .mem = Mir.MemorySib.decode(extra) },
+            });
+        },
+        .m_rip_cc => {
+            const data = emit.mir.instructions.items(.data)[inst].x_cc;
+            const extra = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
+            const mnemonic = mnemonicFromConditionCode("set", data.cc);
+            return emit.encode(mnemonic, .{
+                .op1 = .{ .mem = Mir.MemoryRip.decode(extra) },
+            });
+        },
         else => unreachable, // TODO
     }
 }

src/arch/x86_64/Encoding.zig

@@ -19,17 +19,12 @@ op1: Op,
 op2: Op,
 op3: Op,
 op4: Op,
-opc_len: u2,
-opc: [3]u8,
+opc_len: u3,
+opc: [7]u8,
 modrm_ext: u3,
 mode: Mode,
 
-pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
-    op1: Instruction.Operand,
-    op2: Instruction.Operand,
-    op3: Instruction.Operand,
-    op4: Instruction.Operand,
-}) !?Encoding {
+pub fn findByMnemonic(mnemonic: Mnemonic, args: Instruction.Init) !?Encoding {
     const input_op1 = Op.fromOperand(args.op1);
     const input_op2 = Op.fromOperand(args.op2);
     const input_op3 = Op.fromOperand(args.op3);
@@ -69,18 +64,19 @@ pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
     var candidates: [10]Encoding = undefined;
     var count: usize = 0;
     for (table) |entry| {
-        const enc = Encoding{
+        var enc = Encoding{
             .mnemonic = entry[0],
             .op_en = entry[1],
             .op1 = entry[2],
             .op2 = entry[3],
             .op3 = entry[4],
             .op4 = entry[5],
-            .opc_len = entry[6],
-            .opc = .{ entry[7], entry[8], entry[9] },
-            .modrm_ext = entry[10],
-            .mode = entry[11],
+            .opc_len = @intCast(u3, entry[6].len),
+            .opc = undefined,
+            .modrm_ext = entry[7],
+            .mode = entry[8],
         };
+        std.mem.copy(u8, &enc.opc, entry[6]);
         if (enc.mnemonic == mnemonic and
             input_op1.isSubset(enc.op1, enc.mode) and
             input_op2.isSubset(enc.op2, enc.mode) and
@@ -108,17 +104,13 @@ pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
     if (count == 1) return candidates[0];
 
     const EncodingLength = struct {
-        fn estimate(encoding: Encoding, params: struct {
-            op1: Instruction.Operand,
-            op2: Instruction.Operand,
-            op3: Instruction.Operand,
-            op4: Instruction.Operand,
-        }) usize {
+        fn estimate(encoding: Encoding, params: Instruction.Init) usize {
             var inst = Instruction{
                 .op1 = params.op1,
                 .op2 = params.op2,
                 .op3 = params.op3,
                 .op4 = params.op4,
+                .prefix = params.prefix,
                 .encoding = encoding,
             };
             var cwriter = std.io.countingWriter(std.io.null_writer);
@@ -139,12 +131,7 @@ pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
             else => {},
         }
 
-        const len = EncodingLength.estimate(candidate, .{
-            .op1 = args.op1,
-            .op2 = args.op2,
-            .op3 = args.op3,
-            .op4 = args.op4,
-        });
+        const len = EncodingLength.estimate(candidate, args);
         const current = shortest_encoding orelse {
             shortest_encoding = .{ .index = i, .len = len };
             continue;
@@ -184,7 +171,7 @@ pub fn findByOpcode(opc: []const u8, prefixes: struct {
         if (match) {
             if (prefixes.rex.w) {
                 switch (enc.mode) {
-                    .fpu, .sse, .sse2, .none => {},
+                    .fpu, .sse, .sse2, .sse4_1, .none => {},
                     .long, .rex => return enc,
                 }
             } else if (prefixes.rex.present and !prefixes.rex.isSet()) {
@@ -227,7 +214,11 @@ pub fn modRmExt(encoding: Encoding) u3 {
 }
 
 pub fn operandBitSize(encoding: Encoding) u64 {
-    if (encoding.mode == .long) return 64;
+    switch (encoding.mode) {
+        .short => return 16,
+        .long => return 64,
+        else => {},
+    }
     const bit_size: u64 = switch (encoding.op_en) {
         .np => switch (encoding.op1) {
             .o16 => 16,
@@ -316,39 +307,63 @@ pub const Mnemonic = enum {
     // zig fmt: off
     // General-purpose
     adc, add, @"and",
-    call, cbw, cwde, cdqe, cwd, cdq, cqo, cmp,
+    bsf, bsr, bswap, bt, btc, btr, bts,
+    call, cbw, cdq, cdqe,
     cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,
     cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,
     cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz,
+    cmp,
+    cmps, cmpsb, cmpsd, cmpsq, cmpsw,
+    cmpxchg, cmpxchg8b, cmpxchg16b,
+    cqo, cwd, cwde,
     div,
     fisttp, fld,
     idiv, imul, int3,
     ja, jae, jb, jbe, jc, jrcxz, je, jg, jge, jl, jle, jna, jnae, jnb, jnbe,
     jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, js, jz,
     jmp, 
-    lea,
-    mov, movsx, movsxd, movzx, mul,
-    nop,
+    lea, lfence,
+    lods, lodsb, lodsd, lodsq, lodsw,
+    lzcnt,
+    mfence, mov, movbe,
+    movs, movsb, movsd, movsq, movsw,
+    movsx, movsxd, movzx, mul,
+    neg, nop, not,
     @"or",
-    pop, push,
-    ret,
-    sal, sar, sbb, shl, shr, sub, syscall,
+    pop, popcnt, push,
+    rcl, rcr, ret, rol, ror,
+    sal, sar, sbb,
+    scas, scasb, scasd, scasq, scasw,
+    shl, shr, sub, syscall,
     seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
     setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
     setnz, seto, setp, setpe, setpo, sets, setz,
-    @"test",
+    sfence,
+    stos, stosb, stosd, stosq, stosw,
+    @"test", tzcnt,
     ud2,
-    xor,
+    xadd, xchg, xor,
     // SSE
     addss,
     cmpss,
+    divss,
+    maxss, minss,
     movss,
+    mulss,
+    subss,
     ucomiss,
     // SSE2
     addsd,
-    cmpsd,
-    movq, movsd,
+    //cmpsd,
+    divsd,
+    maxsd, minsd,
+    movq, //movsd,
+    mulsd,
+    subsd,
     ucomisd,
+    // SSE4.1
+    roundss,
+    roundsd,
     // zig fmt: on
 };
 
@@ -374,7 +389,7 @@ pub const Op = enum {
     cl,
     r8, r16, r32, r64,
     rm8, rm16, rm32, rm64,
-    m8, m16, m32, m64, m80,
+    m8, m16, m32, m64, m80, m128,
     rel8, rel16, rel32,
     m,
     moffs,
@@ -423,6 +438,7 @@ pub const Op = enum {
                         32 => .m32,
                         64 => .m64,
                         80 => .m80,
+                        128 => .m128,
                         else => unreachable,
                     };
                 },
@@ -460,7 +476,7 @@ pub const Op = enum {
             .imm32, .imm32s, .eax, .r32, .m32, .rm32, .rel32, .xmm_m32 => 32,
             .imm64, .rax, .r64, .m64, .rm64, .xmm_m64 => 64,
             .m80 => 80,
-            .xmm => 128,
+            .m128, .xmm => 128,
         };
     }
 
@@ -507,7 +523,7 @@ pub const Op = enum {
         // zig fmt: off
         return switch (op) {
             .rm8, .rm16, .rm32, .rm64,
-            .m8, .m16, .m32, .m64, .m80,
+            .m8, .m16, .m32, .m64, .m80, .m128,
             .m,
             .xmm_m32, .xmm_m64,
             =>  true,
@@ -542,7 +558,7 @@ pub const Op = enum {
             else => {
                 if (op.isRegister() and target.isRegister()) {
                     switch (mode) {
-                        .sse, .sse2 => return op.isFloatingPointRegister() and target.isFloatingPointRegister(),
+                        .sse, .sse2, .sse4_1 => return op.isFloatingPointRegister() and target.isFloatingPointRegister(),
                         else => switch (target) {
                             .cl, .al, .ax, .eax, .rax => return op == target,
                             else => return op.bitSize() == target.bitSize(),
@@ -579,9 +595,11 @@ pub const Op = enum {
 
 pub const Mode = enum {
     none,
+    short,
     fpu,
     rex,
     long,
     sse,
     sse2,
+    sse4_1,
 };

src/arch/x86_64/Mir.zig

@@ -38,6 +38,20 @@ pub const Inst = struct {
         add,
         /// Logical and
         @"and",
+        /// Bit scan forward
+        bsf,
+        /// Bit scan reverse
+        bsr,
+        /// Byte swap
+        bswap,
+        /// Bit test
+        bt,
+        /// Bit test and complement
+        btc,
+        /// Bit test and reset
+        btr,
+        /// Bit test and set
+        bts,
         /// Call
         call,
         /// Convert byte to word
@@ -54,6 +68,10 @@ pub const Inst = struct {
         cqo,
         /// Logical compare
         cmp,
+        /// Compare and exchange
+        cmpxchg,
+        /// Compare and exchange bytes
+        cmpxchgb,
         /// Unsigned division
         div,
         /// Store integer with truncation
@@ -70,30 +88,54 @@ pub const Inst = struct {
         jmp,
         /// Load effective address
         lea,
+        /// Load fence
+        lfence,
+        /// Count the number of leading zero bits
+        lzcnt,
+        /// Memory fence
+        mfence,
         /// Move
         mov,
+        /// Move data after swapping bytes
+        movbe,
         /// Move with sign extension
         movsx,
         /// Move with zero extension
         movzx,
         /// Multiply
         mul,
+        /// Two's complement negation
+        neg,
         /// No-op
         nop,
+        /// One's complement negation
+        not,
         /// Logical or
         @"or",
         /// Pop
         pop,
+        /// Return the count of number of bits set to 1
+        popcnt,
         /// Push
         push,
+        /// Rotate left through carry
+        rcl,
+        /// Rotate right through carry
+        rcr,
         /// Return
         ret,
+        /// Rotate left
+        rol,
+        /// Rotate right
+        ror,
         /// Arithmetic shift left
         sal,
         /// Arithmetic shift right
         sar,
         /// Integer subtraction with borrow
         sbb,
+        /// Store fence
+        sfence,
         /// Logical shift left
         shl,
         /// Logical shift right
@@ -104,28 +146,69 @@ pub const Inst = struct {
         syscall,
         /// Test condition
         @"test",
+        /// Count the number of trailing zero bits
+        tzcnt,
         /// Undefined instruction
         ud2,
+        /// Exchange and add
+        xadd,
+        /// Exchange register/memory with register
+        xchg,
         /// Logical exclusive-or
         xor,
 
-        /// Add single precision floating point
+        /// Add single precision floating point values
         addss,
         /// Compare scalar single-precision floating-point values
         cmpss,
+        /// Divide scalar single-precision floating-point values
+        divss,
+        /// Return maximum single-precision floating-point value
+        maxss,
+        /// Return minimum single-precision floating-point value
+        minss,
         /// Move scalar single-precision floating-point value
         movss,
+        /// Multiply scalar single-precision floating-point values
+        mulss,
+        /// Round scalar single-precision floating-point values
+        roundss,
+        /// Subtract scalar single-precision floating-point values
+        subss,
         /// Unordered compare scalar single-precision floating-point values
         ucomiss,
-        /// Add double precision floating point
+        /// Add double precision floating point values
         addsd,
         /// Compare scalar double-precision floating-point values
         cmpsd,
+        /// Divide scalar double-precision floating-point values
+        divsd,
+        /// Return maximum double-precision floating-point value
+        maxsd,
+        /// Return minimum double-precision floating-point value
+        minsd,
         /// Move scalar double-precision floating-point value
         movsd,
+        /// Multiply scalar double-precision floating-point values
+        mulsd,
+        /// Round scalar double-precision floating-point values
+        roundsd,
+        /// Subtract scalar double-precision floating-point values
+        subsd,
         /// Unordered compare scalar double-precision floating-point values
         ucomisd,
 
+        /// Compare string operands
+        cmps,
+        /// Load string
+        lods,
+        /// Move data from string to string
+        movs,
+        /// Scan string
+        scas,
+        /// Store string
+        stos,
+
         /// Conditional move
         cmovcc,
         /// Conditional jump
@@ -185,11 +268,11 @@ pub const Inst = struct {
         /// Uses `rri`  payload.
         rri_u,
         /// Register with condition code (CC).
-        /// Uses `r_c` payload.
-        r_c,
+        /// Uses `r_cc` payload.
+        r_cc,
         /// Register, register with condition code (CC).
-        /// Uses `rr_c` payload.
-        rr_c,
+        /// Uses `rr_cc` payload.
+        rr_cc,
         /// Register, immediate (sign-extended) operands.
         /// Uses `ri` payload.
         ri_s,
@@ -214,12 +297,24 @@ pub const Inst = struct {
         /// Register, memory (RIP) operands.
         /// Uses `rx` payload.
         rm_rip,
+        /// Register, memory (SIB) operands with condition code (CC).
+        /// Uses `rx_cc` payload.
+        rm_sib_cc,
+        /// Register, memory (RIP) operands with condition code (CC).
+        /// Uses `rx_cc` payload.
+        rm_rip_cc,
         /// Single memory (SIB) operand.
         /// Uses `payload` with extra data of type `MemorySib`.
         m_sib,
         /// Single memory (RIP) operand.
         /// Uses `payload` with extra data of type `MemoryRip`.
         m_rip,
+        /// Single memory (SIB) operand with condition code (CC).
+        /// Uses `x_cc` with extra data of type `MemorySib`.
+        m_sib_cc,
+        /// Single memory (RIP) operand with condition code (CC).
+        /// Uses `x_cc` with extra data of type `MemoryRip`.
+        m_rip_cc,
         /// Memory (SIB), immediate (unsigned) operands.
         /// Uses `xi` payload with extra data of type `MemorySib`.
         mi_u_sib,
@@ -244,16 +339,42 @@ pub const Inst = struct {
         /// Memory moffs, rax.
         /// Uses `payload` with extra data of type `MemoryMoffs`.
         moffs_rax,
+        /// Single memory (SIB) operand with lock prefix.
+        /// Uses `payload` with extra data of type `MemorySib`.
+        lock_m_sib,
+        /// Single memory (RIP) operand with lock prefix.
+        /// Uses `payload` with extra data of type `MemoryRip`.
+        lock_m_rip,
+        /// Memory (SIB), immediate (unsigned) operands with lock prefix.
+        /// Uses `xi` payload with extra data of type `MemorySib`.
+        lock_mi_u_sib,
+        /// Memory (RIP), immediate (unsigned) operands with lock prefix.
+        /// Uses `xi` payload with extra data of type `MemoryRip`.
+        lock_mi_u_rip,
+        /// Memory (SIB), immediate (sign-extend) operands with lock prefix.
+        /// Uses `xi` payload with extra data of type `MemorySib`.
+        lock_mi_s_sib,
+        /// Memory (RIP), immediate (sign-extend) operands with lock prefix.
+        /// Uses `xi` payload with extra data of type `MemoryRip`.
+        lock_mi_s_rip,
+        /// Memory (SIB), register operands with lock prefix.
+        /// Uses `rx` payload with extra data of type `MemorySib`.
+        lock_mr_sib,
+        /// Memory (RIP), register operands with lock prefix.
+        /// Uses `rx` payload with extra data of type `MemoryRip`.
+        lock_mr_rip,
+        /// Memory moffs, rax with lock prefix.
+        /// Uses `payload` with extra data of type `MemoryMoffs`.
+        lock_moffs_rax,
         /// References another Mir instruction directly.
         /// Uses `inst` payload.
         inst,
         /// References another Mir instruction directly with condition code (CC).
         /// Uses `inst_cc` payload.
         inst_cc,
-        /// Uses `payload` payload with data of type `MemoryConditionCode`.
-        m_cc,
-        /// Uses `rx` payload with extra data of type `MemoryConditionCode`.
-        rm_cc,
+        /// String repeat and width
+        /// Uses `string` payload.
+        string,
         /// Uses `reloc` payload.
         reloc,
         /// Linker relocation - GOT indirection.
@@ -295,13 +416,18 @@ pub const Inst = struct {
             r2: Register,
             imm: u32,
         },
+        /// Condition code (CC), followed by custom payload found in extra.
+        x_cc: struct {
+            payload: u32,
+            cc: bits.Condition,
+        },
         /// Register with condition code (CC).
-        r_c: struct {
+        r_cc: struct {
             r1: Register,
             cc: bits.Condition,
         },
         /// Register, register with condition code (CC).
-        rr_c: struct {
+        rr_cc: struct {
             r1: Register,
             r2: Register,
             cc: bits.Condition,
@@ -316,11 +442,22 @@ pub const Inst = struct {
             r1: Register,
             payload: u32,
         },
+        /// Register with condition code (CC), followed by custom payload found in extra.
+        rx_cc: struct {
+            r1: Register,
+            cc: bits.Condition,
+            payload: u32,
+        },
         /// Custom payload followed by an immediate.
         xi: struct {
             payload: u32,
             imm: u32,
         },
+        /// String instruction prefix and width.
+        string: struct {
+            repeat: bits.StringRepeat,
+            width: bits.StringWidth,
+        },
         /// Relocation for the linker where:
         /// * `atom_index` is the index of the source
         /// * `sym_index` is the index of the target

src/arch/x86_64/bits.zig

@@ -6,6 +6,9 @@ const Allocator = std.mem.Allocator;
 const ArrayList = std.ArrayList;
 const DW = std.dwarf;
 
+pub const StringRepeat = enum(u3) { none, rep, repe, repz, repne, repnz };
+pub const StringWidth = enum(u2) { b, w, d, q };
+
 /// EFLAGS condition codes
 pub const Condition = enum(u5) {
     /// above
@@ -97,8 +100,7 @@ pub const Condition = enum(u5) {
         };
     }
 
-    /// Returns the condition which is true iff the given condition is
-    /// false (if such a condition exists)
+    /// Returns the condition which is true iff the given condition is false
     pub fn negate(cond: Condition) Condition {
         return switch (cond) {
             .a => .na,
@@ -127,8 +129,8 @@ pub const Condition = enum(u5) {
             .nz => .z,
             .o => .no,
             .p => .np,
-            .pe => unreachable,
-            .po => unreachable,
+            .pe => .po,
+            .po => .pe,
             .s => .ns,
             .z => .nz,
         };
@@ -470,10 +472,11 @@ pub const Memory = union(enum) {
     }
 
     pub fn sib(ptr_size: PtrSize, args: struct {
-        disp: i32,
+        disp: i32 = 0,
         base: ?Register = null,
         scale_index: ?ScaleIndex = null,
     }) Memory {
+        if (args.scale_index) |si| assert(std.math.isPowerOfTwo(si.scale));
         return .{ .sib = .{
             .base = args.base,
             .disp = args.disp,

src/arch/x86_64/encoder.zig

@@ -15,10 +15,21 @@ pub const Instruction = struct {
     op2: Operand = .none,
     op3: Operand = .none,
     op4: Operand = .none,
+    prefix: Prefix = .none,
     encoding: Encoding,
 
     pub const Mnemonic = Encoding.Mnemonic;
 
+    pub const Prefix = enum(u3) {
+        none,
+        lock,
+        rep,
+        repe,
+        repz,
+        repne,
+        repnz,
+    };
+
     pub const Operand = union(enum) {
         none,
         reg: Register,
@@ -96,19 +107,18 @@ pub const Instruction = struct {
         }
     };
 
-    pub fn new(mnemonic: Mnemonic, args: struct {
+    pub const Init = struct {
+        prefix: Prefix = .none,
         op1: Operand = .none,
         op2: Operand = .none,
         op3: Operand = .none,
         op4: Operand = .none,
-    }) !Instruction {
-        const encoding = (try Encoding.findByMnemonic(mnemonic, .{
-            .op1 = args.op1,
-            .op2 = args.op2,
-            .op3 = args.op3,
-            .op4 = args.op4,
-        })) orelse {
-            log.debug("no encoding found for: {s} {s} {s} {s} {s}", .{
+    };
+
+    pub fn new(mnemonic: Mnemonic, args: Init) !Instruction {
+        const encoding = (try Encoding.findByMnemonic(mnemonic, args)) orelse {
+            log.debug("no encoding found for: {s} {s} {s} {s} {s} {s}", .{
+                @tagName(args.prefix),
                 @tagName(mnemonic),
                 @tagName(Encoding.Op.fromOperand(args.op1)),
                 @tagName(Encoding.Op.fromOperand(args.op2)),
@@ -119,6 +129,7 @@ pub const Instruction = struct {
         };
         log.debug("selected encoding: {}", .{encoding});
         return .{
+            .prefix = args.prefix,
             .op1 = args.op1,
             .op2 = args.op2,
             .op3 = args.op3,
@@ -128,6 +139,7 @@ pub const Instruction = struct {
     }
 
     pub fn fmtPrint(inst: Instruction, writer: anytype) !void {
+        if (inst.prefix != .none) try writer.print("{s} ", .{@tagName(inst.prefix)});
         try writer.print("{s}", .{@tagName(inst.encoding.mnemonic)});
         const ops = [_]struct { Operand, Encoding.Op }{
             .{ inst.op1, inst.encoding.op1 },
@@ -199,14 +211,12 @@ pub const Instruction = struct {
 
     fn encodeOpcode(inst: Instruction, encoder: anytype) !void {
         const opcode = inst.encoding.opcode();
+        const first = @boolToInt(inst.encoding.mandatoryPrefix() != null);
+        const final = opcode.len - 1;
+        for (opcode[first..final]) |byte| try encoder.opcode_1byte(byte);
         switch (inst.encoding.op_en) {
-            .o, .oi => try encoder.opcode_withReg(opcode[0], inst.op1.reg.lowEnc()),
-            else => {
-                const index: usize = if (inst.encoding.mandatoryPrefix()) |_| 1 else 0;
-                for (opcode[index..]) |byte| {
-                    try encoder.opcode_1byte(byte);
-                }
-            },
+            .o, .oi => try encoder.opcode_withReg(opcode[final], inst.op1.reg.lowEnc()),
+            else => try encoder.opcode_1byte(opcode[final]),
         }
     }
 
@@ -215,6 +225,14 @@ pub const Instruction = struct {
         const op_en = enc.op_en;
 
         var legacy = LegacyPrefixes{};
+
+        switch (inst.prefix) {
+            .none => {},
+            .lock => legacy.prefix_f0 = true,
+            .repne, .repnz => legacy.prefix_f2 = true,
+            .rep, .repe, .repz => legacy.prefix_f3 = true,
+        }
+
         if (enc.mode == .none) {
             const bit_size = enc.operandBitSize();
             if (bit_size == 16) {
@@ -811,15 +829,11 @@ const TestEncode = struct {
     buffer: [32]u8 = undefined,
     index: usize = 0,
 
-    fn encode(enc: *TestEncode, mnemonic: Instruction.Mnemonic, args: struct {
-        op1: Instruction.Operand = .none,
-        op2: Instruction.Operand = .none,
-        op3: Instruction.Operand = .none,
-        op4: Instruction.Operand = .none,
-    }) !void {
+    fn encode(enc: *TestEncode, mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void {
         var stream = std.io.fixedBufferStream(&enc.buffer);
         var count_writer = std.io.countingWriter(stream.writer());
         const inst = try Instruction.new(mnemonic, .{
+            .prefix = args.prefix,
             .op1 = args.op1,
             .op2 = args.op2,
             .op3 = args.op3,
@@ -880,10 +894,10 @@ test "lower MI encoding" {
     try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } });
     try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
 
-    try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
-        .base = .r12,
-        .disp = 0,
-    }) }, .op2 = .{ .imm = Immediate.u(0x10) } });
+    try enc.encode(.mov, .{
+        .op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12 }) },
+        .op2 = .{ .imm = Immediate.u(0x10) },
+    });
     try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10");
 
     try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } });
@@ -895,10 +909,10 @@ test "lower MI encoding" {
     try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10) } });
     try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10");
 
-    try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
-        .base = .r11,
-        .disp = 0,
-    }) }, .op2 = .{ .imm = Immediate.u(0x10) } });
+    try enc.encode(.mov, .{
+        .op1 = .{ .mem = Memory.sib(.dword, .{ .base = .r11 }) },
+        .op2 = .{ .imm = Immediate.u(0x10) },
+    });
     try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10");
 
     try enc.encode(.mov, .{
@@ -1014,10 +1028,10 @@ test "lower MI encoding" {
 test "lower RM encoding" {
     var enc = TestEncode{};
 
-    try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.sib(.qword, .{
-        .base = .r11,
-        .disp = 0,
-    }) } });
+    try enc.encode(.mov, .{
+        .op1 = .{ .reg = .rax },
+        .op2 = .{ .mem = Memory.sib(.qword, .{ .base = .r11 }) },
+    });
     try expectEqualHexStrings("\x49\x8b\x03", enc.code(), "mov rax, QWORD PTR [r11]");
 
     try enc.encode(.mov, .{ .op1 = .{ .reg = .rbx }, .op2 = .{ .mem = Memory.sib(.qword, .{
@@ -1100,20 +1114,16 @@ test "lower RM encoding" {
     try enc.encode(.movsx, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .reg = .bl } });
     try expectEqualHexStrings("\x66\x0F\xBE\xC3", enc.code(), "movsx ax, bl");
 
-    try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.word, .{
-        .base = .rbp,
-        .disp = 0,
-    }) } });
+    try enc.encode(.movsx, .{
+        .op1 = .{ .reg = .eax },
+        .op2 = .{ .mem = Memory.sib(.word, .{ .base = .rbp }) },
+    });
     try expectEqualHexStrings("\x0F\xBF\x45\x00", enc.code(), "movsx eax, BYTE PTR [rbp]");
 
-    try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.byte, .{
-        .base = null,
-        .scale_index = .{
-            .index = .rax,
-            .scale = 2,
-        },
-        .disp = 0,
-    }) } });
+    try enc.encode(.movsx, .{
+        .op1 = .{ .reg = .eax },
+        .op2 = .{ .mem = Memory.sib(.byte, .{ .scale_index = .{ .index = .rax, .scale = 2 } }) },
+    });
     try expectEqualHexStrings("\x0F\xBE\x04\x45\x00\x00\x00\x00", enc.code(), "movsx eax, BYTE PTR [rax * 2]");
 
     try enc.encode(.movsx, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.rip(.byte, 0x10) } });
@@ -1140,14 +1150,13 @@ test "lower RM encoding" {
     try enc.encode(.lea, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.rip(.byte, 0x10) } });
     try expectEqualHexStrings("\x66\x8D\x05\x10\x00\x00\x00", enc.code(), "lea ax, BYTE PTR [rip + 0x10]");
 
-    try enc.encode(.lea, .{ .op1 = .{ .reg = .rsi }, .op2 = .{ .mem = Memory.sib(.qword, .{
-        .base = .rbp,
-        .scale_index = .{
-            .scale = 1,
-            .index = .rcx,
-        },
-        .disp = 0,
-    }) } });
+    try enc.encode(.lea, .{
+        .op1 = .{ .reg = .rsi },
+        .op2 = .{ .mem = Memory.sib(.qword, .{
+            .base = .rbp,
+            .scale_index = .{ .scale = 1, .index = .rcx },
+        }) },
+    });
     try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", enc.code(), "lea rsi, QWORD PTR [rbp + rcx*1 + 0]");
 
     try enc.encode(.add, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .mem = Memory.sib(.qword, .{
@@ -1303,51 +1312,35 @@ test "lower M encoding" {
     try enc.encode(.call, .{ .op1 = .{ .reg = .r12 } });
     try expectEqualHexStrings("\x41\xFF\xD4", enc.code(), "call r12");
 
-    try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
-        .base = .r12,
-        .disp = 0,
-    }) } });
+    try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ .base = .r12 }) } });
     try expectEqualHexStrings("\x41\xFF\x14\x24", enc.code(), "call QWORD PTR [r12]");
 
-    try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
-        .base = null,
-        .scale_index = .{
-            .index = .r11,
-            .scale = 2,
-        },
-        .disp = 0,
-    }) } });
+    try enc.encode(.call, .{
+        .op1 = .{ .mem = Memory.sib(.qword, .{
+            .base = null,
+            .scale_index = .{ .index = .r11, .scale = 2 },
+        }) },
+    });
     try expectEqualHexStrings("\x42\xFF\x14\x5D\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r11 * 2]");
 
-    try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
-        .base = null,
-        .scale_index = .{
-            .index = .r12,
-            .scale = 2,
-        },
-        .disp = 0,
-    }) } });
+    try enc.encode(.call, .{
+        .op1 = .{ .mem = Memory.sib(.qword, .{
+            .base = null,
+            .scale_index = .{ .index = .r12, .scale = 2 },
+        }) },
+    });
     try expectEqualHexStrings("\x42\xFF\x14\x65\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r12 * 2]");
 
-    try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
-        .base = .gs,
-        .disp = 0,
-    }) } });
+    try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ .base = .gs }) } });
     try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0");
 
     try enc.encode(.call, .{ .op1 = .{ .imm = Immediate.s(0) } });
     try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0");
 
-    try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
-        .base = .rbp,
-        .disp = 0,
-    }) } });
+    try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ .base = .rbp }) } });
     try expectEqualHexStrings("\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
 
-    try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.word, .{
-        .base = .rbp,
-        .disp = 0,
-    }) } });
+    try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.word, .{ .base = .rbp }) } });
     try expectEqualHexStrings("\x66\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
 
     try enc.encode(.pop, .{ .op1 = .{ .mem = Memory.rip(.qword, 0) } });
@@ -1447,18 +1440,8 @@ test "lower NP encoding" {
     try expectEqualHexStrings("\x0f\x05", enc.code(), "syscall");
 }
 
-fn invalidInstruction(mnemonic: Instruction.Mnemonic, args: struct {
-    op1: Instruction.Operand = .none,
-    op2: Instruction.Operand = .none,
-    op3: Instruction.Operand = .none,
-    op4: Instruction.Operand = .none,
-}) !void {
-    const err = Instruction.new(mnemonic, .{
-        .op1 = args.op1,
-        .op2 = args.op2,
-        .op3 = args.op3,
-        .op4 = args.op4,
-    });
+fn invalidInstruction(mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void {
+    const err = Instruction.new(mnemonic, args);
     try testing.expectError(error.InvalidInstruction, err);
 }
 
@@ -1479,23 +1462,13 @@ test "invalid instruction" {
     try invalidInstruction(.push, .{ .op1 = .{ .imm = Immediate.u(0x1000000000000000) } });
 }
 
-fn cannotEncode(mnemonic: Instruction.Mnemonic, args: struct {
-    op1: Instruction.Operand = .none,
-    op2: Instruction.Operand = .none,
-    op3: Instruction.Operand = .none,
-    op4: Instruction.Operand = .none,
-}) !void {
-    try testing.expectError(error.CannotEncode, Instruction.new(mnemonic, .{
-        .op1 = args.op1,
-        .op2 = args.op2,
-        .op3 = args.op3,
-        .op4 = args.op4,
-    }));
+fn cannotEncode(mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void {
+    try testing.expectError(error.CannotEncode, Instruction.new(mnemonic, args));
 }
 
 test "cannot encode" {
     try cannotEncode(.@"test", .{
-        .op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12, .disp = 0 }) },
+        .op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12 }) },
         .op2 = .{ .reg = .ah },
     });
     try cannotEncode(.@"test", .{

src/arch/x86_64/encodings.zig

@@ -4,618 +4,857 @@ const OpEn = Encoding.OpEn;
 const Op = Encoding.Op;
 const Mode = Encoding.Mode;
 
-const opcode_len = u2;
 const modrm_ext = u3;
 
-const Entry = struct { Mnemonic, OpEn, Op, Op, Op, Op, opcode_len, u8, u8, u8, modrm_ext, Mode };
+const Entry = struct { Mnemonic, OpEn, Op, Op, Op, Op, []const u8, modrm_ext, Mode };
 
 // TODO move this into a .zon file when Zig is capable of importing .zon files
 // zig fmt: off
 pub const table = &[_]Entry{
     // General-purpose
-    .{ .adc, .zi, .al,   .imm8,   .none, .none, 1, 0x14, 0x00, 0x00, 0, .none  },
-    .{ .adc, .zi, .ax,   .imm16,  .none, .none, 1, 0x15, 0x00, 0x00, 0, .none  },
-    .{ .adc, .zi, .eax,  .imm32,  .none, .none, 1, 0x15, 0x00, 0x00, 0, .none  },
-    .{ .adc, .zi, .rax,  .imm32s, .none, .none, 1, 0x15, 0x00, 0x00, 0, .long  },
-    .{ .adc, .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 2, .none  },
-    .{ .adc, .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 2, .rex   },
-    .{ .adc, .mi, .rm16, .imm16,  .none, .none, 1, 0x81, 0x00, 0x00, 2, .none  },
-    .{ .adc, .mi, .rm32, .imm32,  .none, .none, 1, 0x81, 0x00, 0x00, 2, .none  },
-    .{ .adc, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 2, .long  },
-    .{ .adc, .mi, .rm16, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 2, .none  },
-    .{ .adc, .mi, .rm32, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 2, .none  },
-    .{ .adc, .mi, .rm64, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 2, .long  },
-    .{ .adc, .mr, .rm8,  .r8,     .none, .none, 1, 0x10, 0x00, 0x00, 0, .none  },
-    .{ .adc, .mr, .rm8,  .r8,     .none, .none, 1, 0x10, 0x00, 0x00, 0, .rex   },
-    .{ .adc, .mr, .rm16, .r16,    .none, .none, 1, 0x11, 0x00, 0x00, 0, .none  },
-    .{ .adc, .mr, .rm32, .r32,    .none, .none, 1, 0x11, 0x00, 0x00, 0, .none  },
-    .{ .adc, .mr, .rm64, .r64,    .none, .none, 1, 0x11, 0x00, 0x00, 0, .long  },
-    .{ .adc, .rm, .r8,   .rm8,    .none, .none, 1, 0x12, 0x00, 0x00, 0, .none  },
-    .{ .adc, .rm, .r8,   .rm8,    .none, .none, 1, 0x12, 0x00, 0x00, 0, .rex   },
-    .{ .adc, .rm, .r16,  .rm16,   .none, .none, 1, 0x13, 0x00, 0x00, 0, .none  },
-    .{ .adc, .rm, .r32,  .rm32,   .none, .none, 1, 0x13, 0x00, 0x00, 0, .none  },
-    .{ .adc, .rm, .r64,  .rm64,   .none, .none, 1, 0x13, 0x00, 0x00, 0, .long  },
+    .{ .adc, .zi, .al,   .imm8,   .none, .none, &.{ 0x14 }, 0, .none  },
+    .{ .adc, .zi, .ax,   .imm16,  .none, .none, &.{ 0x15 }, 0, .none  },
+    .{ .adc, .zi, .eax,  .imm32,  .none, .none, &.{ 0x15 }, 0, .none  },
+    .{ .adc, .zi, .rax,  .imm32s, .none, .none, &.{ 0x15 }, 0, .long  },
+    .{ .adc, .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 2, .none  },
+    .{ .adc, .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 2, .rex   },
+    .{ .adc, .mi, .rm16, .imm16,  .none, .none, &.{ 0x81 }, 2, .none  },
+    .{ .adc, .mi, .rm32, .imm32,  .none, .none, &.{ 0x81 }, 2, .none  },
+    .{ .adc, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 2, .long  },
+    .{ .adc, .mi, .rm16, .imm8s,  .none, .none, &.{ 0x83 }, 2, .none  },
+    .{ .adc, .mi, .rm32, .imm8s,  .none, .none, &.{ 0x83 }, 2, .none  },
+    .{ .adc, .mi, .rm64, .imm8s,  .none, .none, &.{ 0x83 }, 2, .long  },
+    .{ .adc, .mr, .rm8,  .r8,     .none, .none, &.{ 0x10 }, 0, .none  },
+    .{ .adc, .mr, .rm8,  .r8,     .none, .none, &.{ 0x10 }, 0, .rex   },
+    .{ .adc, .mr, .rm16, .r16,    .none, .none, &.{ 0x11 }, 0, .none  },
+    .{ .adc, .mr, .rm32, .r32,    .none, .none, &.{ 0x11 }, 0, .none  },
+    .{ .adc, .mr, .rm64, .r64,    .none, .none, &.{ 0x11 }, 0, .long  },
+    .{ .adc, .rm, .r8,   .rm8,    .none, .none, &.{ 0x12 }, 0, .none  },
+    .{ .adc, .rm, .r8,   .rm8,    .none, .none, &.{ 0x12 }, 0, .rex   },
+    .{ .adc, .rm, .r16,  .rm16,   .none, .none, &.{ 0x13 }, 0, .none  },
+    .{ .adc, .rm, .r32,  .rm32,   .none, .none, &.{ 0x13 }, 0, .none  },
+    .{ .adc, .rm, .r64,  .rm64,   .none, .none, &.{ 0x13 }, 0, .long  },
 
-    .{ .add, .zi, .al,   .imm8,   .none, .none, 1, 0x04, 0x00, 0x00, 0, .none  },
-    .{ .add, .zi, .ax,   .imm16,  .none, .none, 1, 0x05, 0x00, 0x00, 0, .none  },
-    .{ .add, .zi, .eax,  .imm32,  .none, .none, 1, 0x05, 0x00, 0x00, 0, .none  },
-    .{ .add, .zi, .rax,  .imm32s, .none, .none, 1, 0x05, 0x00, 0x00, 0, .long  },
-    .{ .add, .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 0, .none  },
-    .{ .add, .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 0, .rex   },
-    .{ .add, .mi, .rm16, .imm16,  .none, .none, 1, 0x81, 0x00, 0x00, 0, .none  },
-    .{ .add, .mi, .rm32, .imm32,  .none, .none, 1, 0x81, 0x00, 0x00, 0, .none  },
-    .{ .add, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 0, .long  },
-    .{ .add, .mi, .rm16, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 0, .none  },
-    .{ .add, .mi, .rm32, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 0, .none  },
-    .{ .add, .mi, .rm64, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 0, .long  },
-    .{ .add, .mr, .rm8,  .r8,     .none, .none, 1, 0x00, 0x00, 0x00, 0, .none  },
-    .{ .add, .mr, .rm8,  .r8,     .none, .none, 1, 0x00, 0x00, 0x00, 0, .rex   },
-    .{ .add, .mr, .rm16, .r16,    .none, .none, 1, 0x01, 0x00, 0x00, 0, .none  },
-    .{ .add, .mr, .rm32, .r32,    .none, .none, 1, 0x01, 0x00, 0x00, 0, .none  },
-    .{ .add, .mr, .rm64, .r64,    .none, .none, 1, 0x01, 0x00, 0x00, 0, .long  },
-    .{ .add, .rm, .r8,   .rm8,    .none, .none, 1, 0x02, 0x00, 0x00, 0, .none  },
-    .{ .add, .rm, .r8,   .rm8,    .none, .none, 1, 0x02, 0x00, 0x00, 0, .rex   },
-    .{ .add, .rm, .r16,  .rm16,   .none, .none, 1, 0x03, 0x00, 0x00, 0, .none  },
-    .{ .add, .rm, .r32,  .rm32,   .none, .none, 1, 0x03, 0x00, 0x00, 0, .none  },
-    .{ .add, .rm, .r64,  .rm64,   .none, .none, 1, 0x03, 0x00, 0x00, 0, .long  },
+    .{ .add, .zi, .al,   .imm8,   .none, .none, &.{ 0x04 }, 0, .none  },
+    .{ .add, .zi, .ax,   .imm16,  .none, .none, &.{ 0x05 }, 0, .none  },
+    .{ .add, .zi, .eax,  .imm32,  .none, .none, &.{ 0x05 }, 0, .none  },
+    .{ .add, .zi, .rax,  .imm32s, .none, .none, &.{ 0x05 }, 0, .long  },
+    .{ .add, .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 0, .none  },
+    .{ .add, .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 0, .rex   },
+    .{ .add, .mi, .rm16, .imm16,  .none, .none, &.{ 0x81 }, 0, .none  },
+    .{ .add, .mi, .rm32, .imm32,  .none, .none, &.{ 0x81 }, 0, .none  },
+    .{ .add, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 0, .long  },
+    .{ .add, .mi, .rm16, .imm8s,  .none, .none, &.{ 0x83 }, 0, .none  },
+    .{ .add, .mi, .rm32, .imm8s,  .none, .none, &.{ 0x83 }, 0, .none  },
+    .{ .add, .mi, .rm64, .imm8s,  .none, .none, &.{ 0x83 }, 0, .long  },
+    .{ .add, .mr, .rm8,  .r8,     .none, .none, &.{ 0x00 }, 0, .none  },
+    .{ .add, .mr, .rm8,  .r8,     .none, .none, &.{ 0x00 }, 0, .rex   },
+    .{ .add, .mr, .rm16, .r16,    .none, .none, &.{ 0x01 }, 0, .none  },
+    .{ .add, .mr, .rm32, .r32,    .none, .none, &.{ 0x01 }, 0, .none  },
+    .{ .add, .mr, .rm64, .r64,    .none, .none, &.{ 0x01 }, 0, .long  },
+    .{ .add, .rm, .r8,   .rm8,    .none, .none, &.{ 0x02 }, 0, .none  },
+    .{ .add, .rm, .r8,   .rm8,    .none, .none, &.{ 0x02 }, 0, .rex   },
+    .{ .add, .rm, .r16,  .rm16,   .none, .none, &.{ 0x03 }, 0, .none  },
+    .{ .add, .rm, .r32,  .rm32,   .none, .none, &.{ 0x03 }, 0, .none  },
+    .{ .add, .rm, .r64,  .rm64,   .none, .none, &.{ 0x03 }, 0, .long  },
 
-    .{ .@"and", .zi, .al,   .imm8,   .none, .none, 1, 0x24, 0x00, 0x00, 0, .none  },
-    .{ .@"and", .zi, .ax,   .imm16,  .none, .none, 1, 0x25, 0x00, 0x00, 0, .none  },
-    .{ .@"and", .zi, .eax,  .imm32,  .none, .none, 1, 0x25, 0x00, 0x00, 0, .none  },
-    .{ .@"and", .zi, .rax,  .imm32s, .none, .none, 1, 0x25, 0x00, 0x00, 0, .long  },
-    .{ .@"and", .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 4, .none  },
-    .{ .@"and", .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 4, .rex   },
-    .{ .@"and", .mi, .rm16, .imm16,  .none, .none, 1, 0x81, 0x00, 0x00, 4, .none  },
-    .{ .@"and", .mi, .rm32, .imm32,  .none, .none, 1, 0x81, 0x00, 0x00, 4, .none  },
-    .{ .@"and", .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 4, .long  },
-    .{ .@"and", .mi, .rm16, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 4, .none  },
-    .{ .@"and", .mi, .rm32, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 4, .none  },
-    .{ .@"and", .mi, .rm64, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 4, .long  },
-    .{ .@"and", .mr, .rm8,  .r8,     .none, .none, 1, 0x20, 0x00, 0x00, 0, .none  },
-    .{ .@"and", .mr, .rm8,  .r8,     .none, .none, 1, 0x20, 0x00, 0x00, 0, .rex   },
-    .{ .@"and", .mr, .rm16, .r16,    .none, .none, 1, 0x21, 0x00, 0x00, 0, .none  },
-    .{ .@"and", .mr, .rm32, .r32,    .none, .none, 1, 0x21, 0x00, 0x00, 0, .none  },
-    .{ .@"and", .mr, .rm64, .r64,    .none, .none, 1, 0x21, 0x00, 0x00, 0, .long  },
-    .{ .@"and", .rm, .r8,   .rm8,    .none, .none, 1, 0x22, 0x00, 0x00, 0, .none  },
-    .{ .@"and", .rm, .r8,   .rm8,    .none, .none, 1, 0x22, 0x00, 0x00, 0, .rex   },
-    .{ .@"and", .rm, .r16,  .rm16,   .none, .none, 1, 0x23, 0x00, 0x00, 0, .none  },
-    .{ .@"and", .rm, .r32,  .rm32,   .none, .none, 1, 0x23, 0x00, 0x00, 0, .none  },
-    .{ .@"and", .rm, .r64,  .rm64,   .none, .none, 1, 0x23, 0x00, 0x00, 0, .long  },
+    .{ .@"and", .zi, .al,   .imm8,   .none, .none, &.{ 0x24 }, 0, .none  },
+    .{ .@"and", .zi, .ax,   .imm16,  .none, .none, &.{ 0x25 }, 0, .none  },
+    .{ .@"and", .zi, .eax,  .imm32,  .none, .none, &.{ 0x25 }, 0, .none  },
+    .{ .@"and", .zi, .rax,  .imm32s, .none, .none, &.{ 0x25 }, 0, .long  },
+    .{ .@"and", .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 4, .none  },
+    .{ .@"and", .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 4, .rex   },
+    .{ .@"and", .mi, .rm16, .imm16,  .none, .none, &.{ 0x81 }, 4, .none  },
+    .{ .@"and", .mi, .rm32, .imm32,  .none, .none, &.{ 0x81 }, 4, .none  },
+    .{ .@"and", .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 4, .long  },
+    .{ .@"and", .mi, .rm16, .imm8s,  .none, .none, &.{ 0x83 }, 4, .none  },
+    .{ .@"and", .mi, .rm32, .imm8s,  .none, .none, &.{ 0x83 }, 4, .none  },
+    .{ .@"and", .mi, .rm64, .imm8s,  .none, .none, &.{ 0x83 }, 4, .long  },
+    .{ .@"and", .mr, .rm8,  .r8,     .none, .none, &.{ 0x20 }, 0, .none  },
+    .{ .@"and", .mr, .rm8,  .r8,     .none, .none, &.{ 0x20 }, 0, .rex   },
+    .{ .@"and", .mr, .rm16, .r16,    .none, .none, &.{ 0x21 }, 0, .none  },
+    .{ .@"and", .mr, .rm32, .r32,    .none, .none, &.{ 0x21 }, 0, .none  },
+    .{ .@"and", .mr, .rm64, .r64,    .none, .none, &.{ 0x21 }, 0, .long  },
+    .{ .@"and", .rm, .r8,   .rm8,    .none, .none, &.{ 0x22 }, 0, .none  },
+    .{ .@"and", .rm, .r8,   .rm8,    .none, .none, &.{ 0x22 }, 0, .rex   },
+    .{ .@"and", .rm, .r16,  .rm16,   .none, .none, &.{ 0x23 }, 0, .none  },
+    .{ .@"and", .rm, .r32,  .rm32,   .none, .none, &.{ 0x23 }, 0, .none  },
+    .{ .@"and", .rm, .r64,  .rm64,   .none, .none, &.{ 0x23 }, 0, .long  },
+
+    .{ .bsf, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0xbc }, 0, .none },
+    .{ .bsf, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xbc }, 0, .none },
+    .{ .bsf, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xbc }, 0, .long },
+
+    .{ .bsr, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0xbd }, 0, .none },
+    .{ .bsr, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xbd }, 0, .none },
+    .{ .bsr, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xbd }, 0, .long },
+
+    .{ .bswap, .o, .r32, .none, .none, .none, &.{ 0x0f, 0xc8 }, 0, .none },
+    .{ .bswap, .o, .r64, .none, .none, .none, &.{ 0x0f, 0xc8 }, 0, .long },
+
+    .{ .bt, .mr, .rm16, .r16,  .none, .none, &.{ 0x0f, 0xa3 }, 0, .none },
+    .{ .bt, .mr, .rm32, .r32,  .none, .none, &.{ 0x0f, 0xa3 }, 0, .none },
+    .{ .bt, .mr, .rm64, .r64,  .none, .none, &.{ 0x0f, 0xa3 }, 0, .long },
+    .{ .bt, .mi, .rm16, .imm8, .none, .none, &.{ 0x0f, 0xba }, 4, .none },
+    .{ .bt, .mi, .rm32, .imm8, .none, .none, &.{ 0x0f, 0xba }, 4, .none },
+    .{ .bt, .mi, .rm64, .imm8, .none, .none, &.{ 0x0f, 0xba }, 4, .long },
+
+    .{ .btc, .mr, .rm16, .r16,  .none, .none, &.{ 0x0f, 0xbb }, 0, .none },
+    .{ .btc, .mr, .rm32, .r32,  .none, .none, &.{ 0x0f, 0xbb }, 0, .none },
+    .{ .btc, .mr, .rm64, .r64,  .none, .none, &.{ 0x0f, 0xbb }, 0, .long },
+    .{ .btc, .mi, .rm16, .imm8, .none, .none, &.{ 0x0f, 0xba }, 7, .none },
+    .{ .btc, .mi, .rm32, .imm8, .none, .none, &.{ 0x0f, 0xba }, 7, .none },
+    .{ .btc, .mi, .rm64, .imm8, .none, .none, &.{ 0x0f, 0xba }, 7, .long },
+
+    .{ .btr, .mr, .rm16, .r16,  .none, .none, &.{ 0x0f, 0xb3 }, 0, .none },
+    .{ .btr, .mr, .rm32, .r32,  .none, .none, &.{ 0x0f, 0xb3 }, 0, .none },
+    .{ .btr, .mr, .rm64, .r64,  .none, .none, &.{ 0x0f, 0xb3 }, 0, .long },
+    .{ .btr, .mi, .rm16, .imm8, .none, .none, &.{ 0x0f, 0xba }, 6, .none },
+    .{ .btr, .mi, .rm32, .imm8, .none, .none, &.{ 0x0f, 0xba }, 6, .none },
+    .{ .btr, .mi, .rm64, .imm8, .none, .none, &.{ 0x0f, 0xba }, 6, .long },
+
+    .{ .bts, .mr, .rm16, .r16,  .none, .none, &.{ 0x0f, 0xab }, 0, .none },
+    .{ .bts, .mr, .rm32, .r32,  .none, .none, &.{ 0x0f, 0xab }, 0, .none },
+    .{ .bts, .mr, .rm64, .r64,  .none, .none, &.{ 0x0f, 0xab }, 0, .long },
+    .{ .bts, .mi, .rm16, .imm8, .none, .none, &.{ 0x0f, 0xba }, 5, .none },
+    .{ .bts, .mi, .rm32, .imm8, .none, .none, &.{ 0x0f, 0xba }, 5, .none },
+    .{ .bts, .mi, .rm64, .imm8, .none, .none, &.{ 0x0f, 0xba }, 5, .long },
 
     // This is M encoding according to Intel, but D makes more sense here.
-    .{ .call, .d, .rel32, .none, .none, .none, 1, 0xe8, 0x00, 0x00, 0, .none },
-    .{ .call, .m, .rm64,  .none, .none, .none, 1, 0xff, 0x00, 0x00, 2, .none },
+    .{ .call, .d, .rel32, .none, .none, .none, &.{ 0xe8 }, 0, .none },
+    .{ .call, .m, .rm64,  .none, .none, .none, &.{ 0xff }, 2, .none },
 
-    .{ .cbw,  .np, .o16, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .none  },
-    .{ .cwde, .np, .o32, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .none  },
-    .{ .cdqe, .np, .o64, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .long  },
+    .{ .cbw,  .np, .o16, .none, .none, .none, &.{ 0x98 }, 0, .none  },
+    .{ .cwde, .np, .o32, .none, .none, .none, &.{ 0x98 }, 0, .none  },
+    .{ .cdqe, .np, .o64, .none, .none, .none, &.{ 0x98 }, 0, .long  },
 
-    .{ .cwd, .np, .o16, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .none  },
-    .{ .cdq, .np, .o32, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .none  },
-    .{ .cqo, .np, .o64, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .long  },
+    .{ .cwd, .np, .o16, .none, .none, .none, &.{ 0x99 }, 0, .none  },
+    .{ .cdq, .np, .o32, .none, .none, .none, &.{ 0x99 }, 0, .none  },
+    .{ .cqo, .np, .o64, .none, .none, .none, &.{ 0x99 }, 0, .long  },
 
-    .{ .cmova,   .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none  },
-    .{ .cmova,   .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none  },
-    .{ .cmova,   .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .long  },
-    .{ .cmovae,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none  },
-    .{ .cmovae,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none  },
-    .{ .cmovae,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long  },
-    .{ .cmovb,   .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none  },
-    .{ .cmovb,   .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none  },
-    .{ .cmovb,   .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long  },
-    .{ .cmovbe,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none  },
-    .{ .cmovbe,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none  },
-    .{ .cmovbe,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .long  },
-    .{ .cmovc,   .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none  },
-    .{ .cmovc,   .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none  },
-    .{ .cmovc,   .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long  },
-    .{ .cmove,   .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none  },
-    .{ .cmove,   .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none  },
-    .{ .cmove,   .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .long  },
-    .{ .cmovg,   .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none  },
-    .{ .cmovg,   .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none  },
-    .{ .cmovg,   .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .long  },
-    .{ .cmovge,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none  },
-    .{ .cmovge,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none  },
-    .{ .cmovge,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .long  },
-    .{ .cmovl,   .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none  },
-    .{ .cmovl,   .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none  },
-    .{ .cmovl,   .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .long  },
-    .{ .cmovle,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none  },
-    .{ .cmovle,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none  },
-    .{ .cmovle,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .long  },
-    .{ .cmovna,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none  },
-    .{ .cmovna,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none  },
-    .{ .cmovna,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .long  },
-    .{ .cmovnae, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none  },
-    .{ .cmovnae, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none  },
-    .{ .cmovnae, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long  },
-    .{ .cmovnb,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none  },
-    .{ .cmovnb,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none  },
-    .{ .cmovnb,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long  },
-    .{ .cmovnbe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none  },
-    .{ .cmovnbe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none  },
-    .{ .cmovnbe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .long  },
-    .{ .cmovnc,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none  },
-    .{ .cmovnc,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none  },
-    .{ .cmovnc,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long  },
-    .{ .cmovne,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none  },
-    .{ .cmovne,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none  },
-    .{ .cmovne,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .long  },
-    .{ .cmovng,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none  },
-    .{ .cmovng,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none  },
-    .{ .cmovng,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .long  },
-    .{ .cmovnge, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none  },
-    .{ .cmovnge, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none  },
-    .{ .cmovnge, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .long  },
-    .{ .cmovnl,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none  },
-    .{ .cmovnl,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none  },
-    .{ .cmovnl,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .long  },
-    .{ .cmovnle, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none  },
-    .{ .cmovnle, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none  },
-    .{ .cmovnle, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .long  },
-    .{ .cmovno,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .none  },
-    .{ .cmovno,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .none  },
-    .{ .cmovno,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .long  },
-    .{ .cmovnp,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none  },
-    .{ .cmovnp,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none  },
-    .{ .cmovnp,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .long  },
-    .{ .cmovns,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .none  },
-    .{ .cmovns,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .none  },
-    .{ .cmovns,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .long  },
-    .{ .cmovnz,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none  },
-    .{ .cmovnz,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none  },
-    .{ .cmovnz,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .long  },
-    .{ .cmovo,   .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .none  },
-    .{ .cmovo,   .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .none  },
-    .{ .cmovo,   .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .long  },
-    .{ .cmovp,   .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none  },
-    .{ .cmovp,   .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none  },
-    .{ .cmovp,   .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .long  },
-    .{ .cmovpe,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none  },
-    .{ .cmovpe,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none  },
-    .{ .cmovpe,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .long  },
-    .{ .cmovpo,  .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none  },
-    .{ .cmovpo,  .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none  },
-    .{ .cmovpo,  .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .long  },
-    .{ .cmovs,   .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .none  },
-    .{ .cmovs,   .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .none  },
-    .{ .cmovs,   .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .long  },
-    .{ .cmovz,   .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none  },
-    .{ .cmovz,   .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none  },
-    .{ .cmovz,   .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .long  },
+    .{ .cmova,   .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x47 }, 0, .none  },
+    .{ .cmova,   .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x47 }, 0, .none  },
+    .{ .cmova,   .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x47 }, 0, .long  },
+    .{ .cmovae,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x43 }, 0, .none  },
+    .{ .cmovae,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x43 }, 0, .none  },
+    .{ .cmovae,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x43 }, 0, .long  },
+    .{ .cmovb,   .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x42 }, 0, .none  },
+    .{ .cmovb,   .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x42 }, 0, .none  },
+    .{ .cmovb,   .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x42 }, 0, .long  },
+    .{ .cmovbe,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x46 }, 0, .none  },
+    .{ .cmovbe,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x46 }, 0, .none  },
+    .{ .cmovbe,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x46 }, 0, .long  },
+    .{ .cmovc,   .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x42 }, 0, .none  },
+    .{ .cmovc,   .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x42 }, 0, .none  },
+    .{ .cmovc,   .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x42 }, 0, .long  },
+    .{ .cmove,   .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x44 }, 0, .none  },
+    .{ .cmove,   .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x44 }, 0, .none  },
+    .{ .cmove,   .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x44 }, 0, .long  },
+    .{ .cmovg,   .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4f }, 0, .none  },
+    .{ .cmovg,   .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4f }, 0, .none  },
+    .{ .cmovg,   .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4f }, 0, .long  },
+    .{ .cmovge,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4d }, 0, .none  },
+    .{ .cmovge,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4d }, 0, .none  },
+    .{ .cmovge,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4d }, 0, .long  },
+    .{ .cmovl,   .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4c }, 0, .none  },
+    .{ .cmovl,   .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4c }, 0, .none  },
+    .{ .cmovl,   .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4c }, 0, .long  },
+    .{ .cmovle,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4e }, 0, .none  },
+    .{ .cmovle,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4e }, 0, .none  },
+    .{ .cmovle,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4e }, 0, .long  },
+    .{ .cmovna,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x46 }, 0, .none  },
+    .{ .cmovna,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x46 }, 0, .none  },
+    .{ .cmovna,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x46 }, 0, .long  },
+    .{ .cmovnae, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x42 }, 0, .none  },
+    .{ .cmovnae, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x42 }, 0, .none  },
+    .{ .cmovnae, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x42 }, 0, .long  },
+    .{ .cmovnb,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x43 }, 0, .none  },
+    .{ .cmovnb,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x43 }, 0, .none  },
+    .{ .cmovnb,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x43 }, 0, .long  },
+    .{ .cmovnbe, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x47 }, 0, .none  },
+    .{ .cmovnbe, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x47 }, 0, .none  },
+    .{ .cmovnbe, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x47 }, 0, .long  },
+    .{ .cmovnc,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x43 }, 0, .none  },
+    .{ .cmovnc,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x43 }, 0, .none  },
+    .{ .cmovnc,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x43 }, 0, .long  },
+    .{ .cmovne,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x45 }, 0, .none  },
+    .{ .cmovne,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x45 }, 0, .none  },
+    .{ .cmovne,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x45 }, 0, .long  },
+    .{ .cmovng,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4e }, 0, .none  },
+    .{ .cmovng,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4e }, 0, .none  },
+    .{ .cmovng,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4e }, 0, .long  },
+    .{ .cmovnge, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4c }, 0, .none  },
+    .{ .cmovnge, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4c }, 0, .none  },
+    .{ .cmovnge, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4c }, 0, .long  },
+    .{ .cmovnl,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4d }, 0, .none  },
+    .{ .cmovnl,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4d }, 0, .none  },
+    .{ .cmovnl,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4d }, 0, .long  },
+    .{ .cmovnle, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4f }, 0, .none  },
+    .{ .cmovnle, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4f }, 0, .none  },
+    .{ .cmovnle, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4f }, 0, .long  },
+    .{ .cmovno,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x41 }, 0, .none  },
+    .{ .cmovno,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x41 }, 0, .none  },
+    .{ .cmovno,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x41 }, 0, .long  },
+    .{ .cmovnp,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4b }, 0, .none  },
+    .{ .cmovnp,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4b }, 0, .none  },
+    .{ .cmovnp,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4b }, 0, .long  },
+    .{ .cmovns,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x49 }, 0, .none  },
+    .{ .cmovns,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x49 }, 0, .none  },
+    .{ .cmovns,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x49 }, 0, .long  },
+    .{ .cmovnz,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x45 }, 0, .none  },
+    .{ .cmovnz,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x45 }, 0, .none  },
+    .{ .cmovnz,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x45 }, 0, .long  },
+    .{ .cmovo,   .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x40 }, 0, .none  },
+    .{ .cmovo,   .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x40 }, 0, .none  },
+    .{ .cmovo,   .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x40 }, 0, .long  },
+    .{ .cmovp,   .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4a }, 0, .none  },
+    .{ .cmovp,   .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4a }, 0, .none  },
+    .{ .cmovp,   .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4a }, 0, .long  },
+    .{ .cmovpe,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4a }, 0, .none  },
+    .{ .cmovpe,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4a }, 0, .none  },
+    .{ .cmovpe,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4a }, 0, .long  },
+    .{ .cmovpo,  .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4b }, 0, .none  },
+    .{ .cmovpo,  .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4b }, 0, .none  },
+    .{ .cmovpo,  .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4b }, 0, .long  },
+    .{ .cmovs,   .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x48 }, 0, .none  },
+    .{ .cmovs,   .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x48 }, 0, .none  },
+    .{ .cmovs,   .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x48 }, 0, .long  },
+    .{ .cmovz,   .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x44 }, 0, .none  },
+    .{ .cmovz,   .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x44 }, 0, .none  },
+    .{ .cmovz,   .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x44 }, 0, .long  },
 
-    .{ .cmp, .zi, .al,   .imm8,   .none, .none, 1, 0x3c, 0x00, 0x00, 0, .none  },
-    .{ .cmp, .zi, .ax,   .imm16,  .none, .none, 1, 0x3d, 0x00, 0x00, 0, .none  },
-    .{ .cmp, .zi, .eax,  .imm32,  .none, .none, 1, 0x3d, 0x00, 0x00, 0, .none  },
-    .{ .cmp, .zi, .rax,  .imm32s, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .long  },
-    .{ .cmp, .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 7, .none  },
-    .{ .cmp, .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 7, .rex   },
-    .{ .cmp, .mi, .rm16, .imm16,  .none, .none, 1, 0x81, 0x00, 0x00, 7, .none  },
-    .{ .cmp, .mi, .rm32, .imm32,  .none, .none, 1, 0x81, 0x00, 0x00, 7, .none  },
-    .{ .cmp, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 7, .long  },
-    .{ .cmp, .mi, .rm16, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 7, .none  },
-    .{ .cmp, .mi, .rm32, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 7, .none  },
-    .{ .cmp, .mi, .rm64, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 7, .long  },
-    .{ .cmp, .mr, .rm8,  .r8,     .none, .none, 1, 0x38, 0x00, 0x00, 0, .none  },
-    .{ .cmp, .mr, .rm8,  .r8,     .none, .none, 1, 0x38, 0x00, 0x00, 0, .rex   },
-    .{ .cmp, .mr, .rm16, .r16,    .none, .none, 1, 0x39, 0x00, 0x00, 0, .none  },
-    .{ .cmp, .mr, .rm32, .r32,    .none, .none, 1, 0x39, 0x00, 0x00, 0, .none  },
-    .{ .cmp, .mr, .rm64, .r64,    .none, .none, 1, 0x39, 0x00, 0x00, 0, .long  },
-    .{ .cmp, .rm, .r8,   .rm8,    .none, .none, 1, 0x3a, 0x00, 0x00, 0, .none  },
-    .{ .cmp, .rm, .r8,   .rm8,    .none, .none, 1, 0x3a, 0x00, 0x00, 0, .rex   },
-    .{ .cmp, .rm, .r16,  .rm16,   .none, .none, 1, 0x3b, 0x00, 0x00, 0, .none  },
-    .{ .cmp, .rm, .r32,  .rm32,   .none, .none, 1, 0x3b, 0x00, 0x00, 0, .none  },
-    .{ .cmp, .rm, .r64,  .rm64,   .none, .none, 1, 0x3b, 0x00, 0x00, 0, .long  },
+    .{ .cmp, .zi, .al,   .imm8,   .none, .none, &.{ 0x3c }, 0, .none  },
+    .{ .cmp, .zi, .ax,   .imm16,  .none, .none, &.{ 0x3d }, 0, .none  },
+    .{ .cmp, .zi, .eax,  .imm32,  .none, .none, &.{ 0x3d }, 0, .none  },
+    .{ .cmp, .zi, .rax,  .imm32s, .none, .none, &.{ 0x3d }, 0, .long  },
+    .{ .cmp, .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 7, .none  },
+    .{ .cmp, .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 7, .rex   },
+    .{ .cmp, .mi, .rm16, .imm16,  .none, .none, &.{ 0x81 }, 7, .none  },
+    .{ .cmp, .mi, .rm32, .imm32,  .none, .none, &.{ 0x81 }, 7, .none  },
+    .{ .cmp, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 7, .long  },
+    .{ .cmp, .mi, .rm16, .imm8s,  .none, .none, &.{ 0x83 }, 7, .none  },
+    .{ .cmp, .mi, .rm32, .imm8s,  .none, .none, &.{ 0x83 }, 7, .none  },
+    .{ .cmp, .mi, .rm64, .imm8s,  .none, .none, &.{ 0x83 }, 7, .long  },
+    .{ .cmp, .mr, .rm8,  .r8,     .none, .none, &.{ 0x38 }, 0, .none  },
+    .{ .cmp, .mr, .rm8,  .r8,     .none, .none, &.{ 0x38 }, 0, .rex   },
+    .{ .cmp, .mr, .rm16, .r16,    .none, .none, &.{ 0x39 }, 0, .none  },
+    .{ .cmp, .mr, .rm32, .r32,    .none, .none, &.{ 0x39 }, 0, .none  },
+    .{ .cmp, .mr, .rm64, .r64,    .none, .none, &.{ 0x39 }, 0, .long  },
+    .{ .cmp, .rm, .r8,   .rm8,    .none, .none, &.{ 0x3a }, 0, .none  },
+    .{ .cmp, .rm, .r8,   .rm8,    .none, .none, &.{ 0x3a }, 0, .rex   },
+    .{ .cmp, .rm, .r16,  .rm16,   .none, .none, &.{ 0x3b }, 0, .none  },
+    .{ .cmp, .rm, .r32,  .rm32,   .none, .none, &.{ 0x3b }, 0, .none  },
+    .{ .cmp, .rm, .r64,  .rm64,   .none, .none, &.{ 0x3b }, 0, .long  },
 
-    .{ .div, .m, .rm8,  .none, .none, .none, 1, 0xf6, 0x00, 0x00, 6, .none  },
-    .{ .div, .m, .rm8,  .none, .none, .none, 1, 0xf6, 0x00, 0x00, 6, .rex   },
-    .{ .div, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .none  },
-    .{ .div, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .none  },
-    .{ .div, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .long  },
+    .{ .cmps,  .np, .m8,   .m8,   .none, .none, &.{ 0xa6 }, 0, .none  },
+    .{ .cmps,  .np, .m16,  .m16,  .none, .none, &.{ 0xa7 }, 0, .none  },
+    .{ .cmps,  .np, .m32,  .m32,  .none, .none, &.{ 0xa7 }, 0, .none  },
+    .{ .cmps,  .np, .m64,  .m64,  .none, .none, &.{ 0xa7 }, 0, .long  },
+    .{ .cmpsb, .np, .none, .none, .none, .none, &.{ 0xa6 }, 0, .none  },
+    .{ .cmpsw, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .short },
+    .{ .cmpsd, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .none  },
+    .{ .cmpsq, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .long  },
 
-    .{ .fisttp, .m, .m16, .none, .none, .none, 1, 0xdf, 0x00, 0x00, 1, .fpu },
-    .{ .fisttp, .m, .m32, .none, .none, .none, 1, 0xdb, 0x00, 0x00, 1, .fpu },
-    .{ .fisttp, .m, .m64, .none, .none, .none, 1, 0xdd, 0x00, 0x00, 1, .fpu },
+    .{ .cmpxchg, .mr, .rm8,  .r8,  .none, .none, &.{ 0x0f, 0xb0 }, 0, .none },
+    .{ .cmpxchg, .mr, .rm8,  .r8,  .none, .none, &.{ 0x0f, 0xb0 }, 0, .rex  },
+    .{ .cmpxchg, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xb1 }, 0, .rex  },
+    .{ .cmpxchg, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xb1 }, 0, .rex  },
+    .{ .cmpxchg, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xb1 }, 0, .long },
 
-    .{ .fld, .m, .m32, .none, .none, .none, 1, 0xd9, 0x00, 0x00, 0, .fpu },
-    .{ .fld, .m, .m64, .none, .none, .none, 1, 0xdd, 0x00, 0x00, 0, .fpu },
-    .{ .fld, .m, .m80, .none, .none, .none, 1, 0xdb, 0x00, 0x00, 5, .fpu },
+    .{ .cmpxchg8b , .m, .m64,  .none, .none, .none, &.{ 0x0f, 0xc7 }, 1, .none },
+    .{ .cmpxchg16b, .m, .m128, .none, .none, .none, &.{ 0x0f, 0xc7 }, 1, .long },
 
-    .{ .idiv, .m, .rm8,  .none, .none, .none, 1, 0xf6, 0x00, 0x00, 7, .none  },
-    .{ .idiv, .m, .rm8,  .none, .none, .none, 1, 0xf6, 0x00, 0x00, 7, .rex   },
-    .{ .idiv, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .none  },
-    .{ .idiv, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .none  },
-    .{ .idiv, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .long  },
+    .{ .div, .m, .rm8,  .none, .none, .none, &.{ 0xf6 }, 6, .none  },
+    .{ .div, .m, .rm8,  .none, .none, .none, &.{ 0xf6 }, 6, .rex   },
+    .{ .div, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 6, .none  },
+    .{ .div, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 6, .none  },
+    .{ .div, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 6, .long  },
 
-    .{ .imul, .m,   .rm8,  .none, .none,   .none, 1, 0xf6, 0x00, 0x00, 5, .none  },
-    .{ .imul, .m,   .rm8,  .none, .none,   .none, 1, 0xf6, 0x00, 0x00, 5, .rex   },
-    .{ .imul, .m,   .rm16, .none, .none,   .none, 1, 0xf7, 0x00, 0x00, 5, .none  },
-    .{ .imul, .m,   .rm32, .none, .none,   .none, 1, 0xf7, 0x00, 0x00, 5, .none  },
-    .{ .imul, .m,   .rm64, .none, .none,   .none, 1, 0xf7, 0x00, 0x00, 5, .long  },
-    .{ .imul, .rm,  .r16,  .rm16, .none,   .none, 2, 0x0f, 0xaf, 0x00, 0, .none  },
-    .{ .imul, .rm,  .r32,  .rm32, .none,   .none, 2, 0x0f, 0xaf, 0x00, 0, .none  },
-    .{ .imul, .rm,  .r64,  .rm64, .none,   .none, 2, 0x0f, 0xaf, 0x00, 0, .long  },
-    .{ .imul, .rmi, .r16,  .rm16, .imm8s,  .none, 1, 0x6b, 0x00, 0x00, 0, .none  },
-    .{ .imul, .rmi, .r32,  .rm32, .imm8s,  .none, 1, 0x6b, 0x00, 0x00, 0, .none  },
-    .{ .imul, .rmi, .r64,  .rm64, .imm8s,  .none, 1, 0x6b, 0x00, 0x00, 0, .long  },
-    .{ .imul, .rmi, .r16,  .rm16, .imm16,  .none, 1, 0x69, 0x00, 0x00, 0, .none  },
-    .{ .imul, .rmi, .r32,  .rm32, .imm32,  .none, 1, 0x69, 0x00, 0x00, 0, .none  },
-    .{ .imul, .rmi, .r64,  .rm64, .imm32,  .none, 1, 0x69, 0x00, 0x00, 0, .long  },
+    .{ .fisttp, .m, .m16, .none, .none, .none, &.{ 0xdf }, 1, .fpu },
+    .{ .fisttp, .m, .m32, .none, .none, .none, &.{ 0xdb }, 1, .fpu },
+    .{ .fisttp, .m, .m64, .none, .none, .none, &.{ 0xdd }, 1, .fpu },
 
-    .{ .int3, .np, .none, .none, .none, .none, 1, 0xcc, 0x00, 0x00, 0, .none },
+    .{ .fld, .m, .m32, .none, .none, .none, &.{ 0xd9 }, 0, .fpu },
+    .{ .fld, .m, .m64, .none, .none, .none, &.{ 0xdd }, 0, .fpu },
+    .{ .fld, .m, .m80, .none, .none, .none, &.{ 0xdb }, 5, .fpu },
 
-    .{ .ja,    .d, .rel32, .none, .none, .none, 2, 0x0f, 0x87, 0x00, 0, .none },
-    .{ .jae,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
-    .{ .jb,    .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
-    .{ .jbe,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x86, 0x00, 0, .none },
-    .{ .jc,    .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
-    .{ .jrcxz, .d, .rel32, .none, .none, .none, 1, 0xe3, 0x00, 0x00, 0, .none },
-    .{ .je,    .d, .rel32, .none, .none, .none, 2, 0x0f, 0x84, 0x00, 0, .none },
-    .{ .jg,    .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8f, 0x00, 0, .none },
-    .{ .jge,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8d, 0x00, 0, .none },
-    .{ .jl,    .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8c, 0x00, 0, .none },
-    .{ .jle,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8e, 0x00, 0, .none },
-    .{ .jna,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x86, 0x00, 0, .none },
-    .{ .jnae,  .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
-    .{ .jnb,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
-    .{ .jnbe,  .d, .rel32, .none, .none, .none, 2, 0x0f, 0x87, 0x00, 0, .none },
-    .{ .jnc,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
-    .{ .jne,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x85, 0x00, 0, .none },
-    .{ .jng,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8e, 0x00, 0, .none },
-    .{ .jnge,  .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8c, 0x00, 0, .none },
-    .{ .jnl,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8d, 0x00, 0, .none },
-    .{ .jnle,  .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8f, 0x00, 0, .none },
-    .{ .jno,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x81, 0x00, 0, .none },
-    .{ .jnp,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8b, 0x00, 0, .none },
-    .{ .jns,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x89, 0x00, 0, .none },
-    .{ .jnz,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x85, 0x00, 0, .none },
-    .{ .jo,    .d, .rel32, .none, .none, .none, 2, 0x0f, 0x80, 0x00, 0, .none },
-    .{ .jp,    .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8a, 0x00, 0, .none },
-    .{ .jpe,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8a, 0x00, 0, .none },
-    .{ .jpo,   .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8b, 0x00, 0, .none },
-    .{ .js,    .d, .rel32, .none, .none, .none, 2, 0x0f, 0x88, 0x00, 0, .none },
-    .{ .jz,    .d, .rel32, .none, .none, .none, 2, 0x0f, 0x84, 0x00, 0, .none },
+    .{ .idiv, .m, .rm8,  .none, .none, .none, &.{ 0xf6 }, 7, .none  },
+    .{ .idiv, .m, .rm8,  .none, .none, .none, &.{ 0xf6 }, 7, .rex   },
+    .{ .idiv, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 7, .none  },
+    .{ .idiv, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 7, .none  },
+    .{ .idiv, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 7, .long  },
 
-    .{ .jmp, .d, .rel32, .none, .none, .none, 1, 0xe9, 0x00, 0x00, 0, .none },
-    .{ .jmp, .m, .rm64,  .none, .none, .none, 1, 0xff, 0x00, 0x00, 4, .none },
+    .{ .imul, .m,   .rm8,  .none, .none,  .none, &.{ 0xf6       }, 5, .none  },
+    .{ .imul, .m,   .rm8,  .none, .none,  .none, &.{ 0xf6       }, 5, .rex   },
+    .{ .imul, .m,   .rm16, .none, .none,  .none, &.{ 0xf7       }, 5, .none  },
+    .{ .imul, .m,   .rm32, .none, .none,  .none, &.{ 0xf7       }, 5, .none  },
+    .{ .imul, .m,   .rm64, .none, .none,  .none, &.{ 0xf7       }, 5, .long  },
+    .{ .imul, .rm,  .r16,  .rm16, .none,  .none, &.{ 0x0f, 0xaf }, 0, .none  },
+    .{ .imul, .rm,  .r32,  .rm32, .none,  .none, &.{ 0x0f, 0xaf }, 0, .none  },
+    .{ .imul, .rm,  .r64,  .rm64, .none,  .none, &.{ 0x0f, 0xaf }, 0, .long  },
+    .{ .imul, .rmi, .r16,  .rm16, .imm8s, .none, &.{ 0x6b       }, 0, .none  },
+    .{ .imul, .rmi, .r32,  .rm32, .imm8s, .none, &.{ 0x6b       }, 0, .none  },
+    .{ .imul, .rmi, .r64,  .rm64, .imm8s, .none, &.{ 0x6b       }, 0, .long  },
+    .{ .imul, .rmi, .r16,  .rm16, .imm16, .none, &.{ 0x69       }, 0, .none  },
+    .{ .imul, .rmi, .r32,  .rm32, .imm32, .none, &.{ 0x69       }, 0, .none  },
+    .{ .imul, .rmi, .r64,  .rm64, .imm32, .none, &.{ 0x69       }, 0, .long  },
 
-    .{ .lea, .rm, .r16, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .none  },
-    .{ .lea, .rm, .r32, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .none  },
-    .{ .lea, .rm, .r64, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .long  },
+    .{ .int3, .np, .none, .none, .none, .none, &.{ 0xcc }, 0, .none },
 
-    .{ .mov, .mr, .rm8,   .r8,     .none, .none, 1, 0x88, 0x00, 0x00, 0, .none  },
-    .{ .mov, .mr, .rm8,   .r8,     .none, .none, 1, 0x88, 0x00, 0x00, 0, .rex   },
-    .{ .mov, .mr, .rm16,  .r16,    .none, .none, 1, 0x89, 0x00, 0x00, 0, .none  },
-    .{ .mov, .mr, .rm32,  .r32,    .none, .none, 1, 0x89, 0x00, 0x00, 0, .none  },
-    .{ .mov, .mr, .rm64,  .r64,    .none, .none, 1, 0x89, 0x00, 0x00, 0, .long  },
-    .{ .mov, .rm, .r8,    .rm8,    .none, .none, 1, 0x8a, 0x00, 0x00, 0, .none  },
-    .{ .mov, .rm, .r8,    .rm8,    .none, .none, 1, 0x8a, 0x00, 0x00, 0, .rex   },
-    .{ .mov, .rm, .r16,   .rm16,   .none, .none, 1, 0x8b, 0x00, 0x00, 0, .none  },
-    .{ .mov, .rm, .r32,   .rm32,   .none, .none, 1, 0x8b, 0x00, 0x00, 0, .none  },
-    .{ .mov, .rm, .r64,   .rm64,   .none, .none, 1, 0x8b, 0x00, 0x00, 0, .long  },
-    .{ .mov, .mr, .rm16,  .sreg,   .none, .none, 1, 0x8c, 0x00, 0x00, 0, .none  },
-    .{ .mov, .mr, .rm64,  .sreg,   .none, .none, 1, 0x8c, 0x00, 0x00, 0, .long  },
-    .{ .mov, .rm, .sreg,  .rm16,   .none, .none, 1, 0x8e, 0x00, 0x00, 0, .none  },
-    .{ .mov, .rm, .sreg,  .rm64,   .none, .none, 1, 0x8e, 0x00, 0x00, 0, .long  },
-    .{ .mov, .fd, .al,    .moffs,  .none, .none, 1, 0xa0, 0x00, 0x00, 0, .none  },
-    .{ .mov, .fd, .ax,    .moffs,  .none, .none, 1, 0xa1, 0x00, 0x00, 0, .none  },
-    .{ .mov, .fd, .eax,   .moffs,  .none, .none, 1, 0xa1, 0x00, 0x00, 0, .none  },
-    .{ .mov, .fd, .rax,   .moffs,  .none, .none, 1, 0xa1, 0x00, 0x00, 0, .long  },
-    .{ .mov, .td, .moffs, .al,     .none, .none, 1, 0xa2, 0x00, 0x00, 0, .none  },
-    .{ .mov, .td, .moffs, .ax,     .none, .none, 1, 0xa3, 0x00, 0x00, 0, .none  },
-    .{ .mov, .td, .moffs, .eax,    .none, .none, 1, 0xa3, 0x00, 0x00, 0, .none  },
-    .{ .mov, .td, .moffs, .rax,    .none, .none, 1, 0xa3, 0x00, 0x00, 0, .long  },
-    .{ .mov, .oi, .r8,    .imm8,   .none, .none, 1, 0xb0, 0x00, 0x00, 0, .none  },
-    .{ .mov, .oi, .r8,    .imm8,   .none, .none, 1, 0xb0, 0x00, 0x00, 0, .rex   },
-    .{ .mov, .oi, .r16,   .imm16,  .none, .none, 1, 0xb8, 0x00, 0x00, 0, .none  },
-    .{ .mov, .oi, .r32,   .imm32,  .none, .none, 1, 0xb8, 0x00, 0x00, 0, .none  },
-    .{ .mov, .oi, .r64,   .imm64,  .none, .none, 1, 0xb8, 0x00, 0x00, 0, .long  },
-    .{ .mov, .mi, .rm8,   .imm8,   .none, .none, 1, 0xc6, 0x00, 0x00, 0, .none  },
-    .{ .mov, .mi, .rm8,   .imm8,   .none, .none, 1, 0xc6, 0x00, 0x00, 0, .rex   },
-    .{ .mov, .mi, .rm16,  .imm16,  .none, .none, 1, 0xc7, 0x00, 0x00, 0, .none  },
-    .{ .mov, .mi, .rm32,  .imm32,  .none, .none, 1, 0xc7, 0x00, 0x00, 0, .none  },
-    .{ .mov, .mi, .rm64,  .imm32s, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .long  },
+    .{ .ja,    .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x87 }, 0, .none },
+    .{ .jae,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x83 }, 0, .none },
+    .{ .jb,    .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x82 }, 0, .none },
+    .{ .jbe,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x86 }, 0, .none },
+    .{ .jc,    .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x82 }, 0, .none },
+    .{ .jrcxz, .d, .rel32, .none, .none, .none, &.{ 0xe3       }, 0, .none },
+    .{ .je,    .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x84 }, 0, .none },
+    .{ .jg,    .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8f }, 0, .none },
+    .{ .jge,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8d }, 0, .none },
+    .{ .jl,    .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8c }, 0, .none },
+    .{ .jle,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8e }, 0, .none },
+    .{ .jna,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x86 }, 0, .none },
+    .{ .jnae,  .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x82 }, 0, .none },
+    .{ .jnb,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x83 }, 0, .none },
+    .{ .jnbe,  .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x87 }, 0, .none },
+    .{ .jnc,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x83 }, 0, .none },
+    .{ .jne,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x85 }, 0, .none },
+    .{ .jng,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8e }, 0, .none },
+    .{ .jnge,  .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8c }, 0, .none },
+    .{ .jnl,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8d }, 0, .none },
+    .{ .jnle,  .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8f }, 0, .none },
+    .{ .jno,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x81 }, 0, .none },
+    .{ .jnp,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8b }, 0, .none },
+    .{ .jns,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x89 }, 0, .none },
+    .{ .jnz,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x85 }, 0, .none },
+    .{ .jo,    .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x80 }, 0, .none },
+    .{ .jp,    .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8a }, 0, .none },
+    .{ .jpe,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8a }, 0, .none },
+    .{ .jpo,   .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8b }, 0, .none },
+    .{ .js,    .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x88 }, 0, .none },
+    .{ .jz,    .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x84 }, 0, .none },
 
-    .{ .movsx, .rm, .r16, .rm8,  .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .none  },
-    .{ .movsx, .rm, .r16, .rm8,  .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .rex   },
-    .{ .movsx, .rm, .r32, .rm8,  .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .none  },
-    .{ .movsx, .rm, .r32, .rm8,  .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .rex   },
-    .{ .movsx, .rm, .r64, .rm8,  .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .long  },
-    .{ .movsx, .rm, .r32, .rm16, .none, .none, 2, 0x0f, 0xbf, 0x00, 0, .none  },
-    .{ .movsx, .rm, .r64, .rm16, .none, .none, 2, 0x0f, 0xbf, 0x00, 0, .long  },
+    .{ .jmp, .d, .rel32, .none, .none, .none, &.{ 0xe9 }, 0, .none },
+    .{ .jmp, .m, .rm64,  .none, .none, .none, &.{ 0xff }, 4, .none },
+
+    .{ .lea, .rm, .r16, .m, .none, .none, &.{ 0x8d }, 0, .none  },
+    .{ .lea, .rm, .r32, .m, .none, .none, &.{ 0x8d }, 0, .none  },
+    .{ .lea, .rm, .r64, .m, .none, .none, &.{ 0x8d }, 0, .long  },
+
+    .{ .lfence, .np, .none, .none, .none, .none, &.{ 0x0f, 0xae, 0xe8 }, 0, .none },
+
+    .{ .lods,  .np, .m8,   .none, .none, .none, &.{ 0xac }, 0, .none  },
+    .{ .lods,  .np, .m16,  .none, .none, .none, &.{ 0xad }, 0, .none  },
+    .{ .lods,  .np, .m32,  .none, .none, .none, &.{ 0xad }, 0, .none  },
+    .{ .lods,  .np, .m64,  .none, .none, .none, &.{ 0xad }, 0, .long  },
+    .{ .lodsb, .np, .none, .none, .none, .none, &.{ 0xac }, 0, .none  },
+    .{ .lodsw, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .short },
+    .{ .lodsd, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .none  },
+    .{ .lodsq, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .long  },
+
+    .{ .lzcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
+    .{ .lzcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
+    .{ .lzcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .long },
+
+    .{ .mfence, .np, .none, .none, .none, .none, &.{ 0x0f, 0xae, 0xf0 }, 0, .none },
+
+    .{ .mov, .mr, .rm8,   .r8,     .none, .none, &.{ 0x88 }, 0, .none  },
+    .{ .mov, .mr, .rm8,   .r8,     .none, .none, &.{ 0x88 }, 0, .rex   },
+    .{ .mov, .mr, .rm16,  .r16,    .none, .none, &.{ 0x89 }, 0, .none  },
+    .{ .mov, .mr, .rm32,  .r32,    .none, .none, &.{ 0x89 }, 0, .none  },
+    .{ .mov, .mr, .rm64,  .r64,    .none, .none, &.{ 0x89 }, 0, .long  },
+    .{ .mov, .rm, .r8,    .rm8,    .none, .none, &.{ 0x8a }, 0, .none  },
+    .{ .mov, .rm, .r8,    .rm8,    .none, .none, &.{ 0x8a }, 0, .rex   },
+    .{ .mov, .rm, .r16,   .rm16,   .none, .none, &.{ 0x8b }, 0, .none  },
+    .{ .mov, .rm, .r32,   .rm32,   .none, .none, &.{ 0x8b }, 0, .none  },
+    .{ .mov, .rm, .r64,   .rm64,   .none, .none, &.{ 0x8b }, 0, .long  },
+    .{ .mov, .mr, .rm16,  .sreg,   .none, .none, &.{ 0x8c }, 0, .none  },
+    .{ .mov, .mr, .rm64,  .sreg,   .none, .none, &.{ 0x8c }, 0, .long  },
+    .{ .mov, .rm, .sreg,  .rm16,   .none, .none, &.{ 0x8e }, 0, .none  },
+    .{ .mov, .rm, .sreg,  .rm64,   .none, .none, &.{ 0x8e }, 0, .long  },
+    .{ .mov, .fd, .al,    .moffs,  .none, .none, &.{ 0xa0 }, 0, .none  },
+    .{ .mov, .fd, .ax,    .moffs,  .none, .none, &.{ 0xa1 }, 0, .none  },
+    .{ .mov, .fd, .eax,   .moffs,  .none, .none, &.{ 0xa1 }, 0, .none  },
+    .{ .mov, .fd, .rax,   .moffs,  .none, .none, &.{ 0xa1 }, 0, .long  },
+    .{ .mov, .td, .moffs, .al,     .none, .none, &.{ 0xa2 }, 0, .none  },
+    .{ .mov, .td, .moffs, .ax,     .none, .none, &.{ 0xa3 }, 0, .none  },
+    .{ .mov, .td, .moffs, .eax,    .none, .none, &.{ 0xa3 }, 0, .none  },
+    .{ .mov, .td, .moffs, .rax,    .none, .none, &.{ 0xa3 }, 0, .long  },
+    .{ .mov, .oi, .r8,    .imm8,   .none, .none, &.{ 0xb0 }, 0, .none  },
+    .{ .mov, .oi, .r8,    .imm8,   .none, .none, &.{ 0xb0 }, 0, .rex   },
+    .{ .mov, .oi, .r16,   .imm16,  .none, .none, &.{ 0xb8 }, 0, .none  },
+    .{ .mov, .oi, .r32,   .imm32,  .none, .none, &.{ 0xb8 }, 0, .none  },
+    .{ .mov, .oi, .r64,   .imm64,  .none, .none, &.{ 0xb8 }, 0, .long  },
+    .{ .mov, .mi, .rm8,   .imm8,   .none, .none, &.{ 0xc6 }, 0, .none  },
+    .{ .mov, .mi, .rm8,   .imm8,   .none, .none, &.{ 0xc6 }, 0, .rex   },
+    .{ .mov, .mi, .rm16,  .imm16,  .none, .none, &.{ 0xc7 }, 0, .none  },
+    .{ .mov, .mi, .rm32,  .imm32,  .none, .none, &.{ 0xc7 }, 0, .none  },
+    .{ .mov, .mi, .rm64,  .imm32s, .none, .none, &.{ 0xc7 }, 0, .long  },
+
+    .{ .movbe, .rm, .r16, .m16, .none, .none, &.{ 0x0f, 0x38, 0xf0 }, 0, .none },
+    .{ .movbe, .rm, .r32, .m32, .none, .none, &.{ 0x0f, 0x38, 0xf0 }, 0, .none },
+    .{ .movbe, .rm, .r64, .m64, .none, .none, &.{ 0x0f, 0x38, 0xf0 }, 0, .long },
+    .{ .movbe, .mr, .m16, .r16, .none, .none, &.{ 0x0f, 0x38, 0xf1 }, 0, .none },
+    .{ .movbe, .mr, .m32, .r32, .none, .none, &.{ 0x0f, 0x38, 0xf1 }, 0, .none },
+    .{ .movbe, .mr, .m64, .r64, .none, .none, &.{ 0x0f, 0x38, 0xf1 }, 0, .long },
+
+    .{ .movs,  .np, .m8,   .m8,   .none, .none, &.{ 0xa4 }, 0, .none  },
+    .{ .movs,  .np, .m16,  .m16,  .none, .none, &.{ 0xa5 }, 0, .none  },
+    .{ .movs,  .np, .m32,  .m32,  .none, .none, &.{ 0xa5 }, 0, .none  },
+    .{ .movs,  .np, .m64,  .m64,  .none, .none, &.{ 0xa5 }, 0, .long  },
+    .{ .movsb, .np, .none, .none, .none, .none, &.{ 0xa4 }, 0, .none  },
+    .{ .movsw, .np, .none, .none, .none, .none, &.{ 0xa5 }, 0, .short },
+    .{ .movsd, .np, .none, .none, .none, .none, &.{ 0xa5 }, 0, .none  },
+    .{ .movsq, .np, .none, .none, .none, .none, &.{ 0xa5 }, 0, .long  },
+
+    .{ .movsx, .rm, .r16, .rm8,  .none, .none, &.{ 0x0f, 0xbe }, 0, .none  },
+    .{ .movsx, .rm, .r16, .rm8,  .none, .none, &.{ 0x0f, 0xbe }, 0, .rex   },
+    .{ .movsx, .rm, .r32, .rm8,  .none, .none, &.{ 0x0f, 0xbe }, 0, .none  },
+    .{ .movsx, .rm, .r32, .rm8,  .none, .none, &.{ 0x0f, 0xbe }, 0, .rex   },
+    .{ .movsx, .rm, .r64, .rm8,  .none, .none, &.{ 0x0f, 0xbe }, 0, .long  },
+    .{ .movsx, .rm, .r32, .rm16, .none, .none, &.{ 0x0f, 0xbf }, 0, .none  },
+    .{ .movsx, .rm, .r64, .rm16, .none, .none, &.{ 0x0f, 0xbf }, 0, .long  },
 
     // This instruction is discouraged.
-    .{ .movsxd, .rm, .r32, .rm32, .none, .none, 1, 0x63, 0x00, 0x00, 0, .none  },
-    .{ .movsxd, .rm, .r64, .rm32, .none, .none, 1, 0x63, 0x00, 0x00, 0, .long  },
+    .{ .movsxd, .rm, .r32, .rm32, .none, .none, &.{ 0x63 }, 0, .none  },
+    .{ .movsxd, .rm, .r64, .rm32, .none, .none, &.{ 0x63 }, 0, .long  },
 
-    .{ .movzx, .rm, .r16, .rm8,  .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .none  },
-    .{ .movzx, .rm, .r32, .rm8,  .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .none  },
-    .{ .movzx, .rm, .r64, .rm8,  .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .long  },
-    .{ .movzx, .rm, .r32, .rm16, .none, .none, 2, 0x0f, 0xb7, 0x00, 0, .none  },
-    .{ .movzx, .rm, .r64, .rm16, .none, .none, 2, 0x0f, 0xb7, 0x00, 0, .long  },
+    .{ .movzx, .rm, .r16, .rm8,  .none, .none, &.{ 0x0f, 0xb6 }, 0, .none  },
+    .{ .movzx, .rm, .r32, .rm8,  .none, .none, &.{ 0x0f, 0xb6 }, 0, .none  },
+    .{ .movzx, .rm, .r64, .rm8,  .none, .none, &.{ 0x0f, 0xb6 }, 0, .long  },
+    .{ .movzx, .rm, .r32, .rm16, .none, .none, &.{ 0x0f, 0xb7 }, 0, .none  },
+    .{ .movzx, .rm, .r64, .rm16, .none, .none, &.{ 0x0f, 0xb7 }, 0, .long  },
 
-    .{ .mul, .m, .rm8,  .none, .none, .none, 1, 0xf6, 0x00, 0x00, 4, .none  },
-    .{ .mul, .m, .rm8,  .none, .none, .none, 1, 0xf6, 0x00, 0x00, 4, .rex   },
-    .{ .mul, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .none  },
-    .{ .mul, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .none  },
-    .{ .mul, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .long  },
+    .{ .mul, .m, .rm8,  .none, .none, .none, &.{ 0xf6 }, 4, .none  },
+    .{ .mul, .m, .rm8,  .none, .none, .none, &.{ 0xf6 }, 4, .rex   },
+    .{ .mul, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 4, .none  },
+    .{ .mul, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 4, .none  },
+    .{ .mul, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 4, .long  },
 
-    .{ .nop, .np, .none, .none, .none, .none, 1, 0x90, 0x00, 0x00, 0, .none },
+    .{ .neg, .m, .rm8,  .none, .none, .none, &.{ 0xf6 }, 3, .none },
+    .{ .neg, .m, .rm8,  .none, .none, .none, &.{ 0xf6 }, 3, .rex  },
+    .{ .neg, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 3, .none },
+    .{ .neg, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 3, .none },
+    .{ .neg, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 3, .long },
 
-    .{ .@"or", .zi, .al,   .imm8,   .none, .none, 1, 0x0c, 0x00, 0x00, 0, .none  },
-    .{ .@"or", .zi, .ax,   .imm16,  .none, .none, 1, 0x0d, 0x00, 0x00, 0, .none  },
-    .{ .@"or", .zi, .eax,  .imm32,  .none, .none, 1, 0x0d, 0x00, 0x00, 0, .none  },
-    .{ .@"or", .zi, .rax,  .imm32s, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .long  },
-    .{ .@"or", .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 1, .none  },
-    .{ .@"or", .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 1, .rex   },
-    .{ .@"or", .mi, .rm16, .imm16,  .none, .none, 1, 0x81, 0x00, 0x00, 1, .none  },
-    .{ .@"or", .mi, .rm32, .imm32,  .none, .none, 1, 0x81, 0x00, 0x00, 1, .none  },
-    .{ .@"or", .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 1, .long  },
-    .{ .@"or", .mi, .rm16, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 1, .none  },
-    .{ .@"or", .mi, .rm32, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 1, .none  },
-    .{ .@"or", .mi, .rm64, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 1, .long  },
-    .{ .@"or", .mr, .rm8,  .r8,     .none, .none, 1, 0x08, 0x00, 0x00, 0, .none  },
-    .{ .@"or", .mr, .rm8,  .r8,     .none, .none, 1, 0x08, 0x00, 0x00, 0, .rex   },
-    .{ .@"or", .mr, .rm16, .r16,    .none, .none, 1, 0x09, 0x00, 0x00, 0, .none  },
-    .{ .@"or", .mr, .rm32, .r32,    .none, .none, 1, 0x09, 0x00, 0x00, 0, .none  },
-    .{ .@"or", .mr, .rm64, .r64,    .none, .none, 1, 0x09, 0x00, 0x00, 0, .long  },
-    .{ .@"or", .rm, .r8,   .rm8,    .none, .none, 1, 0x0a, 0x00, 0x00, 0, .none  },
-    .{ .@"or", .rm, .r8,   .rm8,    .none, .none, 1, 0x0a, 0x00, 0x00, 0, .rex   },
-    .{ .@"or", .rm, .r16,  .rm16,   .none, .none, 1, 0x0b, 0x00, 0x00, 0, .none  },
-    .{ .@"or", .rm, .r32,  .rm32,   .none, .none, 1, 0x0b, 0x00, 0x00, 0, .none  },
-    .{ .@"or", .rm, .r64,  .rm64,   .none, .none, 1, 0x0b, 0x00, 0x00, 0, .long  },
+    .{ .nop, .np, .none, .none, .none, .none, &.{ 0x90 }, 0, .none },
 
-    .{ .pop, .o, .r16,  .none, .none, .none, 1, 0x58, 0x00, 0x00, 0, .none  },
-    .{ .pop, .o, .r64,  .none, .none, .none, 1, 0x58, 0x00, 0x00, 0, .none  },
-    .{ .pop, .m, .rm16, .none, .none, .none, 1, 0x8f, 0x00, 0x00, 0, .none  },
-    .{ .pop, .m, .rm64, .none, .none, .none, 1, 0x8f, 0x00, 0x00, 0, .none  },
+    .{ .not, .m, .rm8,  .none, .none, .none, &.{ 0xf6 }, 2, .none },
+    .{ .not, .m, .rm8,  .none, .none, .none, &.{ 0xf6 }, 2, .rex  },
+    .{ .not, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 2, .none },
+    .{ .not, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 2, .none },
+    .{ .not, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 2, .long },
 
-    .{ .push, .o, .r16,   .none, .none, .none, 1, 0x50, 0x00, 0x00, 0, .none  },
-    .{ .push, .o, .r64,   .none, .none, .none, 1, 0x50, 0x00, 0x00, 0, .none  },
-    .{ .push, .m, .rm16,  .none, .none, .none, 1, 0xff, 0x00, 0x00, 6, .none  },
-    .{ .push, .m, .rm64,  .none, .none, .none, 1, 0xff, 0x00, 0x00, 6, .none  },
-    .{ .push, .i, .imm8,  .none, .none, .none, 1, 0x6a, 0x00, 0x00, 0, .none  },
-    .{ .push, .i, .imm16, .none, .none, .none, 1, 0x68, 0x00, 0x00, 0, .none  },
-    .{ .push, .i, .imm32, .none, .none, .none, 1, 0x68, 0x00, 0x00, 0, .none  },
+    .{ .@"or", .zi, .al,   .imm8,   .none, .none, &.{ 0x0c }, 0, .none  },
+    .{ .@"or", .zi, .ax,   .imm16,  .none, .none, &.{ 0x0d }, 0, .none  },
+    .{ .@"or", .zi, .eax,  .imm32,  .none, .none, &.{ 0x0d }, 0, .none  },
+    .{ .@"or", .zi, .rax,  .imm32s, .none, .none, &.{ 0x0d }, 0, .long  },
+    .{ .@"or", .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 1, .none  },
+    .{ .@"or", .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 1, .rex   },
+    .{ .@"or", .mi, .rm16, .imm16,  .none, .none, &.{ 0x81 }, 1, .none  },
+    .{ .@"or", .mi, .rm32, .imm32,  .none, .none, &.{ 0x81 }, 1, .none  },
+    .{ .@"or", .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 1, .long  },
+    .{ .@"or", .mi, .rm16, .imm8s,  .none, .none, &.{ 0x83 }, 1, .none  },
+    .{ .@"or", .mi, .rm32, .imm8s,  .none, .none, &.{ 0x83 }, 1, .none  },
+    .{ .@"or", .mi, .rm64, .imm8s,  .none, .none, &.{ 0x83 }, 1, .long  },
+    .{ .@"or", .mr, .rm8,  .r8,     .none, .none, &.{ 0x08 }, 0, .none  },
+    .{ .@"or", .mr, .rm8,  .r8,     .none, .none, &.{ 0x08 }, 0, .rex   },
+    .{ .@"or", .mr, .rm16, .r16,    .none, .none, &.{ 0x09 }, 0, .none  },
+    .{ .@"or", .mr, .rm32, .r32,    .none, .none, &.{ 0x09 }, 0, .none  },
+    .{ .@"or", .mr, .rm64, .r64,    .none, .none, &.{ 0x09 }, 0, .long  },
+    .{ .@"or", .rm, .r8,   .rm8,    .none, .none, &.{ 0x0a }, 0, .none  },
+    .{ .@"or", .rm, .r8,   .rm8,    .none, .none, &.{ 0x0a }, 0, .rex   },
+    .{ .@"or", .rm, .r16,  .rm16,   .none, .none, &.{ 0x0b }, 0, .none  },
+    .{ .@"or", .rm, .r32,  .rm32,   .none, .none, &.{ 0x0b }, 0, .none  },
+    .{ .@"or", .rm, .r64,  .rm64,   .none, .none, &.{ 0x0b }, 0, .long  },
 
-    .{ .ret, .np, .none, .none, .none, .none, 1, 0xc3, 0x00, 0x00, 0, .none },
+    .{ .pop, .o, .r16,  .none, .none, .none, &.{ 0x58 }, 0, .none  },
+    .{ .pop, .o, .r64,  .none, .none, .none, &.{ 0x58 }, 0, .none  },
+    .{ .pop, .m, .rm16, .none, .none, .none, &.{ 0x8f }, 0, .none  },
+    .{ .pop, .m, .rm64, .none, .none, .none, &.{ 0x8f }, 0, .none  },
 
-    .{ .sal, .m1, .rm8,  .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .none  },
-    .{ .sal, .m1, .rm8,  .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .rex   },
-    .{ .sal, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none  },
-    .{ .sal, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none  },
-    .{ .sal, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .long  },
-    .{ .sal, .mc, .rm8,  .cl,    .none, .none, 1, 0xd2, 0x00, 0x00, 4, .none  },
-    .{ .sal, .mc, .rm8,  .cl,    .none, .none, 1, 0xd2, 0x00, 0x00, 4, .rex   },
-    .{ .sal, .mc, .rm16, .cl,    .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none  },
-    .{ .sal, .mc, .rm32, .cl,    .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none  },
-    .{ .sal, .mc, .rm64, .cl,    .none, .none, 1, 0xd3, 0x00, 0x00, 4, .long  },
-    .{ .sal, .mi, .rm8,  .imm8,  .none, .none, 1, 0xc0, 0x00, 0x00, 4, .none  },
-    .{ .sal, .mi, .rm8,  .imm8,  .none, .none, 1, 0xc0, 0x00, 0x00, 4, .rex   },
-    .{ .sal, .mi, .rm16, .imm8,  .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none  },
-    .{ .sal, .mi, .rm32, .imm8,  .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none  },
-    .{ .sal, .mi, .rm64, .imm8,  .none, .none, 1, 0xc1, 0x00, 0x00, 4, .long  },
+    .{ .popcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },
+    .{ .popcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },
+    .{ .popcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long },
 
-    .{ .sar, .m1, .rm8,  .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 7, .none  },
-    .{ .sar, .m1, .rm8,  .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 7, .rex   },
-    .{ .sar, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .none  },
-    .{ .sar, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .none  },
-    .{ .sar, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .long  },
-    .{ .sar, .mc, .rm8,  .cl,    .none, .none, 1, 0xd2, 0x00, 0x00, 7, .none  },
-    .{ .sar, .mc, .rm8,  .cl,    .none, .none, 1, 0xd2, 0x00, 0x00, 7, .rex   },
-    .{ .sar, .mc, .rm16, .cl,    .none, .none, 1, 0xd3, 0x00, 0x00, 7, .none  },
-    .{ .sar, .mc, .rm32, .cl,    .none, .none, 1, 0xd3, 0x00, 0x00, 7, .none  },
-    .{ .sar, .mc, .rm64, .cl,    .none, .none, 1, 0xd3, 0x00, 0x00, 7, .long  },
-    .{ .sar, .mi, .rm8,  .imm8,  .none, .none, 1, 0xc0, 0x00, 0x00, 7, .none  },
-    .{ .sar, .mi, .rm8,  .imm8,  .none, .none, 1, 0xc0, 0x00, 0x00, 7, .rex   },
-    .{ .sar, .mi, .rm16, .imm8,  .none, .none, 1, 0xc1, 0x00, 0x00, 7, .none  },
-    .{ .sar, .mi, .rm32, .imm8,  .none, .none, 1, 0xc1, 0x00, 0x00, 7, .none  },
-    .{ .sar, .mi, .rm64, .imm8,  .none, .none, 1, 0xc1, 0x00, 0x00, 7, .long  },
+    .{ .push, .o, .r16,   .none, .none, .none, &.{ 0x50 }, 0, .none  },
+    .{ .push, .o, .r64,   .none, .none, .none, &.{ 0x50 }, 0, .none  },
+    .{ .push, .m, .rm16,  .none, .none, .none, &.{ 0xff }, 6, .none  },
+    .{ .push, .m, .rm64,  .none, .none, .none, &.{ 0xff }, 6, .none  },
+    .{ .push, .i, .imm8,  .none, .none, .none, &.{ 0x6a }, 0, .none  },
+    .{ .push, .i, .imm16, .none, .none, .none, &.{ 0x68 }, 0, .none  },
+    .{ .push, .i, .imm32, .none, .none, .none, &.{ 0x68 }, 0, .none  },
 
-    .{ .sbb, .zi, .al,   .imm8,   .none, .none, 1, 0x1c, 0x00, 0x00, 0, .none  },
-    .{ .sbb, .zi, .ax,   .imm16,  .none, .none, 1, 0x1d, 0x00, 0x00, 0, .none  },
-    .{ .sbb, .zi, .eax,  .imm32,  .none, .none, 1, 0x1d, 0x00, 0x00, 0, .none  },
-    .{ .sbb, .zi, .rax,  .imm32s, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .long  },
-    .{ .sbb, .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 3, .none  },
-    .{ .sbb, .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 3, .rex   },
-    .{ .sbb, .mi, .rm16, .imm16,  .none, .none, 1, 0x81, 0x00, 0x00, 3, .none  },
-    .{ .sbb, .mi, .rm32, .imm32,  .none, .none, 1, 0x81, 0x00, 0x00, 3, .none  },
-    .{ .sbb, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 3, .long  },
-    .{ .sbb, .mi, .rm16, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 3, .none  },
-    .{ .sbb, .mi, .rm32, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 3, .none  },
-    .{ .sbb, .mi, .rm64, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 3, .long  },
-    .{ .sbb, .mr, .rm8,  .r8,     .none, .none, 1, 0x18, 0x00, 0x00, 0, .none  },
-    .{ .sbb, .mr, .rm8,  .r8,     .none, .none, 1, 0x18, 0x00, 0x00, 0, .rex   },
-    .{ .sbb, .mr, .rm16, .r16,    .none, .none, 1, 0x19, 0x00, 0x00, 0, .none  },
-    .{ .sbb, .mr, .rm32, .r32,    .none, .none, 1, 0x19, 0x00, 0x00, 0, .none  },
-    .{ .sbb, .mr, .rm64, .r64,    .none, .none, 1, 0x19, 0x00, 0x00, 0, .long  },
-    .{ .sbb, .rm, .r8,   .rm8,    .none, .none, 1, 0x1a, 0x00, 0x00, 0, .none  },
-    .{ .sbb, .rm, .r8,   .rm8,    .none, .none, 1, 0x1a, 0x00, 0x00, 0, .rex   },
-    .{ .sbb, .rm, .r16,  .rm16,   .none, .none, 1, 0x1b, 0x00, 0x00, 0, .none  },
-    .{ .sbb, .rm, .r32,  .rm32,   .none, .none, 1, 0x1b, 0x00, 0x00, 0, .none  },
-    .{ .sbb, .rm, .r64,  .rm64,   .none, .none, 1, 0x1b, 0x00, 0x00, 0, .long  },
+    .{ .ret, .np, .none, .none, .none, .none, &.{ 0xc3 }, 0, .none },
 
-    .{ .seta,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .none },
-    .{ .seta,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .rex  },
-    .{ .setae,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
-    .{ .setae,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex  },
-    .{ .setb,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
-    .{ .setb,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex  },
-    .{ .setbe,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .none },
-    .{ .setbe,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .rex  },
-    .{ .setc,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
-    .{ .setc,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex  },
-    .{ .sete,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .none },
-    .{ .sete,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .rex  },
-    .{ .setg,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .none },
-    .{ .setg,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .rex  },
-    .{ .setge,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .none },
-    .{ .setge,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .rex  },
-    .{ .setl,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .none },
-    .{ .setl,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .rex  },
-    .{ .setle,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .none },
-    .{ .setle,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .rex  },
-    .{ .setna,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .none },
-    .{ .setna,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .rex  },
-    .{ .setnae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
-    .{ .setnae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex  },
-    .{ .setnb,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
-    .{ .setnb,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex  },
-    .{ .setnbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .none },
-    .{ .setnbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .rex  },
-    .{ .setnc,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
-    .{ .setnc,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex  },
-    .{ .setne,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .none },
-    .{ .setne,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .rex  },
-    .{ .setng,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .none },
-    .{ .setng,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .rex  },
-    .{ .setnge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .none },
-    .{ .setnge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .rex  },
-    .{ .setnl,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .none },
-    .{ .setnl,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .rex  },
-    .{ .setnle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .none },
-    .{ .setnle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .rex  },
-    .{ .setno,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x91, 0x00, 0, .none },
-    .{ .setno,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x91, 0x00, 0, .rex  },
-    .{ .setnp,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .none },
-    .{ .setnp,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .rex  },
-    .{ .setns,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x99, 0x00, 0, .none },
-    .{ .setns,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x99, 0x00, 0, .rex  },
-    .{ .setnz,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .none },
-    .{ .setnz,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .rex  },
-    .{ .seto,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x90, 0x00, 0, .none },
-    .{ .seto,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x90, 0x00, 0, .rex  },
-    .{ .setp,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .none },
-    .{ .setp,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .rex  },
-    .{ .setpe,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .none },
-    .{ .setpe,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .rex  },
-    .{ .setpo,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .none },
-    .{ .setpo,  .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .rex  },
-    .{ .sets,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x98, 0x00, 0, .none },
-    .{ .sets,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x98, 0x00, 0, .rex  },
-    .{ .setz,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .none },
-    .{ .setz,   .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .rex  },
+    .{ .rcl, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 2, .none },
+    .{ .rcl, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 2, .rex  },
+    .{ .rcl, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 2, .none },
+    .{ .rcl, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 2, .rex  },
+    .{ .rcl, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 2, .none },
+    .{ .rcl, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 2, .rex  },
+    .{ .rcl, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 2, .none },
+    .{ .rcl, .mc, .rm16, .cl,    .none, .none, &.{ 0xd3 }, 2, .none },
+    .{ .rcl, .mi, .rm16, .imm8,  .none, .none, &.{ 0xc1 }, 2, .none },
+    .{ .rcl, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 2, .none },
+    .{ .rcl, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 2, .long },
+    .{ .rcl, .mc, .rm32, .cl,    .none, .none, &.{ 0xd3 }, 2, .none },
+    .{ .rcl, .mc, .rm64, .cl,    .none, .none, &.{ 0xd3 }, 2, .long },
+    .{ .rcl, .mi, .rm32, .imm8,  .none, .none, &.{ 0xc1 }, 2, .none },
+    .{ .rcl, .mi, .rm64, .imm8,  .none, .none, &.{ 0xc1 }, 2, .long },
 
-    .{ .shl, .m1, .rm8,  .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .none  },
-    .{ .shl, .m1, .rm8,  .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .rex   },
-    .{ .shl, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none  },
-    .{ .shl, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none  },
-    .{ .shl, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .long  },
-    .{ .shl, .mc, .rm8,  .cl,    .none, .none, 1, 0xd2, 0x00, 0x00, 4, .none  },
-    .{ .shl, .mc, .rm8,  .cl,    .none, .none, 1, 0xd2, 0x00, 0x00, 4, .rex   },
-    .{ .shl, .mc, .rm16, .cl,    .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none  },
-    .{ .shl, .mc, .rm32, .cl,    .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none  },
-    .{ .shl, .mc, .rm64, .cl,    .none, .none, 1, 0xd3, 0x00, 0x00, 4, .long  },
-    .{ .shl, .mi, .rm8,  .imm8,  .none, .none, 1, 0xc0, 0x00, 0x00, 4, .none  },
-    .{ .shl, .mi, .rm8,  .imm8,  .none, .none, 1, 0xc0, 0x00, 0x00, 4, .rex   },
-    .{ .shl, .mi, .rm16, .imm8,  .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none  },
-    .{ .shl, .mi, .rm32, .imm8,  .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none  },
-    .{ .shl, .mi, .rm64, .imm8,  .none, .none, 1, 0xc1, 0x00, 0x00, 4, .long  },
+    .{ .rcr, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 3, .none },
+    .{ .rcr, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 3, .rex  },
+    .{ .rcr, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 3, .none },
+    .{ .rcr, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 3, .rex  },
+    .{ .rcr, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 3, .none },
+    .{ .rcr, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 3, .rex  },
+    .{ .rcr, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 3, .none },
+    .{ .rcr, .mc, .rm16, .cl,    .none, .none, &.{ 0xd3 }, 3, .none },
+    .{ .rcr, .mi, .rm16, .imm8,  .none, .none, &.{ 0xc1 }, 3, .none },
+    .{ .rcr, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 3, .none },
+    .{ .rcr, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 3, .long },
+    .{ .rcr, .mc, .rm32, .cl,    .none, .none, &.{ 0xd3 }, 3, .none },
+    .{ .rcr, .mc, .rm64, .cl,    .none, .none, &.{ 0xd3 }, 3, .long },
+    .{ .rcr, .mi, .rm32, .imm8,  .none, .none, &.{ 0xc1 }, 3, .none },
+    .{ .rcr, .mi, .rm64, .imm8,  .none, .none, &.{ 0xc1 }, 3, .long },
 
-    .{ .shr, .m1, .rm8,  .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 5, .none  },
-    .{ .shr, .m1, .rm8,  .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 5, .rex   },
-    .{ .shr, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .none  },
-    .{ .shr, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .none  },
-    .{ .shr, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .long  },
-    .{ .shr, .mc, .rm8,  .cl,    .none, .none, 1, 0xd2, 0x00, 0x00, 5, .none  },
-    .{ .shr, .mc, .rm8,  .cl,    .none, .none, 1, 0xd2, 0x00, 0x00, 5, .rex   },
-    .{ .shr, .mc, .rm16, .cl,    .none, .none, 1, 0xd3, 0x00, 0x00, 5, .none  },
-    .{ .shr, .mc, .rm32, .cl,    .none, .none, 1, 0xd3, 0x00, 0x00, 5, .none  },
-    .{ .shr, .mc, .rm64, .cl,    .none, .none, 1, 0xd3, 0x00, 0x00, 5, .long  },
-    .{ .shr, .mi, .rm8,  .imm8,  .none, .none, 1, 0xc0, 0x00, 0x00, 5, .none  },
-    .{ .shr, .mi, .rm8,  .imm8,  .none, .none, 1, 0xc0, 0x00, 0x00, 5, .rex   },
-    .{ .shr, .mi, .rm16, .imm8,  .none, .none, 1, 0xc1, 0x00, 0x00, 5, .none  },
-    .{ .shr, .mi, .rm32, .imm8,  .none, .none, 1, 0xc1, 0x00, 0x00, 5, .none  },
-    .{ .shr, .mi, .rm64, .imm8,  .none, .none, 1, 0xc1, 0x00, 0x00, 5, .long  },
+    .{ .rol, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 0, .none },
+    .{ .rol, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 0, .rex  },
+    .{ .rol, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 0, .none },
+    .{ .rol, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 0, .rex  },
+    .{ .rol, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 0, .none },
+    .{ .rol, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 0, .rex  },
+    .{ .rol, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 0, .none },
+    .{ .rol, .mc, .rm16, .cl,    .none, .none, &.{ 0xd3 }, 0, .none },
+    .{ .rol, .mi, .rm16, .imm8,  .none, .none, &.{ 0xc1 }, 0, .none },
+    .{ .rol, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 0, .none },
+    .{ .rol, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 0, .long },
+    .{ .rol, .mc, .rm32, .cl,    .none, .none, &.{ 0xd3 }, 0, .none },
+    .{ .rol, .mc, .rm64, .cl,    .none, .none, &.{ 0xd3 }, 0, .long },
+    .{ .rol, .mi, .rm32, .imm8,  .none, .none, &.{ 0xc1 }, 0, .none },
+    .{ .rol, .mi, .rm64, .imm8,  .none, .none, &.{ 0xc1 }, 0, .long },
 
-    .{ .sub, .zi, .al,   .imm8,   .none, .none, 1, 0x2c, 0x00, 0x00, 0, .none  },
-    .{ .sub, .zi, .ax,   .imm16,  .none, .none, 1, 0x2d, 0x00, 0x00, 0, .none  },
-    .{ .sub, .zi, .eax,  .imm32,  .none, .none, 1, 0x2d, 0x00, 0x00, 0, .none  },
-    .{ .sub, .zi, .rax,  .imm32s, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .long  },
-    .{ .sub, .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 5, .none  },
-    .{ .sub, .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 5, .rex   },
-    .{ .sub, .mi, .rm16, .imm16,  .none, .none, 1, 0x81, 0x00, 0x00, 5, .none  },
-    .{ .sub, .mi, .rm32, .imm32,  .none, .none, 1, 0x81, 0x00, 0x00, 5, .none  },
-    .{ .sub, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 5, .long  },
-    .{ .sub, .mi, .rm16, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 5, .none  },
-    .{ .sub, .mi, .rm32, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 5, .none  },
-    .{ .sub, .mi, .rm64, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 5, .long  },
-    .{ .sub, .mr, .rm8,  .r8,     .none, .none, 1, 0x28, 0x00, 0x00, 0, .none  },
-    .{ .sub, .mr, .rm8,  .r8,     .none, .none, 1, 0x28, 0x00, 0x00, 0, .rex   },
-    .{ .sub, .mr, .rm16, .r16,    .none, .none, 1, 0x29, 0x00, 0x00, 0, .none  },
-    .{ .sub, .mr, .rm32, .r32,    .none, .none, 1, 0x29, 0x00, 0x00, 0, .none  },
-    .{ .sub, .mr, .rm64, .r64,    .none, .none, 1, 0x29, 0x00, 0x00, 0, .long  },
-    .{ .sub, .rm, .r8,   .rm8,    .none, .none, 1, 0x2a, 0x00, 0x00, 0, .none  },
-    .{ .sub, .rm, .r8,   .rm8,    .none, .none, 1, 0x2a, 0x00, 0x00, 0, .rex   },
-    .{ .sub, .rm, .r16,  .rm16,   .none, .none, 1, 0x2b, 0x00, 0x00, 0, .none  },
-    .{ .sub, .rm, .r32,  .rm32,   .none, .none, 1, 0x2b, 0x00, 0x00, 0, .none  },
-    .{ .sub, .rm, .r64,  .rm64,   .none, .none, 1, 0x2b, 0x00, 0x00, 0, .long  },
+    .{ .ror, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 1, .none },
+    .{ .ror, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 1, .rex  },
+    .{ .ror, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 1, .none },
+    .{ .ror, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 1, .rex  },
+    .{ .ror, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 1, .none },
+    .{ .ror, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 1, .rex  },
+    .{ .ror, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 1, .none },
+    .{ .ror, .mc, .rm16, .cl,    .none, .none, &.{ 0xd3 }, 1, .none },
+    .{ .ror, .mi, .rm16, .imm8,  .none, .none, &.{ 0xc1 }, 1, .none },
+    .{ .ror, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 1, .none },
+    .{ .ror, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 1, .long },
+    .{ .ror, .mc, .rm32, .cl,    .none, .none, &.{ 0xd3 }, 1, .none },
+    .{ .ror, .mc, .rm64, .cl,    .none, .none, &.{ 0xd3 }, 1, .long },
+    .{ .ror, .mi, .rm32, .imm8,  .none, .none, &.{ 0xc1 }, 1, .none },
+    .{ .ror, .mi, .rm64, .imm8,  .none, .none, &.{ 0xc1 }, 1, .long },
 
-    .{ .syscall, .np, .none, .none, .none, .none, 2, 0x0f, 0x05, 0x00, 0, .none },
+    .{ .sal, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 4, .none  },
+    .{ .sal, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 4, .rex   },
+    .{ .sal, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 4, .none  },
+    .{ .sal, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 4, .none  },
+    .{ .sal, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 4, .long  },
+    .{ .sal, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 4, .none  },
+    .{ .sal, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 4, .rex   },
+    .{ .sal, .mc, .rm16, .cl,    .none, .none, &.{ 0xd3 }, 4, .none  },
+    .{ .sal, .mc, .rm32, .cl,    .none, .none, &.{ 0xd3 }, 4, .none  },
+    .{ .sal, .mc, .rm64, .cl,    .none, .none, &.{ 0xd3 }, 4, .long  },
+    .{ .sal, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 4, .none  },
+    .{ .sal, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 4, .rex   },
+    .{ .sal, .mi, .rm16, .imm8,  .none, .none, &.{ 0xc1 }, 4, .none  },
+    .{ .sal, .mi, .rm32, .imm8,  .none, .none, &.{ 0xc1 }, 4, .none  },
+    .{ .sal, .mi, .rm64, .imm8,  .none, .none, &.{ 0xc1 }, 4, .long  },
 
-    .{ .@"test", .zi, .al,   .imm8,   .none, .none, 1, 0xa8, 0x00, 0x00, 0, .none  },
-    .{ .@"test", .zi, .ax,   .imm16,  .none, .none, 1, 0xa9, 0x00, 0x00, 0, .none  },
-    .{ .@"test", .zi, .eax,  .imm32,  .none, .none, 1, 0xa9, 0x00, 0x00, 0, .none  },
-    .{ .@"test", .zi, .rax,  .imm32s, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .long  },
-    .{ .@"test", .mi, .rm8,  .imm8,   .none, .none, 1, 0xf6, 0x00, 0x00, 0, .none  },
-    .{ .@"test", .mi, .rm8,  .imm8,   .none, .none, 1, 0xf6, 0x00, 0x00, 0, .rex   },
-    .{ .@"test", .mi, .rm16, .imm16,  .none, .none, 1, 0xf7, 0x00, 0x00, 0, .none  },
-    .{ .@"test", .mi, .rm32, .imm32,  .none, .none, 1, 0xf7, 0x00, 0x00, 0, .none  },
-    .{ .@"test", .mi, .rm64, .imm32s, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .long  },
-    .{ .@"test", .mr, .rm8,  .r8,     .none, .none, 1, 0x84, 0x00, 0x00, 0, .none  },
-    .{ .@"test", .mr, .rm8,  .r8,     .none, .none, 1, 0x84, 0x00, 0x00, 0, .rex   },
-    .{ .@"test", .mr, .rm16, .r16,    .none, .none, 1, 0x85, 0x00, 0x00, 0, .none  },
-    .{ .@"test", .mr, .rm32, .r32,    .none, .none, 1, 0x85, 0x00, 0x00, 0, .none  },
-    .{ .@"test", .mr, .rm64, .r64,    .none, .none, 1, 0x85, 0x00, 0x00, 0, .long  },
+    .{ .sar, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 7, .none  },
+    .{ .sar, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 7, .rex   },
+    .{ .sar, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 7, .none  },
+    .{ .sar, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 7, .none  },
+    .{ .sar, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 7, .long  },
+    .{ .sar, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 7, .none  },
+    .{ .sar, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 7, .rex   },
+    .{ .sar, .mc, .rm16, .cl,    .none, .none, &.{ 0xd3 }, 7, .none  },
+    .{ .sar, .mc, .rm32, .cl,    .none, .none, &.{ 0xd3 }, 7, .none  },
+    .{ .sar, .mc, .rm64, .cl,    .none, .none, &.{ 0xd3 }, 7, .long  },
+    .{ .sar, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 7, .none  },
+    .{ .sar, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 7, .rex   },
+    .{ .sar, .mi, .rm16, .imm8,  .none, .none, &.{ 0xc1 }, 7, .none  },
+    .{ .sar, .mi, .rm32, .imm8,  .none, .none, &.{ 0xc1 }, 7, .none  },
+    .{ .sar, .mi, .rm64, .imm8,  .none, .none, &.{ 0xc1 }, 7, .long  },
 
-    .{ .ud2, .np, .none, .none, .none, .none, 2, 0x0f, 0x0b, 0x00, 0, .none  },
+    .{ .sbb, .zi, .al,   .imm8,   .none, .none, &.{ 0x1c }, 0, .none  },
+    .{ .sbb, .zi, .ax,   .imm16,  .none, .none, &.{ 0x1d }, 0, .none  },
+    .{ .sbb, .zi, .eax,  .imm32,  .none, .none, &.{ 0x1d }, 0, .none  },
+    .{ .sbb, .zi, .rax,  .imm32s, .none, .none, &.{ 0x1d }, 0, .long  },
+    .{ .sbb, .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 3, .none  },
+    .{ .sbb, .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 3, .rex   },
+    .{ .sbb, .mi, .rm16, .imm16,  .none, .none, &.{ 0x81 }, 3, .none  },
+    .{ .sbb, .mi, .rm32, .imm32,  .none, .none, &.{ 0x81 }, 3, .none  },
+    .{ .sbb, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 3, .long  },
+    .{ .sbb, .mi, .rm16, .imm8s,  .none, .none, &.{ 0x83 }, 3, .none  },
+    .{ .sbb, .mi, .rm32, .imm8s,  .none, .none, &.{ 0x83 }, 3, .none  },
+    .{ .sbb, .mi, .rm64, .imm8s,  .none, .none, &.{ 0x83 }, 3, .long  },
+    .{ .sbb, .mr, .rm8,  .r8,     .none, .none, &.{ 0x18 }, 0, .none  },
+    .{ .sbb, .mr, .rm8,  .r8,     .none, .none, &.{ 0x18 }, 0, .rex   },
+    .{ .sbb, .mr, .rm16, .r16,    .none, .none, &.{ 0x19 }, 0, .none  },
+    .{ .sbb, .mr, .rm32, .r32,    .none, .none, &.{ 0x19 }, 0, .none  },
+    .{ .sbb, .mr, .rm64, .r64,    .none, .none, &.{ 0x19 }, 0, .long  },
+    .{ .sbb, .rm, .r8,   .rm8,    .none, .none, &.{ 0x1a }, 0, .none  },
+    .{ .sbb, .rm, .r8,   .rm8,    .none, .none, &.{ 0x1a }, 0, .rex   },
+    .{ .sbb, .rm, .r16,  .rm16,   .none, .none, &.{ 0x1b }, 0, .none  },
+    .{ .sbb, .rm, .r32,  .rm32,   .none, .none, &.{ 0x1b }, 0, .none  },
+    .{ .sbb, .rm, .r64,  .rm64,   .none, .none, &.{ 0x1b }, 0, .long  },
 
-    .{ .xor, .zi, .al,   .imm8,   .none, .none, 1, 0x34, 0x00, 0x00, 0, .none  },
-    .{ .xor, .zi, .ax,   .imm16,  .none, .none, 1, 0x35, 0x00, 0x00, 0, .none  },
-    .{ .xor, .zi, .eax,  .imm32,  .none, .none, 1, 0x35, 0x00, 0x00, 0, .none  },
-    .{ .xor, .zi, .rax,  .imm32s, .none, .none, 1, 0x35, 0x00, 0x00, 0, .long  },
-    .{ .xor, .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 6, .none  },
-    .{ .xor, .mi, .rm8,  .imm8,   .none, .none, 1, 0x80, 0x00, 0x00, 6, .rex   },
-    .{ .xor, .mi, .rm16, .imm16,  .none, .none, 1, 0x81, 0x00, 0x00, 6, .none  },
-    .{ .xor, .mi, .rm32, .imm32,  .none, .none, 1, 0x81, 0x00, 0x00, 6, .none  },
-    .{ .xor, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 6, .long  },
-    .{ .xor, .mi, .rm16, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 6, .none  },
-    .{ .xor, .mi, .rm32, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 6, .none  },
-    .{ .xor, .mi, .rm64, .imm8s,  .none, .none, 1, 0x83, 0x00, 0x00, 6, .long  },
-    .{ .xor, .mr, .rm8,  .r8,     .none, .none, 1, 0x30, 0x00, 0x00, 0, .none  },
-    .{ .xor, .mr, .rm8,  .r8,     .none, .none, 1, 0x30, 0x00, 0x00, 0, .rex   },
-    .{ .xor, .mr, .rm16, .r16,    .none, .none, 1, 0x31, 0x00, 0x00, 0, .none  },
-    .{ .xor, .mr, .rm32, .r32,    .none, .none, 1, 0x31, 0x00, 0x00, 0, .none  },
-    .{ .xor, .mr, .rm64, .r64,    .none, .none, 1, 0x31, 0x00, 0x00, 0, .long  },
-    .{ .xor, .rm, .r8,   .rm8,    .none, .none, 1, 0x32, 0x00, 0x00, 0, .none  },
-    .{ .xor, .rm, .r8,   .rm8,    .none, .none, 1, 0x32, 0x00, 0x00, 0, .rex   },
-    .{ .xor, .rm, .r16,  .rm16,   .none, .none, 1, 0x33, 0x00, 0x00, 0, .none  },
-    .{ .xor, .rm, .r32,  .rm32,   .none, .none, 1, 0x33, 0x00, 0x00, 0, .none  },
-    .{ .xor, .rm, .r64,  .rm64,   .none, .none, 1, 0x33, 0x00, 0x00, 0, .long  },
+    .{ .scas,  .np, .m8,   .none, .none, .none, &.{ 0xae }, 0, .none  },
+    .{ .scas,  .np, .m16,  .none, .none, .none, &.{ 0xaf }, 0, .none  },
+    .{ .scas,  .np, .m32,  .none, .none, .none, &.{ 0xaf }, 0, .none  },
+    .{ .scas,  .np, .m64,  .none, .none, .none, &.{ 0xaf }, 0, .long  },
+    .{ .scasb, .np, .none, .none, .none, .none, &.{ 0xae }, 0, .none  },
+    .{ .scasw, .np, .none, .none, .none, .none, &.{ 0xaf }, 0, .short },
+    .{ .scasd, .np, .none, .none, .none, .none, &.{ 0xaf }, 0, .none  },
+    .{ .scasq, .np, .none, .none, .none, .none, &.{ 0xaf }, 0, .long  },
+
+    .{ .seta,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .none },
+    .{ .seta,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .rex  },
+    .{ .setae,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .none },
+    .{ .setae,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .rex  },
+    .{ .setb,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .none },
+    .{ .setb,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .rex  },
+    .{ .setbe,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x96 }, 0, .none },
+    .{ .setbe,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x96 }, 0, .rex  },
+    .{ .setc,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .none },
+    .{ .setc,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .rex  },
+    .{ .sete,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .none },
+    .{ .sete,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .rex  },
+    .{ .setg,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9f }, 0, .none },
+    .{ .setg,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9f }, 0, .rex  },
+    .{ .setge,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9d }, 0, .none },
+    .{ .setge,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9d }, 0, .rex  },
+    .{ .setl,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9c }, 0, .none },
+    .{ .setl,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9c }, 0, .rex  },
+    .{ .setle,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9e }, 0, .none },
+    .{ .setle,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9e }, 0, .rex  },
+    .{ .setna,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x96 }, 0, .none },
+    .{ .setna,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x96 }, 0, .rex  },
+    .{ .setnae, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .none },
+    .{ .setnae, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .rex  },
+    .{ .setnb,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .none },
+    .{ .setnb,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .rex  },
+    .{ .setnbe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .none },
+    .{ .setnbe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .rex  },
+    .{ .setnc,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .none },
+    .{ .setnc,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .rex  },
+    .{ .setne,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x95 }, 0, .none },
+    .{ .setne,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x95 }, 0, .rex  },
+    .{ .setng,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9e }, 0, .none },
+    .{ .setng,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9e }, 0, .rex  },
+    .{ .setnge, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9c }, 0, .none },
+    .{ .setnge, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9c }, 0, .rex  },
+    .{ .setnl,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9d }, 0, .none },
+    .{ .setnl,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9d }, 0, .rex  },
+    .{ .setnle, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9f }, 0, .none },
+    .{ .setnle, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9f }, 0, .rex  },
+    .{ .setno,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x91 }, 0, .none },
+    .{ .setno,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x91 }, 0, .rex  },
+    .{ .setnp,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9b }, 0, .none },
+    .{ .setnp,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9b }, 0, .rex  },
+    .{ .setns,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x99 }, 0, .none },
+    .{ .setns,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x99 }, 0, .rex  },
+    .{ .setnz,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x95 }, 0, .none },
+    .{ .setnz,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x95 }, 0, .rex  },
+    .{ .seto,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x90 }, 0, .none },
+    .{ .seto,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x90 }, 0, .rex  },
+    .{ .setp,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9a }, 0, .none },
+    .{ .setp,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9a }, 0, .rex  },
+    .{ .setpe,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9a }, 0, .none },
+    .{ .setpe,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9a }, 0, .rex  },
+    .{ .setpo,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9b }, 0, .none },
+    .{ .setpo,  .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9b }, 0, .rex  },
+    .{ .sets,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x98 }, 0, .none },
+    .{ .sets,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x98 }, 0, .rex  },
+    .{ .setz,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .none },
+    .{ .setz,   .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .rex  },
+
+    .{ .sfence, .np, .none, .none, .none, .none, &.{ 0x0f, 0xae, 0xf8 }, 0, .none },
+
+    .{ .shl, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 4, .none  },
+    .{ .shl, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 4, .rex   },
+    .{ .shl, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 4, .none  },
+    .{ .shl, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 4, .none  },
+    .{ .shl, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 4, .long  },
+    .{ .shl, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 4, .none  },
+    .{ .shl, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 4, .rex   },
+    .{ .shl, .mc, .rm16, .cl,    .none, .none, &.{ 0xd3 }, 4, .none  },
+    .{ .shl, .mc, .rm32, .cl,    .none, .none, &.{ 0xd3 }, 4, .none  },
+    .{ .shl, .mc, .rm64, .cl,    .none, .none, &.{ 0xd3 }, 4, .long  },
+    .{ .shl, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 4, .none  },
+    .{ .shl, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 4, .rex   },
+    .{ .shl, .mi, .rm16, .imm8,  .none, .none, &.{ 0xc1 }, 4, .none  },
+    .{ .shl, .mi, .rm32, .imm8,  .none, .none, &.{ 0xc1 }, 4, .none  },
+    .{ .shl, .mi, .rm64, .imm8,  .none, .none, &.{ 0xc1 }, 4, .long  },
+
+    .{ .shr, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 5, .none  },
+    .{ .shr, .m1, .rm8,  .unity, .none, .none, &.{ 0xd0 }, 5, .rex   },
+    .{ .shr, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 5, .none  },
+    .{ .shr, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 5, .none  },
+    .{ .shr, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 5, .long  },
+    .{ .shr, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 5, .none  },
+    .{ .shr, .mc, .rm8,  .cl,    .none, .none, &.{ 0xd2 }, 5, .rex   },
+    .{ .shr, .mc, .rm16, .cl,    .none, .none, &.{ 0xd3 }, 5, .none  },
+    .{ .shr, .mc, .rm32, .cl,    .none, .none, &.{ 0xd3 }, 5, .none  },
+    .{ .shr, .mc, .rm64, .cl,    .none, .none, &.{ 0xd3 }, 5, .long  },
+    .{ .shr, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 5, .none  },
+    .{ .shr, .mi, .rm8,  .imm8,  .none, .none, &.{ 0xc0 }, 5, .rex   },
+    .{ .shr, .mi, .rm16, .imm8,  .none, .none, &.{ 0xc1 }, 5, .none  },
+    .{ .shr, .mi, .rm32, .imm8,  .none, .none, &.{ 0xc1 }, 5, .none  },
+    .{ .shr, .mi, .rm64, .imm8,  .none, .none, &.{ 0xc1 }, 5, .long  },
+
+    .{ .stos,  .np, .m8,   .none, .none, .none, &.{ 0xaa }, 0, .none  },
+    .{ .stos,  .np, .m16,  .none, .none, .none, &.{ 0xab }, 0, .none  },
+    .{ .stos,  .np, .m32,  .none, .none, .none, &.{ 0xab }, 0, .none  },
+    .{ .stos,  .np, .m64,  .none, .none, .none, &.{ 0xab }, 0, .long  },
+    .{ .stosb, .np, .none, .none, .none, .none, &.{ 0xaa }, 0, .none  },
+    .{ .stosw, .np, .none, .none, .none, .none, &.{ 0xab }, 0, .short },
+    .{ .stosd, .np, .none, .none, .none, .none, &.{ 0xab }, 0, .none  },
+    .{ .stosq, .np, .none, .none, .none, .none, &.{ 0xab }, 0, .long  },
+
+    .{ .sub, .zi, .al,   .imm8,   .none, .none, &.{ 0x2c }, 0, .none  },
+    .{ .sub, .zi, .ax,   .imm16,  .none, .none, &.{ 0x2d }, 0, .none  },
+    .{ .sub, .zi, .eax,  .imm32,  .none, .none, &.{ 0x2d }, 0, .none  },
+    .{ .sub, .zi, .rax,  .imm32s, .none, .none, &.{ 0x2d }, 0, .long  },
+    .{ .sub, .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 5, .none  },
+    .{ .sub, .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 5, .rex   },
+    .{ .sub, .mi, .rm16, .imm16,  .none, .none, &.{ 0x81 }, 5, .none  },
+    .{ .sub, .mi, .rm32, .imm32,  .none, .none, &.{ 0x81 }, 5, .none  },
+    .{ .sub, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 5, .long  },
+    .{ .sub, .mi, .rm16, .imm8s,  .none, .none, &.{ 0x83 }, 5, .none  },
+    .{ .sub, .mi, .rm32, .imm8s,  .none, .none, &.{ 0x83 }, 5, .none  },
+    .{ .sub, .mi, .rm64, .imm8s,  .none, .none, &.{ 0x83 }, 5, .long  },
+    .{ .sub, .mr, .rm8,  .r8,     .none, .none, &.{ 0x28 }, 0, .none  },
+    .{ .sub, .mr, .rm8,  .r8,     .none, .none, &.{ 0x28 }, 0, .rex   },
+    .{ .sub, .mr, .rm16, .r16,    .none, .none, &.{ 0x29 }, 0, .none  },
+    .{ .sub, .mr, .rm32, .r32,    .none, .none, &.{ 0x29 }, 0, .none  },
+    .{ .sub, .mr, .rm64, .r64,    .none, .none, &.{ 0x29 }, 0, .long  },
+    .{ .sub, .rm, .r8,   .rm8,    .none, .none, &.{ 0x2a }, 0, .none  },
+    .{ .sub, .rm, .r8,   .rm8,    .none, .none, &.{ 0x2a }, 0, .rex   },
+    .{ .sub, .rm, .r16,  .rm16,   .none, .none, &.{ 0x2b }, 0, .none  },
+    .{ .sub, .rm, .r32,  .rm32,   .none, .none, &.{ 0x2b }, 0, .none  },
+    .{ .sub, .rm, .r64,  .rm64,   .none, .none, &.{ 0x2b }, 0, .long  },
+
+    .{ .syscall, .np, .none, .none, .none, .none, &.{ 0x0f, 0x05 }, 0, .none }
+,
+    .{ .@"test", .zi, .al,   .imm8,   .none, .none, &.{ 0xa8 }, 0, .none  },
+    .{ .@"test", .zi, .ax,   .imm16,  .none, .none, &.{ 0xa9 }, 0, .none  },
+    .{ .@"test", .zi, .eax,  .imm32,  .none, .none, &.{ 0xa9 }, 0, .none  },
+    .{ .@"test", .zi, .rax,  .imm32s, .none, .none, &.{ 0xa9 }, 0, .long  },
+    .{ .@"test", .mi, .rm8,  .imm8,   .none, .none, &.{ 0xf6 }, 0, .none  },
+    .{ .@"test", .mi, .rm8,  .imm8,   .none, .none, &.{ 0xf6 }, 0, .rex   },
+    .{ .@"test", .mi, .rm16, .imm16,  .none, .none, &.{ 0xf7 }, 0, .none  },
+    .{ .@"test", .mi, .rm32, .imm32,  .none, .none, &.{ 0xf7 }, 0, .none  },
+    .{ .@"test", .mi, .rm64, .imm32s, .none, .none, &.{ 0xf7 }, 0, .long  },
+    .{ .@"test", .mr, .rm8,  .r8,     .none, .none, &.{ 0x84 }, 0, .none  },
+    .{ .@"test", .mr, .rm8,  .r8,     .none, .none, &.{ 0x84 }, 0, .rex   },
+    .{ .@"test", .mr, .rm16, .r16,    .none, .none, &.{ 0x85 }, 0, .none  },
+    .{ .@"test", .mr, .rm32, .r32,    .none, .none, &.{ 0x85 }, 0, .none  },
+    .{ .@"test", .mr, .rm64, .r64,    .none, .none, &.{ 0x85 }, 0, .long  },
+
+    .{ .tzcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },
+    .{ .tzcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },
+    .{ .tzcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .long },
+
+    .{ .ud2, .np, .none, .none, .none, .none, &.{ 0x0f, 0x0b }, 0, .none  },
+
+    .{ .xadd, .mr, .rm8,  .r8,  .none, .none, &.{ 0x0f, 0xc0 }, 0, .none  },
+    .{ .xadd, .mr, .rm8,  .r8,  .none, .none, &.{ 0x0f, 0xc0 }, 0, .rex   },
+    .{ .xadd, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xc1 }, 0, .none  },
+    .{ .xadd, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xc1 }, 0, .none  },
+    .{ .xadd, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xc1 }, 0, .long  },
+
+    .{ .xchg, .o,  .ax,   .r16,  .none, .none, &.{ 0x90 }, 0, .none },
+    .{ .xchg, .o,  .r16,  .ax,   .none, .none, &.{ 0x90 }, 0, .none },
+    .{ .xchg, .o,  .eax,  .r32,  .none, .none, &.{ 0x90 }, 0, .none },
+    .{ .xchg, .o,  .rax,  .r64,  .none, .none, &.{ 0x90 }, 0, .long },
+    .{ .xchg, .o,  .r32,  .eax,  .none, .none, &.{ 0x90 }, 0, .none },
+    .{ .xchg, .o,  .r64,  .rax,  .none, .none, &.{ 0x90 }, 0, .long },
+    .{ .xchg, .mr, .rm8,  .r8,   .none, .none, &.{ 0x86 }, 0, .none },
+    .{ .xchg, .mr, .rm8,  .r8,   .none, .none, &.{ 0x86 }, 0, .rex  },
+    .{ .xchg, .rm, .r8,   .rm8,  .none, .none, &.{ 0x86 }, 0, .none },
+    .{ .xchg, .rm, .r8,   .rm8,  .none, .none, &.{ 0x86 }, 0, .rex  },
+    .{ .xchg, .mr, .rm16, .r16,  .none, .none, &.{ 0x87 }, 0, .none },
+    .{ .xchg, .rm, .r16,  .rm16, .none, .none, &.{ 0x87 }, 0, .none },
+    .{ .xchg, .mr, .rm32, .r32,  .none, .none, &.{ 0x87 }, 0, .none },
+    .{ .xchg, .mr, .rm64, .r64,  .none, .none, &.{ 0x87 }, 0, .long },
+    .{ .xchg, .rm, .r32,  .rm32, .none, .none, &.{ 0x87 }, 0, .none },
+    .{ .xchg, .rm, .r64,  .rm64, .none, .none, &.{ 0x87 }, 0, .long },
+
+    .{ .xor, .zi, .al,   .imm8,   .none, .none, &.{ 0x34 }, 0, .none  },
+    .{ .xor, .zi, .ax,   .imm16,  .none, .none, &.{ 0x35 }, 0, .none  },
+    .{ .xor, .zi, .eax,  .imm32,  .none, .none, &.{ 0x35 }, 0, .none  },
+    .{ .xor, .zi, .rax,  .imm32s, .none, .none, &.{ 0x35 }, 0, .long  },
+    .{ .xor, .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 6, .none  },
+    .{ .xor, .mi, .rm8,  .imm8,   .none, .none, &.{ 0x80 }, 6, .rex   },
+    .{ .xor, .mi, .rm16, .imm16,  .none, .none, &.{ 0x81 }, 6, .none  },
+    .{ .xor, .mi, .rm32, .imm32,  .none, .none, &.{ 0x81 }, 6, .none  },
+    .{ .xor, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 6, .long  },
+    .{ .xor, .mi, .rm16, .imm8s,  .none, .none, &.{ 0x83 }, 6, .none  },
+    .{ .xor, .mi, .rm32, .imm8s,  .none, .none, &.{ 0x83 }, 6, .none  },
+    .{ .xor, .mi, .rm64, .imm8s,  .none, .none, &.{ 0x83 }, 6, .long  },
+    .{ .xor, .mr, .rm8,  .r8,     .none, .none, &.{ 0x30 }, 0, .none  },
+    .{ .xor, .mr, .rm8,  .r8,     .none, .none, &.{ 0x30 }, 0, .rex   },
+    .{ .xor, .mr, .rm16, .r16,    .none, .none, &.{ 0x31 }, 0, .none  },
+    .{ .xor, .mr, .rm32, .r32,    .none, .none, &.{ 0x31 }, 0, .none  },
+    .{ .xor, .mr, .rm64, .r64,    .none, .none, &.{ 0x31 }, 0, .long  },
+    .{ .xor, .rm, .r8,   .rm8,    .none, .none, &.{ 0x32 }, 0, .none  },
+    .{ .xor, .rm, .r8,   .rm8,    .none, .none, &.{ 0x32 }, 0, .rex   },
+    .{ .xor, .rm, .r16,  .rm16,   .none, .none, &.{ 0x33 }, 0, .none  },
+    .{ .xor, .rm, .r32,  .rm32,   .none, .none, &.{ 0x33 }, 0, .none  },
+    .{ .xor, .rm, .r64,  .rm64,   .none, .none, &.{ 0x33 }, 0, .long  },
 
     // SSE
-    .{ .addss, .rm, .xmm, .xmm_m32, .none, .none, 3, 0xf3, 0x0f, 0x58, 0, .sse },
+    .{ .addss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x58 }, 0, .sse },
 
-    .{ .cmpss, .rmi, .xmm, .xmm_m32, .imm8, .none, 3, 0xf3, 0x0f, 0xc2, 0, .sse },
+    .{ .cmpss, .rmi, .xmm, .xmm_m32, .imm8, .none, &.{ 0xf3, 0x0f, 0xc2 }, 0, .sse },
 
-    .{ .movss, .rm, .xmm,     .xmm_m32, .none, .none, 3, 0xf3, 0x0f, 0x10, 0, .sse },
-    .{ .movss, .mr, .xmm_m32, .xmm,     .none, .none, 3, 0xf3, 0x0f, 0x11, 0, .sse },
+    .{ .divss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x5e }, 0, .sse },
 
-    .{ .ucomiss, .rm, .xmm, .xmm_m32, .none, .none, 2, 0x0f, 0x2e, 0x00, 0, .sse },
+    .{ .maxss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x5f }, 0, .sse },
+
+    .{ .minss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x5d }, 0, .sse },
+
+    .{ .movss, .rm, .xmm,     .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x10 }, 0, .sse },
+    .{ .movss, .mr, .xmm_m32, .xmm,     .none, .none, &.{ 0xf3, 0x0f, 0x11 }, 0, .sse },
+
+    .{ .mulss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x59 }, 0, .sse },
+
+    .{ .subss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x5c }, 0, .sse },
+
+    .{ .ucomiss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0x0f, 0x2e }, 0, .sse },
 
     // SSE2
-    .{ .addsd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf2, 0x0f, 0x58, 0, .sse2 },
+    .{ .addsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x58 }, 0, .sse2 },
 
-    .{ .cmpsd, .rmi, .xmm, .xmm_m64, .imm8, .none, 3, 0xf2, 0x0f, 0xc2, 0, .sse2 },
+    .{ .cmpsd, .rmi, .xmm, .xmm_m64, .imm8, .none, &.{ 0xf2, 0x0f, 0xc2 }, 0, .sse2 },
 
-    .{ .movq, .rm, .xmm,     .xmm_m64, .none, .none, 3, 0xf3, 0x0f, 0x7e, 0, .sse2 },
-    .{ .movq, .mr, .xmm_m64, .xmm,     .none, .none, 3, 0x66, 0x0f, 0xd6, 0, .sse2 },
+    .{ .divsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x5e }, 0, .sse2 },
 
-    .{ .movsd, .rm, .xmm,     .xmm_m64, .none, .none, 3, 0xf2, 0x0f, 0x10, 0, .sse2 },
-    .{ .movsd, .mr, .xmm_m64, .xmm,     .none, .none, 3, 0xf2, 0x0f, 0x11, 0, .sse2 },
+    .{ .maxsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x5f }, 0, .sse2 },
 
-    .{ .ucomisd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0x66, 0x0f, 0x2e, 0, .sse2 },
+    .{ .minsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x5d }, 0, .sse2 },
+
+    .{ .movq, .rm, .xmm,     .xmm_m64, .none, .none, &.{ 0xf3, 0x0f, 0x7e }, 0, .sse2 },
+    .{ .movq, .mr, .xmm_m64, .xmm,     .none, .none, &.{ 0x66, 0x0f, 0xd6 }, 0, .sse2 },
+
+    .{ .mulsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x59 }, 0, .sse2 },
+
+    .{ .subsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x5c }, 0, .sse2 },
+
+    .{ .movsd, .rm, .xmm,     .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x10 }, 0, .sse2 },
+    .{ .movsd, .mr, .xmm_m64, .xmm,     .none, .none, &.{ 0xf2, 0x0f, 0x11 }, 0, .sse2 },
+
+    .{ .ucomisd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0x66, 0x0f, 0x2e }, 0, .sse2 },
+
+    // SSE4.1
+    .{ .roundss, .rmi, .xmm, .xmm_m32, .imm8, .none, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .sse4_1 },
+    .{ .roundsd, .rmi, .xmm, .xmm_m64, .imm8, .none, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .sse4_1 },
 };
 // zig fmt: on
-

src/codegen.zig

@@ -608,7 +608,6 @@ pub fn generateSymbol(
             const payload_type = typed_value.ty.optionalChild(&opt_buf);
             const is_pl = !typed_value.val.isNull();
             const abi_size = math.cast(usize, typed_value.ty.abiSize(target)) orelse return error.Overflow;
-            const offset = abi_size - (math.cast(usize, payload_type.abiSize(target)) orelse return error.Overflow);
 
             if (!payload_type.hasRuntimeBits()) {
                 try code.writer().writeByteNTimes(@boolToInt(is_pl), abi_size);
@@ -639,8 +638,8 @@ pub fn generateSymbol(
                 return Result.ok;
             }
 
+            const padding = abi_size - (math.cast(usize, payload_type.abiSize(target)) orelse return error.Overflow) - 1;
             const value = if (typed_value.val.castTag(.opt_payload)) |payload| payload.data else Value.initTag(.undef);
-            try code.writer().writeByteNTimes(@boolToInt(is_pl), offset);
             switch (try generateSymbol(bin_file, src_loc, .{
                 .ty = payload_type,
                 .val = value,
@@ -648,6 +647,8 @@ pub fn generateSymbol(
                 .ok => {},
                 .fail => |em| return Result{ .fail = em },
             }
+            try code.writer().writeByte(@boolToInt(is_pl));
+            try code.writer().writeByteNTimes(0, padding);
 
             return Result.ok;
         },

src/register_manager.zig

@@ -305,40 +305,32 @@ pub fn RegisterManager(
         pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {
             const index = indexOfRegIntoTracked(reg) orelse return;
             log.debug("getReg {} for inst {?}", .{ reg, inst });
-            self.markRegAllocated(reg);
 
-            if (inst) |tracked_inst|
-                if (!self.isRegFree(reg)) {
-                    // Move the instruction that was previously there to a
-                    // stack allocation.
-                    const spilled_inst = self.registers[index];
-                    self.registers[index] = tracked_inst;
-                    try self.getFunction().spillInstruction(reg, spilled_inst);
-                } else {
-                    self.getRegAssumeFree(reg, tracked_inst);
-                }
-            else {
-                if (!self.isRegFree(reg)) {
-                    // Move the instruction that was previously there to a
-                    // stack allocation.
-                    const spilled_inst = self.registers[index];
-                    try self.getFunction().spillInstruction(reg, spilled_inst);
-                    self.freeReg(reg);
-                }
-            }
+            if (!self.isRegFree(reg)) {
+                self.markRegAllocated(reg);
+
+                // Move the instruction that was previously there to a
+                // stack allocation.
+                const spilled_inst = self.registers[index];
+                if (inst) |tracked_inst| self.registers[index] = tracked_inst;
+                try self.getFunction().spillInstruction(reg, spilled_inst);
+                if (inst == null) self.freeReg(reg);
+            } else self.getRegAssumeFree(reg, inst);
         }
 
         /// Allocates the specified register with the specified
         /// instruction. Asserts that the register is free and no
         /// spilling is necessary.
-        pub fn getRegAssumeFree(self: *Self, reg: Register, inst: Air.Inst.Index) void {
+        pub fn getRegAssumeFree(self: *Self, reg: Register, inst: ?Air.Inst.Index) void {
             const index = indexOfRegIntoTracked(reg) orelse return;
-            log.debug("getRegAssumeFree {} for inst {}", .{ reg, inst });
+            log.debug("getRegAssumeFree {} for inst {?}", .{ reg, inst });
             self.markRegAllocated(reg);
 
             assert(self.isRegFree(reg));
-            self.registers[index] = inst;
-            self.markRegUsed(reg);
+            if (inst) |tracked_inst| {
+                self.registers[index] = tracked_inst;
+                self.markRegUsed(reg);
+            }
         }
 
         /// Marks the specified register as free

test/behavior/atomics.zig

@@ -33,7 +33,6 @@ fn testCmpxchg() !void {
 
 test "fence" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
 
@@ -44,7 +43,6 @@ test "fence" {
 
 test "atomicrmw and atomicload" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
@@ -73,7 +71,6 @@ fn testAtomicLoad(ptr: *u8) !void {
 
 test "cmpxchg with ptr" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
@@ -162,7 +159,6 @@ test "cmpxchg on a global variable" {
 
 test "atomic load and rmw with enum" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
@@ -180,7 +176,6 @@ test "atomic load and rmw with enum" {
 
 test "atomic store" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
@@ -194,7 +189,6 @@ test "atomic store" {
 
 test "atomic store comptime" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
@@ -424,7 +418,6 @@ fn testAtomicsWithType(comptime T: type, a: T, b: T) !void {
 
 test "return @atomicStore, using it as a void value" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO

test/behavior/bugs/12984.zig

@@ -14,7 +14,6 @@ pub const CustomDraw = DeleagateWithContext(fn (?OnConfirm) void);
 test "simple test" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     var c: CustomDraw = undefined;
     _ = c;

test/behavior/bugs/13068.zig

@@ -8,7 +8,6 @@ test {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     list.items.len = 0;

test/behavior/bugs/13785.zig

@@ -3,7 +3,6 @@ const std = @import("std");
 
 const S = packed struct { a: u0 = 0 };
 test {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/call.zig

@@ -329,7 +329,6 @@ test "inline call preserves tail call" {
 test "inline call doesn't re-evaluate non generic struct" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn foo(f: struct { a: u8, b: u8 }) !void {

test/behavior/cast.zig

@@ -655,7 +655,6 @@ test "@floatCast cast down" {
 }
 
 test "peer type resolution: unreachable, error set, unreachable" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
@@ -1206,7 +1205,6 @@ fn cast128Float(x: u128) f128 {
 test "implicit cast from *[N]T to ?[*]T" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     var x: ?[*]u16 = null;

test/behavior/error.zig

@@ -451,7 +451,6 @@ test "optional error set is the same size as error set" {
 }
 
 test "nested catch" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
@@ -731,7 +730,6 @@ test "pointer to error union payload" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     var err_union: anyerror!u8 = 15;
@@ -876,6 +874,7 @@ test "field access of anyerror results in smaller error set" {
 }
 
 test "optional error union return type" {
+    if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
 
     const S = struct {

test/behavior/if.zig

@@ -130,7 +130,6 @@ test "if peer expressions inferred optional type" {
 }
 
 test "if-else expression with runtime condition result location is inferred optional" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/maximum_minimum.zig

@@ -5,7 +5,6 @@ const expect = std.testing.expect;
 const expectEqual = std.testing.expectEqual;
 
 test "@max" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -52,7 +51,6 @@ test "@max on vectors" {
 }
 
 test "@min" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/merge_error_sets.zig

@@ -12,7 +12,6 @@ fn foo() C!void {
 }
 
 test "merge error sets" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     if (foo()) {

test/behavior/null.zig

@@ -29,7 +29,6 @@ test "optional type" {
 }
 
 test "test maybe object and get a pointer to the inner value" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -138,7 +137,6 @@ test "optional pointer to 0 bit type null value at runtime" {
 }
 
 test "if var maybe pointer" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/optional.zig

@@ -91,7 +91,6 @@ test "address of unwrap optional" {
 test "nested optional field in struct" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const S2 = struct {
@@ -109,7 +108,6 @@ test "nested optional field in struct" {
 test "equality compare optional with non-optional" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     try test_cmp_optional_non_optional();
@@ -227,7 +225,6 @@ test "assigning to an unwrapped optional field in an inline loop" {
 }
 
 test "coerce an anon struct literal to optional struct" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -247,7 +244,6 @@ test "coerce an anon struct literal to optional struct" {
 }
 
 test "0-bit child type coerced to optional return ptr result location" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -299,7 +295,6 @@ test "0-bit child type coerced to optional" {
 }
 
 test "array of optional unaligned types" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -336,7 +331,6 @@ test "array of optional unaligned types" {
 }
 
 test "optional pointer to zero bit optional payload" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -450,7 +444,6 @@ test "Optional slice size is optimized" {
 test "peer type resolution in nested if expressions" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
 
     const Thing = struct { n: i32 };
     var a = false;

test/behavior/ptrcast.zig

@@ -18,7 +18,6 @@ fn testReinterpretBytesAsInteger() !void {
 }
 
 test "reinterpret an array over multiple elements, with no well-defined layout" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/return_address.zig

@@ -7,7 +7,6 @@ fn retAddr() usize {
 
 test "return address" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/struct.zig

@@ -1149,7 +1149,6 @@ test "anon init through error unions and optionals" {
 }
 
 test "anon init through optional" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1456,7 +1455,6 @@ test "struct has only one reference" {
 test "no dependency loop on pointer to optional struct" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const S = struct {
         const A = struct { b: B };
@@ -1509,7 +1507,6 @@ test "no dependency loop on optional field wrapped in generic function" {
 }
 
 test "optional field init with tuple" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
 

test/behavior/switch.zig

@@ -228,7 +228,6 @@ const SwitchProngWithVarEnum = union(enum) {
 };
 
 test "switch prong with variable" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/tuple.zig

@@ -263,7 +263,6 @@ test "initializing anon struct with mixed comptime-runtime fields" {
 test "tuple in tuple passed to generic function" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const S = struct {
@@ -283,7 +282,6 @@ test "tuple in tuple passed to generic function" {
 test "coerce tuple to tuple" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const T = std.meta.Tuple(&.{u8});
@@ -298,7 +296,6 @@ test "coerce tuple to tuple" {
 test "tuple type with void field" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const T = std.meta.Tuple(&[_]type{void});
     const x = T{{}};
@@ -335,7 +332,6 @@ test "zero sized struct in tuple handled correctly" {
 test "tuple type with void field and a runtime field" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const T = std.meta.Tuple(&[_]type{ usize, void });
     var t: T = .{ 5, {} };

test/behavior/union.zig

@@ -1227,7 +1227,6 @@ test "union tag is set when initiated as a temporary value at runtime" {
 }
 
 test "extern union most-aligned field is smaller" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/while.zig

@@ -341,7 +341,6 @@ test "else continue outer while" {
 }
 
 test "try terminating an infinite loop" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO