From 1dd26042cc11e15eb7406e750c9454a2e2289a0f Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Sat, 25 Feb 2023 10:41:37 +0100 Subject: [PATCH 001/725] docgen: remove line support in printShell Line support in printShell breaks the support to terminal colors, when a color is active over multiple lines. An example is when printing reference traces. Remove line support in printShell, since they are not really necessary like in printSourceBlock. In genHtml, remove an empty line when printing the execution of the executable generated by `zig build-exe`. Update the "shell parsed" tests. Remove the call to log.emerg, since it is no longer supported. Add an extra space after "--build-option1 \", as documented. Fixes #13280 --- doc/docgen.zig | 75 ++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index 277316dd37..e7663cdfe2 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -1224,41 +1224,39 @@ fn printShell(out: anytype, shell_content: []const u8, escape: bool) !void { while (iter.next()) |orig_line| { const line = mem.trimRight(u8, orig_line, " "); if (!cmd_cont and line.len > 1 and mem.eql(u8, line[0..2], "$ ") and line[line.len - 1] != '\\') { - try out.writeAll(start_line ++ "$ "); + try out.writeAll("$ "); const s = std.mem.trimLeft(u8, line[1..], " "); if (escape) { try writeEscaped(out, s); } else { try out.writeAll(s); } - try out.writeAll("" ++ end_line ++ "\n"); + try out.writeAll("" ++ "\n"); } else if (!cmd_cont and line.len > 1 and mem.eql(u8, line[0..2], "$ ") and line[line.len - 1] == '\\') { - try out.writeAll(start_line ++ "$ "); + try out.writeAll("$ "); const s = std.mem.trimLeft(u8, line[1..], " "); if (escape) { try writeEscaped(out, s); } else { try out.writeAll(s); } - try out.writeAll(end_line ++ "\n"); + try out.writeAll("\n"); cmd_cont = true; } else if (line.len > 0 and line[line.len - 1] != '\\' and cmd_cont) { - try out.writeAll(start_line); if (escape) { try writeEscaped(out, line); } else { try out.writeAll(line); } - try out.writeAll("" ++ end_line ++ "\n"); + try out.writeAll("" ++ "\n"); cmd_cont = false; } else { - try out.writeAll(start_line); if (escape) { try writeEscaped(out, line); } else { try out.writeAll(line); } - try out.writeAll(end_line ++ "\n"); + try out.writeAll("\n"); } } @@ -1511,7 +1509,7 @@ fn genHtml( const colored_stderr = try termColor(allocator, escaped_stderr); const colored_stdout = try termColor(allocator, escaped_stdout); - try shell_out.print("\n$ ./{s}\n{s}{s}", .{ code.name, colored_stdout, colored_stderr }); + try shell_out.print("$ ./{s}\n{s}{s}", .{ code.name, colored_stdout, colored_stderr }); if (exited_with_signal) { try shell_out.print("(process terminated by signal)", .{}); } @@ -1884,7 +1882,7 @@ test "shell parsed" { \\$ zig build test.zig ; const expected = - \\
Shell
$ zig build test.zig
+            \\
Shell
$ zig build test.zig
             \\

         ;
 
@@ -1892,7 +1890,6 @@ test "shell parsed" {
         defer buffer.deinit();
 
         try printShell(buffer.writer(), shell_out, false);
-        std.log.emerg("{s}", .{buffer.items});
         try testing.expectEqualSlices(u8, expected, buffer.items);
     }
     {
@@ -1901,8 +1898,8 @@ test "shell parsed" {
             \\build output
         ;
         const expected =
-            \\
Shell
$ zig build test.zig
-            \\build output
+            \\
Shell
$ zig build test.zig
+            \\build output
             \\

         ;
 
@@ -1919,9 +1916,9 @@ test "shell parsed" {
             \\$ ./test
         ;
         const expected =
-            \\
Shell
$ zig build test.zig
-            \\build output
-            \\$ ./test
+            \\
Shell
$ zig build test.zig
+            \\build output
+            \\$ ./test
             \\

         ;
 
@@ -1939,10 +1936,10 @@ test "shell parsed" {
             \\output
         ;
         const expected =
-            \\
Shell
$ zig build test.zig
-            \\
-            \\$ ./test
-            \\output
+            \\
Shell
$ zig build test.zig
+            \\
+            \\$ ./test
+            \\output
             \\

         ;
 
@@ -1959,9 +1956,9 @@ test "shell parsed" {
             \\output
         ;
         const expected =
-            \\
Shell
$ zig build test.zig
-            \\$ ./test
-            \\output
+            \\
Shell
$ zig build test.zig
+            \\$ ./test
+            \\output
             \\

         ;
 
@@ -1980,11 +1977,11 @@ test "shell parsed" {
             \\output
         ;
         const expected =
-            \\
Shell
$ zig build test.zig \
-            \\ --build-option
-            \\build output
-            \\$ ./test
-            \\output
+            \\
Shell
$ zig build test.zig \
+            \\ --build-option
+            \\build output
+            \\$ ./test
+            \\output
             \\

         ;
 
@@ -1998,15 +1995,15 @@ test "shell parsed" {
         // intentional space after "--build-option1 \"
         const shell_out =
             \\$ zig build test.zig \
-            \\ --build-option1 \
+            \\ --build-option1 \ 
             \\ --build-option2
             \\$ ./test
         ;
         const expected =
-            \\
Shell
$ zig build test.zig \
-            \\ --build-option1 \
-            \\ --build-option2
-            \\$ ./test
+            \\
Shell
$ zig build test.zig \
+            \\ --build-option1 \
+            \\ --build-option2
+            \\$ ./test
             \\

         ;
 
@@ -2022,8 +2019,8 @@ test "shell parsed" {
             \\$ ./test
         ;
         const expected =
-            \\
Shell
$ zig build test.zig \
-            \\$ ./test
+            \\
Shell
$ zig build test.zig \
+            \\$ ./test
             \\

         ;
 
@@ -2040,9 +2037,9 @@ test "shell parsed" {
             \\$1
         ;
         const expected =
-            \\
Shell
$ zig build test.zig
-            \\$ ./test
-            \\$1
+            \\
Shell
$ zig build test.zig
+            \\$ ./test
+            \\$1
             \\

         ;
 
@@ -2057,7 +2054,7 @@ test "shell parsed" {
             \\$zig build test.zig
         ;
         const expected =
-            \\
Shell
$zig build test.zig
+            \\
Shell
$zig build test.zig
             \\

         ;
 

From 7ca052a6fb017882de1ebcf86226f3b2927a11c7 Mon Sep 17 00:00:00 2001
From: Manlio Perillo 
Date: Sat, 25 Feb 2023 11:14:43 +0100
Subject: [PATCH 002/725] docgen: improve the termColor function

Make the termColor function more robust, ensuring that the generated
HTML classes are more consistent and that they are supported in the CSS.

Add documentation about the ANSI codes generated by Zig, and remove the
previous comment with the supported colors.

Improve test coverage for the termColor function, and move the tests at
the end of the file before the printShell tests.

Rename the "term color" test to "term supported colors" and add an
additional "term output from zig" test, using test data generated from
the Zig compiler.

Update the langref.html.in CSS to use the new names and remove incorrect
or obsolete colors like .t0_1, .t37 and .t37_1.

Fix support for the 1m color. Change font-weight to normal for the kbd
element, to avoid both the command (like `zig build-exe`) and the first
line of the error message having a bold font.

Rename the "shell parsed" test to "printShell".
---
 doc/docgen.zig      | 258 +++++++++++++++++++++++++++++++++++++++-----
 doc/langref.html.in |  20 ++--
 2 files changed, 243 insertions(+), 35 deletions(-)

diff --git a/doc/docgen.zig b/doc/docgen.zig
index e7663cdfe2..6f2438c857 100644
--- a/doc/docgen.zig
+++ b/doc/docgen.zig
@@ -793,28 +793,37 @@ fn writeEscaped(out: anytype, input: []const u8) !void {
     }
 }
 
-//#define VT_RED "\x1b[31;1m"
-//#define VT_GREEN "\x1b[32;1m"
-//#define VT_CYAN "\x1b[36;1m"
-//#define VT_WHITE "\x1b[37;1m"
-//#define VT_BOLD "\x1b[0;1m"
-//#define VT_RESET "\x1b[0m"
-
-test "term color" {
-    const input_bytes = "A\x1b[32;1mgreen\x1b[0mB";
-    const result = try termColor(std.testing.allocator, input_bytes);
-    defer std.testing.allocator.free(result);
-    try testing.expectEqualSlices(u8, "AgreenB", result);
+// Returns true if number is in slice.
+fn in(slice: []const u8, number: u8) bool {
+    for (slice) |n| {
+        if (number == n) return true;
+    }
+    return false;
 }
 
 fn termColor(allocator: Allocator, input: []const u8) ![]u8 {
+    // The SRG sequences generates by the Zig compiler are in the format:
+    //   ESC [  ;  m
+    // or
+    //   ESC [  m
+    //
+    // where
+    //   foreground-color is 31 (red), 32 (green), 36 (cyan)
+    //   n is 0 (reset), 1 (bold), 2 (dim)
+    //
+    //   Note that 37 (white) is currently not used by the compiler.
+    //
+    // See std.debug.TTY.Color.
+    const supported_sgr_colors = [_]u8{ 31, 32, 36 };
+    const supported_sgr_numbers = [_]u8{ 0, 1, 2 };
+
     var buf = std.ArrayList(u8).init(allocator);
     defer buf.deinit();
 
     var out = buf.writer();
-    var number_start_index: usize = undefined;
-    var first_number: usize = undefined;
-    var second_number: usize = undefined;
+    var sgr_param_start_index: usize = undefined;
+    var sgr_num: u8 = undefined;
+    var sgr_color: u8 = undefined;
     var i: usize = 0;
     var state: enum {
         start,
@@ -845,7 +854,7 @@ fn termColor(allocator: Allocator, input: []const u8) ![]u8 {
             },
             .lbracket => switch (c) {
                 '0'...'9' => {
-                    number_start_index = i;
+                    sgr_param_start_index = i;
                     state = .number;
                 },
                 else => return error.UnsupportedEscape,
@@ -853,13 +862,12 @@ fn termColor(allocator: Allocator, input: []const u8) ![]u8 {
             .number => switch (c) {
                 '0'...'9' => {},
                 else => {
-                    first_number = std.fmt.parseInt(usize, input[number_start_index..i], 10) catch unreachable;
-                    second_number = 0;
+                    sgr_num = try std.fmt.parseInt(u8, input[sgr_param_start_index..i], 10);
+                    sgr_color = 0;
                     state = .after_number;
                     i -= 1;
                 },
             },
-
             .after_number => switch (c) {
                 ';' => state = .arg,
                 'D' => state = .start,
@@ -874,7 +882,7 @@ fn termColor(allocator: Allocator, input: []const u8) ![]u8 {
             },
             .arg => switch (c) {
                 '0'...'9' => {
-                    number_start_index = i;
+                    sgr_param_start_index = i;
                     state = .arg_number;
                 },
                 else => return error.UnsupportedEscape,
@@ -882,7 +890,15 @@ fn termColor(allocator: Allocator, input: []const u8) ![]u8 {
             .arg_number => switch (c) {
                 '0'...'9' => {},
                 else => {
-                    second_number = std.fmt.parseInt(usize, input[number_start_index..i], 10) catch unreachable;
+                    // Keep the sequence consistent, foreground color first.
+                    // 32;1m is equivalent to 1;32m, but the latter will
+                    // generate an incorrect HTML class without notice.
+                    sgr_color = sgr_num;
+                    if (!in(&supported_sgr_colors, sgr_color)) return error.UnsupportedForegroundColor;
+
+                    sgr_num = try std.fmt.parseInt(u8, input[sgr_param_start_index..i], 10);
+                    if (!in(&supported_sgr_numbers, sgr_num)) return error.UnsupportedNumber;
+
                     state = .expect_end;
                     i -= 1;
                 },
@@ -893,10 +909,16 @@ fn termColor(allocator: Allocator, input: []const u8) ![]u8 {
                     while (open_span_count != 0) : (open_span_count -= 1) {
                         try out.writeAll("");
                     }
-                    if (first_number != 0 or second_number != 0) {
-                        try out.print("", .{ first_number, second_number });
-                        open_span_count += 1;
+                    if (sgr_num == 0) {
+                        if (sgr_color != 0) return error.UnsupportedColor;
+                        continue;
                     }
+                    if (sgr_color != 0) {
+                        try out.print("", .{ sgr_color, sgr_num });
+                    } else {
+                        try out.print("", .{sgr_num});
+                    }
+                    open_span_count += 1;
                 },
                 else => return error.UnsupportedEscape,
             },
@@ -1874,7 +1896,193 @@ fn dumpArgs(args: []const []const u8) void {
         print("\n", .{});
 }
 
-test "shell parsed" {
+test "term supported colors" {
+    const test_allocator = testing.allocator;
+
+    {
+        const input = "A\x1b[31;1mred\x1b[0mB";
+        const expect = "AredB";
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        const input = "A\x1b[32;1mgreen\x1b[0mB";
+        const expect = "AgreenB";
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        const input = "A\x1b[36;1mcyan\x1b[0mB";
+        const expect = "AcyanB";
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        const input = "A\x1b[1mbold\x1b[0mB";
+        const expect = "AboldB";
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        const input = "A\x1b[2mdim\x1b[0mB";
+        const expect = "AdimB";
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+}
+
+test "term output from zig" {
+    // Use data generated by https://github.com/perillo/zig-tty-test-data,
+    // with zig version 0.11.0-dev.1898+36d47dd19.
+    const test_allocator = testing.allocator;
+
+    {
+        // 1.1-with-build-progress.out
+        const input = "Semantic Analysis [1324] \x1b[25D\x1b[0KLLVM Emit Object... \x1b[20D\x1b[0KLLVM Emit Object... \x1b[20D\x1b[0KLLD Link... \x1b[12D\x1b[0K";
+        const expect = "";
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        // 2.1-with-reference-traces.out
+        const input = "\x1b[1msrc/2.1-with-reference-traces.zig:3:7: \x1b[31;1merror: \x1b[0m\x1b[1mcannot assign to constant\n\x1b[0m    x += 1;\n    \x1b[32;1m~~^~~~\n\x1b[0m\x1b[0m\x1b[2mreferenced by:\n    main: src/2.1-with-reference-traces.zig:7:5\n    callMain: /usr/local/lib/zig/lib/std/start.zig:607:17\n    remaining reference traces hidden; use '-freference-trace' to see all reference traces\n\n\x1b[0m";
+        const expect =
+            \\src/2.1-with-reference-traces.zig:3:7: error: cannot assign to constant
+            \\    x += 1;
+            \\    ~~^~~~
+            \\referenced by:
+            \\    main: src/2.1-with-reference-traces.zig:7:5
+            \\    callMain: /usr/local/lib/zig/lib/std/start.zig:607:17
+            \\    remaining reference traces hidden; use '-freference-trace' to see all reference traces
+            \\
+            \\
+        ;
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        // 2.2-without-reference-traces.out
+        const input = "\x1b[1m/usr/local/lib/zig/lib/std/io/fixed_buffer_stream.zig:128:29: \x1b[31;1merror: \x1b[0m\x1b[1minvalid type given to fixedBufferStream\n\x1b[0m                    else => @compileError(\"invalid type given to fixedBufferStream\"),\n                            \x1b[32;1m^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\x1b[0m\x1b[1m/usr/local/lib/zig/lib/std/io/fixed_buffer_stream.zig:116:66: \x1b[36;1mnote: \x1b[0m\x1b[1mcalled from here\n\x1b[0mpub fn fixedBufferStream(buffer: anytype) FixedBufferStream(Slice(@TypeOf(buffer))) {\n;                                                            \x1b[32;1m~~~~~^~~~~~~~~~~~~~~~~\n\x1b[0m";
+        const expect =
+            \\/usr/local/lib/zig/lib/std/io/fixed_buffer_stream.zig:128:29: error: invalid type given to fixedBufferStream
+            \\                    else => @compileError("invalid type given to fixedBufferStream"),
+            \\                            ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+            \\/usr/local/lib/zig/lib/std/io/fixed_buffer_stream.zig:116:66: note: called from here
+            \\pub fn fixedBufferStream(buffer: anytype) FixedBufferStream(Slice(@TypeOf(buffer))) {
+            \\;                                                            ~~~~~^~~~~~~~~~~~~~~~~
+            \\
+        ;
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        // 2.3-with-notes.out
+        const input = "\x1b[1msrc/2.3-with-notes.zig:6:9: \x1b[31;1merror: \x1b[0m\x1b[1mexpected type '*2.3-with-notes.Derp', found '*2.3-with-notes.Wat'\n\x1b[0m    bar(w);\n        \x1b[32;1m^\n\x1b[0m\x1b[1msrc/2.3-with-notes.zig:6:9: \x1b[36;1mnote: \x1b[0m\x1b[1mpointer type child '2.3-with-notes.Wat' cannot cast into pointer type child '2.3-with-notes.Derp'\n\x1b[0m\x1b[1msrc/2.3-with-notes.zig:2:13: \x1b[36;1mnote: \x1b[0m\x1b[1mopaque declared here\n\x1b[0mconst Wat = opaque {};\n            \x1b[32;1m^~~~~~~~~\n\x1b[0m\x1b[1msrc/2.3-with-notes.zig:1:14: \x1b[36;1mnote: \x1b[0m\x1b[1mopaque declared here\n\x1b[0mconst Derp = opaque {};\n             \x1b[32;1m^~~~~~~~~\n\x1b[0m\x1b[1msrc/2.3-with-notes.zig:4:18: \x1b[36;1mnote: \x1b[0m\x1b[1mparameter type declared here\n\x1b[0mextern fn bar(d: *Derp) void;\n                 \x1b[32;1m^~~~~\n\x1b[0m\x1b[0m\x1b[2mreferenced by:\n    main: src/2.3-with-notes.zig:10:5\n    callMain: /usr/local/lib/zig/lib/std/start.zig:607:17\n    remaining reference traces hidden; use '-freference-trace' to see all reference traces\n\n\x1b[0m";
+        const expect =
+            \\src/2.3-with-notes.zig:6:9: error: expected type '*2.3-with-notes.Derp', found '*2.3-with-notes.Wat'
+            \\    bar(w);
+            \\        ^
+            \\src/2.3-with-notes.zig:6:9: note: pointer type child '2.3-with-notes.Wat' cannot cast into pointer type child '2.3-with-notes.Derp'
+            \\src/2.3-with-notes.zig:2:13: note: opaque declared here
+            \\const Wat = opaque {};
+            \\            ^~~~~~~~~
+            \\src/2.3-with-notes.zig:1:14: note: opaque declared here
+            \\const Derp = opaque {};
+            \\             ^~~~~~~~~
+            \\src/2.3-with-notes.zig:4:18: note: parameter type declared here
+            \\extern fn bar(d: *Derp) void;
+            \\                 ^~~~~
+            \\referenced by:
+            \\    main: src/2.3-with-notes.zig:10:5
+            \\    callMain: /usr/local/lib/zig/lib/std/start.zig:607:17
+            \\    remaining reference traces hidden; use '-freference-trace' to see all reference traces
+            \\
+            \\
+        ;
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        // 3.1-with-error-return-traces.out
+
+        const input = "error: Error\n\x1b[1m/home/zig/src/3.1-with-error-return-traces.zig:5:5\x1b[0m: \x1b[2m0x20b008 in callee (3.1-with-error-return-traces)\x1b[0m\n    return error.Error;\n    \x1b[32;1m^\x1b[0m\n\x1b[1m/home/zig/src/3.1-with-error-return-traces.zig:9:5\x1b[0m: \x1b[2m0x20b113 in caller (3.1-with-error-return-traces)\x1b[0m\n    try callee();\n    \x1b[32;1m^\x1b[0m\n\x1b[1m/home/zig/src/3.1-with-error-return-traces.zig:13:5\x1b[0m: \x1b[2m0x20b153 in main (3.1-with-error-return-traces)\x1b[0m\n    try caller();\n    \x1b[32;1m^\x1b[0m\n";
+        const expect =
+            \\error: Error
+            \\/home/zig/src/3.1-with-error-return-traces.zig:5:5: 0x20b008 in callee (3.1-with-error-return-traces)
+            \\    return error.Error;
+            \\    ^
+            \\/home/zig/src/3.1-with-error-return-traces.zig:9:5: 0x20b113 in caller (3.1-with-error-return-traces)
+            \\    try callee();
+            \\    ^
+            \\/home/zig/src/3.1-with-error-return-traces.zig:13:5: 0x20b153 in main (3.1-with-error-return-traces)
+            \\    try caller();
+            \\    ^
+            \\
+        ;
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        // 3.2-with-stack-trace.out
+        const input = "\x1b[1m/usr/local/lib/zig/lib/std/debug.zig:561:19\x1b[0m: \x1b[2m0x22a107 in writeCurrentStackTrace__anon_5898 (3.2-with-stack-trace)\x1b[0m\n    while (it.next()) |return_address| {\n                  \x1b[32;1m^\x1b[0m\n\x1b[1m/usr/local/lib/zig/lib/std/debug.zig:157:80\x1b[0m: \x1b[2m0x20bb23 in dumpCurrentStackTrace (3.2-with-stack-trace)\x1b[0m\n        writeCurrentStackTrace(stderr, debug_info, detectTTYConfig(io.getStdErr()), start_addr) catch |err| {\n                                                                               \x1b[32;1m^\x1b[0m\n\x1b[1m/home/zig/src/3.2-with-stack-trace.zig:5:36\x1b[0m: \x1b[2m0x20d3b2 in foo (3.2-with-stack-trace)\x1b[0m\n    std.debug.dumpCurrentStackTrace(null);\n                                   \x1b[32;1m^\x1b[0m\n\x1b[1m/home/zig/src/3.2-with-stack-trace.zig:9:8\x1b[0m: \x1b[2m0x20b458 in main (3.2-with-stack-trace)\x1b[0m\n    foo();\n       \x1b[32;1m^\x1b[0m\n\x1b[1m/usr/local/lib/zig/lib/std/start.zig:607:22\x1b[0m: \x1b[2m0x20a965 in posixCallMainAndExit (3.2-with-stack-trace)\x1b[0m\n            root.main();\n                     \x1b[32;1m^\x1b[0m\n\x1b[1m/usr/local/lib/zig/lib/std/start.zig:376:5\x1b[0m: \x1b[2m0x20a411 in _start (3.2-with-stack-trace)\x1b[0m\n    @call(.never_inline, posixCallMainAndExit, .{});\n    \x1b[32;1m^\x1b[0m\n";
+        const expect =
+            \\/usr/local/lib/zig/lib/std/debug.zig:561:19: 0x22a107 in writeCurrentStackTrace__anon_5898 (3.2-with-stack-trace)
+            \\    while (it.next()) |return_address| {
+            \\                  ^
+            \\/usr/local/lib/zig/lib/std/debug.zig:157:80: 0x20bb23 in dumpCurrentStackTrace (3.2-with-stack-trace)
+            \\        writeCurrentStackTrace(stderr, debug_info, detectTTYConfig(io.getStdErr()), start_addr) catch |err| {
+            \\                                                                               ^
+            \\/home/zig/src/3.2-with-stack-trace.zig:5:36: 0x20d3b2 in foo (3.2-with-stack-trace)
+            \\    std.debug.dumpCurrentStackTrace(null);
+            \\                                   ^
+            \\/home/zig/src/3.2-with-stack-trace.zig:9:8: 0x20b458 in main (3.2-with-stack-trace)
+            \\    foo();
+            \\       ^
+            \\/usr/local/lib/zig/lib/std/start.zig:607:22: 0x20a965 in posixCallMainAndExit (3.2-with-stack-trace)
+            \\            root.main();
+            \\                     ^
+            \\/usr/local/lib/zig/lib/std/start.zig:376:5: 0x20a411 in _start (3.2-with-stack-trace)
+            \\    @call(.never_inline, posixCallMainAndExit, .{});
+            \\    ^
+            \\
+        ;
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+}
+
+test "printShell" {
     const test_allocator = std.testing.allocator;
 
     {
diff --git a/doc/langref.html.in b/doc/langref.html.in
index 907464867e..c88c91560c 100644
--- a/doc/langref.html.in
+++ b/doc/langref.html.in
@@ -71,19 +71,19 @@
           text-align: left;
           font-weight: normal;
       }
-      .t0_1, .t37, .t37_1 {
+      .sgr-1m {
         font-weight: bold;
       }
-      .t2_0 {
+      .sgr-2m {
         color: #575757;
       }
-      .t31_1 {
+      .sgr-31_1m {
         color: #b40000;
       }
-      .t32_1 {
+      .sgr-32_1m {
         color: green;
       }
-      .t36_1 {
+      .sgr-36_1m {
         color: #005C7A;
       }
       .file {
@@ -114,7 +114,7 @@
         line-height: normal;
       }
       kbd {
-        font-weight: bold;
+        font-weight: normal;
       }
       .table-wrapper {
         width: 100%;
@@ -232,16 +232,16 @@
         table, th, td {
             border-color: grey;
         }
-        .t2_0 {
+        .sgr-2m {
             color: grey;
         }
-        .t31_1 {
+        .sgr-31_1m {
             color: red;
         }
-        .t32_1 {
+        .sgr-32_1m {
             color: #00B800;
         }
-        .t36_1 {
+        .sgr-36_1m {
             color: #0086b3;
         }
         code {

From c579a23c5d997964a67883db5677c290bdbd7da6 Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Sun, 16 Apr 2023 13:27:38 +0100
Subject: [PATCH 003/725] std: add CCRandomGenerateBytes macOs native api.

---
 lib/std/c/darwin.zig | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/lib/std/c/darwin.zig b/lib/std/c/darwin.zig
index a46337b721..4ede0b3891 100644
--- a/lib/std/c/darwin.zig
+++ b/lib/std/c/darwin.zig
@@ -897,7 +897,35 @@ pub extern "c" fn pthread_attr_get_qos_class_np(attr: *pthread_attr_t, qos_class
 pub extern "c" fn pthread_set_qos_class_self_np(qos_class: qos_class_t, relative_priority: c_int) c_int;
 pub extern "c" fn pthread_get_qos_class_np(pthread: std.c.pthread_t, qos_class: *qos_class_t, relative_priority: *c_int) c_int;
 
+pub const CCryptorStatus = enum(i32) {
+    /// Operation completed
+    kCCSuccess = 0,
+    /// Illegal parameter
+    kCCParamError = -4300,
+    /// Provided buffer too small
+    kCCBufferTooSmall = -4301,
+    /// Failed memory allocation
+    kCCMemoryFailure = -4302,
+    /// Size alignment issue
+    kCCAlignmentError = -4303,
+    /// Decoding issue
+    kCCDecodeError = -4304,
+    /// Call not implemented
+    kCCUnimplemented = -4305,
+    kCCOverflow = -4306,
+    kCCRNGFailure = -4307,
+    /// Unspecified error
+    kCCUnspecifiedError = -4308,
+    kCCCallSequenceError = -4309,
+    kCCKeySizeError = -4310,
+    /// Invalid key
+    kCCInvalidKey = -4311,
+};
+
+pub const CCRNGStatus = CCryptorStatus;
+
 pub extern "c" fn arc4random_buf(buf: [*]u8, len: usize) void;
+pub extern "c" fn CCRandomGenerateBytes(bytes: ?*anyopaque, count: usize) CCRNGStatus;
 
 // Grand Central Dispatch is exposed by libSystem.
 pub extern "c" fn dispatch_release(object: *anyopaque) void;

From 6248ac535b9bfc7113101c1ff0f0bb1e41c394d4 Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Sun, 16 Apr 2023 15:49:42 +0100
Subject: [PATCH 004/725] os: getrandom wrapper favoring it for macOs/iOs only

---
 lib/std/os.zig | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/lib/std/os.zig b/lib/std/os.zig
index 13b0c62455..c2591ac355 100644
--- a/lib/std/os.zig
+++ b/lib/std/os.zig
@@ -512,7 +512,18 @@ pub fn getrandom(buffer: []u8) GetRandomError!void {
         return;
     }
     switch (builtin.os.tag) {
-        .netbsd, .openbsd, .macos, .ios, .tvos, .watchos => {
+        .macos, .ios => {
+            const rc = darwin.CCRandomGenerateBytes(buffer.ptr, buffer.len);
+            if (rc != darwin.CCRNGStatus.kCCSuccess) {
+                if (rc == darwin.CCRNGStatus.kCCParamError or rc == darwin.CCRNGStatus.kCCBufferTooSmall) {
+                    return error.InvalidHandle;
+                } else {
+                    return error.SystemResources;
+                }
+            }
+            return;
+        },
+        .netbsd, .openbsd, .tvos, .watchos => {
             system.arc4random_buf(buffer.ptr, buffer.len);
             return;
         },
@@ -991,7 +1002,7 @@ pub fn preadv(fd: fd_t, iov: []const iovec, offset: u64) PReadError!usize {
     if (have_pread_but_not_preadv) {
         // We could loop here; but proper usage of `preadv` must handle partial reads anyway.
         // So we simply read into the first vector only.
-        if (iov.len == 0) return @as(usize, 0);
+        if (iov.len == 0) return @intCast(usize, 0);
         const first = iov[0];
         return pread(fd, first.iov_base[0..first.iov_len], offset);
     }

From d339e86fb1308f8df37fc351ee01007492e72f25 Mon Sep 17 00:00:00 2001
From: Koakuma 
Date: Wed, 4 Jan 2023 10:46:05 +0700
Subject: [PATCH 005/725] stage2: sparc64: Skip unimplemented tests

---
 test/behavior/array.zig           |  4 ++++
 test/behavior/bugs/12051.zig      |  1 +
 test/behavior/bugs/12450.zig      |  1 +
 test/behavior/comptime_memory.zig |  3 +++
 test/behavior/packed-struct.zig   |  4 +++-
 test/behavior/ptrcast.zig         | 10 ++++++++++
 test/behavior/tuple.zig           |  5 +++++
 test/behavior/union.zig           |  1 +
 test/behavior/var_args.zig        |  2 ++
 test/behavior/vector.zig          |  1 +
 10 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/test/behavior/array.zig b/test/behavior/array.zig
index ec31d4a3fb..6cf0ab9e1d 100644
--- a/test/behavior/array.zig
+++ b/test/behavior/array.zig
@@ -47,6 +47,7 @@ fn getArrayLen(a: []const u32) usize {
 
 test "array concat with undefined" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
@@ -70,6 +71,7 @@ test "array concat with undefined" {
 test "array concat with tuple" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const array: [2]u8 = .{ 1, 2 };
     {
@@ -103,6 +105,7 @@ test "array init with mult" {
 
 test "array literal with explicit type" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const hex_mult: [4]u16 = .{ 4096, 256, 16, 1 };
 
@@ -203,6 +206,7 @@ test "nested arrays of strings" {
 
 test "nested arrays of integers" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const array_of_numbers = [_][2]u8{
         [2]u8{ 1, 2 },
diff --git a/test/behavior/bugs/12051.zig b/test/behavior/bugs/12051.zig
index 5e2087d422..2b2765358b 100644
--- a/test/behavior/bugs/12051.zig
+++ b/test/behavior/bugs/12051.zig
@@ -6,6 +6,7 @@ test {
     if (builtin.zig_backend == .stage2_x86) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     const x = X{};
     try std.testing.expectEqual(@as(u16, 0), x.y.a);
     try std.testing.expectEqual(false, x.y.b);
diff --git a/test/behavior/bugs/12450.zig b/test/behavior/bugs/12450.zig
index 89e5c774e0..4d0c7cd9a1 100644
--- a/test/behavior/bugs/12450.zig
+++ b/test/behavior/bugs/12450.zig
@@ -13,6 +13,7 @@ test {
     if (builtin.zig_backend == .stage2_x86) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     var f1: *align(16) Foo = @alignCast(16, @ptrCast(*align(1) Foo, &buffer[0]));
     try expect(@typeInfo(@TypeOf(f1)).Pointer.alignment == 16);
     try expect(@ptrToInt(f1) == @ptrToInt(&f1.a));
diff --git a/test/behavior/comptime_memory.zig b/test/behavior/comptime_memory.zig
index 98e6e65725..5c3012d1dc 100644
--- a/test/behavior/comptime_memory.zig
+++ b/test/behavior/comptime_memory.zig
@@ -76,6 +76,7 @@ fn bigToNativeEndian(comptime T: type, v: T) T {
 }
 test "type pun endianness" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     comptime {
         const StructOfBytes = extern struct { x: [4]u8 };
@@ -376,6 +377,8 @@ test "offset field ptr by enclosing array element size" {
 
 test "accessing reinterpreted memory of parent object" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+
     const S = extern struct {
         a: f32,
         b: [4]u8,
diff --git a/test/behavior/packed-struct.zig b/test/behavior/packed-struct.zig
index ac4b62b0b8..8a0ef72382 100644
--- a/test/behavior/packed-struct.zig
+++ b/test/behavior/packed-struct.zig
@@ -563,7 +563,7 @@ test "packed struct passed to callconv(.C) function" {
 test "overaligned pointer to packed struct" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
-
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     const S = packed struct { a: u32, b: u32 };
     var foo: S align(4) = .{ .a = 123, .b = 456 };
     const ptr: *align(4) S = &foo;
@@ -583,6 +583,7 @@ test "overaligned pointer to packed struct" {
 test "packed struct initialized in bitcast" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const T = packed struct { val: u8 };
     var val: u8 = 123;
@@ -595,6 +596,7 @@ test "pointer to container level packed struct field" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const S = packed struct(u32) {
         test_bit: bool,
diff --git a/test/behavior/ptrcast.zig b/test/behavior/ptrcast.zig
index becdee6b05..6a5ae726ae 100644
--- a/test/behavior/ptrcast.zig
+++ b/test/behavior/ptrcast.zig
@@ -4,6 +4,8 @@ const expect = std.testing.expect;
 const native_endian = builtin.target.cpu.arch.endian();
 
 test "reinterpret bytes as integer with nonzero offset" {
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+
     try testReinterpretBytesAsInteger();
     comptime try testReinterpretBytesAsInteger();
 }
@@ -74,6 +76,8 @@ fn testReinterpretBytesAsExternStruct() !void {
 }
 
 test "reinterpret bytes of an extern struct (with under-aligned fields) into another" {
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+
     try testReinterpretExternStructAsExternStruct();
     comptime try testReinterpretExternStructAsExternStruct();
 }
@@ -96,6 +100,8 @@ fn testReinterpretExternStructAsExternStruct() !void {
 }
 
 test "reinterpret bytes of an extern struct into another" {
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+
     try testReinterpretOverAlignedExternStructAsExternStruct();
     comptime try testReinterpretOverAlignedExternStructAsExternStruct();
 }
@@ -191,6 +197,8 @@ const Bytes = struct {
 };
 
 test "comptime ptrcast keeps larger alignment" {
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+
     comptime {
         const a: u32 = 1234;
         const p = @ptrCast([*]const u8, &a);
@@ -199,6 +207,8 @@ test "comptime ptrcast keeps larger alignment" {
 }
 
 test "ptrcast of const integer has the correct object size" {
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+
     const is_value = ~@intCast(isize, std.math.minInt(isize));
     const is_bytes = @ptrCast([*]const u8, &is_value)[0..@sizeOf(isize)];
     if (@sizeOf(isize) == 8) {
diff --git a/test/behavior/tuple.zig b/test/behavior/tuple.zig
index 1cf68a0769..11cc8b2dce 100644
--- a/test/behavior/tuple.zig
+++ b/test/behavior/tuple.zig
@@ -296,6 +296,7 @@ test "coerce tuple to tuple" {
 test "tuple type with void field" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const T = std.meta.Tuple(&[_]type{void});
     const x = T{{}};
@@ -341,6 +342,7 @@ test "tuple type with void field and a runtime field" {
 test "branching inside tuple literal" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn foo(a: anytype) !void {
@@ -355,6 +357,7 @@ test "tuple initialized with a runtime known value" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const E = union(enum) { e: []const u8 };
     const W = union(enum) { w: E };
@@ -368,6 +371,7 @@ test "tuple of struct concatenation and coercion to array" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const StructWithDefault = struct { value: f32 = 42 };
     const SomeStruct = struct { array: [4]StructWithDefault };
@@ -381,6 +385,7 @@ test "tuple of struct concatenation and coercion to array" {
 test "nested runtime conditionals in tuple initializer" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     var data: u8 = 0;
     const x = .{
diff --git a/test/behavior/union.zig b/test/behavior/union.zig
index 37b1f0cb49..025eb4f6bd 100644
--- a/test/behavior/union.zig
+++ b/test/behavior/union.zig
@@ -1495,6 +1495,7 @@ test "packed union with zero-bit field" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const S = packed struct {
         nested: packed union {
diff --git a/test/behavior/var_args.zig b/test/behavior/var_args.zig
index cdceea96c6..968b15b218 100644
--- a/test/behavior/var_args.zig
+++ b/test/behavior/var_args.zig
@@ -95,6 +95,7 @@ fn doNothingWithFirstArg(args: anytype) void {
 
 test "simple variadic function" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
@@ -143,6 +144,7 @@ test "simple variadic function" {
 
 test "variadic functions" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/vector.zig b/test/behavior/vector.zig
index d549588dee..f6521f04c3 100644
--- a/test/behavior/vector.zig
+++ b/test/behavior/vector.zig
@@ -1282,6 +1282,7 @@ test "store to vector in slice" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     var v = [_]@Vector(3, f32){
         .{ 1, 1, 1 },

From 1794a45572a30a493a5c10fcdc45ba0fa3ac4218 Mon Sep 17 00:00:00 2001
From: Koakuma 
Date: Wed, 4 Jan 2023 10:46:27 +0700
Subject: [PATCH 006/725] stage2: sparc64: Add stub for c_va_*

---
 src/arch/sparc64/CodeGen.zig | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig
index cc5c9e9832..58b384c832 100644
--- a/src/arch/sparc64/CodeGen.zig
+++ b/src/arch/sparc64/CodeGen.zig
@@ -720,10 +720,10 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .error_set_has_value => @panic("TODO implement error_set_has_value"),
             .vector_store_elem => @panic("TODO implement vector_store_elem"),
 
-            .c_va_arg => @panic("TODO implement c_va_arg"),
-            .c_va_copy => @panic("TODO implement c_va_copy"),
-            .c_va_end => @panic("TODO implement c_va_end"),
-            .c_va_start => @panic("TODO implement c_va_start"),
+            .c_va_arg => return self.fail("TODO implement c_va_arg", .{}),
+            .c_va_copy => return self.fail("TODO implement c_va_copy", .{}),
+            .c_va_end => return self.fail("TODO implement c_va_end", .{}),
+            .c_va_start => return self.fail("TODO implement c_va_start", .{}),
 
             .wasm_memory_size => unreachable,
             .wasm_memory_grow => unreachable,

From cc2a5185d631b8b33eb5d466cc52ceb092435fd4 Mon Sep 17 00:00:00 2001
From: Koakuma 
Date: Wed, 4 Jan 2023 10:58:40 +0700
Subject: [PATCH 007/725] stage2: sparc64: Implement airStructFieldPtr

---
 src/arch/sparc64/CodeGen.zig | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig
index 58b384c832..d682ce8c2b 100644
--- a/src/arch/sparc64/CodeGen.zig
+++ b/src/arch/sparc64/CodeGen.zig
@@ -595,7 +595,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .ret_load        => try self.airRetLoad(inst),
             .store           => try self.airStore(inst, false),
             .store_safe      => try self.airStore(inst, true),
-            .struct_field_ptr=> @panic("TODO try self.airStructFieldPtr(inst)"),
+            .struct_field_ptr=> try self.airStructFieldPtr(inst),
             .struct_field_val=> try self.airStructFieldVal(inst),
             .array_to_slice  => try self.airArrayToSlice(inst),
             .int_to_float    => try self.airIntToFloat(inst),
@@ -2425,6 +2425,13 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
     return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
 }
 
+fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
+    const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
+    const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
+    const result = try self.structFieldPtr(inst, extra.struct_operand, extra.field_index);
+    return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
+}
+
 fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
     const result = try self.structFieldPtr(inst, ty_op.operand, index);

From 486ab3852e22f8d5ba474691a5b068f1f1729f2e Mon Sep 17 00:00:00 2001
From: Koakuma 
Date: Wed, 4 Jan 2023 12:53:11 +0700
Subject: [PATCH 008/725] stage2: sparc64: Factor machine offset calculation

---
 src/arch/sparc64/CodeGen.zig | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig
index d682ce8c2b..3d82cad4ec 100644
--- a/src/arch/sparc64/CodeGen.zig
+++ b/src/arch/sparc64/CodeGen.zig
@@ -141,6 +141,8 @@ const MCValue = union(enum) {
     /// The value is one of the stack variables.
     /// If the type is a pointer, it means the pointer address is in the stack at this offset.
     /// Note that this stores the plain value (i.e without the effects of the stack bias).
+    /// Always convert this value into machine offsets with realStackOffset() before
+    /// lowering into asm!
     stack_offset: u32,
     /// The value is a pointer to one of the stack variables (payload is stack offset).
     ptr_stack_offset: u32,
@@ -3651,7 +3653,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
             return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa });
         },
         .ptr_stack_offset => |off| {
-            const real_offset = off + abi.stack_bias + abi.stack_reserved_area;
+            const real_offset = realStackOffset(off);
             const simm13 = math.cast(i13, real_offset) orelse
                 return self.fail("TODO larger stack offsets: {}", .{real_offset});
 
@@ -3783,7 +3785,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
             try self.genLoad(reg, reg, i13, 0, ty.abiSize(self.target.*));
         },
         .stack_offset => |off| {
-            const real_offset = off + abi.stack_bias + abi.stack_reserved_area;
+            const real_offset = realStackOffset(off);
             const simm13 = math.cast(i13, real_offset) orelse
                 return self.fail("TODO larger stack offsets: {}", .{real_offset});
             try self.genLoad(reg, .sp, i13, simm13, ty.abiSize(self.target.*));
@@ -3817,7 +3819,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
             return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
         },
         .register => |reg| {
-            const real_offset = stack_offset + abi.stack_bias + abi.stack_reserved_area;
+            const real_offset = realStackOffset(stack_offset);
             const simm13 = math.cast(i13, real_offset) orelse
                 return self.fail("TODO larger stack offsets: {}", .{real_offset});
             return self.genStore(reg, .sp, i13, simm13, abi_size);
@@ -4252,6 +4254,17 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void {
     }
 }
 
+/// Turns stack_offset MCV into a real SPARCv9 stack offset usable for asm.
+fn realStackOffset(off: u32) u32 {
+    return off
+        // SPARCv9 %sp points away from the stack by some amount.
+        + abi.stack_bias
+        // The first couple bytes of each stack frame is reserved
+        // for ABI and hardware purposes.
+        + abi.stack_reserved_area;
+        // Only after that we have the usable stack frame portion.
+}
+
 /// Caller must call `CallMCValues.deinit`.
 fn resolveCallingConventionValues(self: *Self, fn_ty: Type, role: RegisterView) !CallMCValues {
     const cc = fn_ty.fnCallingConvention();

From 83e6223192acd635275e67614e55b7a4c579a969 Mon Sep 17 00:00:00 2001
From: Koakuma 
Date: Wed, 4 Jan 2023 16:38:15 +0700
Subject: [PATCH 009/725] stage2: sparc64: Implement airByteSwap

---
 src/arch/sparc64/CodeGen.zig | 156 +++++++++++++++++++++++++++++++++--
 src/arch/sparc64/Emit.zig    |  10 +++
 src/arch/sparc64/Mir.zig     |  30 +++++++
 3 files changed, 189 insertions(+), 7 deletions(-)

diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig
index 3d82cad4ec..a4fa7e179c 100644
--- a/src/arch/sparc64/CodeGen.zig
+++ b/src/arch/sparc64/CodeGen.zig
@@ -22,6 +22,7 @@ const Type = @import("../../type.zig").Type;
 const CodeGenError = codegen.CodeGenError;
 const Result = @import("../../codegen.zig").Result;
 const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
+const Endian = std.builtin.Endian;
 
 const build_options = @import("build_options");
 
@@ -30,6 +31,7 @@ const abi = @import("abi.zig");
 const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
 const errUnionErrorOffset = codegen.errUnionErrorOffset;
 const Instruction = bits.Instruction;
+const ASI = Instruction.ASI;
 const ShiftWidth = Instruction.ShiftWidth;
 const RegisterManager = abi.RegisterManager;
 const RegisterLock = RegisterManager.RegisterLock;
@@ -615,7 +617,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .clz             => try self.airClz(inst),
             .ctz             => try self.airCtz(inst),
             .popcount        => try self.airPopcount(inst),
-            .byte_swap       => @panic("TODO try self.airByteSwap(inst)"),
+            .byte_swap       => try self.airByteSwap(inst),
             .bit_reverse     => try self.airBitReverse(inst),
             .tag_name        => try self.airTagName(inst),
             .error_name      => try self.airErrorName(inst),
@@ -1200,6 +1202,90 @@ fn airBreakpoint(self: *Self) !void {
     return self.finishAirBookkeeping();
 }
 
+fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
+    const ty_op = self.air.instructions.items(.data)[inst].ty_op;
+
+    // We have hardware byteswapper in SPARCv9, don't let mainstream compilers mislead you.
+    // That being said, the strategy to lower this is:
+    // - If src is an immediate, comptime-swap it.
+    // - If src is in memory then issue an LD*A with #ASI_P_[oppposite-endian]
+    // - If src is a register then issue an ST*A with #ASI_P_[oppposite-endian]
+    //   to a stack slot, then follow with a normal load from said stack slot.
+    //   This is because on some implementations, ASI-tagged memory operations are non-piplelinable
+    //   and loads tend to have longer latency than stores, so the sequence will minimize stall.
+    // The result will always be either another immediate or stored in a register.
+    // TODO: Fold byteswap+store into a single ST*A and load+byteswap into a single LD*A.
+    const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
+        const operand = try self.resolveInst(ty_op.operand);
+        const operand_ty = self.air.typeOf(ty_op.operand);
+        switch (operand_ty.zigTypeTag()) {
+            .Vector => return self.fail("TODO byteswap for vectors", .{}),
+            .Int => {
+                const int_info = operand_ty.intInfo(self.target.*);
+                if (int_info.bits == 8) break :result operand;
+
+                const abi_size = int_info.bits >> 3;
+                const abi_align = operand_ty.abiAlignment(self.target.*);
+                const opposite_endian_asi = switch (self.target.cpu.arch.endian()) {
+                    Endian.Big => ASI.asi_primary_little,
+                    Endian.Little => ASI.asi_primary,
+                };
+
+                switch (operand) {
+                    .immediate => |imm| {
+                        const swapped = switch (int_info.bits) {
+                            16 => @byteSwap(@intCast(u16, imm)),
+                            24 => @byteSwap(@intCast(u24, imm)),
+                            32 => @byteSwap(@intCast(u32, imm)),
+                            40 => @byteSwap(@intCast(u40, imm)),
+                            48 => @byteSwap(@intCast(u48, imm)),
+                            56 => @byteSwap(@intCast(u56, imm)),
+                            64 => @byteSwap(@intCast(u64, imm)),
+                            else => return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{}),
+                        };
+                        break :result .{ .immediate = swapped };
+                    },
+                    .register => |reg| {
+                        if (int_info.bits > 64 or @popCount(int_info.bits) != 1)
+                            return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{});
+
+                        const off = try self.allocMem(inst, abi_size, abi_align);
+                        const off_reg = try self.copyToTmpRegister(operand_ty, .{ .immediate = realStackOffset(off) });
+
+                        try self.genStoreASI(reg, .sp, off_reg, abi_size, opposite_endian_asi);
+                        try self.genLoad(reg, .sp, Register, off_reg, abi_size);
+                        break :result reg;
+                    },
+                    .memory => {
+                        if (int_info.bits > 64 or @popCount(int_info.bits) != 1)
+                            return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{});
+
+                        const addr_reg = try self.copyToTmpRegister(operand_ty, operand);
+                        const dst_reg = try self.register_manager.allocReg(null, gp);
+
+                        try self.genLoadASI(dst_reg, addr_reg, .g0, abi_size, opposite_endian_asi);
+                        break :result dst_reg;
+                    },
+                    .stack_offset => |off| {
+                        if (int_info.bits > 64 or @popCount(int_info.bits) != 1)
+                            return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{});
+
+                        const off_reg = try self.copyToTmpRegister(operand_ty, .{ .immediate = realStackOffset(off) });
+                        const dst_reg = try self.register_manager.allocReg(null, gp);
+
+                        try self.genLoadASI(dst_reg, .sp, off_reg, abi_size, opposite_endian_asi);
+                        break :result dst_reg;
+                    },
+                    else => unreachable,
+                }
+            },
+            else => unreachable,
+        }
+    };
+
+    return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
+}
+
 fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
     if (modifier == .always_tail) return self.fail("TODO implement tail calls for {}", .{self.target.cpu.arch});
 
@@ -3583,6 +3669,34 @@ fn genLoad(self: *Self, value_reg: Register, addr_reg: Register, comptime off_ty
     }
 }
 
+fn genLoadASI(self: *Self, value_reg: Register, addr_reg: Register, off_reg: Register, abi_size: u64, asi: ASI) !void {
+    switch (abi_size) {
+        1, 2, 4, 8 => {
+            const tag: Mir.Inst.Tag = switch (abi_size) {
+                1 => .lduba,
+                2 => .lduha,
+                4 => .lduwa,
+                8 => .ldxa,
+                else => unreachable, // unexpected abi size
+            };
+
+            _ = try self.addInst(.{
+                .tag = tag,
+                .data = .{
+                    .mem_asi = .{
+                        .rd = value_reg,
+                        .rs1 = addr_reg,
+                        .rs2 = off_reg,
+                        .asi = asi,
+                    },
+                },
+            });
+        },
+        3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}),
+        else => unreachable,
+    }
+}
+
 fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
     switch (mcv) {
         .dead => unreachable,
@@ -3942,6 +4056,34 @@ fn genStore(self: *Self, value_reg: Register, addr_reg: Register, comptime off_t
     }
 }
 
+fn genStoreASI(self: *Self, value_reg: Register, addr_reg: Register, off_reg: Register, abi_size: u64, asi: ASI) !void {
+    switch (abi_size) {
+        1, 2, 4, 8 => {
+            const tag: Mir.Inst.Tag = switch (abi_size) {
+                1 => .stba,
+                2 => .stha,
+                4 => .stwa,
+                8 => .stxa,
+                else => unreachable, // unexpected abi size
+            };
+
+            _ = try self.addInst(.{
+                .tag = tag,
+                .data = .{
+                    .mem_asi = .{
+                        .rd = value_reg,
+                        .rs1 = addr_reg,
+                        .rs2 = off_reg,
+                        .asi = asi,
+                    },
+                },
+            });
+        },
+        3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}),
+        else => unreachable,
+    }
+}
+
 fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue {
     const mcv: MCValue = switch (try codegen.genTypedValue(
         self.bin_file,
@@ -4257,12 +4399,12 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void {
 /// Turns stack_offset MCV into a real SPARCv9 stack offset usable for asm.
 fn realStackOffset(off: u32) u32 {
     return off
-        // SPARCv9 %sp points away from the stack by some amount.
-        + abi.stack_bias
-        // The first couple bytes of each stack frame is reserved
-        // for ABI and hardware purposes.
-        + abi.stack_reserved_area;
-        // Only after that we have the usable stack frame portion.
+    // SPARCv9 %sp points away from the stack by some amount.
+    + abi.stack_bias
+    // The first couple bytes of each stack frame is reserved
+    // for ABI and hardware purposes.
+    + abi.stack_reserved_area;
+    // Only after that we have the usable stack frame portion.
 }
 
 /// Caller must call `CallMCValues.deinit`.
diff --git a/src/arch/sparc64/Emit.zig b/src/arch/sparc64/Emit.zig
index 7e71492af7..c0dbab5a14 100644
--- a/src/arch/sparc64/Emit.zig
+++ b/src/arch/sparc64/Emit.zig
@@ -91,6 +91,11 @@ pub fn emitMir(
             .lduw => try emit.mirArithmetic3Op(inst),
             .ldx => try emit.mirArithmetic3Op(inst),
 
+            .lduba => unreachable,
+            .lduha => unreachable,
+            .lduwa => unreachable,
+            .ldxa => unreachable,
+
             .@"and" => try emit.mirArithmetic3Op(inst),
             .@"or" => try emit.mirArithmetic3Op(inst),
             .xor => try emit.mirArithmetic3Op(inst),
@@ -127,6 +132,11 @@ pub fn emitMir(
             .stw => try emit.mirArithmetic3Op(inst),
             .stx => try emit.mirArithmetic3Op(inst),
 
+            .stba => unreachable,
+            .stha => unreachable,
+            .stwa => unreachable,
+            .stxa => unreachable,
+
             .sub => try emit.mirArithmetic3Op(inst),
             .subcc => try emit.mirArithmetic3Op(inst),
 
diff --git a/src/arch/sparc64/Mir.zig b/src/arch/sparc64/Mir.zig
index f854152a2f..f9a4056705 100644
--- a/src/arch/sparc64/Mir.zig
+++ b/src/arch/sparc64/Mir.zig
@@ -15,6 +15,7 @@ const bits = @import("bits.zig");
 const Air = @import("../../Air.zig");
 
 const Instruction = bits.Instruction;
+const ASI = bits.Instruction.ASI;
 const Register = bits.Register;
 
 instructions: std.MultiArrayList(Inst).Slice,
@@ -70,6 +71,16 @@ pub const Inst = struct {
         lduw,
         ldx,
 
+        /// A.28 Load Integer from Alternate Space
+        /// This uses the mem_asi field.
+        /// Note that the ldda variant of this instruction is deprecated, so do not emit
+        /// it unless specifically requested (e.g. by inline assembly).
+        // TODO add other operations.
+        lduba,
+        lduha,
+        lduwa,
+        ldxa,
+
         /// A.31 Logical Operations
         /// This uses the arithmetic_3op field.
         // TODO add other operations.
@@ -132,6 +143,16 @@ pub const Inst = struct {
         stw,
         stx,
 
+        /// A.55 Store Integer into Alternate Space
+        /// This uses the mem_asi field.
+        /// Note that the stda variant of this instruction is deprecated, so do not emit
+        /// it unless specifically requested (e.g. by inline assembly).
+        // TODO add other operations.
+        stba,
+        stha,
+        stwa,
+        stxa,
+
         /// A.56 Subtract
         /// This uses the arithmetic_3op field.
         // TODO add other operations.
@@ -241,6 +262,15 @@ pub const Inst = struct {
             inst: Index,
         },
 
+        /// ASI-tagged memory operations.
+        /// Used by e.g. ldxa, stxa
+        mem_asi: struct {
+            rd: Register,
+            rs1: Register,
+            rs2: Register = .g0,
+            asi: ASI,
+        },
+
         /// Membar mask, controls the barrier behavior
         /// Used by e.g. membar
         membar_mask: struct {

From 75a1360cdd1745ac267c9a25b84ec7ee765d9262 Mon Sep 17 00:00:00 2001
From: Koakuma 
Date: Mon, 20 Feb 2023 22:40:08 +0700
Subject: [PATCH 010/725] stage2: sparc64: Implement ASI load/store ops

---
 src/arch/sparc64/CodeGen.zig |  6 +++---
 src/arch/sparc64/Emit.zig    | 39 ++++++++++++++++++++++++++++--------
 src/arch/sparc64/bits.zig    | 32 +++++++++++++++++++++++++++++
 3 files changed, 66 insertions(+), 11 deletions(-)

diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig
index a4fa7e179c..7c9476161c 100644
--- a/src/arch/sparc64/CodeGen.zig
+++ b/src/arch/sparc64/CodeGen.zig
@@ -1254,7 +1254,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
 
                         try self.genStoreASI(reg, .sp, off_reg, abi_size, opposite_endian_asi);
                         try self.genLoad(reg, .sp, Register, off_reg, abi_size);
-                        break :result reg;
+                        break :result .{ .register = reg };
                     },
                     .memory => {
                         if (int_info.bits > 64 or @popCount(int_info.bits) != 1)
@@ -1264,7 +1264,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
                         const dst_reg = try self.register_manager.allocReg(null, gp);
 
                         try self.genLoadASI(dst_reg, addr_reg, .g0, abi_size, opposite_endian_asi);
-                        break :result dst_reg;
+                        break :result .{ .register = dst_reg };
                     },
                     .stack_offset => |off| {
                         if (int_info.bits > 64 or @popCount(int_info.bits) != 1)
@@ -1274,7 +1274,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
                         const dst_reg = try self.register_manager.allocReg(null, gp);
 
                         try self.genLoadASI(dst_reg, .sp, off_reg, abi_size, opposite_endian_asi);
-                        break :result dst_reg;
+                        break :result .{ .register = dst_reg };
                     },
                     else => unreachable,
                 }
diff --git a/src/arch/sparc64/Emit.zig b/src/arch/sparc64/Emit.zig
index c0dbab5a14..7d16105348 100644
--- a/src/arch/sparc64/Emit.zig
+++ b/src/arch/sparc64/Emit.zig
@@ -91,10 +91,10 @@ pub fn emitMir(
             .lduw => try emit.mirArithmetic3Op(inst),
             .ldx => try emit.mirArithmetic3Op(inst),
 
-            .lduba => unreachable,
-            .lduha => unreachable,
-            .lduwa => unreachable,
-            .ldxa => unreachable,
+            .lduba => try emit.mirMemASI(inst),
+            .lduha => try emit.mirMemASI(inst),
+            .lduwa => try emit.mirMemASI(inst),
+            .ldxa => try emit.mirMemASI(inst),
 
             .@"and" => try emit.mirArithmetic3Op(inst),
             .@"or" => try emit.mirArithmetic3Op(inst),
@@ -132,10 +132,10 @@ pub fn emitMir(
             .stw => try emit.mirArithmetic3Op(inst),
             .stx => try emit.mirArithmetic3Op(inst),
 
-            .stba => unreachable,
-            .stha => unreachable,
-            .stwa => unreachable,
-            .stxa => unreachable,
+            .stba => try emit.mirMemASI(inst),
+            .stha => try emit.mirMemASI(inst),
+            .stwa => try emit.mirMemASI(inst),
+            .stxa => try emit.mirMemASI(inst),
 
             .sub => try emit.mirArithmetic3Op(inst),
             .subcc => try emit.mirArithmetic3Op(inst),
@@ -378,6 +378,29 @@ fn mirConditionalMove(emit: *Emit, inst: Mir.Inst.Index) !void {
     }
 }
 
+fn mirMemASI(emit: *Emit, inst: Mir.Inst.Index) !void {
+    const tag = emit.mir.instructions.items(.tag)[inst];
+    const data = emit.mir.instructions.items(.data)[inst].mem_asi;
+
+    const rd = data.rd;
+    const rs1 = data.rs1;
+    const rs2 = data.rs2;
+    const asi = data.asi;
+
+    switch (tag) {
+        .lduba => try emit.writeInstruction(Instruction.lduba(rs1, rs2, asi, rd)),
+        .lduha => try emit.writeInstruction(Instruction.lduha(rs1, rs2, asi, rd)),
+        .lduwa => try emit.writeInstruction(Instruction.lduwa(rs1, rs2, asi, rd)),
+        .ldxa => try emit.writeInstruction(Instruction.ldxa(rs1, rs2, asi, rd)),
+
+        .stba => try emit.writeInstruction(Instruction.stba(rs1, rs2, asi, rd)),
+        .stha => try emit.writeInstruction(Instruction.stha(rs1, rs2, asi, rd)),
+        .stwa => try emit.writeInstruction(Instruction.stwa(rs1, rs2, asi, rd)),
+        .stxa => try emit.writeInstruction(Instruction.stxa(rs1, rs2, asi, rd)),
+        else => unreachable,
+    }
+}
+
 fn mirMembar(emit: *Emit, inst: Mir.Inst.Index) !void {
     const tag = emit.mir.instructions.items(.tag)[inst];
     const mask = emit.mir.instructions.items(.data)[inst].membar_mask;
diff --git a/src/arch/sparc64/bits.zig b/src/arch/sparc64/bits.zig
index 0446a84d6d..7c943626f9 100644
--- a/src/arch/sparc64/bits.zig
+++ b/src/arch/sparc64/bits.zig
@@ -1229,6 +1229,22 @@ pub const Instruction = union(enum) {
         };
     }
 
+    pub fn lduba(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction {
+        return format3i(0b11, 0b01_0001, rs1, rs2, rd, asi);
+    }
+
+    pub fn lduha(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction {
+        return format3i(0b11, 0b01_0010, rs1, rs2, rd, asi);
+    }
+
+    pub fn lduwa(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction {
+        return format3i(0b11, 0b01_0000, rs1, rs2, rd, asi);
+    }
+
+    pub fn ldxa(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction {
+        return format3i(0b11, 0b01_1011, rs1, rs2, rd, asi);
+    }
+
     pub fn @"and"(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction {
         return switch (s2) {
             Register => format3a(0b10, 0b00_0001, rs1, rs2, rd),
@@ -1417,6 +1433,22 @@ pub const Instruction = union(enum) {
         };
     }
 
+    pub fn stba(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction {
+        return format3i(0b11, 0b01_0101, rs1, rs2, rd, asi);
+    }
+
+    pub fn stha(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction {
+        return format3i(0b11, 0b01_0110, rs1, rs2, rd, asi);
+    }
+
+    pub fn stwa(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction {
+        return format3i(0b11, 0b01_0100, rs1, rs2, rd, asi);
+    }
+
+    pub fn stxa(rs1: Register, rs2: Register, asi: ASI, rd: Register) Instruction {
+        return format3i(0b11, 0b01_1110, rs1, rs2, rd, asi);
+    }
+
     pub fn sub(comptime s2: type, rs1: Register, rs2: s2, rd: Register) Instruction {
         return switch (s2) {
             Register => format3a(0b10, 0b00_0100, rs1, rs2, rd),

From ccc9b8caf632bf97f7765c1b1d7821118cf34008 Mon Sep 17 00:00:00 2001
From: Koakuma 
Date: Mon, 24 Apr 2023 13:23:17 +0700
Subject: [PATCH 011/725] stage2: sparc64: Implement
 airPtrSliceLenPtr/airPtrSlicePtrPtr stubs

---
 src/arch/sparc64/CodeGen.zig | 36 ++++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig
index 7c9476161c..226a0c6fc9 100644
--- a/src/arch/sparc64/CodeGen.zig
+++ b/src/arch/sparc64/CodeGen.zig
@@ -667,8 +667,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .slice_ptr       => try self.airSlicePtr(inst),
             .slice_len       => try self.airSliceLen(inst),
 
-            .ptr_slice_len_ptr => @panic("TODO try self.airPtrSliceLenPtr(inst)"),
-            .ptr_slice_ptr_ptr => @panic("TODO try self.airPtrSlicePtrPtr(inst)"),
+            .ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
+            .ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
 
             .array_elem_val      => try self.airArrayElemVal(inst),
             .slice_elem_val      => try self.airSliceElemVal(inst),
@@ -2238,6 +2238,38 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
 }
 
+fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
+    const ty_op = self.air.instructions.items(.data)[inst].ty_op;
+    const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
+        const ptr_bits = self.target.cpu.arch.ptrBitWidth();
+        const ptr_bytes = @divExact(ptr_bits, 8);
+        const mcv = try self.resolveInst(ty_op.operand);
+        switch (mcv) {
+            .dead, .unreach, .none => unreachable,
+            .ptr_stack_offset => |off| {
+                break :result MCValue{ .ptr_stack_offset = off - ptr_bytes };
+            },
+            else => return self.fail("TODO implement ptr_slice_len_ptr for {}", .{mcv}),
+        }
+    };
+    return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
+}
+
+fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
+    const ty_op = self.air.instructions.items(.data)[inst].ty_op;
+    const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
+        const mcv = try self.resolveInst(ty_op.operand);
+        switch (mcv) {
+            .dead, .unreach, .none => unreachable,
+            .ptr_stack_offset => |off| {
+                break :result MCValue{ .ptr_stack_offset = off };
+            },
+            else => return self.fail("TODO implement ptr_slice_len_ptr for {}", .{mcv}),
+        }
+    };
+    return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
+}
+
 fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     const result = try self.resolveInst(un_op);

From 0c9c9117bacfcff2b11ab658b89f192e27fc7c3d Mon Sep 17 00:00:00 2001
From: r00ster91 
Date: Sat, 29 Apr 2023 03:35:16 +0200
Subject: [PATCH 012/725] std.builtin.CallModifier: add missing word

---
 lib/std/builtin.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/std/builtin.zig b/lib/std/builtin.zig
index 3523c5ac12..4a5e8a28d6 100644
--- a/lib/std/builtin.zig
+++ b/lib/std/builtin.zig
@@ -646,7 +646,7 @@ pub const CallModifier = enum {
     /// If this is not possible, a compile error is emitted instead.
     always_tail,
 
-    /// Guarantees that the call will inlined at the callsite.
+    /// Guarantees that the call will be inlined at the callsite.
     /// If this is not possible, a compile error is emitted instead.
     always_inline,
 

From bd8b5c25ece2fa158196381004db6e655226576c Mon Sep 17 00:00:00 2001
From: r00ster91 
Date: Sat, 29 Apr 2023 04:19:58 +0200
Subject: [PATCH 013/725] Sema: emit error for always_inline call of noinline
 function

Fixes #15489

This also lays the groundwork for exposing the whether or not a function is
noinline in std.builtin.Fn as an `is_noinline: bool` field if we ever want to do that.
---
 src/Sema.zig                                  | 10 ++++++++--
 src/type.zig                                  | 13 ++++++++++++
 .../compile_errors/bad_usage_of_call.zig      | 20 +++++++++++++++----
 3 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/src/Sema.zig b/src/Sema.zig
index 8b47f1877b..207ccb41a9 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -6565,7 +6565,10 @@ fn analyzeCall(
     };
 
     if (modifier == .never_inline and func_ty_info.cc == .Inline) {
-        return sema.fail(block, call_src, "no-inline call of inline function", .{});
+        return sema.fail(block, call_src, "'never_inline' call of inline function", .{});
+    }
+    if (modifier == .always_inline and func_ty_info.is_noinline) {
+        return sema.fail(block, call_src, "'always_inline' call of noinline function", .{});
     }
 
     const gpa = sema.gpa;
@@ -8784,7 +8787,8 @@ fn funcCommon(
         if (!is_generic and block.params.items.len == 0 and !var_args and !inferred_error_set and
             alignment.? == 0 and
             address_space.? == target_util.defaultAddressSpace(target, .function) and
-            section == .default)
+            section == .default and
+            !is_noinline)
         {
             if (bare_return_type.zigTypeTag() == .NoReturn and cc.? == .Unspecified) {
                 break :fn_ty Type.initTag(.fn_noreturn_no_args);
@@ -9002,6 +9006,7 @@ fn funcCommon(
             .addrspace_is_generic = address_space == null,
             .is_var_args = var_args,
             .is_generic = is_generic,
+            .is_noinline = is_noinline,
             .noalias_bits = noalias_bits,
         });
     };
@@ -19217,6 +19222,7 @@ fn zirReify(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData, in
                 .cc = cc,
                 .is_var_args = is_var_args,
                 .is_generic = false,
+                .is_noinline = false,
                 .align_is_generic = false,
                 .cc_is_generic = false,
                 .section_is_generic = false,
diff --git a/src/type.zig b/src/type.zig
index c7b2844970..82896800ed 100644
--- a/src/type.zig
+++ b/src/type.zig
@@ -666,6 +666,9 @@ pub const Type = extern union {
                 if (a_info.is_generic != b_info.is_generic)
                     return false;
 
+                if (a_info.is_noinline != b_info.is_noinline)
+                    return false;
+
                 if (a_info.noalias_bits != b_info.noalias_bits)
                     return false;
 
@@ -1074,6 +1077,7 @@ pub const Type = extern union {
                 }
                 std.hash.autoHash(hasher, fn_info.is_var_args);
                 std.hash.autoHash(hasher, fn_info.is_generic);
+                std.hash.autoHash(hasher, fn_info.is_noinline);
                 std.hash.autoHash(hasher, fn_info.noalias_bits);
 
                 std.hash.autoHash(hasher, fn_info.param_types.len);
@@ -1454,6 +1458,7 @@ pub const Type = extern union {
                     .alignment = payload.alignment,
                     .is_var_args = payload.is_var_args,
                     .is_generic = payload.is_generic,
+                    .is_noinline = payload.is_noinline,
                     .comptime_params = comptime_params.ptr,
                     .align_is_generic = payload.align_is_generic,
                     .cc_is_generic = payload.cc_is_generic,
@@ -2069,6 +2074,9 @@ pub const Type = extern union {
 
             .function => {
                 const fn_info = ty.fnInfo();
+                if (fn_info.is_noinline) {
+                    try writer.writeAll("noinline ");
+                }
                 try writer.writeAll("fn(");
                 for (fn_info.param_types, 0..) |param_ty, i| {
                     if (i != 0) try writer.writeAll(", ");
@@ -4863,6 +4871,7 @@ pub const Type = extern union {
                 .alignment = 0,
                 .is_var_args = false,
                 .is_generic = false,
+                .is_noinline = false,
                 .align_is_generic = false,
                 .cc_is_generic = false,
                 .section_is_generic = false,
@@ -4877,6 +4886,7 @@ pub const Type = extern union {
                 .alignment = 0,
                 .is_var_args = false,
                 .is_generic = false,
+                .is_noinline = false,
                 .align_is_generic = false,
                 .cc_is_generic = false,
                 .section_is_generic = false,
@@ -4891,6 +4901,7 @@ pub const Type = extern union {
                 .alignment = 0,
                 .is_var_args = false,
                 .is_generic = false,
+                .is_noinline = false,
                 .align_is_generic = false,
                 .cc_is_generic = false,
                 .section_is_generic = false,
@@ -4905,6 +4916,7 @@ pub const Type = extern union {
                 .alignment = 0,
                 .is_var_args = false,
                 .is_generic = false,
+                .is_noinline = false,
                 .align_is_generic = false,
                 .cc_is_generic = false,
                 .section_is_generic = false,
@@ -6367,6 +6379,7 @@ pub const Type = extern union {
                 cc: std.builtin.CallingConvention,
                 is_var_args: bool,
                 is_generic: bool,
+                is_noinline: bool,
                 align_is_generic: bool,
                 cc_is_generic: bool,
                 section_is_generic: bool,
diff --git a/test/cases/compile_errors/bad_usage_of_call.zig b/test/cases/compile_errors/bad_usage_of_call.zig
index c0b632bef6..3669cda3bf 100644
--- a/test/cases/compile_errors/bad_usage_of_call.zig
+++ b/test/cases/compile_errors/bad_usage_of_call.zig
@@ -14,14 +14,25 @@ export fn entry5(c: bool) void {
     var baz = if (c) &baz1 else &baz2;
     @call(.compile_time, baz, .{});
 }
+export fn entry6() void {
+    _ = @call(.always_inline, dummy, .{});
+}
+export fn entry7() void {
+    _ = @call(.always_inline, dummy2, .{});
+}
 pub export fn entry() void {
     var call_me: *const fn () void = undefined;
     @call(.always_inline, call_me, .{});
 }
+
 fn foo() void {}
-fn bar() callconv(.Inline) void {}
+inline fn bar() void {}
 fn baz1() void {}
 fn baz2() void {}
+noinline fn dummy() u32 {
+    return 0;
+}
+noinline fn dummy2() void {}
 
 // error
 // backend=stage2
@@ -30,7 +41,8 @@ fn baz2() void {}
 // :2:23: error: expected a tuple, found 'void'
 // :5:21: error: unable to perform 'never_inline' call at compile-time
 // :8:21: error: unable to perform 'never_tail' call at compile-time
-// :11:5: error: no-inline call of inline function
+// :11:5: error: 'never_inline' call of inline function
 // :15:26: error: modifier 'compile_time' requires a comptime-known function
-// :19:27: error: modifier 'always_inline' requires a comptime-known function
-
+// :18:9: error: 'always_inline' call of noinline function
+// :21:9: error: 'always_inline' call of noinline function
+// :25:27: error: modifier 'always_inline' requires a comptime-known function

From 440b3df702f1c8bfddabc1c594a3f49cf0011a63 Mon Sep 17 00:00:00 2001
From: kcbanner 
Date: Sat, 29 Apr 2023 11:54:21 -0400
Subject: [PATCH 014/725] main: parse --dynamicbase linker arg

---
 src/main.zig | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main.zig b/src/main.zig
index 66b207aa43..45b0cf30b8 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -2072,6 +2072,8 @@ fn buildOutputType(
                     linker_tsaware = true;
                 } else if (mem.eql(u8, arg, "--nxcompat")) {
                     linker_nxcompat = true;
+                } else if (mem.eql(u8, arg, "--dynamicbase")) {
+                    linker_dynamicbase = true;
                 } else if (mem.eql(u8, arg, "--no-dynamicbase")) {
                     linker_dynamicbase = false;
                 } else if (mem.eql(u8, arg, "--high-entropy-va")) {

From 4f248e1b519b001cee67e461068245c142d38e73 Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Sat, 29 Apr 2023 22:44:32 +0100
Subject: [PATCH 015/725] std.c:complete further more netbsd's mmap flags

---
 lib/std/c/netbsd.zig | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/lib/std/c/netbsd.zig b/lib/std/c/netbsd.zig
index f70549775c..11126e7115 100644
--- a/lib/std/c/netbsd.zig
+++ b/lib/std/c/netbsd.zig
@@ -603,6 +603,17 @@ pub const MAP = struct {
     pub const ANON = 0x1000;
     pub const ANONYMOUS = ANON;
     pub const STACK = 0x2000;
+
+    pub const ALIGNMENT_SHIFT = 24;
+    pub fn ALIGNED(n: u32) u32 {
+        return n << ALIGNMENT_SHIFT;
+    }
+    pub const ALIGNMENT_64KB = MAP.ALIGNED(0xff);
+    pub const ALIGNMENT_16MB = MAP.ALIGNED(16);
+    pub const ALIGNMENT_4GB = MAP.ALIGNED(32);
+    pub const ALIGNMENT_1TB = MAP.ALIGNED(40);
+    pub const ALIGNMENT_256TB = MAP.ALIGNED(48);
+    pub const ALIGNMENT_64PB = MAP.ALIGNED(56);
 };
 
 pub const MSF = struct {

From 3fb93fc8f2d3e755492a495fa69f65ae6615cab6 Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Wed, 26 Apr 2023 22:04:55 +0100
Subject: [PATCH 016/725] std.c: freebsd add procctl exclusive x86_64 flags

---
 lib/std/c/freebsd.zig | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/lib/std/c/freebsd.zig b/lib/std/c/freebsd.zig
index d3fa8f51c7..e5cdc0e715 100644
--- a/lib/std/c/freebsd.zig
+++ b/lib/std/c/freebsd.zig
@@ -2172,6 +2172,30 @@ pub const PROC = struct {
     pub const WX_MAPPINGS_PERMIT = 0x0001;
     pub const WX_MAPPINGS_DISALLOW_EXEC = 0x0002;
     pub const WX_MAPPINGS_ENFORCE = 0x80000000;
+    pub const PROCCTL_MD_MIN = 0x10000000;
+    // x86_64-only constants
+    pub const KPTI = switch (builtin.cpu.arch) {
+        .x86_64 => struct {
+            pub const CTL = PROC.PROCCTL_MD_MIND;
+            pub const STATUS = PROC.PROCCTL_MD_MIND + 1;
+            pub const CTL_ENABLE_ON_EXEC = 1;
+            pub const CTL_DISABLE_ON_EXEC = 2;
+            pub const STATUS_ACTIVE = 0x80000000;
+        },
+        else => void,
+    };
+    pub const LA = switch (builtin.cpu.arch) {
+        .x86_64 => struct {
+            pub const CTL = PROC.PROCCTL_MD_MIND + 2;
+            pub const STATUS = PROC.PROCCTL_MD_MIND + 3;
+            pub const CTL_LA48_ON_EXEC = 1;
+            pub const CTL_LA57_ON_EXEC = 2;
+            pub const CTL_DEFAULT_ON_EXEC = 3;
+            pub const STATUS_LA48 = 0x01000000;
+            pub const STATUS_LA57 = 0x02000000;
+        },
+        else => void,
+    };
 };
 
 pub const PPROT = struct {

From ab086b62cf585aa7fa73f2df3332e87f60226f86 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Sun, 30 Apr 2023 00:09:51 -0700
Subject: [PATCH 017/725] update zig1.wasm

Looks like I might have messed up the wasm kernel in my recent branch causing some sporadic failures on the CI.

This file was built the following way:

1. check out d0311e28b397d173f0d60c403985047ec952a172
2. do the cmake bootstrap
3. check out 57ea6207d3cb2db706bdc06c14605e4b901736dd
4. `zig build update-zig1` and stash those modified files
5. check out 440b3df702f1c8bfddabc1c594a3f49cf0011a63 (master)
6. do the cmake bootstrap
7. `zig build update-zig1` to produce this commit
---
 stage1/zig1.wasm | Bin 2454149 -> 2455067 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/stage1/zig1.wasm b/stage1/zig1.wasm
index b00227e45d31187e03274df3b2c07a91832b35c7..600e273ec172ff7a813830f128187f5c04ecd69a 100644
GIT binary patch
delta 222707
zcmb5X2YgjU(?6cw(~{hj8u_S~C9^nE|S|HqH)-QC&Q+1c6I+1c5XF{h`q
zuP#n!nJ-4LAnV159$AYbI&l8mKI^hOjvdM}qH>jp-1(E-HF;1o9^?r!bF9NLt@-rb
zR)yGUkJhY>oe`h9aJ1VDan~~0&<^1W8pI`e43BQ+SwqS$t~T-3s6egSwxsgFv`KDv
zZ7&bgT;_!N)??-Ru|s0{3|>joJSv+0SgtakaL77XE|FV@vO?n$RCda$USS|899LmD
zcNMLk?yk>*yrJd;E3$GUb|9-=omOpxa`48)X*J@k_nVc~@7ODdMG
z!?Z*ODlR&`5~?459qBg91u67EXV5Cc&vVKA{f%dn%XGVN_jh;Pn%@Ts}XCZ
z`ZTL?lX7f@)uqW>Y@M~L$>Nv?*NeOC0O6roxn~V+`W5Ckyjf@c*Q4mR3J}%dmyXIg
zzJ648xvyDyfLsMg?lFMWl_0krF}P1+P$7Xk=%!pM=1-u9!&&>A*JaAgOPJz@diEvE
zX;sU7EzWxJvGVLp*87hgV=*_DjtxjlpxGhowN{OJ>in!(tqd0PV?i-{0{~mCRgX93
z)8<>}9v3`qzBRJ-N71)0qWf@%*|p^O@-5dB&7<;nOm!z=0h(&3a0a0}Wj+7I|LBJ&
zfpJe1Gj7i0V#dX_@v)csXm#u3GMweb&bYT
zx{LX7JB)AP3H81x>vmFUbfDDm#&|yU*BtqFjN1$i)P$mjgz+4e`lqjjELm&S>A4g#
z{AbTD<$hcZruY{n_3@O1$gx`WiZA!|ioe#?mv!^4A-zVFTTW76j5XAoDC^c)%1bYn
zTYee>Q;LqpvhGLgy_cSf8*v=egq6@(SJq8AZtZ+2F0Sb7zYd_DtQuUIR3$Hv?ir3XT-=6qot=v|LxTOoa3f`Gizr?dV!2}nH%NL@!fej)MblKBP5
zjd9Rd^B7)_XArs|6x#`@U#StT9$9*bO3V1kF`2iw=imOCP1f+2D@E_6MOaKO;aO%a
zd$|@nV&%Q;i=Mw5vr)`(s{O?Z?^`Q+8I>2$5|yvBp6Xkj-LT&8+Xz~Ie&4G6zj@Y{
zzSY=H>k5iiSiXK;*gWgqe!bXsYjeNq?0|KtUn6$V^7QY*(|)kt=|7y0`oX%`zbF6v
zl+}JfHFn=h8qkP;b;_DEpmDk14`J8=GQsPPa&Q~1`~ferKddJP_AWQ?>bSsG8{Ptp
z>E(x5Ht?yq*}MKa$7N;RvRzi>pw6|n?7~=_9Xz}RIP`?9JVB+!@2H06~yDoi21`Q_;`Pb%{^M~Brt
z!5+;lu=uMV^3Q*?{`G1VY>HpK`nJAnKO`WrSmlk#lgg~mYbLX;*{_w4&bR|wA_AgD
zG=6Jsd95m&ZC!q?4V!D#d%Y^#Xg&Y>YQHQ>#EiDjW}%mZ@f_#W`OmE7xS|E
zjjHUtb>NKw(0eW4tjf+<1Kykw@7M8-$gJ^iJ{PWlc5a!2V?%io`r<=OD93?=W36^?
zoryU`wIS}(4Gvk45B8Cg95{F^Y0mt?#k#UISYFn!q=pRr{&z@=sFdIQy5At(A7U0+
zSBCtLF`ab(&q2kUINnQzx;$E{;;`;?poldM|P6L{J}%lFP0zITds^y5bS&XnRk
z;`Te;^ogGXi>DaL($9-)XS_R7pG6yUWz^Qu*U-=MWYzGzi2v4aevE61j<`|J|4wxSw}u@%)YeJYm}Ad
zL*gg(_AdIdkrNua8qbz;}7w?BJ=O&8u_k+D|JXZ6{1tKhT7eCc?}nwG;G
z!;*V4kMMhE-qbO;ZEQTxz3HN17)%>M=qycEBIH(K!-=7Fro
z{`0lMezx|HSivq^6Gm>t*4cGbcZH9iku`U08O1Ym_H=hGC8)WQ@dA)#?g>irg_%KS
z3M+Cs*mq6n0hLl^;`{8Pl{WEN{60Cc8c*G8g-)u#f7xsGZ(PA@KWP^`XO*42lby>t
zJGn6fXu_01pnmw2ihTAVz*NiHFy%$ow3Oip;aAP^)}%o1Va(-u#uIG*e5P1$N|4N1
zR+rQ+kr|7Gv|YR*9%g1P%vzWl&#FvLL-U}Z=0RsW0NEISPB})o!W}_L=A1O^_|)Ms
z|2+T(GWY9#OVXp5v&UH8
zSp&-KB;&LMCAAAptH<_OkEOk(7fpdM#}*4$+7|&?7w_v#wQi<0LdlZ=?cu}aN{(LT)
zteI7+$q&|>GoFrrlwZ^4V4788+twL1z_x)R!3q~;u%zpX_H}H$Ma01Z+;2A%c0`E2R@8Hwd(Ag&dp1QFV_@054igc2;1F=>hPaUnXK29(-9m4LN~nc;c&9*lBCt
zSB?3&;uVoq@YRfPeZeLe3Hz3KKfq
zJ;YgsHE_h_fxZiW2}*Lm-M%2?RBRz$xqD(z2Fo
zc?kQ)YM5EsI{MwR(4xEJoVp-u(8^eA$&wH&d1bY@DZEwC?QY=fIkc5x{;(@@CuF;*
zrSCod)3ouk+Mu}VkRFXm^%S)Hc3Se9`&D!u?CF{V_&{o?6$cvHu>Mv1>
zk^EG;WOe|FEeUyIS?J>vEI)z8I7a(RN^lgFCZre#4$UCM=XM3?-C7##i$GUDOE4B`
zjb73tI)Z>{`TCf->>lQB+C8h&@-B~GDBV=D<7DX)$&Q1iOXdd9|KYIX;_}eP=lB7n
zLXb#A=_V4%7o|()1dxn#kaW)st+fXeDpgo92e+f7cw(jon*3s=tgLK(lUX@D|L{0B
zb`dkq%W!$JkBl)$Bg7DA#Al
z#bxY+y73=Ag35am%^<9%2y=hdt%S)TPzE3D8OvY|d-xZj$+p$l7S+@gh!Sq`D>v3Yk
zt{qjajk&|hj=Tz5s|RT19*zN`y6cu2{JX2xM_X~&yMo?DTsg>VKv+xV@=s%
z67a4aUzW`wLWhO|T&WqW=PW1l>I9Mph+${*L`)6COIp9|2K?WMeFhg
za%XMS!rG{Ium^)E}ImM4q^zTjXh
z9xIm{+2uiIbVs!<&aQ;N?U8C#18Yx~$n_TmHU|~2zep?xYr>Hh)Yrx%tpk1KSNls}
z5uOS}Q-!0I0hiVK=u3>xoNtL^Ur@7}$5&8t@c3{_`kYwg|3$3`bj|1q%v+we+4~nad)z|HV;9-I_^?H3At?5kE_TQTI
zRDKWrz%@v{5m-Tx_22WQShhYlnfe@hu?G9mx_)j8@U1&v8~^5??+o`^!wcy8q|^0D
zYr}PqWnZXG4T{^{1?=0GsD-%HnWvqwj$f*el@@imF+XrZws_;RzeRa#$>k3Mz3sI+
zUpa_YRST$Xw}Q?_?s3q*fsV6v*kp-dnuDP7)qg^eYq4vm(DckTLJ)Pmv+>KR5(ML|
zhDD9}l+&Q#Y3efkMqr%Q&o^$Mfqk-7>Ws>*w>tC1XRI&I
z#LIK)o-hX(WK
zX`&)4rRa*vm)#OW8EeYIHzv+JUOu?GBe$7yRxk2jHf0cUE_PhoY~elbB|Z~D*H
zWpPr=weKlxEYoMv(U%VR((|sbDLDya6k@N&y6FFdSQ;Av|)DBko;xELSN~~d2
z{ugM(N^U(0#r;aG8ox+qNI5dyZe5v`WyTc(%n1@dRbq)zD^vfm)%UUo;)gEsJgk7-
z5?j2iDbFnClWxcR*mlm=*k`J;@wn=mCEl;jCa|Av2-2xY%*@ed5Rd88+Lo7sypA0I
zQjj(Kv*v7ANHCr0z2W8oQP`GMjq;!H2ZV*WOx+ua;zW~GWe3D_ChG+O72lbx4bnvy
zO~$#nCN4k8+Qj6}1Nphke}>E_Lt(#dwrA5I8up$J%*Xhwhjw8{Hj(kem#9s7droII
zK_iT{o@J|%+Q*({Ll|2xp6SjSL7tPlvxXplMRy2R;hzw!`XpGobL5vkC>-)J7W>m4
zY!TPz7eI>Z(#~%`{sKG1!!mHN#JLv=eZe?!WOZx{(YF^f`I2#BZZGyYUph`4?!~H8
zs=dV8Q`+SvRu`%L$xG;&T@>H-X3hD6QTD0c>@mh3h)OTBvM~$4hcUz5BRM>PaNxjj
zGq;J(FS8c)mz={V@dxtP+&r{^HbGqGexJvyFWcIMo17NyJ#%uJykZ?cIZ-5-5zP8E+38
z;-WpTFRR32Xj?7mL}Ixm{^-YQ@iDi=gMMJu)VX%u{*Xp}1v%qN9Ef|!85eJ74rKQ@
z+b*J5e2ks*D)WTuxnjE8+#}As!|p`R%ujQCM_{6vnr~lzmwm<_EnW1<`%Hzi%l`K;
z)>w&1A%7DOYA$b}3&q-hvGSOw{r_UE91Ks{cm>HB*G{@htpH6033Y6Z1k*qOK`4gbg
zqT?tQ7rg5Csc!RevrxP-igk*d`JLZMhWlGHeX&?Oigl#HtE1Ush7-A1K89^;CXeJi
ztd+}zw3rKi9*6thu$C?}6uB=goCkQ=V@__?by?TKWoG1w%(1NOt831UqWY#Tl0!3<
z4);8)iHn5Q{C05)QOtrI*`SWg3`Xwi1zA_ysmq%+o{-gYI$D({Nvi2GLjd{5P3R^bR>R4C
zu~A~F?sWS@k*urcboSZG+qL7IC-HuO2Ub3e(P*Wtl`oen)t-8iR<
z%1&w^gDj>(o6E$>UkVi_;uBqn!bMbxl%T#>w4tcpl|6zv0Z4+v)Pe1_>GI
z)Qxc_BEqRVFRlP}a)txqatc!%_JlgQFP&~foZQk2a;U)$vIUDJoraUkE|l!i9jZ4M
z$Ziq2K{LMF=|Odw5yUnJ$wFi?nZpd5TA+@U70QkCQI_F@DqooKux|}j3U@ylvz?__>11D$LBve{RPk*WTooRT8NUv#RZ2wahbji=+XGY~l4WzmuX;hVy$K-+el!aT0j
zWsV}z%Qa={XTqGsdfVkv*$K`wj*ENKSR1xdw6fTXNTyn>e*8GP_BGs=
zM*ABpZ!(V}J$>iIpBC%H?h0=j8_pLWveVPpc7;!~#k(_E*>WRncwuyZGq^O!p~1}+
z;;Wf}#vWRJ7JHM!-W&N9`1DWR?*@O46`;siQH|Wjm~1%B@0}
z6|i9Sh9b9Rugqcfx{%vITY(aW)k~0DoF3%r<>tuR7iA@uFYzgRCX}lZhUH5gZX(^P
zkYA2CQzR0Xuteqf>Cqx{411n<-Z6QJqYES}74w>-3&2X+u&z;0QZ@rn{;R>ToGd3a@M6(3s^TfT5S|OoB7vf4gS*19+Sxyum%#XVjkm3KEwT~
zDUD5Xp?G+LDN{Ycr<-(_f?Etgh^s%dSG?mD-1++p6?42&d|lwijX9NzLn~QtIu0D)
z#Kus%K8sZi9s>b}kvc^D+hz$uTZQxLelgc(b=ht4!zy-8C%35A5K~vNvZC!87KQSi
ztC=^}Pi2ad*>9yrH7tkiCxW_eV4qRCZ3C;y$LEQw8?eV9kLipTmw#sDYZ84og6aoi
z#75TBUni$};4kdoGQY){`qwu3^L|lyYcp`p*=wv_(w&kyA-r9=Z
zPKwXBvLASQzUaJ-c@^ASlAekeW43`sE5!0`teTr{|FE{8`~YEMUp%#q)nr9>_;xm(
zMO{BBx0y?n*?E;$|T4um|&Lxo|d{=v#=?2w(2$F`{vdANG@Z-0upQ;aywZn4Ya<0I@%wo4p4
z!iGoW0)oy!xK>ik95Li5To^NEiIyNhXeVnaiKZ~#vtg3$O2=?{0qS*;nkL?t%
zpJ1IHh5qpbtCKA4Z`ILAp{6vI)iCTP8e(dt0j$z)q9Z3QSTzi4%0NyYLsZ&L6y(Mt
zD%7hIL-()5rmBWvBcV3(9H7!h5`vtx=T%Z^aspIbbBNbYvI+(@Hz%GJ<4(eKSu1`!
zDP0aC@)R5?kfLU%n2*vvrx^Ly#iyqn>B`AxHALiTM*ehxOhr-t919iClrBj=?X;PD
znmtQkS5C8P!EynRs=9oR4HCo8un?z6F*8NmbIdE2pK*|GI>TO}b6BmjY#>`8CZ1&p
zbtK0OFrgfcD`zcsV*~6eN8^%*3nvG4y-@5t>rj5boUonv(EC6=9Yn(sZwMp3@cdDbdGak`|~UVom2vPWjv-gkk$p74sNrghT
zEckcWKPGlndv_toXMc+4?y+HPi}>{(xN}z2EMlA3@Al;)Hj1$V`-2B8p3=z=QCp^P
zj{hXz!V_+`9b%@;UHyPpV$nGWsg@`FT#kJoO3=&|^*L`7m7Ok4K%Yvd0+*SYE|NKa
ztMTZ$e)vLyhblHFqr|_0TxQX9gvXG>V6=kj$*3MZLzGweQ*68an!?A%mPrjDsELiJ
zL6SLhu3Z$zQ$l&l9r2$kyh=#w9YG$N3rw(@yuA3O3SaR=29e3md=JmQBa*B0>S2@9DVAJ=(L6p~tfcFK0u;UBZe^lPPjDKoE$X?6Kxp1M`EPUNj4
z$8Vh~>EplPw#qytCM5C)?5sV$9zSfrJ@-a4K8f+vbaA~o&*00x7GJmERgkxTZoxky
zPwitZ`BvQY*;*@}_!kEaC%29HIQNoI)>Pd<&(n!WgAg#XoEJcjY~
zt#I%0cpi~A!Y>(gE4o0{^%#pH69HKPN@rBR2Qr!>I7_{=7w8?hS|8kVDmU
zBiaLb5;$?JKd)eC^yi%!pR`i`y6ggd(QhjiX+YZfd8OCsFOb855kcS
z9?f4;iR6vr`8=fd8xwdiV;k-FCh`NpW%t5k(2S>iK@x!jc82_sD+XA6v|~DPvmiXl
zr@{aU!Bqk96k7a4$KsJy0ZSn|P(LP(x2A*MFKM6_^x@Tp5A(kp3IFO(acc%|&wdfD
z1aANh@wVWVOQw`~lP6{g-hyoty9I9q1*FXc;4Qn^OrGLmYlJp&pSE#%Meh{6e_qmn0Hl)Pp>Y8{Fm?m<}rBi-E#g4jVLmcJ3Cz_^Z@d`GC5sd3GWJ?Fk^*NhTkoz
ztltVg7mU8Yf^WyXZ(Ye-M^y1EOH!5G9LMb07qBv(pZ9OM!6JJkE#k2@mb*$9Wjzlh2CzCwN0{
zofZ2|@Wy=RS>Za#oATLbMLSB<&x%h@@+LSpEIY}`mOgP3`eNA?QSlTM+pnU{DZU(i
zTsg%T;O&Z-ewtrqTkO$ic$&fIEfnQ1^O|AzaQyM=6s3`NOFVm-FVXWTEPLCGE4+J!
z{bwPV*$7hfaep9&i7nS{#*!CXtQ2CeHCpEiZPU&5IIQrh)bdfmHM1Z=sHAPE#NVt
z+ch4-GcJif*Z6HrQ1glY+UGga@LJklduK~Zb4%l8znsV!2c7X*=;_9ofbQ8L!Dd`1xWO9
zqk*`r!0ke;NG#;R95#LPLh$HE@opiwuwN{u+zoN85YVH?0J>bsIQZ-_qQ)KcojpeM
zN6Pk!v^yaC+8A-XkjIJ79%5)C$2wz}G8T*
zU|Wz=B4>Vnp*ZjWGybQDc_<~e+e3_H)*UhKAsDp!j#!VB9ThjIIOXp2x(XKB-RZ9?
zP1)Azvy^7+?)075cD}kNf|>Fdyq1@kQcb+cl=2YGkxZ%2eitj5g2zck;xbcOunnRH
zR~mrZFL9+N&n&V>a%BpGKt!oZkMMch$=_qDVep4v6z{1@x#Z&rCPR@P2FLd|XUSD#
zYUEA3f6xyKU)t(1bY}nj
z&n?^k8|R&CcO?GWA#~)~JxVqXCFak^?I1b}rencsd7FaYVEl1WUsD>8A|IqFRm#uK
zhH9umOVUvm3_cs|Yi7zdF-KE8Y^(i~rhG>`{E%qn5(`|fy%Vc!;QHj-a18IUo5U%b
zWAwaSTrKUHk*G8Z0Ch@KdPK4P_N*tA|H6>lR?5+ux=lRWRFt3rbht2viqQ^=Kmr&OEET(z>jFBlbn(`kCI$EQK`7Xcg6z@-7g3PJrFCEBH=EL$O`!_EriEe1+2R
z(LEKjYpF47W;V(Lq<}O41~`&Q%~C>>%pkKay^zq%oq#O@RvHs-z^UklDAP;vm7axT
z)GQy6n1!c%DOGvO*W&$NN~f3wU;F9F4go|?NRqkaYq71DvZrZgpk_9iw+qlV8zp{j
zDghISrZ|HDG9Bc5UQ((?{8Aclt06^%7K1E?u3Cf|;cYQUmar
zfNuwrG8d`n-bbl2WI<`T*#WqnO>MQ;zZD}WL+3$aS~h-rP5j_zX)v@`F)us!noD!%Ab0q`_~jy_lq7e&Q>$}}jw
z%znynB=!0$-y`{TfKo$*4Zv=7MR*1(6-Dm>$~#W^OHR>5B;O5${9du|3{>7Cz4^*3
zNUw-(uPB?5418UwFE+fY^u_Oj*OiKPtJjpO1cr()&L-&7Qw_Qck~
zfXWkB1}mMDv$yyaf}CvVm>d)oPuxWmw+qAQq&+iBOC*
z4F!e%CJ9(FxTQczL1`qU&GPMCNy;e|bbtRI7QTLL21S3^?T0EiR2*3!`$&1cBF?vS
zEu{^PSw}5pQ!uI6|ISv%BDL#%t-Nl~#o+V>N=129xaz;kMjn%k=LJb}yFth+`c^-J
zk7&|j_CX*Or?bg|5lkttuwz>UXeGBG0T+KxRGXoh*dw?fO%13-)Ad7A=1*G
zu|`>{@U*RVho6+PfAMq3@RXz+B^BGPeQ$#@sSLYlkK3axWK9bxh=u%1Y0zutY#gjo
z&NP9<)!nV?@q|KgZ8f-Aqfj2Om+Q{2RJQY=4z~WJeuS
z8b#@GNwk2(Q&*J)tpYSxGkfq=WthUV&WfWql=^(#S>e8^Od!^MbyJ!W_Kus%pE7JF
zRFO>xZvu0E*|f^&vCh0;%g)jM#2e;A^E%J|&f;!3FM
zfi8Bt)Utf+Ls8SEKBLdwjnl<(@u^Gg$nwPomzqFTM_g(>c3gyqsjs8-!!XrX!QKyR
z!f;>aaL~wa0pM;M2W0s6Chiq0!qg|I-PJJlGqig*Ty0WiJ7IR0YJi(>BQYovm-XB%
z1P>SgAvT4puffEx9ijSgJnI^v#`B4L?YAOS&apCvxYcu*tQVuytFDSGbQYXk1)r>7
zLws6Q((u7axS$TI$%B0%;?;_3_4rT}y77SuH!fkcn!JM7;B7rxVo*)7prY!94|+2t
zn?!y^40esEUrD8~qPHrkRYUNg4JHV~t}(rmYDQ2a_kRt-66AA<;{b^}v=8G6hf${o
zj5-%i#b9&A%Ao$DXj@tRLz8&yIFAZ9&G*9VRY&ke_r(IQ+LSN3FAjRuHhjf>yMj+m
zVj=ShVU+pBKKqL*>KJks#a2^sbh{>Asjl`!^B=0KP1#}lN_F*H#uw}pU)59-SfQ|M
zs%!aoW5oZ|Qv0wyVrMP&9RF^dSW{bVhU}Tu@h^%_b<|Eg<$IA@M_tSAi7)D^vEoHL
zK2of#tHy+;YG70<**R2GmH-#kRo?C9+0TM#o0AGx4t@xqxCb5)DD=9X^qtL
zEXN++7`=iT`_Cq79|cr4X`v2bN9^<#>S{@4ua;_K^}B0h+#;u?T24&f6d%F<5UX0M
zb>Zzf-clW>FQ9>Qk=jy?EuS9-TJpn^%muiRHc#Pt*kdHXQ+t_I>>;Z7ix?L4{hWyN!?RXj9ypx@1nHvzrk3AK{g
zksB{f!DpTTJ^uPBPhbLN{YSKHYPAVeMT^00)L0s9N*lEjCt}h~HJ#T0%d}f;Hu`fP>
zr>2RyUDSoF(C+Yz8qU}%@qAbHYd&SKeXXl{l$>~T55&L6azy08_|Epk?rKF|?(Awm
zuL5hCEI*8V{c~zVc4<+;b8zNPnIYC4i|=GV)>B=tDA)E(5Lf%d-5JqG-4Ex{nLcVe
zxC&~$tRBKZ^uDUaF4K%0jl6&u{A2coHh=Qjail*@EQhu|1(%z*VX)~e1Qs;gmT_tZK(ZH{>TJ#_+k$qU|7+i}>KP2N{i
zc={ai{rl<@m9tBANjt#}a&87S3!1JX&75OL55?-`OGb$S;V!wdU#NIbEQ9bN9I7x;ZM|h2Em}p
zF7v5+iScii+UzrRgc30uheuD8dmc!s4+Z8;QR|8I$?CH_W1Ah3qCTtgRom?Tqt(7@
ziHnn5Jb5B_y!t=1RN73CJB|JO1ogJuhjXW@s@#Kqn*nBE|L-l-Ph?=wo|$T#{Be1f
zx>v;+?w2{xZpZAqb8s1nEnYlzEh5v-UZggTVAt)7t5wz2GBM~Rt^^W;Ch<<;aKYB(
zhmgg)YbTo+behrYBmiJa@b7q(Q}|Y5&?%9yK}~qgAm|&+-#C}SpAzI{_+M4p;sj%K
z^y%S%f$k5cPqO&v{U?kOYE{5IOqOYI0HdrX--*4SEEf7*=>LuWX4EJX6Vs+6qKGwf
zQ&2ZGS#BE8uwu;ywX%zuYLb_%29{*s-2gqqC*Ky8HewY{zHPVOs6H)iq6wSSd}-c%
zxJ6ye*-pFSc6BOOw|jY%Sos@PcZ_)HceN3cNxwr*V#LqCtBsM|`W;(XjHtg8`$>%b
z8O-_+V7KO{A
zU!0=KvM53pZFGt%$s(NAf$L|dsG=;2ltmlx$tdcwf-H*?IeWo|GJ@|@TNu%pI5>`l
zxlbJgYUl4$d-X5Vj2BT0y%{g0)S+8Iv{bbXoM%p=TQX
zbm*8$Kcnof`=O<OSZyFC98$f1D?Ii0!gmP>dzhqY++nq6=|(FqVK04f7}~BB;S&A)
zBmaQ@!9Nt*y^pG+B6;T5;>cw+j^m75cv)Rd`Q=yC8fBJ1NsuMWg79nr4lC5>Ur`sA
zF2~{uGBdxn=NG6W7&WYZO|3zQx?NM7Rmdzs0NW5I)Qcvg&`gPH-Zgb$tw$hCJhtyC
ztBZT4m{Ig>zjs~j8;nl}ZNG=B;?Y8^=ow#8?Ddv$+YO7~O6<0!y}s
zYaLjoT_!?{rn`~qZj{~-Jt-*^W8Khk$T;z(`nv4}k|vE*ZF(mcP3YIU?lFuixw(Oy6U
zGf}G~+Sb)VU{8xKbu|P$JQU08;*o>|sU!v@YPzVKs730lF2dg^TU>x)lUPq{&CrMa
zc6}{Hg%|!(V+4`zx9d02V1#0L2b*el6`r1M?|)2F7|%$z54F_Z<*@A#yVf2_YHN+&
zpWENH)&_ASgRrmqNuK8HE^W26Fi>#p87-x9W*Ro7d^*SlxeK^Y^Pei9Drt+x*{&-_
z0!v}fvsw%IsKdKy9(GvN>!wxpWKR62rW77VO)=sRx?vPsL}oXwJ&duN-L&rUsrVdN
zGky%Wf}UWc6k>b~R~6sh7X7aJC*yqzw7R64~t
z1a=5#h=`$w+tk%*(XfYB$&>Ou=vhldDY(yhbSNYDi}!kH4M0eG4{b)=f?dGq!a2e}
zVvd)xA!L0$`<&)O0PoP}v>uO*r8lp1sZ$Ij$u1zW3qZCXNikV)@cOabdrqqYER~Ft#ae=qr3(A!U60ptsRng-onTsC{x}7
z-)@LM-qNb#xK#L-HVJ|-da%~qMUi@%>GB%*BE{yx+FU@}ZzgH3za6#3h#}hXM<0UN
zL*LfES0N+#*h)FkW~kN>zlRRh#?v~yH&p9gsw!9v`#`JZ)O+mpA7E6y3dl8nEJ#Lw
zBLEvm`nx3s2*1YX=?Y52v6#uwz<9}d&DFw9fykF*4`
z;QD-|HICd436@si+u%LSx!E6SpYzc>>^lF_YV(Las3ollPk~`>7q5Pz-G?1I=u@p{
zHNPF|1iQe)hxb&aE>IPG-X+PrAr5`2)vOrs6}y|mFo7S@f1%5tK?a7|zqP^8HhkVhoBMZMuACE?2v4(q>xV~b(OgNnRc;4>0UibEv2R}3f~sN
z4od+z4~J{LB1@UfghaglA8jn|hA;j{dz-N=(K|(ZA4_m+iq->Ku)+u|uI-}F2xzR4
zmY6X@%Z8@vHd4!=X}mKMyj=9P-DH&3T*hL&GukQ07_D_tah$&~UW*LH<+;!-?LY1e
z_+HEw^355(6NV@z%hJAe*T}eD%vQ}zM}}rV*{MU-?bMU-FS+TLT(@6a04*Oe8ZIOM
zBxoia9vRwDH6jqvA-%!5;@T2zP>C5UKVm5cE!74R%^R0$RVdA0s^L>PiZ3@RdS4eYw_`A+qIurgkfo?AM!XwBO-U+O$^NsX%N4k-UG2%LOv~
zUJh+0XYI(1T5OaqJz0C~+IzGnF7nX~JgOzep2C3L=W&A?@H8+J>+GAOkV+!QeM9?H
zJa$4WN15kNXxEAhb51&if1WH=*y*%WIPkQ#wf^sP%PlVw!@Yk6E=CtquW9{7r?c>ofQX@I5m)4-(c(k&C-Tl}n>&3g
zpS$q3umP(MiQsAwI-2XenZacmEbL)sG(IdUJ5r&uQ)B4%2r`NZm4rnFyx9I(Cr#pRMPd
z*OpT+?u(iOr`*OBb<`!yo>N*KZ|B63b6Nv&`hpf6S4bU|E!`1aCzS%7dtR#{23*v}
z1)}^45akC}$q}p;k(adCM{49aeivOYY3(b0zX$%2e?X7;Yz6lx5|bZ(=Ot}&Qw79-xdqscqJQFpX7{>3a2e^+b#53L8w`SZ7uBYW|lmJ4&C
zQjwPU5AapR@FJ}(HTtPY8-dhrcVFAhjl%O#U8{atouIFw^}L_!@A2r;m#*-tKmyL&
zy)``?=JG}REnRQUpc~Q+{b@GczF_E#M@}yBH3yla&xv~>`bO$3CseOR>4i{Tx`lTR
z_EZ+rU3!8eBr0x0lY`Z2;;>6!8+#EuBR-Gft!(D|e|gMP_OdWNwK}QVwvF}vEL+TN
ztezNncuIGIfR^t4RM*4lcTyMmwCr7h8&D|fs(qn
z@bYBb1d1e}c5k}O!=sCHZ@A38B{*MqnOEW1^h3VpGFOVcReG)DErn3dPV<2-b45vR
z0D>oQ@9(eckGKiE=|YZ3>gO`gXBFrAy3BJj7xpqp#bLtVrVm1FOGxeQGG{p$h}4%{
z=1j^ZI|zHZ%=8k_o-Q-rp_Mv&(Pb``bzv_c=wcRPTAVh|BkV$QFzh*(Ngs6eciY2d
zeqDmAyURRXf~%X$T)n%qS$ewYT-dxSvgSwN;TvtM7M~u|L+`(mzFPZZ8h=yM?HaS4`lP>cLUL^a+(9UIkS<i7EwGP{=&i4*kx`{krPI3@Obzq_#ZJuJoTc-
z!*lkI5}(<6eDa}rR2)_hvVD=>HNk99!V+EP#N9H7Csko}UFI6+T}BsMVm{#U*iTN+cq>K^$TmqtvU#-2Jmd6Cb$INX@|$c&IRrPIkwHXZ
zc<6giwD>`<+U0Ab#{Ok<}{H_)53<+4i(ZlbzhizFn%o9M0gwapR|0Z#Ou
zd(mYv_XoY~lh<}i7~LryYdgLEnMhKK6zP&A-?5lXK`Z~iHBn0_+%bIJBrX!;|N`wBSy@DOko?vYY+4x+}dUDdZ6ocCd_-NSLLa@#Djribl^Wsk6|x|8p30H6XY4
zd>k>04V`0RRYjvVJ1>q_G@ih5tx_cepEAEF{;X_linvi4@}gMcF}gPNpDlfefegZg
z{52Q=Qj9}yt%05`%7|2Sj
zTymP_iEd4dZlUF1Q_}$(qMOmg_)Ju4WJq(mNh701>C?1|NR9WJ8h_Z2H#W3?LM1;j
z0~9hCuw~P~wYJf^Gev7;w=kM&xa_*z+W1~!XY7Th!KKe&aXVvD*>AVw)+;&?2*?q-
zD1-2yeiEXVW54&LF^J)au&ce{2`@u366}k0|IB@%;z4_(W%G*7m0%nZkl=cZl2l?y
z221jU*5onxi2!<^d||+n+<(%OoovL=91`z!Fk<5>;_n(@ZKKeQ2LP+^)7%cXL#UYD
z!7%ZL>qrNq{8O>G8N!97nMpWt&Eq~mSSVIUx`PnpfS;v2D+uC8Jk4LGITZz?=LoZ-
zQ6u03YOWy8u+*#~tPry*#&k4Zh)ykaiD>4dEp8t#b~IYK$D=}eNV16_+(kC;WK^z6
zCqw4`LG|DAVm*KXdXjr0hAp|^!@({XapV8#WHi92mUl8LR3h&9%guB)q$B*D@SA~l
z2q3KJbDfO(21cF(45D&pqh*;&%@rw!7&I=;=mCNl(%GoMDv9x(jmpsK8OT-2Bns($
z)p+7+Hc!GOC;|+@Jv@;O7biL!^`Al4@b=H7Nr96GOV*1StjT}Ty+lB|l7xR};+-zS
zJHQf5UkFl6BLOW?+=sN1frf~A&l{C0{%?ahr6>6!#g|WkOb60c3Q`<<%BXCD?YKMS
zcpiNufR{81%*?&t?)}sgoZyW>?>S16seXX4>maWSt~&z|cr60beZivh(?<2`Dsdg>
zKs>1&mV_ztfn7Pof26s1FXC0<9;qbKo;G|;7539cqFZ$zgCNyshZLWJ*8kg3LgWW9
zYVz?2a1-^r7;zCGIGygt_^>1wX0=BbqtZVOZ&nw>Tb8b!kAas?S2zBDbme-+s0tcS
zNE(YuH*8soiH__9W5>c+L0Ae$Ff_wrN|D(j+N41zRpsj-Ry<>PUWx=!DOgA+f!i4v
zrK!dc2?`HU5D)}W*bTl6f=ep+knopVTsUOol2wNTmJRV5*cn_<)VyU-U8iGju!!u6
zZ;AfHO!w|;bS#fa6{x&L|_S&)Q#Fef_wd82fyh^7JmB_^<
zr2#+Fd6HM*$wWbs;R}Q4Ktyyii1w{8Gi(UH3$7-Sh!ffJ_8CrfunKpd=|JTP!`&36
z1{MU}Rzvc^m@}G>M9d%o3nSd85_O-H)QdVWLd5*yX(S|>Rq0dKVJHND9HBKt&a+0X
zPu&O||YsRDSBsVW8Y=uNQ=z!d1s#G$`GDln$lvc$@P*em>nmN~*Vz^5M8k_UAK*%7|us38kLqN^&-S
z5MT}!RBo*h`VqF*Qt1dQ-Q)LF5T%O?_`
z#7W?HQq4{|sPB`JWy}<>-7u2?nMjV4Z#<5sP{l)gomMhadWSH6pFV
zK|sd@pl3SJ^ddfAL_B9amV^js^oE-;5;5JqDzJGFWrUyBu?N}{;o>R&mFx?L*#Lt&
zLax$qgxm6Az6gxL6FkI?XkE0zaz_CzR6b1l`Vm?uRy=3K`4?sU-_{ttJI9w4G{K%X
zynd*^uSA1hbT})qQf+e%&X8thcO5KU@FK7?q;P17l7ajm8kH<#v@61D4ikr-H|qN^
zF>=T{=JAGHXV{z#IZkI(x=4J%Sn~2J3eX94U%+7NvjsX$BrwsgL>fvqOjo=H*0By9
z<?XjDzc
z7P?A`A8A&+>+=6T1*)(HC`WcPR0Up)l4$0tjvbVt;_qIdZ}(M_0N4p0#r&|~B7}k=~GN^}NzrHT#9ziw1KKLQY;^3H7
z;$FT^D+5YShanINg8>#evKPr7h=eu@*jc%V1ocn2dk3V>*=$L%)Mq<`OKqaH{w8J`
zK6XGjw+B1=CpVa8%TrN2*UM;8O@;a-cc(wF0o{j{nA=j76X^0@M(y(8B*a|K^d1$r
zwthLM`s=;0w>x`Du?KCx!;Bvewk_eH~+lZ|e
zf_=eU#VCLWJ41*$S%yH#pO6=Ssra$C(J=%j_DoU|6-7uN*j0w8-p8mOgC1!|A*(n8
zdWp1(AqMs_%C&+t;y#a)9hiqSph{sNVZw#w5t>1Cr+0>s-az~7sJeMpiEdBtN#r2{?JlBD757tQ(_
zEur#<_BE>3K_Ac?Bq1*#!8SUuxzbr4eelGPduXzJYAx(%zKSBJuTk#pF!x!&)o0ld
zNg#!YE>X#-P7GS5(t-{)i)fHJ@=D!>Y#i)j(3Fl-LOx`{Y~o*J!FU+tjgZUy@g$l=
zvJ|+qnsBOzwFQD3W@+)EG$k;HaTzV%=x5X_ouAdus9ie0qn}YX7!i5+G#o5g)*s?s
zQ6%;^W+cP-AzosflffYmUi74aXmh5K6TObU_z$f-;hbK|YMkd-5DXkXD=KzkhNpV4LPg&v)lmK}Uki$d@mR8=Ol0NeA
z(=XvWfwoMbkIB$T5q_K)N`%B9W)3iFCaWHVxlzAGIKN3r0@cAue(h$if{5vX%-Z5|
zS|#vek(q)&JA>Bg&`ZDgzga-&sL2AVSW6!^-~gYA9C#rw`fG7jvge{Ppt#clISk1X
zY(&2Bc4Q-lN*znXNTIy}FG75DRwkqTy0q6vxisc;LQSDo3W5+%$>Zh14&-06H!z4>DqD
z$4k*Yl1{8TsA#%Azoi9YXV4?H8c#}QJoJ#qjbmLgHPY_p_Kv}HurAfs~RJbZot`jBjHX3pI&jt?@*hn1!EO3%U|rXsNs
zE_=JauEvJSpim46=1uEYut-Yf-+u**q*Q*vE5?B2zw!e9*We`l`7MYzK8!_IjHT2P
zw7Q)AbUH=D{)Bpju*eT+5!<9GhJRD=>X~vkz-S6+bp)}vt^d5hC=NTw6O0t!BgFnP
z@l~UquF_vk(8c;!ak_VlBd;2sWMEDVD&jOlnj^gQJVX9MBp@ijQ4?zVZt2S~^aF{z|!0u>aLDu7U!$&BdllHjdi26;owWL!2d
zZ#ku{r<|ukM`DT#io?Jjgc}r(?qv!YOr${}0*a@6k}nEc6T&kLe}*88etbmX#2yVZ
z+0HY;*Otypfb)EBAH7w80llDMnjy3#Vl$yg=F=BP
zl;R-0Kxm)>d4Nbuev~Ej`NmTqLn<05jk2=KJ@_%iT`7PWe8KQAui>6D{#h&IY-Duo
z_60eWL7p;!9L-)(0KyFjP!7Tf!kv^Xv$V}X86lyaJq#y!w};pYbaTOUasaoN9+Svx61(GCUCPEKwtIM_y_0W2Z<+z5Rzjb3C?rYGg(d0(XpI1fvW
z!NEYDd1qLaJVAb>c-uv!Ir}tiU9iev#MeSfXd=X%w~R)?xUgvk2X3&)d&{WZN9KZI
zYLVGYos#W>;Z`M;>%PUY(6Dhg@dY`%2T8sT7b{I6v?J9u0CWq2HsjuQuVFmY4LF$v&>qt1K~6)Q+Kvo15?^2mSXYkg
zB}gicA|S^fwSgYMf$Xm)pLA4EkeE8e@Odc8lbDayR#$Ge%o7V6`VM+F
zgNup!<85Od?pLP1V@&)1czX}{sEY1?IQJ&m-P`YONLfOX%~C_}y)7N-q9}+IQBj(r
zh!qlgkN^RJ0cing(viS`Qi6aKks?((2#Nv%N)u52-*fNoCipzh@Av%Q&-=deF|#vw
z?%XM7&YU?F&hmM&Avn18jNxIWzz9fXo*C*q{m-)c>CTn^EPIgd9Q@C+p&8B+`W5(^
z3QlkY+;V)8;Vh%yE?9PrT}7EU6W+?g^)is5xxkgqU_z85hBq4Kyc+qB5AZspzzPvg
zuxCa)fBi?Jy!9Ao(1%fyz?pOU(aFYMCXTOXEc7I^Len_
zf12p5BCejp60@EANDR5j6JLyJlbj`u)yKlots(-SALETDIU5RM#w9jtvhxcl`quo!
znIz5_$DjMexn6Leg`6DNjoe27dr1(DBu{l7f{OUfsZMwbI>9ELFrmCQ9c24iHhsGD
zZI{JWiV7U(F*BSpjDuP7Y$w7A^QE($h=g&P{W!wHJp#9v(Jyc}I<6O_lz+bYx<
zH3Aa>=g}tK=__Xy6;(doywI1w=B6fF-U4Bo)y%d@k%{$q&HO`-;8#0CRPn&3)i32Mz4+9^k-@
zf-x#H9y-fovg$u{#?hj7dWg+>jeYRYSsj5a);x4pg+HU89y+_Y93(P_)V^7it2wl7
zCq%iDT}W4`u2)=+Z0w1^5uOn1%8GQ%vNKB)Tt&$EbeGrFF_MfqiX^$7C*zL6Nv@A6
zFDAQ|7PJDf6s_@mOR}pr)q9Za8bbBnPjMv`tY_|D$sAdVcaOegOhBNR|jXdayE8!yK
z&vRdCS8{av{ep!z!x`w4*e_*5#W37xMmPgE)AsY-;UeVo!WrnC*w3eji;&L_XP|{*
zJ+By85-vkQPB;VQ6uW^9;UeT$|H`jda5ah4pZqLuP{q|wNMC)&T1cpoK$#{toPjRQ
zhHwVXnKy+q@XeeT&Op0nM>qrJn%&_HbZhp7Gf=HL7|uYmCO?x7&6@lilxy-cP`Jp?
zz?IOEa0^hmIT6l4AD!MGC6fc<>U&UC`SAr`tBegp-a+u;lp
zFz$yl(7$*T&Ol8i<8El?pnx+noPqL1=3RENn(Ks^oy#{>cR2<2dTp1&_t(HrsOKN6
z>8dKI+3TP$0umRae9Yo%yNZgdb6K6*uJTN#%7bdVN^0pDWW+$?SAtGJV`exHcjhQU
z%u*9c-qk0o?GEQ{kPKEk7%sM3I}y&?kQuCYK3q)kwrCIV)E2I11^3PH1H1e_AJY=J
z09FK!R<5~F2w2q0RW0>FdcsU41==5#sE7TDv!Ck~L~A;aS4uK>U)N=}W`OH;VFyoq!?jruKfljUzUkWM
zirKtwglQftWcm9BNm?N=ejVTVzN;aOYe%w&LtP(1kN3TF7xX255|wl{+gLqjWmrC|PiI72cAIHv+cz!+RH<6Liu)2}Y>mgN$}oU6RYcnlo`$~O~S55y%`+1C?Y
zv#{3Pvt2`@g$?}dR2O_sZeV+u>o0nJ_jK2bRITI;O!6cC;S5)0QCz)_ubSzaj`nA>
zmUCSnBIV!Ab;ViMh-Y(Lv*^J`^ISVAZ9L!Am(tbqU9ptzn(x|52UX^0uCZzXTSS>1
zoa;(p)fTuGqZ&WEz*W;~FJY0(vTh74a#ctvU><25fCciHLV~VjXqM
zXY;&8u1{r{=)bVWRgA2v~r0}&Dr3(8=0S-
zwaL}fQjvMQ+4V)-_-x!j&5()==oFAZ0OMU-T*btkY<7JM=4?eaPu=S3>=M4{Jo}LAeF5G;<-^z$
z6OQx7hg~~F@v|jNI_9b_Y-0_Np&JMIpkpq0sy)H49(Tn><8ASbE06Z%jx(-RaX(DQ
zXPu#eLOc#}G@gw;>mujiE6-X3xp>w!LJ+4N;JwegwicMe@)un&Rs4fhy^LW$Wc@C?
zs))0vaP+kpb(0o@pZUpED@yo@#a~4yKbg(iU&YkHv+c;MuD+4t+$rqVZ?5W&#Z%A?
zKO1=2RbKd(cfH}75pF=c<%)?ey#SU-q*DQFurx4iiS7q&>I}dg4`@!&mi!W%WdnQa
zuIr-&k^~2q!xEBQqfB-t`em#kp7OJIT?@$>1Rr(ZH6~hIzL4pEqxW}tv%g*IMd6pJ
ze@WnuPvF%BslQ8Hd5A5Pq=hhDYbC=4$!XqCmO6`$c^80YO0zkJl)!&frF978#Ag^%
zt;m>j8z8V8wh<2h%|t)=QZ$*DJyN{_5A;O8j;k>qREv=;^Nrmx5;=h9)eA{~!PUn)
zRxM7d4w|)JoK!`)%yZ(ToG4g#1`?$jnC~RXRS6K}oYE7BW`}w3-qr_>`*^y%KqQ7D~Q;SQLVRzb~xU}6e>6D`L+QmGw$kotaG6dxP{&0UJFRY;)ZiPE1CZ+_#U1m
zvaCvyBrfN?bt5TN6vxkDZ#2PZpry6A2}ZMppKKzP5XGEaCN{^+t!K5GQ=fSM=F)`x
z&iJ3`Oj*{prPMWU`B(U=5&EhTa@d>K5D)LObgerL|M1yD`?g~zv(Mu+-i8OH5sFU_LKn$-bK
z4(T9~l^jRLh!!X1uoa!q=9C8XQoz?@@uCcvH#3^I>;~r9J
z5p+W7Ug+t0OT&Ir8FaBMWch!tz3kFJ8
zY2=gMlBPxe)Au5;K3HlhirLrrkoTllq8;nuJvKFx32THD<{T=u0L-@^DkY|mIBTio
zf^wudEQi*Ily@l(%jVxg%Da?XL@`1l$pJsr{MJ?h2JuL_JC{SsyOjI%pp^qXL%DMs
zL(039TlkO^dvVI!F8=thL!~~V@F#nA7_G}X_VF+&PaMCFwH}TIn6Zv!442M|GuN>m
zBar`W9h)*j%AmE68!3&ATeuGRDvbW2%4B1Rbt9!>;+l2*)JSPhBvoFNDUFI-OO?Yx
zWF4W(fw6#(uh#L`$4d8PtZmOJ(wjm8`+kb_y|9ceoGO(Pmht^lr6>W%{^4oTwJ1ED
z_qmkD-k2j*=P9!YyzCN=5?EmdAK#fPO}Bq1@I^V&$Ha4}G+(0daI@!2f7_4Bveikt
z%8q;{l}q0Qdd55f4aksb;3^@o8)ZSV5jha82T>5@=ZqAfTN!?g93Ia?LGbZP#CxO1
zkrDpu7RM3=s1f2MKn6uzeK&M6;*hG4@~z1!%YR)h9SUa3@n6|*`>;MX;|>fW!blI5=IVlwfocfl#_{f`+O^~4M!_v8#N|K|*%z?wlmai;{AT3$B&8!0WZ
z!0gbJ*+i)|xn6$y8|h$h29lWXJtdkQ_!d)hf;ZkL4GqK1)&0`T!OhRy)I+{vgc-uKAPwbW*Ab
zjFfUpdLY2>RpB#IEnp;M<5-_F(rw`g?|v5O0qM>kr8L4bH!ew8k;s2^MM4ZvtMLTh
z^QtsD($1u@>rJpibBfCbt9V0t)-El}oFjCNZM`AAD_mhUZc4oY=5udKLy&aXrGlKj
zC2bE?#o{Zx;_nh7VqRe{{eg4R`b=To{kF7%Je6s8fjBXWyV7v1{!F	?qw2>+eek
z1abI6cIZ!F_@xWE^gxP4%&}EmePrku(&P)l-|)C0KXis3q{hj+g|+li=
zLf+!Q18keJ?A-fU@aN0P?Zh&-M*x;rKo_opFfz_BLv;euIB@*q{9dSL#n`fuBW{#>
z+5J{}S^jqgd7~iVVGbS!PP4p9@=%ar4Jyld!F`s@lWWS6)PLHqyg@CwH}!OJZTaiq
zpnUA(y7EZ7H)*_PJ-HiVtufX>zG-*8+_c-8%IZv(O?XfGzMgD8Py8<0Psn*r4>GQnjPvdm*7oW%L(ux!Cq`9pB479y6xqnnBX4mu{K`zQF{#A
z%O19u&)4*tnfk|}zSRW<54xnbz{g6HJY_U{kvTBZ3r&&zKL!UO(OXZa)`iEn#BPNnq6
zu9ybh;&?*_k_o+B7#c}EPx-P)4&7$IEWe7bN4+9HMsd`ucr^ts
z>CkK6b)k{)5zu=eBGvFg&&`q=%04fz^E$eUg}%~9t|8oEzP@r5ar~$J*}n4Y)b`eX
zGW<-Q=2(*ykZ%0e0C@qOchlaKj|2S-7$|=Le8qoxOMXtk4Jj~4UV*1K2FcGOY4MJH
z6z?VvmbcUW<>I^Y2yh@?eNTQb3XpK_BW#+}{Ok}pInonWaECO#(JDS!+sq1sa)ND9LIWniaO`n#81)16MXBZax?6E?wBQ?
zaM8(set}#h3Z(U4OXRU&CXM-A9*#s_D(4_swp8xs+YNb(&t;yHd{JsE=ye2^KwOMH
z8=waPzTy-Z$CM0&lI@+@cY89%>Lo=UDQIv^q}
z!N~RUb5XGCIsK))E)o~C$y?<17AbjYn>;0s(}fHk@Ir)|4($+3*MzU-Vz`K}{Ti;q
z9A4bBGi6rfj=Y4=-ys*GR{2h#C*1l@eIu^|Opn_o_d+qdv0MHb$?-k%E_7k;UOKXF
z@(=gPhXiryue|Mk`JljB@0Z=Y!a?~BdSmV(I%wGAqjC}}{k{Ae5xG;oN6#+pK+lTj
z_sp<46ZH?v#l)H-3{%QTS>7ExAdgSV!Q^Zp)J?_RJ4=W$-qC=jHCp
zUVN7~|5KhR2;2CN4*)@Et?(nlZS37gm~k(k^+=vV+p5xI*>X;>>9PC?1~j#U(j28H
zy-Lcu4azVlYJvxiEp~#f2j4u-3mt?R5T97fNP8M;AX7x=
zmD)vUMKj!e%%ytp6{yT$r{}-Kx5x@EGpBi)rbq(tUKL%rLHVL?rL=&%Xj_j`0hdlb
z6cQC0#oSn>F_Ir+mE!1|6sL@!Qj8*rEsa;K@OB9a%E`FV467REH<4n21#d2|0;5O9
z%6k=DRvCK>t>RaTDXk*K?4_()DWy8LQ@>KmIR_$$w63815Je0o
zHuoJho;R$hfK~(la7QJjlEosqS52{Q>qV+7+3}a3U>JN&EHm<>&mLD-iXfc4BE{dM
z8l^&ziyA>M{HRe}oWF^esG&58!pilnqg+PPp{{b+P7+wXdP*js@tbxMr5WSEV3ME_&l+A)6M4y|XahC1<_d9N0?n1ixP$UhEtR4`3M=^1r}9XFQ9eQPR~uyrk`LP|`^coSWjp0V2e<}1pH=!qV`?hCplq{PmVdmc
z-nqH0ZO@O%W%WI0;8(vK&G>gx)TSV)d}mvDm(nPvYPP9s6omgr2OF^1tzstjU
zO?vQi!OIIM-crH4QX2nlq_SS*KW8XsLiY|Tni?dJ(-<-@aU)CV7fB?|kcoegRwH`#f)@-e)V&SsZaW5DPJTch}cV@u;dexW!aCmF-8ait`su^W^X
zaes{ibfksA9SmCgNItPaDHiv6){~GU+mPeg2*_E>YHS41?qhv7D$|m3AzcTZY6MaS
z2H>z4`i_B2>l7@61CS?wN${LGWpU%L6l+U1-D+>iK3kRf)Dt~VvEn@;Ey7>TQ6jIwDSyD(H4VFZO
zSoIV%aFjST7X*J^oLY}=zJC{1UlW8MSnnb#_z!$u5f!Jtbpsy8cGESuUy6DJV1p|$
zt{DmJaeuuztL9gWhJNSRzv*VoZ~E2oB5dY|7FWR^Uc-2CwHH2_SVFxF>Psl8b`{0N
zi+T4nb-%UmDpXK^k6ZomlY1(7{?w?b7DJ4ut`*fPxISi6a+U3(7d6c8~*-
z#@AL+-xnblDpy@CDF~iBEooW>!{#)
zZDLXNG4A86etq?bEj-hpmsGfc`b=;Z%JEqNbwC)oJ$pCYxyvq2_>a1Y*5iPfqs_+vp
zsGkK(ER^fubGxZ+MYvP=t%phsml8cy%SQk!`;xjLZq+hOS=izo)<*V`{oo}`*-n0SXpl!~)Hq2B>R-2SZr~fKd2>w^SCK18Z<8Z18T?
zWNY77zXiuE{R6cw7-MYHM;Pd3e)}U8{JXR4uOaGp!FKt*p=vb|$7H!-sPd4%GEA)l
zP~`K5tM3ry&l5(gxa%I_!^f!I2%BBVR5#l1rtuwP)xSh%P>jWPXQ}H6y=IJ8*T;Re
z3bPUBQHK$I>j{{}U2Nb4%;Gt|Xo8wWIy_af0Vk(f|7>(6V
zD3!IJtDljpgXdPLsUgs2^H-^N?ST~MTUM)YM`HAqbJeB66RZYPAAr;2Tdy|8=wDv1
zPH-UoW0Tr1e8i32qCORDt~mcQ58IMX!^+#$9#M2cvDMv`c(!7@It!%AQ#;h>a3-{Y8{
zGUx48tJ>QO><@lsulg17JZJ1vYdKCIz;4l4yYE#mt5{g~u?8hEkrlpEKNBEkJ-i=X
z9+%D41L}7qj5v8nttDoE$zs1(tBP~JNEkXC%ALc5eP>kb3V!j7x`P^6a#o#^bnF|zSbxKq#
zAzOM$U0`j-%j$N!o|l#T3G;A-sXv4Gvk5!mRQpM)p|RvE>fTU;o?lc?7+%``s&=N`
zYjHp?TvvCCnIqV;-_#&$u{B&htJCZ^6BxUxy8j*H=gXVwq<>-jbh)KQ795z$hX0Ob
zvASl#Q+3w-5A`SwBl)&kj?y-_LkD3R-+5b|Ex49|-wH}9hn?E0dZCg`3>_FN7lP0{
zfEPZwtM+sijR<6bCS<2t4x0(Sn3rFBIeA-&p;tro|S&xT`M#3W^6=hHg
zE6Gfx0pNne7%~kor=ON=*%>q#^)OIU>xj`!MIyox_b+MM96Zb{t^JM9jwz$Hmb@z(W#GTO
zW&Cs*ZLA=USkg4if0uD2;r(l!Q>nPinTuNwaTEGohGry~uVW*4ej->*0idoEMYNzN{m(v{%!iLq?PE*U>8frB_WKCPXJj@k-QoVu8`?SwDp^3k2NTh!)z&ucZsi8=g>
z=e1GMv0k54E+XSqe{3MyVlTti0U*@6o3;_o$Q|9anKD2hidMdu>LTlK1q|IC-L;P}
z{6G)wZ6xb@Xs?szkgum!Ly7i+UFGFn`vKN4m<#=})4lw?0UEAnUN-g(tvZstH?)WG
z-pGcQwS+eylKB(N!!IH#pO=6Crq)eVyyg*111aKrS)I4B+Fq74Nb7{Q1`Wav^#l5R
zqep`21Qdr&-4_;{ZKxGrx$5GjUP!$x_Z_t0Wq04vz>s8>2Wwv-xjb0ghGg-(S_+a~
z?`q2&Ub6H;AK%kDMNtM)3V!4x%_C6Ekt;*A-<`tO{M0B-Bi_|7W3)E33^g;gnQ=EL
ztW08k3rR`E64+SBC
zyH#U13JM6^rNBNEJSlUR0!L5=@lmx2+Fzi4s!YVbLX^bY{_ZOFW46Xe($vQQt-^T
zbO93BoU_22R?Q#nN(pRrJw2K|e_lI}f_yUWg$vp+yGCJt?}7$VVjeRuYdN@stho%F
z29iAfXRR*ybI<&&L6z2~BlwW(+Iq0&ud`}5uw^##em69TG1su4Zek~IqA{O>=tQ^XWJ`A{n-2w!p6
zUs|sqy<~}9*Dch6g%j8?QGa@xsAt$e7u%n-sHYf~8L2M}R$&=Y`e^(HOWLy?>aj;JM+zzBWArA_7J55I
zFOFnJjNXDue~i&HaX0Q)Nbf1!X8Q|as&Df)u{y;5cUkLreN@U_n7QCU+a$oTNMZ7Q
zc%STE9#E;47d+k;3bRc
zZ`0LxW^uhAssK7(Wdlm+0sGZ*{7ea5hct&>E2Y}u;mfi*ooz&WTV=8;K
zwq7~%*I-RK8+uQH>^MaCU|-dSn@Rg6pE&3E`Z_wJX(w3cz#Jj11)mZfI3Hu{&R$^`
z>+3s$Be!%R&NtAv+W8vnw+8t7Zw+Q&=lAe@!oqr2T7pUjonMfOB9Vx;b9j^_t#@CB3dUhMV5bko(wbCr!0D
z)^nVatiU1>rDPiwJf^4h(RYcHzv12c>ScqQ&pI|VwzHpJ32GMXen0&!O8fQK%K@=}
z+FvgcMe&1JhqqKe*sti8#iM)8-nwag(g1z9C@z}BVh8Hgg`K?hK>Z%#4X`GIfUibQ
z;zI`E%o9gXVt>5@CwB`b@oIzh5v0w%@m>8ap@x~m^rCFjd-^6q;UB!Oe=ES4quGZ*
z^auFR5A|N6aF1Oag2pC~=ZQlxAL5kp?B#TPY&D;mu6L%k?qui-f^D6#ni;NZL6laU
z1C1^wKHRfo=$M$xCydc!Vam%FjMWR%Bv^+W)b4cV7^g2o0nWlCc3_(&ui$y50
z{IN(^D1E#{2iGr;4_K;yPtOC(^-07bJ-h-3^Bu&QLGl-?vr<2Y?WU^th+7dHi2=x|d{p;PUxoBB}*K&@zww}o$6{XcLL0tScv
zp*Ip29_L^Gp-+kexBHp<`d%SYA=>qUo(At`tsdwV9ppt9MZ+HGz2NC=%OgGBF9*FK
zL?+@e@X&J|*Cx@-bf&66Y`RZj{)c*7Hw60%3U-)qPCM+O{wf;2@=$-uBUgxs
z@WVb*7>NH#YAdrQk91#gIM5#KS6h-+3BqSt{xabnOG%Z~2qhw#d_ls$C>#4oulj${
zpNEh1_D}W)n91r-6PELrUKE|$@t6Ko7wGj^oq`@4p?|AeKBLcqKq`W22*;UDFX**
z!)eM?ConZF=DsMy>ygcA@a0%RSP}(P3@?HxzZYdRbr7>CRbh>yjXNNhYC4Rb;%dfb
zI*iJZYuQv5nCULeesvh_-B86PrH>(005|Y?(JeoiYr8>8
zp)-3Wqhy&Va|V$Bw``2d5t{
z-cvS4L<%?AWzBe3Tyl-2x{WwCNH_N33K=kr1nlfChOtu!K3gJAJ<7gy8)=ERFN^@=
zTlK+iu!4~3sw22?t;}}8q4|1dc#OuO#&{o(ks{zVJ}$=C8A*?Sg`{E)qa~6L
zY8Vw0R*goa6M<-xhS0yj*swkO62bI!4a2&!KB{3{#4Yb!P2&}6saY-KLqaawYZ-lw
zk-0&)`4krD{#w@XyX07RJj|Vu6#{z@|0AFWHnLu|jhSfTac!eJE;8Ng7?p*`Y;qlg
zqTp?;W6UKL)-H98HOQ*`p!%B(BV~VTs@<%gPa4IOW5C?j6`;&o-rtL
z)zVRH^(f(e*1ej?!Oqq<-b%WiX~k4PkPUc!3S6*r2(m#D7S^)<4U8I1uG)nalz<0?
zC+u8#yULe;Siv0-PJ;qJ{Bg%>tgIc>;Ws;1MhUFEz^*hfa(i7qN^v~kK_Nil9CD6Z
zxgvH5iBWf~9D-dq@oe)g7zbH4`jq>QVgQILjPxmYYZ`?sQDLP|t$c2`0yBNs4+YM!
z#|@1%bGaQUA_mHQD4cqW)eIPLD|&#v888aN7I0j^NUXK*01THz_@@rUqS87$pat~*
z3L&@Ceg^X=1-Uu)GZ@ax$gO7Q@H)tihe*tmc1;*HQE!&6VI>+F)4;CX+{oxp%U-rI
zzHFxn?4!oUR?5_CVi&&C#K=OtL$N7P7S^&-Gh+afoMuKz3UIimnUM?m+R)}kKhWBt
zXoN1-ZZI&J6_T(J8$a0SN-ZvGVQfv_NFmG3z+xokoI*;h4OhFVRE8sw5iJcUivh^b
zJY~!Z0(m92yp?eiusyr={{-8YTN{=}P-+{aIy(G(8>13+_~W+5U?f62qZGwhEYr>a
zKk+sz-#!fbe#YAyozX+?>0rQ$;wG!`G{$_NZF<_MPn-48(?HaCC+!&v*2g|$Q~|7S
ze#R(i!|z5!RySaT{;W|jSa*-QMhaX0tT8@R$z9JVN84aPC!7brut}XTMXT6Xos4>*
zS^n&V?Mt)N@{;=G@gT^h}z86kBUP`YCKePh&h%p7ausr44(nkgS>(YlRp&HpqC^E=^!f-og3-7~jF{;LYRj
z0LK0oVC7c_8%887ydS=A90Xyv?*n5Ml6kki-Pp+wjZYDwo)7-Ws1^m*>GBLCAdcC<
zZe$plkhfrtHw+MiYl+p1gtW9&U7i0e9gMMorumI*u@g;r*i{P&k{F
z90?RUpLH1tusFiHJo0wnM@Ir-qQ<>Z_`>9)y!L2AfMEfz`?0ZHa9V9HpTcg8Gs=h~
z$MPas2E183;>E@rk4ZzM*hHh6`0+9J!bA`SXIY1Aqna?6jmb8aJN~={odTI7H6>y|V_hkHu3M{$6yKFOxDzu~1_rbXsO!?>S
z!$8SRz`pN{rJCyzMSZOp$m2s#88L-wZ<=da1Csn&2%HDGnMdSiAg^HYO%$KEig$#I
z1H1XRH;lQ#*r*9?#BEx8zWTPYg^a4+cw`)o6b`IwY#xF;q|@SKeJ=i}M0E^|#&L{1mXZIDIN(~kq_(941+8F}MWp0yz@`T&`3?b4!@|f`Lu;h0(I>xMrl<$l&
zvqbUZT-GtpJQ_beH*6c}Yy{^caJ%NDsH6?z&CZTZxby~UvNgRD3$X)t>up+7EaDFev
z1TY=pk9{WO12@?Fe)EJlcOLt|Z|3knikkPF5XfIGV{Q<0XYqw)%>#mw34Iwd_=JE7
z+UB$Qi1KC=(Xj(v4^*X=KV`iunU^tUJ8B<8)V^YTQ)ROSWW1a?s+twVxf@uGs^$cM
z{5Mt260})=t7_K5^`}%dvm21sm}++Es%mB}F=sRTwVL^&z*&pxcJZ6lQTZHOS{;?Y
zV^R&fv{DU}-ei4in9t)x+z~9jMx|?6iJGW;k&UWp*IiK)4G-JKuGB=sdq5c3#RF=g
z@?Ey17Ao&#&f0cqh1w{+2v;&@H*xuPwkKG8i;54iGIh+iacIn~V^)rzy`9cC^Kn!t
z5@}^XHAZ)(4nDVZJ1bh(d{JDmi4Cu7x4g72n%KoI)kVt_x3VVn?Bag)P`r*Us)ue|
zVA1tWODVT}eKayYkG)>stN=T=8TIL_LIbm>IQe_FrGfdJIDa#X
zZ)iVm&=8OF*!vC9=GW{*L$fqZdsF~zW^G{&0+Y6z@hcSP5byRKCP|HnK~z
z8c_$fvR#cZiV0g-iN-MGMeX*DQF}Z4xG`$4W4|=EYZq#Q(qpV`6LfRLHny^fUHJqR
zui~Dj=1^jROloG=-qZ}mkNMqZW+y?++0NRw0GLeLz|vcoJ;a3z*@YJHV!Uz;Ti6nV
zzr&8UG%JEB<9Ny(D^6d^COu_0{?$|H;qs;I?o+6>pOt834i~nu&s*8wIMT|j6~7bb
z9vH{@OC-Ua3kFfFH97zd^w#F|_z}CYjHw0cz~LX&>D~qo+={g^{och?Bl3wF2$>bE
zQ%hX2n|;~_1IXORzD0_0K5T;yuV!W2B7rR+5^>Q(HnFYQ1Cqy^ZB6Dq@%2Ao$MSwn
zP3E*izc2smW9xf2@etr%y1n@l)`CxHZ=QGH3vWCJR_lF!`#D^UvFz2mn6HDalGDXp
z3GKV#tbJD#@n?tg@m)=w3i#|7&0N^lvZuS7RmI6y*reHJBLA$r36?uM!*hF@GqCaV
z4ZERQG?jPlW!`tVuB=AC{ed04YajD1kvMe+`sLAN?VawCa(J3eB7P1Vsa213mmdjP#
zb-)pzYcDxiV;wTgK`_Nbl(?b30(D0U1uMhTyjA(-VP;usdGauG7@c>q!_Dc$$U!*!
zLVWvhL_o)e`eKyXxJLnlcm!k;`V_SH#_>^I{0O|5kth#|?s1$(2C`64Ky;7j)F=pv
z?x74wb>NAYrtR6$=E;K7hKs=*UVn@^#(?{XZ>E`dv6nY9vousVt}rt)3IgD3GtD=%p+(kZ>!lsRp~J-YblsL6RIW^
z(wb3$#hMTTVCLnSRZ0<&1sB>sL&uz>V`W
zUz_u3x$l2%UK38T%iA%ZXZf=`OpUG(y>^-v9e?eGj4GN{I-PKe4f@7>KBV?QtO(1A
z;Rbf*8}lh|*krT_$Oka~%T^cbq5^a<8tyXJNHd}EoQ_G&V-I$jZPVYxSZr5_gU~~`
z;1AF0P0Z?985ejAVUqyqLAoyr9(ZZiHlW9m3Yipx(6Z(qe_egdWx-i}&5!AM4`uh!
zhP`Uw!dR1*fd+yfgz!d6f8vt}$!=!B5!(OqN#amgTnhUG`$brT_~9iLF|I{{c!@9#
zfgD8&P`EK0NI+?sC#}_3j-il;e@q$%04^qtATwAIqLjdj`@=!-n7a`gqJoemOxt4?
zs+$Qnz97gX+9Kf=QNapwBEf%`dC;-g=QQU6{Uf3TA~BHbAxLSUjl$aPF_Wt0SS8kU
ztq!+=sa!&M8No(i*!h4Q!Q$oBRFz)M9yD$7~76SEi
z_h+-5n0=iUzhbT>lz#Gx*&AJ^iv8-9${y>41s|Ib}F={CiF
zx^5DKi2Hss8w>jVf{6Vl8+Ze!=*Veo`3*tfBO@sGxf02L$fiZXwTwVp=vC1J;HHyg2g{FE61JQ`JbD-yVGit
z&wT{`7(>
z;SC+`??o|ZBrhbnuSN=I*K2ND%VUxWwrd65ofY@PVsxs28Dwyrhje#Q7?t1A-Ic`0
zi`kk^-X!M8GEM%h;jV2B7fmdRTTgF?WyIlGvfkdq2D$aK50Vdw7EI11s6k-Aed@zunM1Spc!+Xyk_70;}K1
zT}AwDtv&w}8o4V+LHT4`GdH3wOk)q4xqIO_>DJu65@$8nTDp^>9499LMvJp;9U&9!
znpw!j+P87fz!-jQgN9GDA#L67K&|tSw(eO`ID)b}y5Xvb?LZ>VKh6K_==O->=O@{u
zj_%5!J{xwz2@<;k7hPeqAS*Reh?;=M)d_}mwGCD!129D!g8q7MH~hw}XC->O
z!TDUzyYzPN6ae$S*WIr~k_;zZXgHqR#l*GVBG$!es6bSam9Lu4|ZT(`n$W(QMA6l
z`z?ak(gWPEdB%^{Z@ED^<+7Z2+?~N7dhm|>Lt2G*2fMp?j+^iiPc8w-&o{2`<_Y%W
zVD~D*C{x~bm#VmYI9A1JW1oXithBIlU_JaT1ysBAe5jO|;Mdu;cipQ%=+1i2J&@3m
z@xHr1e2@%z-~CCnIC?nuq`Q+vbfVHQ_s@>&gq9!yhL0#-b+~({3}ASW}Bp1BG7C7mGC<>
zYy}^@()|JUI=i*T{T}!=eZFv8a=UR~xNACSKkeM${vvMuL~Pum`OraX>UcJOqq`U&
z_=}Bz$xHmFjqW1WUXS|9or72TqOaVaMnRkz$a7Z{zx$jeh=sij1kv5&&fYk7;IKP7C5L=WBf3Q3Qv@y#
z)cKD!fz^4^A$MUR+EUqL!ccb{d-{8<+zB@Pd-q71Y1d)<%jFKcEj_%Ahuzj0_Qzp&
z0H*A$+EKR;z~1SoJC~qbJ?3tPRy!VZ_a#Net;gL%MRCHvK9Btmv*cQ77(idRE}r
zA*u-P;Tar+NFJA%xx+eicyGxw`d=?{k7Umc%Ut-n?pZ)1{KW8hq8$&WW0!hyuz1Tj
zrp_RAz<#6ObLe*!cHs|qJR4io9m`sM;2p$P#Ck3i^kxlOitb79a_Gr~AJW;ZwAa&>
z(oCJRFBbMZL;0GCc79Nzr!HNzzDV@+r}ESy
zo;j3m3#L7jJf$f=KFPDEcmbcH4kZEsE~zgza}5_hkdW3L?!#oyd-TekWY0vTd~}NE
zAY}Mc+4HF$+F0XLJ(b`BVr!~rf&d(^^aATl*F
zu=SRe>uu-G-?nnE+PSMct=ubi?)w{-9O`8|_pvRb>SgC{Q7mdveaX&^wt^Rx%xCXc_dK0`$QH9bXN$Zh+qq7*0BW0^d)5|C
zt+R8_C;=6NxnK+3I@;ZsYYWw$wsT{Cx4PKD&i!n6vAvzUV++;V**#oQ)9zwhTP(QA
zmL9aRAFhNz79(wK=O9z0Tq`?wgAK3YXFL<1A4eVi7uOW@3+=DY#uCy)fTX>CisO2g3EJrW_jHQGvXE|!O4fTr{
zys_7AyO_O9mzP;Gn4)%TSy!z0pu0g*&`DdYp!KPC&4X-zEzgkV`$2D^X^1IF&bq_S
zK@LRXoxkiEfj@Ct!Efwbp^!k)YC6Ww?PQZ{d(w;?dt`?F`dqfRwkNUHXTMu;1W^)+
zX}{04z6Ws;-QCKP>v$5M{yi)``qipTgC^v_(Ku6rx7St|y1@}3Pn
z*-1e8#GztRH_u$KAV2Ts`98!*f;UZ1sq$Gz1hIt8*vN?K>sN|_Z3LcwY7xlWOA%!h
zX-To@%;@S3qznsTJ`dlYQOOlZh*RtCu`JmUqM@RrFW3bU;Z)fiQpNB3$sV
zWtrnZkWnl(+EM{^*fIF3psBCfu-=};^gOcgtGpQ8YFi5udWDuQhy0yms`ab69iT;izjopxOIT#R%75Wml
z{P0{+*cEGvU>O6*qZYC?sK-b?#D6i7R3XJv=(dpJDJXo)+Jy#`r`Cx8XUhmcQyYt6
zg|H$8&>AC(B)tl*1(-FdbTMASPS_776A6YOaOp+
z`u89p+aN$2=${}^U`?x8Yf9`mdqw~I1&yQyE%86EXy`+1!YiJ5g+kZ_r=G2T#S>fa
zi8+_;IhXA@hn^A5Ias|k=dv~D63w|BoN|esza}T715^_57_eN(ClSZX+AWk&lpcut
z07C#rns&d-oI@J8HateswF#@T!$@^f3c!RdAZ-v*P%0%Bp+7!9!s0g6-7^3?L|JO(Eu1Bys0RYmA!9Le=V`&(2_{pr1vEcN!?$4}pF9qFu{5
zDm1ZJrSl18(@1gJiIXcOnp7DecPq3qAMpS%ACq0NMLtSQTUh~AiMrbj^_LFDf
z638410JI&#>EGd!f5J5Yn&8^je9OlrA-Gom6|QZVD1c9hf$B?y;<^&3omvJb2I7>G
zcDwx#7dGcWKSI>?)uSV#5!jswc>oq*&HNhLmQjQTjX)Gxi3_Ra0(DZ$;5dE~qFDP+
zW1_0#=m}aV1aT7?Cp>vDc`SIsbtMc>5(`Z56t)o4zdM698N^xKWe;SoINP>e+
z?+;SF8;IPqXkpU)ebb)+&%ihjfQ%qw3f9ZE3m!#K*U7Q)a=QRBD=H|Bqr@(_Z5Q~f
zi#--l2^wCFlt@+l&B0%ENHtGIW44H}203}3o+G`9ln7pn5+xec>iRqvqF`erxq=mn
zKJZa$YQUoW9PI7>p0skQO))VJ_(xJw<9!K1{c&$%Y6VQR+9%aV@9vONK{kU@fLmZ3
zmL9BML~0CBXaVf&t6=FplUYz87Qx{oAo@c%47`P?gC-TH>O>QwWh=z{V=*z|%Sn#xkHZ{6UmAv3zy|~xiJ%Dp8xVR87=T`_w%$g-VG@U0FW{oL4MHCVg4Q?JU_=nuAS3RwrTgS7`Y%`ou_0vhcbeMWc7vdJ3beLQPBm1WU{w$7
zAwRw`O*JPSD^)DI-<ZsPhghf%vnWj!s5~P`9D;D87@z(m
zALgBWV(LDl521N@F1CI}0ID#jY9Bw$Fz{)FfA%Gz(-g#A@x`s64mY|Kk+Ilc2;r1f
zjBU9!HY}Pow!-*P7-PdLsmX8$N>ONIP~x&m;xNHRBv1&xmrOEm1EF)v0<H`VQnLoYA}DoVs`r&hvC8Y)$4R121c4hV@VEgtAMQ$0|8Q+*z4#UI<$r^B=i
z0hE2QfyWU8n;9T{LPU1n6t4@uvr46#DKXCEgEnHun2m#M8Mox+uxHONGA*)Yp86ed?lYZ6ypLk>b!|jOhhR5~u`wj&>WWD85p#zA%vDVxC87N_FuzrSU+M
zl%`e}uOYQ=NVqXY*fJm5n&xH$v}eN)pt_A2Wb
zrUeibT@C-B=K+jUT|6k`m4nu=SNWh5M?+dV@`q)8n1N}5!+5(t06k}@zCkFe@mqH-98uVdb#Mywj28fC9
zl+SGGk55E=?cjd1;MN|u4bi~o^{e|%qDi)|qG?9hf3owhYg>wiE2w&Ul37FKg#IAP2xY0``Uv@x=?9XNkV_
zA@X*hVPAiuIU*9vUD%?*(BnI12qZ8liz`Wkv-~Hei&OC1rf`9kGbjT{QZQNB^c%$D
zKEgt!fzl`iX=q_3keRgLgvmER?VCU)884mySHq%n{f-drL93RHr9DVU4D3w&1u6>>
zl7PR!YVdw6CtpfnIey|(g#VV{C%#5MbMez|#ma6%)@q(|=mD5MNR#0aRn5$IE!)^(@mn(?(5XqoA@i4zXq=q
zv|Ta`Kk<3(_wMVUk#%c0IC#K8*ZKtR2L)zUZ%?2gfj)LK
zs2nbyMi;{BiXf;CB2S&(L0vw-ESI*C7i8qVN
z-|!OV#<5V8$rKigo)r*KHuH0>07Y)pUOvUB2y>P%zNg!ym~{zoyI|NCh+->en&B=R
z#QgG~%gTSQL%bObo<~_4$9Wz6Gecw$KM}Aw__i24Zv`_pv7}Sne(=$l>@(b4Nb`_2
z=o1%UE&&E^SQcjBi}c23Ned7L&eb`D@ad8r|Rw#p{Wr;otg_Y
zT~CPv0ZU6TDH}yP
z$B}SW_!^ZO?>+=A01i*M@0e81ycwIVkO8LX`q35(ZV(^?#US`CSVtCPkPW0$fwG7Y
z83U#OHI&5k6C@iyi+e16t48nMv5XNyKHs%s*TP7`vL1GfiW{N>E
z08_>21!v5PN^?&WaCHhDWI+cnfkt4s87;&VlrD}>MYy~(%$$}4=H(17c9`i9oKDG~
zmIHJ`l6w)PS?GJr(y~##gQh~KkFE%Q)@yIITr|Y5%BN+%
z-Bg8^P!`C)rkv{mrUOLq*#MXXoShVcZ!>q`iM1H{P_o-QcE_JoVZILrj&V-CkX@b_
zTwE^rIfM(v1Hatt;UM)+mXxsxg2Tt<8NZ+>4WvV&%JdsnOK>&!pfA9G2$02{EW~t#
zK#cn9@$NT~S=$;Hu&2(1tr|8ivibq3%0Gd_?Gh!pV
zzZ4sBK5ry$6l5X~Fv()cGER9(05*#lP76d)Y%ly?1XwK=dvIf*=%4QXTMBslqdFGS
zQ=swSconRRK(FFNul{Np#Ha+o#ZdcU7zubicd;B`5N#aIuj=vy2~Lo2)nWlmaFyt{
z0Z9-^VoC&rKHi1(g-!uNgV`_*tmp^_#3ytImzi>RB3OO`@YKSQKksRl;87vtOx69Z
zbUW8IJ|>|0!|$A!R96D@dm+qD)s&|IKoAot#f8RP{s@Ft17FKJ431)fvq$%@?r2A7
zphapl+>;=A$?UBAKv_i_$nIr5K^pWCBy0Q4y3eybqze^6Ty&I8+dZ2+7zM6N4S%9i9G{&7Uu89$W9<
zXujmHnr8xMrvKIGFO)Tp?fP#t|KndY&ysN^_k?z`zdBIBCk}rN2(V=`t~mOK3Mudn
z`<|tZatIW_kr74y5uW<#@xn(XLZ~SMB6K`25@g>L4>Te8&5}4oW56SGA?$n_Tt7bf
z1>TIZIRhu>33qPXn`=VB(A3thxB
znun^eL(yA-JI{#69Z}_6X|Z^-xx$+w@ur~s4fCmaQQ+#i!V-M-bMyI8seX4J@I(Wt
zRCLXT{)UVjFC(W=f&*KD5eX>6FGfM{g69(GWdNdXOj`rLhlY&w80mtwg82kbQ>u}LBSBpUe;}}j+#4`!RYfZb=2@~VI8%Evkov0TtBE`dA)T+#Xv*A
z#lQe*Dd4_DJRPC3f_{Y&f}lDZqp1BTPbJvr_8jF&O0VMgBfC4_8I)GTa0&nuAYM?k
z0u8cIneZy#iq5hE#BUSfI;}%;SQ>@4IEz!P*c7B@E<_9_b6MaKY^*vzTCBV{$iE#G
zCqU2@C@43&4m4U7s#c)ulV83Z8N%MWA|Ni-Ud#)4UGk6i_|ofk;`1Uij|CaD1B`Fx
zVoBrOzhSMg&e1qn=KynYPy$1Dhm&hmQ9&EGIvko2+(wmx4HEra8d$2>)*<0oEhzAu
zNQCm6mAMS$&^)C<86dQOwhs7aKQJSoD}%_#c4Ai5EaBU~Mc9@gIE9H|XaR#5<|Jy$
z7sG6zF9}Q0>Dp+VUI;!dAO$%630=h|goj@yhQ}rZBRIvWTn3Iuc+asA2Z}b9UxRxG
zRy-QgIRTr-?_wm1Rp5Eaf$kLZ5?~rR-V3lMfD7(ZzYE4-MtCav?eyOj3&>+XCe*~^
ziRXzbuuFrqF+96>;T^oQL~%{LlnUDVCtc1ww=oTRp{0+o+{}~cyDMGC=Z7+
zh(P0vhQ1Hri!BYlG`yUdu2756YgDuv!r5Nnc&0GeJJ?g8YogjiqFQXAK($G74%b_@
zbWUDLHs)aHkE8ixJY7Jm&W-W3PDj8)RN)64Gnn#J7jcTIKh86G2CXq17h&pbI6fDi
z>cA=mtY5IM0f+|J1R?rNI4r9KKs&`=(Ln`mfCeP8N{$Wwpy9*+>d7*sLLoAyMu5G8
zds(Oh9gpCHfDkhT(}kwu>65XZ2O1@FVVA$2t)^xwBY@w%DAJw=nBbIo&|z*fW9Xi7
zp7sskC5X-3gMI4&BB6eF6plS%D&dqR$-%LB{K=h=f*M0u*DUea))0$4s26s6L9+tZwzHD;|6y6PIl?Ub7EWaOr$<2P
znvm)+!%ZNe1laotZxYB_IIb9MuTU{#8Ys3*-ye?U|6V&isN7(_gAG-o4|Y7qfHsOb
zjta&c(+<%66FlB~g4SOc{9hT~Mf$_p@UF}VOxRSwf`-8|lDEtvPX!||%q|;&#XG^W7>tq6w8%Uqr&WxR55sG`2r7O>j@@{1om#jMFM>YV
zLjPHW7mGl>1y8`0mcaH!L)%rsmFDv0L8;kdE~=6CD|
zR`#LGy>V17nL)L@+?(4O#9XI8iJi#NK1|$GnMT2YHlm_39~pI#lA#y$i`gMoEK798O@cp56qNFb0Nmbh#Ngmx6mTlPYn<6>pYrv;=`
z=Ig0>`PghY%sk}~u^D2rJq-|Ww_UcUTJs=K$;TZ$Dnr;8gUu|(sQu9CU*r^%QN$?@
zf|j8o_=V=&ZH
z(G34=SO-OD6&_6?y3OgK(2D0dSc-oOEZGVf`c)t%fj|D;?HmA0Y{bhf`$8hT1PgbF
zyLy>!kVOLw5nEblB@kl}8bG`$5de7^LuaG@DD%8{F66&#Q~@?O%DjZwsF&PO+c7Vc
z8jHcR*vEI)l))r&Q{6cp_rD01+TT1MxzjzZQZrn9S9vH#`ez1y7~W;NG~JU_^M5jU
z_{7|>;0_xPIsO?Q0uFcBcnJ4y|D6kN+u@Ys@32rvjZjO(v7(1ny>b*>M$O70DL5`?
z9qT2WL_(TFFhhS*y^y-5qYIvH7}5U(vpy2Utm5h`+tTcBgyTyyJt^VgeE5^(52vcL
zJXPXvlsqG8f5V7GC=JVG%)hebj>O8dj2X*kVt?)nRzXPnqXhCFv$&tO2qJB{Y!R@y
zk2P)IAawh$q{CQSI*hd~?hzK7)l;#ybQs%}vFv8yBNQ97xaTuIJ{OPe=xhOlBatMc
zgpf93ZLcA=rLE|#bt71W*nl4L&j9_d7=H)@`ril&ptMSQz`O%oS%D1A1j91l8O6wp
zwJt!OF4lSo#Ks8gIO0s$7~DQTo#IM$0GLER;dF_gId+kn^0`oVmLN?tg1PWXIAaF!
z8r7KN@&1b_jeVH@?)i}Zg&hkV6ySya<{sUMgCL~-8IPv?6_ZxpF^-D#H%`@!9bYcg*(~?p#+YG?$B*FfRgZR;7v0Bv-oK
zUnXP)L@c0;{(*zLNHgGF5Fs)}r#gfc$q8uqWGMvaVUa9^F->&fBm%mk5t|K`cKaYdo3XYK?YWr2#B83*X6Nx(NrxZDQ&m|gi)SQP
zl_8)2WAq~{*!3%7pd0^h43saM*$N2YAs664WwVQom3D~HKxlKNxz`FTwj09{xDKD-
z7K7Y7aD4#eT@DkQQVD1h6=eV!a(bDsGX!67-U)`z#A%b$pD3IylWn7KFU0u*0Viw*
z1oab4IdEbpte2xq9sHqtX*i_l;BXj$1GN5|u@iLt$pKqx9QMFQ9UcWZ$xKH;
ze?Ao&0LI966^L*!_#_97)5Bq$E`jbu+-RIG=7eq9gy9Y-5LjmHL*7?7LUOM_E9C~{
z!Y~qm$cB-u98U4_d?5?D7JC%k8wdf2g9R*WvKifvEAmpq#qe-0}
zc>(qR&9KQs;Qd;P-JKBzU~};~g6zR1NZ8OUxy1`Bl8i7~z0_0F?E#4Du$f?n2)RA<
z-BOP)G!sXT(Nt%-rxL6Oo$gb7UIhUYiXoi+gt>=@Qm^HnD%5M4N2_Pg$)AzyzoORV
zDwV$(x0Aly(hBXJhZCvUtwRpH2LMFh8;
zc=<1Eng84rw|URwZ^9Od<{7~}yT1+%bXI7V#9<#G&I$W6KFs~UuxI$B71|lec2)a3
zdW;j#|12MhE<^32LQ*%F0W)neFy39uwrcn{#?Uj!I;6C!(Aug!5WdSoR0`~fkS7oW_=G$~9P-0M3+AOz+QT@&T<$_N^!^V}6$UpR)j4$Dc2~l~nF#gILUBy33PuBhkbSz0_%F;=%eI<9
zw%z
zJmmCK!#SSle~riZ+rj=ufdiIP5Fy#72eZO>6pB>8&v2guyg+;mWAK2D$NvGSU|s_k
zRzVRc&cXscj-pX3JlelD`6BMie^7jfS9Ne=D>5tz|MNqjznf#>Lm(Uz*%;8##
z17k=t0mqQFkzh83hYEvyKFrwy#x4-VVV+U|2CgFxhj$=lEM6+X=a!i|;oT-560gn8
zVX%uS{RX3Yhh=
zZ5R~j=r63U0e}5!kV$I7Kj7v5izBS9K8c8!t0x8=EKgq1})v-X;fne)(mG%m}*{Wp6B373*7))
zps|L4=G-2orAHvRKq?!*Mn9J-4q{wsh&50aVIY>>VwmJ|JOU0d0Fr|f3H%F3Hb4n0
zC=kENe&E$bbc!?%k%4w=n-!xBL{x|}0G$s!zM;EF4}=-O#^fjhioXU%l$wko1=Ii>
zY@#+KZTJ({h9V%vFXZl2wuwUs3@6k?{3{M+BOM(uE0Hih2i!#3ssTF!K^(Mc7+x$q
z!jMA~`-fE}Fb{)@2BvBLR9j5~Y!L#3KG?K}3uS{d7?%K$?)29d%7%na{)m9G0p_r_
zCwxM{6aIPy4339UANu$giUvsendvdCVX#kwkQ17L@By%S42cSXjVi1y*{Gkp8g#0y
z*wYIA4D_#JP=oTyc$3@^)PVa0YhHiNsxmAnoc>f{&|vQ_e=1fQ9p{B+OBCpfT77`H
zF;&qX6jtyGV4D=Ep$hE42llmy!b~&DZ!fH{EZz!((L@GBcrn#L{n+R=7!Wtd@jixX
z41gz=5#R%_Za^15ajSqS0lw^6%HS&)N~{!{SA?tM
z&H4ekA%Y)6LhJzS$dqvUd=tFE61lF8w)gx-I1J)`XEX%++HR?_h$s;Bo)_asGxp3z
z1UCq8H7uJ`*pi}dp590RMQnlq1bo5#W1@kd%GLl&RTRcc#nTjt&|B;l$T;@jP%{08
zDWtF%FCzxF8WpA(Gk7O|ZGsJcZ`B;4^H~~n2;*2{$t#8C{5kj2`pFIRC~D-pGsK
zsO^UCIqlP9=l|pM*!ee~9)GaSlZ30(j&JkSsZv)6nx())w)EBzMj3Dn0jU6+8V*Dh
zEfcOfRH)s@p7=kQO-BMm;DU3bd958svZ572bRh0uG=RNAL|TSuwnF$PasSwQis>-J
zH&_S3IhYau%`qh3=NOU^0V`T?MddLh)5Wm^Uh$r~-VA;x7gEa$@knnBDv3kn@~C#0
zy1%l72(>LC%3=?Ku5;|cu;x1O2L$wBgxX;2!DuRrJ(vJ(hzd}+KK3AYH$$=d@9^x7
zm#6Qs+JENh>5z_DIiGvNaf6^W=alE56kdRk`G`ebU=2C#8Sg}3^@#60%_Zp@>TnK$
z)qk;GK8IYm0QS}&@c1naK_X{NwpRY&IqN{~=|6gw;C2nG^F>d-EDxJYmw)lJ^RGB0
z!crp!^oi9O99kOgAPWwOAkzqqHn!O6`Ku?)DbHU)Ph3K4AJMc+o}IBL5MLV?xUupO
zx(B*24YBOoUG>zX{L7xh
z;&n8_O9i)<{h}+~L*+3$sZ)~DKpwx71|}(u<=mar;c4%e*7#)Q9vPQz75kMP;wFnF
zsY(e)?VnLaSt;WNj613;9i;;6t?Ejo%r{-ktf?I1bZ{-@L^!>N*w=~XZ&vTx%2*j!
zRee`i*@KH|*VR+PoS6G9jT9p;6y7r1;N$|&9PGc8$??nm)PJ$m9>7KWcP2d$Q4^AF
z%`|h^L0Z&UNel;0fYU54IoZumE%DSc&HRl@@DzdbKv-*=%Pi8GDAmzIyC%xAxLYwt
zu&Bki9k_9716{a9@wKTEoeHNR)=Y!nyc6GW@rXH+FYJqP_el-0_eN@TN_nOSkqKvU
z<{Y}cDcb6dM;Hm4%V+=|8}!8^Op$Tl8Wy`k6_wf7IoM8rWI~v3bIB6g*i=by4Z?G2
z2%TuERIZx=3qH29HYal@wDL|QMkkclkjc3+sa7+^-w=0B**CRKM~Htjw7f(m+!R1Se09RE=~q{HwsZhx#Mf*bxB|v9765
zc9Z~nRA{G^>wpYl&ga{q%D2?ieg7;1d6|xd$;ZrdSJdO(O=5Wy*6vfTX
zAGA>TX6KVFlqxNUAHBZQ8n=yu6}IiPn@xpmZxYYu*`aR8RsU*8`#)l3TN$xMP1~56NNw+B8YUL}wgY)k|{u{fc
zM^W!RNtIK0-A#6I7<&MU{IK%L)^QayKJBKCt(1G^x%(-nmC{2Vvy0BQQX0w=c2PoW
zcrCoOi|%QyG{ohuBU&qsnDS;iFtQLB!ly>JaFp^e@v>}~Khe?F7$)Zn4)ZESwNYA1
zpVKXE>>3tTtRaUVD%LQIsLc&E%%xA-DEGqB-QQNJ5sS>F0A3RuiaG2jy05KLo!^yp
zVdPIVrmfOi&iskCwN)OLLMf%4l7OULJEdmr8(|La*x?MO&J<>(cZYOMjfAUT2<(Ar
zQadG4o_LU!w^M3F^Y^kZ#qAU?y7NmrWhm->xxI1+lKlsglt(JUc^+@)Jm(+b9Dbr<
z*S8rJXcKSwrT}~U*l4kUYbii2=t+~Cly1_8lzx*^Lv;90i@{j27-cYk9XBbRT{fs&
zY6k@ukIlB|PE)xH7gf$RmAj?O^s}kd4dvUW4C;M2sYZCBjUNoUucOiogBphP^IPgbknmO|wa4xQLP~)(xG7tk|%<{-W1Ii+>
ziS*OWqJHQSL#h5P%6&T6&QSM6L})i*kIY*OOAQlqdgm79Fs8hJC&ibFcIt(|r~}e@
zXR+%rVWfL77XO
z=%us_lZP!L<0;^X`HL#t-PNvpQn)qrDFru`7tw&*k}6sBv~n^Gw;jIrywXXM$G>T9
zenGj{f$R0WFDcU{q$^)mJaPPC+nb6r#7b1bTFi5~01%87f@4#M&i0LsfbERxtzC2XmB5
zPI+7&ZCHSL0sklNIgDMXYzjXy4aF02VdUY3N`{2#xPP(oJ9=*2wOmo?FRd_8X|H1I^@Mit=${2E2JY<{QJssC$93_v_)wlowa`5qRepN&a){y(8?j|xl`GO
zr1CChRPctxrMr}0!i5{J(><8>f28!HM-D2nr0iCn;VB)mTUmx8we~2tV;u(WQQAnZ
zjQ2ozYEZZR%FHO{0HXytIz*4{1LJeZ8nRE>DapeNsLOuk3d(jmpmYy3XXPAF+~Nkc
zB?pzGT>qPgM9*$LtgH{;yavKW9BxUCJc9kXnHnBZ=Agav=@Q;>0FUgmBJp2h@;A%DiJi{bLBIs*y?*iQ2?Yg
zet<|(Q#skPAFgOWOs?^#9(olZN{Md8QiB8SJ^VNG?a3*#UQv{%*s
z64Lj)>S!Sov^3P33HgYu=e`P^B_e&fQe_0`ieyu{P+V(9uN!JL$P#l5bxRcI15%H~
z87`TIszv2Moywk8oN=O0)p;}veO$oW<5SxTsU#}l`WA3tA|tp%B%h~<7Vbz8Exeqd
zuH*6z60grcKRl2>H{8zuVxnjuCs7?28e?|}7%|c;4YfN}5^(?GPTiGicM1_jsq#dK
zoxtWW)08VgkC*Lc5pgW15D>>twft4o$x`q>H(Fm+UB>!Og(jcl%z}6utw_Ll04ya>u_T=M~=Z6+5mqTG$!cri`L7{)UO#oJk&z{MRxsi0IeOR
z`&+An82$@et8+Phxs6&Ccdburqc*{vmV4T$$7FfM8Cud_Z7zLkoouf@EbakpZ>n!;
z^Uey!FK}mHw4eDhRPnW`R+X2ZwW2zzJ+c4JT2J4smP)t?@v%;7E9p0y)=8}vw~R09
zQu!_>BSkll)1FRhOX*#5bylm%lV(%B&T0ZK-R#;~-ORm;>Y@sS+OUh7#LqptsN*<2
z-bLk$`#4RYMz@NO)}6Ov5#*(Nt*^VPT_tRq2HjLE;v9l7v6^xKo$98x*3R=IPfe%R
zAwTeM9Y4Oen%=Gs5O`r>cXa|Td~bG#+CnQVz*IFU$KYTuc$~xUz&J;(vliW<-sXai
z>!*9wSHSo@d!PDS^ts^xlu#iu9=D#qUmfI-r>&qfJu$0Wt;B~gA2KZ`t(V$F-nN{Q
z9#ZSkpkC@~6tDY;nvBZ2J%W{AW+gnTI?Aq)YW0}k$TxB*T>X;nj_-0V**
znVNafdh~U*yCfCTvLR}b8t|Rwp#D&x2zjx9a)zqagLlBECbkgr8sCOoI8=Se75>cx
zte$G_r^X}HUEyE6g(uD2ZaGJ)16;WDec(8?sWgtTCQYp`#;JoP*MSw_2P*R|vQSm@
zQe&+N6V$I9;omGl$pmv34SEX`a-3q`RvE`mcw5a6|8WL(Lb!Rx>N;875GEa^(CJv&
z$;T(hKde~Yr>n2?m76@YjDOUOAxCa1^X>g
zbMb7&ELFb?OI$G@R~^q$qkIC;AV4_GNi!LpnWJV}>T0zOQ~EpCs_&Pr7_I>aLu}?>
zzhW#pUpEy{0og0I;
z%*tG+j$jlXvq7ED>8g$DoM@!??gY3Y&HP0DEtC=BoBP!X+Pv+6SYJ8b;;{bsR{a1_
z{3fg00rmMX408Ra6$XhoOaJ;Hlm4|qzI04&SaFaCj;oJ3p~G!_QeBQ!u!>KqHw&Cx
z?X>!~wsjL?PDhpxiVgWOs6D4KsQuQDr_~~$+EujsTM(@Emg75hwnI8Y^L|jrFe$Eo
zUR@*=EIfK1Xtl^H`cYjT3S>K~RPBP<+gqw83+nd6@0h(UJbOXsKq!I&4ZU><4IQ`E
zUQ&1Q%nu9q))RM<=0$pw7`*mHdT){9Sj!8`WY$GFUo;O0QQ^2*WEnF5cERwQ|oxuLyD-lv+B*8<%IUm5`Huh`hX
z1dJ8t4xHsjYLFt-Gp8Q2`XqaA4ht8O4a+cj)x13%+4ZkYY$=)=|oz@^Eq)kYy85)(!MxzmFX;_D_m#}T7B4{0=HpznMQpz*lCdvQ2
z02MytHC#*g;!wmvCy$515pHW}82S2ojXDd~;$~9^qKKKHDY7|Qj%)D2v1zV^qTjnh;%3AQ%{N^CQHAPXv2tpOQ8L%6B@_BXwi
zrE}JEZ+fdU;P0@eS_{2a11a1#I>eC<
zxLjBiF#92y7p<-1y$^=QWsbGOr-PJ)m>FZK&fDI3cjQ!jnN{|{#!(?KO)(dhy~1;D
z*@Mj?;(O_s3YC^n;$&~7@Lw%l1@16^wAxShR&{{QO?f%48dYrP5b^3n9pT5t8(
zN@Yb>$6yO)_W(1H$8a{LxaIM03Q41%yPbjm@kUv!G44*&0vIV{%q&r
zO@XIT-&;R^=)D5{omH~U`=~=6V_A3Z^g<0$N_{@@uJY}kjSb;(U*jvd#1#=CuvZFW
zV%%`I7uQ)^Pw)0V5Y{XwzbxEQD4Q!TV!>O
zH`by5MOjNf>YXS5;VZ5+;+n1fpL$ay*P)&0UR4@$%v*yhAH!Dqjb1(GoiF`LiO0PI
zNb;kptkJ?
zm;s!V@jzer(%T9oW$~BZM$#X;6BsC6_{!T=&daB^r@Ujr#`C0&pKa|v<)u(xV0@`C
z`GGjYvxAtYVhlsR_0EMxG~e=n53DZzYTfk%kdUWr-VjQIcTG+{-BRKmgEiV);{DPg
z<9XFZZx1;upBz7XpOo`wSbcx?&Xo8=*stE9{9)9u=t6T=C
z;(P98Fb2mLeskF?NtdlNR}dr-#k1>Z{55-#2KHyo9_^eYU9r~7TCM|QvwFF-Wm4JT
z;=;AXVShOv`{J}`rDAI3*1E~LGia1s>n!K2roC=0M>=3V6R)k3pz@*wMSGd2?JY&C
zf$^
z0&-pN)3!+mD5H|5F}~!tu8j34B-19Jb_*pWXuV|m#Ck13OOm9mM2Xr$GymVo5p
zB&|kt!Q04Ql>?zhJ7}M-8KI2m3TSDP_NhG5vPLCqTfwkiq1*jhA{d8f{aRgz1R`HW
zI}TnkpZb+(iB?v1?ZXIuy0@WrGX|K`PBJ`5
zAV6}?(&e?1_F-NOn^=_&~OTNOh=#jyr4&2tlTQQPHeZDzH{hhU`nWjs=YyUKx
z{LZ&l?H1Y+hik{LsLoF(=V*RH18%zGL9GF`YNIuiw$Tf1w88SY&#mv;Xvva4X9kGE
zrnum2Gm*)~Rm2it9CeJJO|{x-!{xkb*7A1R^gm>06O&>D<|@zL_a^PQlPjg#?X?to
zwu5%3j9NBy(AK-`wkUU$Rx@!CY@9_?!G_R{S#kc*2GX(^8ZZh)i%soKZn#fJ%`aws
z^etKsdBkcex<$Ja^IfZxhJAtO9i6qlkb@d^(c0qm^Ifzh;&r92+KYHSp{v#b$=xiU+UE#>+p8XG=hHN+05dlxMq9@O5DCw^lM?Wsk}Kp|rv*4kf(
zLZGt{DCDzV8l#Y>9?>9O7Sq{BwE?cJZvyGn;|3I}(_4F@H
z_?mKF(nd=kTmF}|SMZ&tzM^%K-#$&Bzk=5ntUd#^Ns?>nX$%=8ucv0vf|mi6se`ne
z8kGW8*tQwr^+L^(;KOFVrwuRIQa&?y>kH?|m}$K|2)GKUl{yU8&Z9F8U)QE_qq~M^
z?Oa>Ak(%6yLjIxJFxU6|!e92{cFXmK_OkKJyF;aWqN>rEc6uY9=HuuLsQ
zmdEB>^T%q=KhlQHUi2j^pEixt9+xxrQtEiE8KWa}0=8=rO&+f;D}S_JpP>CNxr(l#
zCqBC;6DMhNq#vz@S(*oo8;HQ$+FSDIucj{6bo$|KZG_{UjH%WulQG*B-%1Cqlx(eY
zXxN;uP*qU~*Hzz|@s1WDxqjzYVs`5j-qe^nL%Z2E`4nDO7ukX@oiuj_U_AeQIy^&r
zO+lM%U*P#nz>pX
z={veN7n|WzYhNx9n!I{AeLe^D;4%%Is|}KKKCnvWB1|GClsxmbBY&VR!TG#4UmJ);
z8ThW2>99xs0m#&*#oAmKmwhr%dzwk%CwZ85a0>NT0u)?da*qVMQ>>p>Y2XR2jlvtT
zoC;58j1{>S)WCJ}OH4;Kd)6MdfKi!#y!ejjk^IH>E_K^&i^Q5^cHO{=!2EFMboj-Xvas|WCr8MKGv#G##U_sV7bONtvSQC
zg_rLzwy-X3*VZz0uiB|C;SZ1Q(qJ%GXnnUAwa7D|zTK~#7WnbpLG4Q%WbHeoEplCD
z^wNs~#lGX-}=c&{_fszVrpw{bTEgFEpQYcgeJJX3<=v
zq?u>%qHKE?2Oo09ZMC6h9&#MNvB;rdF5d6Et)3^fZ+LuG$5XswD>u-?UjuOGeL+XQ
z*7ov;1*f&UBzf2d`t2L-HP?v~=x1d*-$vU)tK~blfji@Ae5BlP3O}CuiYn>$55*~Q7vVpgwWj|=Ud42|;2W7$p
z&N#0%0&lVRJSb#=l~$rv#aZfl%KAz3G7?z$6L7+I>*P<`6gh7Eyb2REVxBeP7i}Cb
zue+4_GOJmsHZ|~=7?FpIx>zzhZe93I>k1xk2DQBeJOdi?(k1OlJX=MVw7L=Un4c_H
zsNN%t)%p)a>u(C>@9|jubyoiNjME=P?^ea>O;~CCO`M(#wTa8E_msY|db{;%Qe57V
zOn0r2kQO1#2Q5BgO^Mf|rNrFz5P|EKYwFnQge~&K@%KiLUKQ1x^yt-ZEKZuDSC^;n
zruK?{kS9P<^|^^TlTcfHd2Q@U!p;t&w*4v$*_KVBpH#hpoIi=JBdb|my?UfGW-6C+
znG-p=WcBvxoh3b}jS)6oe0RESeQCX4NgojdhOJ{w{WvRwD%a76IBl&=C0g4+Zx)_0
z2h}tNRQ=RIA1+H1t$~g81`^T*P4o`@{7o}`Ez^5zb_=~3*KxR&zE{SH!2Gs)KGXl0
zoAk!k_;z}_$b~0EhVG`isXxy{ooeb$BWy=M=H3pP`U;dC-chf|Usra#?kgU>mg8o<
zdJH#M=T3d5tWUuC^8nVs;qsP={2Qk|QRX@8i@WuAWN8u&zE5w4B=0`GzBGZpxKD2a
zU4nAI-dP@XjUK*V?+<0^?)&vST*6K@i~JAhcLFI4ctEdYo33tww_1?1hNWR5EDf)Y
zWhOj6(NJi^1OKa{`?;uQ7hUq8{)F^1#q`u)kmrx0H+t$m^(}YB#?EN4;k-tT-qm&b
zu&3Ur``b8-#ck4ZDo$~w=vAQju$V;bT-MyU%zAG7`wafRT8dsNzUP*GpT^n)Xe|yx
z2jg&0rVbD3NpaWm#miM+23|hLFPGRkM->Z8-^rBokls?7Li-=mJ3`p*a79m`1`q2^
zal;zj^}Al1COxbx(p1{82EX?_thaUL{fO$KC@a*^sd_KHw~S%LyYX=TVLeNfrv+9J3IzoGwTE({5MU>nE05b9u!rryuvSRX%j{v7<^4GaCmZij_9J?A
zRJrjHJt^|gG#-r^;?AM3?AMWxim#3AeEFxjyutfIa2g&(T!r@zoAU{m9OnGRJk{v#>~L_gyLJL-oQN5sh^#^#ox|q+fuy(~i>dhTt?kVK^NpI{3#g99F((6*`<9aHVG~o%o
zhm=MAp3vK49pf+RPfHsq{z<*DYyN1ggo}n=#F{_#q+VZ!5$EQ4x|b$i)Lrx*vRwb>
zD!;K;TcMs8fftGHeoDW`#bD>2K}(;~lU)pL_`MImYcr6!p%npOBajp(C`*cdTTBq*
z2?gf1=rrdcy^rp9jW~ecGpSu4Ovg-mvX6cjMz^Mq0MH-$0NZ6#v#0g@%3yQt(}2xc
zwCicTdf1LBkeX(@>4j(X>Xh^hu6~$v-G<9dhNc*M>P$kKDe%E8T8ON!jiMY4ds|m1
zv9BISKR=__ljcyBzWNK&T$OQa6mdQHu~+n1nvWOUlT$D1kMkh9yo8F0UU*4whr-KW
z5{1vbq`!v3Prs}`#Cbn_8FU+Ye|=fM1MA-T6}=<%TB%1EF>zvTi;~!oJ0}S%9Ds)Y
zujmFA^TSv4fvC0BKvCCXn2
z;?irPo@MXpN##pXn;safs~E|1gY_ORyU(;`uwIEK<>|O_VX$7eMnx#*t7>3u69grP
zi2<7#Ygt`DEAz1UJzm%2!Y)s?cf#z~^%QZ38wvz7nx`gHyOnxn%t^uP*tu)4gziNY
zKLopVviNQ9)(GCMgQJHq+!jUz_o`$D^7ll3mz|%64$%|C`8RO#Vw!`ex}1xj?1_rx
zx&}rM5)>^mSeGavp~P7(FG}!J%us!yw1if?q1Tm`(w9T^Ps*Q;y`fvuIhvdfNXeoN
z>0mQ3>2)&nE({beXXw+UWfVJ1uTp85Tkaquz8X7E!|wB5T!HQ|3qW#zpl-wT?(xgr
z@DR191%TthmRvjx+r5Z(F^P3B04z|OxF8W{3sp9Fw4k0
z8c2E>wHyt%+Fxn#XyB0d=-tu!-8j`N9gS^1h3?A4ep*P+W$J0jc{EcL_&rk;Xgfv}
zcxDW?@e~?41^|~w=f~*3A?Mk#`aDM06UXWIugt9uq&mp-Q#-qQDCD!|Vu*;kB2`Uem@&=BzZ
zl4L$4nI#yA>1Rc|bP
zOzWrW2hh0*gudcv2T9_h3|x^K=Keu0=MKYQ+aUF0p7|WuAT`YW5CIzET_BKKsm=Z>qq=lVvc@x*RcgM04$T)`0TWBwrmTw-w!XsVzU
z%=@7`2EDLfj|zk5MRI^z)TR%n=`Pm-Zp2IbXX>@01^O|+eG_kD`HfCp4(M*${GyJ6
zL7ukDJNm<@8Z38x+W(HOMrD^%W0D@wl~884k5Kq@@S4Fw_e}>S4A8E#)AcwRs?!1U
zK$uS)*2CgJyETS7fz51MR-f*g0f^c}{b!&WurCW{=xxiYyEFqpw}!gU)VpIvLHg{$
z$Qui3zzncC?svfEoSCU7a!J=Lz)`RxPtVdzr1z``W@CP!r=OLJRGznlPS1fOialWo
z6$XSZ1-n)@arNo8c|ei@N?tNWuLMd-W9NYyme3|7WsOD8*N32gn5xFKVZNTsP=#_=BCII6?8Gr>d%sWbVF99LVra4P=pHf}}Qy8w{GupERm=C5V=a=ZZG3puw
z9JvCdO1SY07jRPYQoVVADqhq${s#?s8B{TSsh%3Xf_cnP_YdUG*S$JOGvk;*PlN6J
zMu(T`o9%gRNFOiLJ?IlCWlC_>sx1do02{D8@M{4o*&$E&F_nxm1o8lVL#MHq%~DW)
zF=&3xt4oa7|b>|G_Ol*J_Ke7^#x|ZHssV5@Yy%I2O(RX$d
zxk}GKSH`W<>qM6GCp4giq0yb!(ETq~q32WR+$z9EHf?$zm@13%+8ecL^lHIf%vcRR
z>7jf*nwBjATaobrM1VQ;aK63<#p2h1^#dqD*sDVW*MQUrhLP)l%-0?x{(TLK_uhz>
zI<3{~IhoO>Hfu3?S!*F#%%S47s3@DJT7P2)on`#|<<zf6Cx_+~Mi!{l)wprgG@iKg<-|jkh0XRFFI(!6Kt>#C%
zM_xLb+I|Er!)2QNk)W*?KGOfeh^+Tk{S`I8Y?Hyp!7PBx1JrhQtNu9Z>$VNJXvIi+
zW*g`_3M|{EcXeGEiG3Qu_3PyMSl8q^SE=^L;Kj1&z{fTz^l#VOFlBpWJ2q4ny|G@eQkgf#bhYP?s!4QCre_ddrBMV
z*gkB!4fMO6_zJ~S%R(Tbjr35Vp6DuF1AC>7G=zUoSqdR{Bh9MtdvhVg2!S{nQqBDs
zVK&{mUx+>+JoRbmepD!QPZ|~P*KZBVo%Ii3I6-#dwF8)h_i5t+)~%$Hl{XTWLQvIs!HX97N9}fU_g~s8H4sJ)Ymq
z<9EjhM8%*lj_A8l-Lj)%`p+K~(|^Hl)S`|>P}G5l3@EbM+yzBotY*=u?b%J$>~lAaV|M{tQz-i_$*>6F-}leHP5M_cI|{U;a%05pAA7rgxK8
zQk&!8WmeJC$FW*jwDP#Vhl_r$Pejq!6Daz~3AkxCJ|Ph7qbDGMZ=z`@^n2hs^8E?D
zP1F|L#%?o-4!XSj`q!6uMNbR|yj^vTO=fJE@Q1?U8p9WMmobJgJY0&BeF@UuVFHhPXeZN4^9Z
zMi&29`psbdp886Eh1c@fSKxgLtrn;BC|;_YzZScC_}BWYu2qMy&5qNxudzBK4pEKM
zkle={qI*v3b>tB^$v>_8!X~ru4kX1v#lj96xY^Ns!<76@I&KNM
zH#HQ2c+5qIsKGbT6^|;W-rqnpS$>FK{|1C4i*mlvZ%?*4kP2iQ{%~xaQJRR694Gk<
z)}(+Mp3!fSGpEqNGvG`h#Vk9czrktUv-*RajykJ%V@}}USs)FTa=x|Szxb{Fe$%%Y
z3C8~2x6s`{&Gen#Ct@S(4&u==J$?>oCE|NMsuufPB2;)09D>avaQZWuKaeW)&WZST
z5zPDU`cAJR&%vqxcYr@kGSWnZM8MkS(~xuEwR33&3?~?%KRyS5UP9lV)1Tm+cYF_&
zoKFkB*I(mRPWb@}>dQ3e2aIfscub@hegHF=bY2hVKG!%8;3evE9_sEXboIPV5LX^W
zmM2O;RI^BIhyx`Mva+e9M1LMG1dm++r#9v=rC-1%|A_K0Kx{ZfNk8g!<&jsY+mHGb
z)K&bW&|N_<*A!3C$}zV!lb?h_u5~E1QEpo)$31=RBH;1zK~59stBb%4=&yL_^qn40
zc{PkE%Dbp*5oN3xb-xJcBpUUz{v29>)YF8j{~}Zk&3}P}IEe=RqTgMnFoQBTg0B6d
zM`DRsqZ=D)))LO4%Tu=s`MUaG)spTDGc
z2;U1-4o}eq)Z#L7ETD(&B;8Kt*~#|H0-;~Kj9Czhap;U_#}z@z!mffPV(!x$lt(Zd
z7?~MBj2*z80?)j8Re#wz1-gi-SM<8@6B9x+>t`EX6TI5caH9^*yQV)Q&B~54s)vcv
z+uYgD8!6d64QL^=M;lMr%gVxRHT;BJTAR8{#)C}37fMDo_2V*9&!Q2lH?akZ|MH7Z
zHXaQUpP{l*-Pj&1kKo@NzSFJ)?7{}wcoAo%^&Lh7Ou(ZKqXUkFXFH5!Bp>io1u46}
z(_ksPqto~jvm`p%Bh+xYHgJpvclqkdqes*DP@_-c`xmeT5f#=<=u0|Zq(0i4|WISzRnaHhK*8WF2ZBVYJpR2W)2H*}5_ClVNFMgmH)T1AQK0bOlJ&h&1ky$Bw2~
zBaME4F%+nNc_>#SjV4e0=}^#Z0X$ii)7-m)Yry&{)*1(+K3qR~Wc5eiFnT*PkJ?L%
zq6{I+iicxShQU*Svl+!F8gqYA-VAgfw(yn`Z47kf;FQ9>n`TBEiLS{%gNfZuo1zWo
zXMhbGQ1uwGx~*f34vb0!W?T?s)bItA?m>Z$J6~Xno~L7se}K5(8*BJ$luLXNJsJCF
zRpwFhZV;+?xD5)9Ys~ny$pSK6IEw2RXVePp^VYP+^wf=
zamH=(*vmZ97V@Oaw(8|^x50Yw!*0XGGVAe1GieWXj~8G&k&|rN7jN8;6Upix!<4h8
z(z71pZrBoT#G^yHN>{zcBY5nk88Az+4pV=_xD^cII>YD$M1RdNhVx0~n?9qr6wn)&
z3%`DfOE8p_autD4+6ERJqBVxk_;MEAoM3dM4GF;OSqTP9kQ)<>O7h#6sVKn!)eDTJ
zYNEj>Ap;W)*3+y@G@8I2`TIn}fPs@M$!IDMKWep2GT3dBx+EL@aTr~YjD5O~j&f2+
zzu1Ylvatyj9IlL}rVuWO?0B78rvENIiTe4C8Y%nT2-j9dsCmhDFu}G#*69wyz~_Fz
z#Z;z_H!(VDoMNQOV@A-d6r(!`$GH@vF*tO~pK3TH*99i@PI|m5@O+0VhUPlS5>2H2
z;w65ulyiVr;2aC87!9FQJ6Z)WaGBgyjc(8$_Nt0KxR91s#geY4@2g@x4p8%IMpF>`
ze$|XF5I9y>Guj21RbGoj9NV6j;sh*LFWfqx_(ku0hfzg{`M~beOLW8%S
zeyt}asY89E<@Fy%)aMV@p8Cc@$!mwx9pS^0Ms&owqWruDU(TnOdGotByQzJF;!*a}JFN;lgF~&K-M@_L45Qn0f
zF;e=3b~gj!IzrCoNIs*s&Bd}VX^tIqkS;gJVq{aB7GgR3wlMC*R$kY_=vjeFwl&T$
zHL!_c5%AdsYS_|v60D+?-4bAiGl{wUHr7hiP6L
zP?y8hs;yC#k;P+e0l@N@EA(DFUd=0Xs-4l}uG1_NL{%h(LQhl-Wfq_9;K&AlqzVeW
zpx*2c^Bz>ljcAVzE_6V*ubJ8J(YU&GoGYcFByGEd&W2*zcY`%eFpHk
zc%GHg*Vrw|OO9LTpEW*`QOoM*jI%Oq%4WV`bY=_7Qv-}64(8!EzG4gmhHgL5@Cmtf
zAmIM&TDE@aLL=rHPgyly1ukG8-Ny$RgB^ZbXOBxGD-SRuy9~e$6YzOQ2q*h}j7(dG
z8U`stj5!JRD;Sc**_mT-_5+7|&P@A;7;ga4+%eQ(HGkis#_PU7&4J7Vu)x90mxmfl
zxd)5iFj_j;ryW01Eaz}zVyJrDvVCn*qnab~DdLqn&C9f4v>|-H`(_&X+N5`|&K2U@
zai}^Byzv+S$MkpT!7)IW`R~{h38ha5ewn<`!i>5&kV(Nz@O7oo
ze*+Xp@1zTFCv~x&nq{m&&2L(2uJMZmd>S
z8p!(=aQUUyJ?|PrWeD^87oyTHX4{cHQWhD%f(H2(8(&Kot!s-7xA3`+&ok=D<&JNi
zR~hd!KseVJpNC4PtS{Fa_o%#STI>T;BTt!X-M8Nu?+mBC7+Z>Y(K>tBSSdNCXN;hk
zM~yyRE}w)`h!f$O%;*&GiI@~Td861%#N2)iA@H2hYLw~0)8a93opMGi_P71th$l~!
zsp8u~>RM#n1H3V_$VjNO>9VK=Zd~#BHgnntE(bR*t|e=Uc!C=je>+BB6&baX7R?rw
z!Q%vuC+4oJqBx{>?!;QE^{MejMBZ%l-)+vCZRLFmwnf@bmyUtf?X&70H*ot70HVza
z<8^JsK4e4m4O_DTfyNG`vik(KMdm*1{0U<}`(bY>HU_{{uE|N`K}kAIUQST_1%43-A@}FC%eN-{l>L}*b*SxY8je|erLhmj)97uLroIUfCRaA~Dg5|xQ(slr{!PHTitVr+
z*35_C3%jkmn)_}Jk6oUHYH@P|Clj))x$S+)VLh@rwn#nmG}>^#dtEO+w!aV({!-@d
zm~4Kt4?FkzE*s{KLZH+Sthmm;-f)2#K?Az_?t`n`_O8Abk>D8M%c)!1y}ogx?NU>O0&GnB)(gvdbvQQWs~t`S(f8b$W-
zN700beA}VDyzgP(ricTqHHF9L2D($hSy$mSW0+Og%Qx039kCvM+_&C^Dc1V>sz%FM
zzgTTw_8oWdk4l4lGbJFZg4cWvaI*;x2dmcOYyRSSPNTqg=ZDJOrMeH=CL^uvkj5O
z5VywNQ5u3c^$rg5=b%<&d{t=K0e5(^ljnd(2dvBVrx%}&N`fjGLr-d)=9I$A3W2UbKz+ml6
zUlZVV6#C7rUV0B!B60431Fu6#44^X<9+Pvp`75o~!1aa2&9%6;%
zZ4RxC7kA`I$$(eO$n)Vm^hbJZrSCr1S{5iG==@4wb+||RBBal)Z&&&jOG#zX>;k@h
zyyEQKQv#3nQp#KH+bAy`Nzdo|dV(7)tmS-^s;}{7@}WZB8ecQKJH5s?gX!&vwZ4rI
zu}sU?K{~~W3d6>=G|loogDol*_$(QY4C~hW;1F_&+Pv?(1v@t5ecu@jWoQ#;Gi%ld
zzG_lAX)Y;Yw+w6lW$XcrWkuOa;VU7KWHzxEL9k%%z!`9%XZN3`dT&HjI)xp5G
zZ}W}fk3By2rMTv^S>h=g__41Rznk;1uOIeF%y!>R5MR4*2bM0SN!xvbpIWlrR~K)O
z+HZY3FoLV}&<@5e^wv&a67AXHdyG*){hhu#;a6D&D7pL4u=Y-75|HB+{usWi;>SmC
za{AG*^~f$?5<;IHL0~JNfvuOl+XuG?Oy9gczBcmYuc>$sdWh$!y}nkM&VStO>`Als
z`no1ri-GK%?mj6lxLSk8!BZJGE!eIuKaz8w??p_zHE5qNQcAtVY%+%{4Xh$=2NtVX
z3|C7TAZ?b?{6b%CEX=+_j2Y#=E%fylL6>?R@O}M<@@)6QP*95Kk%9709z^*fYvMuQ
z7Ptw*^5%$7oD0nE=KTL?dk^?3iZ6cj-c2Wg1VSM9Zo5e+q4!=dy-G(EsZyjD1;IiB
zqI8IqoJ%hvy|)b=6qP2@2|ZD&bPxqWc;7R-n}Yt{|GoG5y!erGXXd+S&di)SXJ&S0
zb{hS2)|D@W-!5Yd-}zkByqSLt)i~$67QE#I1oJ~>b!_ghQNvW%$I{d@#=cb7752~z
zFSrId_Zi_AT|K3IH@BM5So1@QP`rq9_y<4c0tZL*
zOn%p2p&Qp+2jQaDUq`EAuy);b5zX@R4c81j*X^dOjU-KeOxtd|26H>a|L$6ssbzXs
z|F<#zs(8os9(F^_xZ_&GhpN4IU4t_YRl^AIp=!U3L)C}wy6y<~uT$UlgMff)YCg*Sr6@<*;uofz21J#qblwmzC)C22{nWKy!eUY5XRg}PxD=Z74BZXtQsg-pouf)FrE91Yi9i2?Y^R^0@K>&78E=1fy>j7=%mm~1D_4*cM@JvMcKyva
zq}_G8%S#i_Qmo{Dj9`?XyFA`gHSu$=418`bXteNi7b4x?J&NB>F7$T~;TFjg;4T8q
ztRCR5T=nmn*oEU4(a%-_fA6Bq%Z&IQ1NIaS{`M12=Hhon_z#IM6242n%+_7>Q-Hg6
z&WuMK?0@qHy5~!iC(!0V_xpwJiCsPsYr{Me?u5jRoFKt50a9%{ADrgKGY@jOU6{$X
z4{{d;86D&<_XgAbLGIzsd$d2?U0ND9i~0t;dvH;X2fIt4D6fLu)&8Fq#XoCN;!%|N
zjG{~qao2ex*NY+U$N8nHt+?@VDigGEJJ2vD4WWcu@iz3BR`%ri8
zH*)wW)cvtpu6SI-Eb%qWD4rVT?#0FXGR$2G#oHa`uB-naT8i|XZa3;qv`U+tZtoiz
zEzapqmbOfwD!JV6Bg6T*+^#}ayNMs^oUZWX}QTm)%XB
zSnc1OC!1RFRinHMkktxb7R|0!)guMVw6=x?R`q6_(
zw@lcz{ATU9iOyP#WS1LfsJ0m;w`-&PjNcV>EnK5)1ZeJ5;Tf-b++1dk^X!FG=6$!!*8Q#b-DRW|n8d&D{vh}%Hcfb=y%*@w`|i3?
z^a7J#ai{!=6!8n^axPKh9_|7KuOZinO?mlx7JJ;7$Gg^tr||-eNltH)&zRoB9p}&I
z#*7c#O_}=laevQ;&<^!+>j9OH;=Ti%l#9*FM`5ug6enK~SW!`CR1%_gSy8xG2^
zq}fKd8E%|w!$It6^W5>!rrE~!1#S#%P;12&xeIVat3~c&Qn5rn60c-EIqwy_ta)=a
z20E{uL%bCm$Xxh87ZgTa
zPrFq*yVQM)4{h%I*4-Qgo-YJKfbK-z5s)
zhQRP$a9t^v=+Z9Ovy)VNkNbU^wA_YUVnqr@Ti1%J#Ff{(eUqQ2;u
z`&-w!-yvQI*9$HRdhN&4Bzz#1>!sZh{liz87Dzm`~}yxg)cl
z?+x`nG^A_Ld_IU=;|45w(iUUI4L6o25s~i}bSY^I^||FP$_F)P-Exn^m`uIx#`Oc2
zjrVT5bHVa#G5X(eH}VS{&1Y5f(7J0{e(qqW?z_tcCUcxb&g)uDqsasJg8*gUmW&RX
z$D4wHb)YtgFIs8$)Ll_nl_}5N355?{H!V@dic7fn{F_nn-rH2}Z};rjhiGSJg|U`{
zKhyBc4#SM`2qEzsy8pNP09t1A3wH;+cME&zK8nfrI{hFe!*XET8o*fgrw1fMVnb=l^)_q4nIe|9CDnRu0Fa@jT;p
zoP2|qaE}+_LZM-+#iJ0Ygfd0tt6V6AHKTMfxltHUBP+>Hf%>enJj7pGv)Oo1T|OLu
zLG_M0ay*vjeybz*4B)zI(nyZsv&91%$;Yq>zj|X?}*
z>j59$7JbILK7?Nki0easYbsaC`{G~8-g8vCnJiW<8#I%vNaJ46=w@<9wdw%I4V_hFV65f@2qi=3D
zZgrF|OM$cbZh)vP54`9sFJ#=jF7jGu;I$VhdkHgnP&auYrkvs3<@1>1-R>@bC+&Gb
zuNTS%==+v(LtJt$*G0m6J>+|~K@xHd>M3^*T+J3LGJ{b#jp`}qBc+#|CXJtCyy_(<
zVX`@gl0J}U!6FvwBTtex%%RPFfo*GQpqsJ!oJiPfsT&Y
zAtU6IOzV%7XYmycmqyBEI2`^7!l27Ok*9zf6+V^s`k_{yj>6-+Xlw%XX%`s@^4`Gd
zFH!qNGg>*AmL|%LIOqF`Ca;*$sCR%zn_1o&je5UJ!^fDxRb%9doKwXwOmv?w%j|A5_@!J1CUMc1@=Ot~I1Uda&ZR!%!JkjZEGwYxS~moJ~L!o
ze1n9jS!O~blNz7Sk|&6SKr$1mShT-MjfEtSaiT6_XUi$f;L>d1R?MaFIr6r^-CU6|
z85LQVO3sDh0IJ?x`C8yrPL0iy`t>|Bb@@DbUEr91-gtJ;d^7d*e55Yr)VM6q-dTY5
zJ4o6>`6_zEtA+9~2=&n-c~fBWD?sD3K&LI1b0Ts6VtJFa_Z8cekDR-VAD77a{SsDS
z#4nZ**o`ehK=?QZ7Td%{bz8C~ulZNtIC_(XCtd#3uZF2fbd8st|C#t*(Bi4mK(Wk5AG18Ks==CaS(SBOGS}x_hK&Mv&
ze3SC8krVN1+P6mjR{G{3)mjUqx&9yxT8n3Q9i&Zb<;BvTgVb)FToM;tj9rI^&e2cn
z!n-y%29c0njUV=68PeYsVh#&?)b*(Q$%HS!yB^=v@+W4?csdZx(V@W3a>cGwNj
zireKeflHQSfi!={Yqi`vWEb7rE(a^SI3YaqMGQ~Ic}6a7#RV)T@E9Y^+pF=CZ6clD
z0S_{EB~{;vPIZNr?3CBX*_TJ)wI=Q^$S<(2*7dl2hTp(cJSjhy{sUk8jQr()
z#h*PVpAN!&c=c6zb|7yF^PV=Eq`@Bs&*#rV#Cx9@htlPyfEZD?J
zA(zbQ&%3wjQLqwetO{0UhRoXG%9-oOrFguMcL7MmqP#|pJjzNxoUgbUrM&HrZNs(P
z$|Y%+L8`2D!yc6cSt$*&K~`#E4De7^_;7YBMQQ6yre%sEcJbs=m5xxmzN*q3d#`A_
zsuT^YkHV*TrhAmC$o6NCQUlqB#+ccD7^CcGJ{4mHpZ2lJ$NA1i#0HDW0WTzB
zu2ww;Q;cNew^*er_FIrvP$`NsHZG{NmR94y*kmPdE_E}g#=g-;!EToj!ys6^wv
z%e{h%ILVy5kkUl@=A6-~ka8T?528}BX}4Uw(kAeGE@l8N-lgQFMe#~y^Gt=cgYUO^
zrJY1)sA>^hT|OUs=87t{X>$=pGdJw{^W+D+k}elfI!KFf(PvS>mI+v!Y-cl$6$N0`
zJ&G>|z{Y#jt(c+*ZoP+`owRVhl8@#UQ%0iw^A%SnVNH8UaTIbU{ajor3G%YI(g;;p
zzXYmxD@`w<)JILGmQb1_vQSCXTrxE%sSM!VSX)af8+py_gHmWnP@|Occ6pqg$+&MX
zP<)TFBL-e)whjM55u?<=EJN2dRj
zQF_5n_b97$l~#Gw$
zq*N8XCbg>QH94z^UemUk!uy^QtD*B?gL^gfnn&a4T6Lv44X&;X#pLa`>dITe`^D`F
zIFmTHy3#Pq?ltShlDPK|pI~8%551wnm();NB`W#BhUDB$otT)t`B9HNf+uXEg?+$N9cFn2Iyi*G`InD7*gau2`PSWptRw)Ftr*g
z4FkXA55M87rS;f6G*n7qYv%EWN~6HJ{4y&`eBku#z??L`krEp?H9HJTGmVvqQ1~W)
z??GtR>{s-(k@7W)IIgiWlucpjL69J_394%*b!noM1DVkT-E9^fZ=&=@|F7Is^#4&!
z(Ib;-cT=*bA^wNoNlhvbDK5|ulq_laqUhp%xe&T
zciKPVTPpjOlE6NC^IOV0EJdys!XTGyp)}8JZv?Y1U4BWETOjY5bfksouK_KU&XB7|
zOQkky^4pd|SJPUWa8YHQ;@^)Tykc``Cp^Bn|(16w=t0}FO`O->*inmebgZJS!
zg13KLI3jT&L=shKt4!xMI@DHq%RG$w1F7v4**=(B6TkUHPseu32vJs8I2;nrk|Vr3
z%66}vGTF+(g$>PF(knwz%gJ2^T(2s-A
z{TH$I$V+7gE0xSM(?8I_!Ae}Lbu)^+qKMI_KY#jCxEFj_l=suVY^1p8=wM|SM5!}G
zDAs2~gk@d9p#}8o5DZHek!L784=zz1s#KO2W@rXDijTzz`&k+cjqipk7q|-NeuA0)FBQO&p;#%)s6mp%e);
zT@Kr;{G^Rgg8Xnuf=y&0quWS`i33!=Pn3Z{ICzbX5NoLOr;0*FK7|pSMjbv?+Mv?c
zdTp!<};-*=lJetN=N&UEl_cEbn0`Z682Swj#4&jFL=K}e%>!^E$aGt?_yg*
zOc0j#ym)mu3d%i&o{v(h1o2k~BK8N-FJCGuwNFs02jlV@6y$&(Zk|Pyk)X7N8XGwi
zl_Ji-yDahnsyp#=W?5fO|YXMY6?fl~tMUnyNeGET$@(emPJ3=G73Ti95$<|0H&4y1qq
zOdtH~IE-l)zcQTT6&I_MJONh!81=64LOKXUfw>j~x=|Yy
zzFDT#CTQABnz~7;26A|lk_@BTZ8MzlEZVVIc#?OvC|$s6`xd1=y|Wed6R}$Hry^U?
z-is+>EBe8ptx8~E8qf2rx}UaHndRI~@9k9FRBM}Qd(o~6twVqYZ8L|#qFzw8AlkSM
zjboR#>l&EkXWN8H<`$wfu+N^FXea?v#Mfr){AY5A^S@H}8qgj)9|S_F>273joTVz2
zqVNHzw^HD97SQw*l(y7%l$P21%~<#qwM;>2E~F^AOtru*Hrp}ZIZYk6WA46yrfgSA
z+r7HX4tR$P+m$dXy#p*>Zbv^}Oo)v$2U`p1-5o%i7jJQVerbarwb%*^`{NF~>f+dA
zjot|(Xi6;8yq!vp9BJ?smhnKt=L=$I%r6^Jn=+N&1@R8EMr2P2fpLY;4ndv$D-Hff
zq0YHDcfIcWl-}mNK9E1gjCuW;eag&i^Z1YUD;;@!a$>*oJXb1~bj;;PtQ;Mu
z)bEss(9uKRD~nj$g;^bS0EXo|T6+L4^)Gg*;pR*t{0DT2#jmK&56Y+r98Tl!j&2ZB
zG@hp%|3OSp;O}{e-}Of&2{J$WQAy>4>PLP8f7qb`2jTi5%C>`wC-6DXnZnrw<)w!Q
zv8rMEBQv7YA?0J+RSU%ZLr^nJ?0AG>R5*+Qx3oNhoJSR%CLUM9slv}_X#J=X5c0Fw
z0tVO2sVbE|h6%|78g*0|_pc`^<4o1SpA{_0J*E>sV+K9pG37X>jK#oq+A*b~YkNdB
zXBdi^-r0?kB)&!C1Tq
zhLSF)<0q6_Tzm|bhG)zLBXP_Z6FpIZ{@zry9p;6_)++JFrIWBkuodB_l^FE+GN;8<
zYT#+bgQl8xS}d#}4VQhM#w7Rz{e$1A?5HzJ6?Swj&nN{0t&z{CXW(|0QjOJcJL}IV
z`5?@XXO!}G&58x2yk}u3(5^MlqAL-9#Ss%(Zj?{YDtYk0)U!%yZnS-8l^!9>U&2@h
zbLH+^?24kDD}ZZ%PWdMI0lNvwd&h95V)O`cTAs(E-6Ps~UKwMJj`LB|3u2jM&IRRD
zu8;f|F)e>YRWB-EFl6sVr2rnhb`fUw6zP{x&~*CX5_-r#wDyuxhlPA}33KzUM!Cz%
z4`TZN_Z6%}?4!7=%Fw_E=HBC1S707CUWLitXH-eU>>@PdA{PJJ6p;>{7@4lP;X{pC
z>B@EfZr7*RmBxN0Z=i4B>o(OK=FZz)&()Va%0Zr%I$x8oxs*5=K=
zQ~QOHS7$Qig99`yLKUxA`J{lcEJDSV!BeMF;XEq7{YoA4sAZ(*Q^l74Xv+u
z)?kvJj+DQ`QqV!rv?k#&}j
zk_KT}6%Ti92Qg0>U$#@fli&p#bx<7CuxfcjjTf1A1vR2w0B28UHY-z`oH%WQ-!XJ$g|Gc$h~LlV>or}KAXOrjd@44;XEpBeA%CU2ufqg5?#
z`+6|ayt6^@9wY1L9v7@dip%@OT1p?SMi$(^2EjP*JP-!TVjYE#aG+5(%|IFsMa?i=
zW7J_0cmXzTx>`yax0&`aow3=#Nj|*^!pr?DKd3Wsgm37N
z>JewcGt4bnL??e_%H!nEtvkGbWQ%lovw%cz&5|xUOqO)fQL==^d$G(gzn)s;1^gHImB#%IRj
z01t-$G~}?lFticgd$DuGos=XWQDYKX;wPpBf>TW9GaV3}is&3*#Yc9nY`Y?w7wf_h
zt!GAmfN0*B%;IR7(Ss4q^V<+atC`Uw5Y0>C8ln@F%=koWVM&x#&Ws<=@w|!@i3i-7
z@kt!di}C@8cV)(BY;+Apd~{~~BF3Yhg_!xx_yFF6zJh8WRoz(j?|4)#kg$Pa7-o8l
z;PHG610;nb_lZR>3}PdWL*e43qfT?H@Ifa=ZGI&j7oAWPyQ#
z>lZN+r>n3VGG031j1liQ_c>z%{c7{I+`)ABs9M4qMDCweB^oOw4zJ!6>ka^)c6uDc
z&1DE`^t0MG$9a({bD@-<)rQ!_cT8;{r8=p}F|}YmY_)`7sm_RBAp?H{m3ic)(Z|%r
zFs(<9sZ}K{AYwW`3C*QUS!Dr>Yv~V
zK!Bo7sU2fE-i|?+35A*E!(Q0nUr|cq8SrcD9q7#SIx5KS2IQkUEB>Av!dR^Jbe8Ht55_;e{ns-T6f~}tSoc19U
zj`H)yx-~p_gLNdOT%-rN<-Dw}3D434F&p1W`|qo+YH?w)nL1*3$uC!#ekayp?vRWP
z`;QK~n3OB364Dhpa>ralUIa_o9rEBTEEL0a*(~(`>Yl4s(c~+t9BB4ST6aa&i)Zx3
z87u?}ZF5k<{?#+-`4!buDWf;eJ&Z-2fXpY{|JgTxVpJ&VCb2?fRE~_Uxt#`GRTXqa
z^vHY-pg-Of=q%&-svRYhC2sB%Rt(>j2#)v;pHar=F&6~cBIZYri=_y#
zEJa3z;FHrZrFi%kUT6(Q^_Qp=i9tVZW+{b0TUy+n({T
zJMH;E0yWprF-L}rxv2`#^dGBeID^DDY|O|2Y!c`RB5
zC*!GUHIvbbtc-jd$jWE|^}hZFqa`%=y4n`z&{I9Ka4vwKL9e~fFbp64gbLnJ3vpbA
zBws$EcW%H1jDJG2Zm1od0rcpGDudLh5gA|fO?AwJle4VB{%~9V!WHCzD{x>qOw(#a
zD$>+b^!ZJ-a2Tg&q@>k|jDk2n-c&ya-+H$&(1cYzsYZ6n+66XnPsf`2zq&wfY&p%I
zCp#O?6S0}j#_aAX%q0a}Ac@~?wJDZcTHIC(3xWUhG22J^lkc|LAkb8QGSzRNVnh!Mey|11@l@G2T>54@@B^=iR@oLlAK=daR`58PUcfvoI`qS$A
zk*bvRgyuX@`Fo{TAE;V9QYT|b!`e2UDUS_o8iMhNIZBuam3pYw4xTFBVnS`_H;Akn
zJezO5lMuSvAX3BGg<}uZBEj=Gk~>bRhLN$s=7(AkdZ%Gz9ICGQBefJetpkngdoQj5nzFXuLjl(8-`?y>5{V}~BA1$o{4;bS#_!%TJJ9+)avu&QTB
zdxT#Hz~fj}oCo!&25Y?F;HneTx>iqAIa}X6PY)ieQM9LVWd7hiY&Owcbh&Y4fnc9y
zfhnX(WN};o5%`Ci#I5f8L;ad*v8QUoH>W$s=%?ypG07_MTusK&>YdM3+~WCDB#JsIC~M`UsTb&7
zSxZO3j2eoT&tDmT$NEGsEAvS3$!@!!6XRqBcS7T&<{;
zU<%M_&xQrC;1j>V&G)A8-lzt#PT3pe4e@}W8D3X{^qan-@|Cp@nA?KN+5vvLM->gH
zQ%J9>mBtp{miV-ZlcWEvrj-UY;;U;_oYMMqdcTG?)AevCswjW9z4F#nKB6YrPuxW~
z9+l7NSX29mxfmyEX$Sl;wN9?9)e0Mr!yNG9G11-$TPVG*HjuYnwyLLnjBS^P>S@FL
zaFC!@BkhLkZ{7e~oO7|Z4~5}?P6QXCW@D|2wDL8LY^?2at$7WY_FS?q#coF;>BJkRNtHwiten%@h*-U
zowaz>a<9%>FAg5-tPRIdxPBL{GHPd37o3Q}XR)b^Ruqe6-*M~-dcoujmG7#RlqRQ8
zkFHu7X?hCoG1IibC2VUJ(9W(}0VySg(z|Mf@eVwsoAwTWwYx_*tsb90T-8l0>>n!L
z*Ft7|=G(Z`Q+vl5e31u|Vcw%gjoz9Y$3Kk@A83(G`}fhX5QlJot+o>v+|C}LJ35k=>(d{IyiG=Awg|9(ptfJAN
z$nEIY057J;Ox95Jbb!Ca-qYz2)OU({g5nEM6A_KKt{a$ar?b;F0vR|1*eR1})(jNp
z4;nF3>xJd5t24EZypUaemX?T((7R^=G%1Cy%>tu)6p{=M&!|+g)>Qi9C4H7GVi$9)
zK|e8hO!tztPuT+ZC5`VOe^pA4p-3N81sA0Gw2IuIANs%?FBkrpt(9>eq(XD>;2vr=
z2a5FVSXw>@M!}%eIob%^yVhzhFz4xuxmsU*gXI2PZ6L0%>ogBWXw^ZQJWuO@;lcHJ
zz`UmX^Py+I(UAFO;-dLliZuQNy}JNCZ|Vt}x&X-nUwmSo++&Ok2rvF4x*)iE+zvWO14NR%l)Lg-?$akoqv~ULj(`R+_N`R-!U*
zV}m*>;{c_s)LNqSA*-|sSWc|9O5=|SeY^^3Culy1H0cN(ScNCf(Ct;)E|zKeYOPnk
zUA)r`ml)XdCF@X8+!_sg45`5ySb&?fdyUo!A$ct#Z&J;*+8Mj7ld7Yv)z^u#wqIwK
zb=5j;Fur3buh(+Jl8M8}Df7-2`6~~Yl?$iFvatzr`TlZ_@7+Ui?ZKi;p{!yDJ
zW!@d}{DRhtt{l=P+D|>WsJUtUVHD)66l!n;<%0$dIHJW!6H_SZh*pckCpbKN9Q|=b
zt6TIp9_(hG3p}9t
zCs5o;X;kQ>_PK@f&<`gyHwV&BY8R}!z!~mp=d^ry&2!Rdld8=ZmG}69W}o3UPz|Y4
z^p9hcBY5WNl|Etzjx|932=tz@boGiBE$tnPNxxP~Ny(1F7MH$Q$YA95u~hG>)&M$#>q#a|E)4+M>SaPRn9ie^0)6fNfpyg>=T@2*!q`{4R
zPkGZ3+D9GIwK(T~iI2r!aHLbifWe5FpO>0*w>6UrSGINy~oAs9Oz@t&#
zX&@0BGY*^=^se~Q7=2xvga!63My}i1+z`C8SoK)z;U{g`Yef91ofa>NZsgEMnHC8f
zEkgCu_`NPv7Z&M8sNOmVBateRdQLx-reQw)d#g_v=>P3RNqa}@;tQ1LqV?A(s*&o_
z2Z*c68fyA?cnJLo_tty$5GtM!<*q8AK`(Fxy{X#m=PmFtbvJrLNW9LD>!*0&;JEI`>%wtWEdmP1
z^%W?jI8g-L7Smr$){t6MFBoPox`A0eT=k~A*@g8j3Q^|LGytH`TbXR9i$!%DXqk?@
zqeT&N2crnnis>Qj(B>3F;s2n|i|g#rt`*mXL-Ur<*`a+`0#d=D-7W#P_b69M@Owt(
zOX|X*jV>u-mvO8?M?u2G-cBs3L8xMMA{X+BD5ZB|O&?TB7aycvR!V={R4YCo03WB&
z;ZO(<4M`}COr1ND%j#vq4zdBl+zgCjDJdtaY7ccNs|%O6xh%BNplfAyc6pu3>Fn~R
zmDAbf{asFHm)EB}-k!ka`O53;@*b53<~7At(AnjEUcpRUQ$ZIluUAD~xV$+Pb+|m-
zPpWU1W}G0SlKu)`mfBWXkCT#4(8bDndkhcDR?!P%0Qhbd{QL=v56gzzK@0sk8U{sHV=|Z&gj5z2C1j(JBWhxE9E5
zDp?C9KTP9m>Fiq1)zXD)ty~+=!|&Ec$>4Wui$p)Knb^atFG7|9b@cbRao(?ka&tpY
zuB(5>G)FyszFCF=6jfhuZA&f&07>ISLS~b#31WQYIQ({YfDrXSUN-v7t%L`gzNc3JSJ>2;x^>3x)FyjeY*a|HONVgAl
zUHa+-J!qxZM*Eg-jc8=orM1o%{Y+@B7q{;p$!HtNz9|Jqbue8qFZ%JLA#dxEIBxl%
zwO$U29@j=Mlqchs7&u6N@r3+3w$T%ie6ful2lBcNs^l&eZVUPE(eSqV8h*5$J{e>~
zJJC1JwA0H2lB2y|6OoPE>z_f)AKRN^K5Z|=)Zf;N>(+fi8TS#rAtv>G8(rWowOAtL
z8`1$i>nKG2FCibjErbMyh3xW;5OQ}1A!OJ)LP+!Zcl40}h~MwLtLMh=e(wr#7riUQ
z6)C&E)vHs8Zr@$olXbe)0ND~^@t3$aXk^1;O|oDt~wT|EEK6zvx-%HD#TiO1{^|BoG#hYMseCNyqX+_w8h!d_8|?v
z&(%%HSh<@XD@{K|?{yOe_yj>%micDlE>4_rj?&D;SDbjAV!DebYj=liU$AW9$fHD3
zR@s^*A^*o`krsipLD|x#a@r++E;rIPW=lJcH0j&(RP=ozO{4db;YI5AzQ}MQhxXB0
zGjx0KdwPoET;kyTQ^~zVNb3dY
z4f6I94|YW;_!x_ZtM{n+%&emIpNZ@b*`iI$mbQ-5=CWv9q!ZcF9wH69tQz+gIegF?
zx#P^>8ew~^+Wp&cChfi9b&mmE4rdS8L8*%u+~
zv^7JC2nBEE!bc&40X{uHG~%s+xVvqEPY(%xa0)5WNLg>DByma_rz}5>l)OlJgp^Dx
z5HTC-H~uvB?I&tsd_QEii&pm&Ss&&QPHvi^u>J_0r&9ez+FJ;D7I6z{LjADk!nSq~
zF|+VK3}W?50xI|qu8&A${rMarZs)-eYTXQI8Z1q5xw{Z)gCO^pn|*@kuDmnm&Lf|
z)L?y4@K-4qY4OE`&5vk&KPk-}qI-B^ynBdVlf(aT7!zZCs9vXtH8Jk_f1Vdl9IAV;
zKYaC2UEJmQBM0x(&7r!ueGwy(zk<(R$4hJpStp**6pZaZ|49ETcoKi)og+(})HEtG
zOz&WiTxiKK!OPe^OrIvkR&RZ*-{91WBlJacg%kky_
zsMrK^0Mu-PUOr;wLNo?uDY4eYmNewG_64h`gF%)*X3&57N7=&Zp
z$SBMGGU~tG?-QV;Rp%)GR3Tx7sR*t-L+_Zuj}gSq_c>FA6x%tppDvl97Yv14)u)Lk
zYfMApCF(j&Kob!1%wT(pH$cd|coq?{e?V+MzDxTVb)9Vr&r1Emie;i6G76`CU{m~(
zkkJ?CsBn_Vs6K*Na{M4kWHgRLmuQt4I>b;o*Lx;3^lKzuq>^6?Xmbwjqrqlq8bY4C
z=8%W;E|8gb8oSdp;Z9F5%5u7l`fsNjIUSt0QpM>apLQJDOC!t>AtZgBN;{_u>CbcG
z0eWdB$}W1loSVC+U
z$*{|edMHqFGm+6TYA{n|)RRMpXuKI(j*v7rm5$65+1}>FQxrB!K#R>nB3x>VSt4-&
zf+%T{nYbPy&vJ{rk9)XI?L_K8GLxC!n49@qdeX_`Q
z7>6+NHA4nM{DHKK$wG+doOqrT63}Xd&_?QPhCb)e546Y(?M29Qjw>gZ$T}=D>$E@?
zFpvdw`hYsb{Ry{VpbD`aBclo#^-!RGnF}xW9$YJC)4hBm>)i7!KMX6m!i@R}yrku)
z>5fl`8ZsL}xZ4u5MX)J?aJK_zi^r#M=oW1-L%8}g)^nbvFq<2j9BnDm*uka=2b+5i
zP?j5JS^wJ&U!4!Z`orU5O=S2Sky*?F%L(J`U&L;7;-C#zeLSj9a@$CYPa#<973C9-rID6#d@W^)wbrhT9B%^XL3Lh=kqmXwS
zL_?C{^F$7_ICPIv%n)kHMGxoc@xY;uTr_A9R(arcYs?q5$d{yM^STLUJnAN!*WEW?
zFh^y?dKTY+1TG<`ltoVV#rP;a-mECQxB8YaSbKY#&voj4OA>}R|Sw6hQ>l!cro5N)SHxSJY5G*ndj>a
zkkiAS3=hR`)GKC*$3tfP;Ej58mUzbFgiA8b*{JJT;;}W?wHm&_9Hu<@ar^g>aY
z`6AwnU7LbcgH3wjsH}23y&-+^)*I+CoAhFDJcTz4xWOZtzWX=B9DEiPN&o!&sW2<4
z#%4YC4Tik`lArnrNyW2En9EAaQlk+=aT62oY{L`Yt$Gw!e2Fc3e*a*O;a@?)W?gf@
zWhE<2n_A&sTXZb};S8=$Z$MvMd6zv3n^Z&Ih!X9X9fi#@*aO2&o0T1kl;hvAsJRe3
zmi)Hrd5ULy3XSmwBMI%0y|mfHX7!3eFGtI>f@NhUG>_l!APIwIltr|3mJ*{Zq8YP9
zLB4;9Ahaa2guLB=HJdN`FonIoRiBh6>4@2n#I3SO8h^wXvrUi0U7beqPAv8XZ?-q@
zELx;vR}fA??b6$YJUpC?%Tgm2N0Po6>rY4da(>?7QybRA_;$D6sUF@R_(`7b&OeM^
zk|%`u4G!{LcS9MvrcE4ksR$A14=HpM|IvZD!Dy-
zYOsm>>ut~MP;qBL5H_<%44~rOJuYKQ2hU+~x+1ZoXAckJYrp4d=l|yJn&nHodc=@_
zSvSuiMh)ri*~}3}p&lOd^hiiAPb-diub1b%U|6Y-hY&%>`*?1KhZ)Q(92YL$Ud*gtEbsC1I4fGVHj|O@c
zqmuFs@+=Q&{NF&_vgnVU#W)QRdj3h|HQC`!OiYXj%~NrpQwnl895(ex4(}FD**M(@
z9O;QW>nYr=|Hs`%e%PhY!2W#(_AKmqzgz$QoD^kgM>(p2K?%wL!+f
z7BTtf84K^k?Dpj^U9wD>lBG-Z=sCP(*(zl#7b{=BboX*SdKvGGjs2xlkj2I*_-pKm
zE|vWuN&x;3ba?MipA-=qVg?+JAcuGC3`97KIJ`@f`A1oYch_wGQO<#lm8I&$)slR3
zXmFjle0g@wLA?Je{5ib0=HYidPHV;$3GyzTpAiW5rg314laoeJ%Q|tA!*S<
z+E_QPLebvOoKi4E>1@&Hc!v~-@Ww|DiKS|fc&<7BXB1H{u22H~BuTj)-n2#F>db?)
zV~hDm3a*>yA7M^F`OlOph2v}qMS9}Hza;_a`LpWqE?L44J00F}-&jAU^ACS;LF5Yl
zt+Z(3l>G3>Z$%uEzazK9ar2f#NMvSloPMkqci6Q8cP83-PnRX}1c!Uj%KCAoaxOq}
zD1uz^Mgf?ehwIn3ipAr`E?{MA
z(^CmnwjPmfssOSUsF%!aZCcRI*0U%E3|BKl)+X0(6KB)a;K%g>e!!(NP7gs?x8$?w
z1Mp+{q8(^KuWt=-8b%|Ijxa=A?2eS)FfJ0Wl%6(>i;wlKMY<~t@xH%JZMA9B;|{3;
zGQ!TVI8TCa9piEVr#*H^oUcu5^*3<>5RQag_e`y@Y130Ikk_}KIYaboUk9VD%t%2vhgDRY#+yY=6#x~0QmAiwze!wVf^Qo?q#^Dpgk^2Wpy3WF
z4!H|}j)aOioD!>rO^pc-sS>g^SUe9BN)I#pnN8o#W9pq>+TP#5NZ?|Cn|j+RtRW~q
zC54D#P|zZvex6-lfME*dXc|{IAvFbz)i}i82{k*MP5&P0kXWMHFp%2;Qa-w6nlqbD
z9p;d@ocN1C&Wv$65r}n~jBFY#7s=%=lb-HQd
zY}yPNvP42A-%b{s>r%xaM;ZpMBjlE;(Tq`8g2qo1%O?_hxbi@O`o?gIqanu&EoWg)V<7!J6C56
z#W*jU{$>I9F@TLqo{J{Hrrk|IssON2*|W(6*tD6&VLx+Vqmnz^ERIcEwG0%(;CO2D
zI8UtaI|j1p_{&l+n^v<*Md`pL_`YXcUf||GH0_*CXRUQev8X@+V9SxR)pYYVZPwi(
zxlw-ym_t4Q%04k0(WX9V7dN5+ut}+Q$IQZ}``=A)ND5f|z%1CNT!65w4>sKj6=oI!
z;6g(Ehi1Llv|>kxgdv;nM{w}4nTe3_mxVR~2eumcs|`j1$JVAggq0k4!u;WIoQJd)
z5dP5j6S-Q%>2MoWTEuxI-$Ck#U_$CamX!NK5j@X&XVYPL9M$DJ!~hQxMxfU+z@}@^
ziMT!mfDOfMI7SB8G{OY<4l{>XB&4G}g4z`et!@hB`i)g#QJ
z01|HEI(5#&rgOSFq(a~z00of{oy!E+w3!L;9c2!M0Jvgt5HtZ1A6hH~f*o{-g|O+g
zUJj`=g!q|R!~?MJh3VdG+M%cLZm9ymmNDEC!lnf+A&xPJq5w3nV)_pHt7TkxT&jpK
zhWN;r4vD){tVO57XtV46g^IKijn}vpH{NmThhW{*2}MYvd%KdRSt>owsH!WYtt+5
zibn9AVnBHS3JoxGv+0yMJoNOY3P1$_HlXWqJDZdSw~h;m^_}L&D*RI{58T|MzDsPcQK-wnCHT%V4wG2NU}!GqX4C#qOdf&>0Pa74
z@o@^pw5dN@9D3DZ0}v
zlucuMIK)_+0a4EfnbmI7kkh6qI?qqDE8N-KRBM|S`M@E(0j0O$qF!K}A8?!UnS&gg
zUc|VH+gSkEoh&dMVixd&Mf-PgNW~%NMQ~8L(!U?-6e@VqqKk)^PLMsb0C6$%L!Hjq_F1%Jb*EGdz~2Dyuv>hJ
zc?7G{Q(D+QE(#7~Yx}sulJ6Q_0i%S}Ys`pU+aFd_*z^U$>@cn~fL&XuZ_E*Bb&IwG
z&5I%m0K2vyp*<|~b&EP?nTGZTb6}qqThP>cn^rEE;N+Qisz9*5&4e1W_UyFiNB=k^
z%*cH=nFWt2AM9ecm`O)seMbWa0pNjj+CFp8XVVk_ba1%E9N00Qh4ylN2+9uQs~CYu
z6$lR^`apFVVbcaCz;~NjuxE5zdDzrv=8-A@>=}zdPnm;F-?4=FojJrI;RMu_0SWex
zLKX{wV9(eID#{3(23jocFbnpKv!R^~u;~rmEQSCSLc%@hB?D~Q%PPiQ=D?2e@$VRB
zGXg&ZePszD5bPP7KnvN**t8*x92?Sm%z{0mis1~mS4oTR>fn^}W5q!LCSwT5iSsSG
zR3wUZ5Z`@pP@NcNJ4WGC(2Sa9(a9EyyR|@ZbH5BTK{+kZM;3_9!~nkhX(eicBL8|8iVeg=E*D$PEm$RCRo-jS`OCSNc~b>|Ezn?$;utW~qQgvp
z?-6rg=allq>)$E<0b0;v@t9d~jSRuCN#tSCf0mg-2mmJp
z>@fjui=Ht7z9-Cq4d_f=6NTIU<1Z6n)A04?c=kC1SWnkOKyF-{mH_}OaH;gsd+_V?{^nfdon~*d=^k=O
zje+4rk;{9v*yv%D;x*}r7}Jivuye^}t&u1eE#1Z;6$1wW;GQt$6%P{}jvW@Qj?y-A
z6!ZFCf<+8>g9FHe5#J&YrrU4=q#G#zanGZgo#I?o9!EK)j%Fms`u^d^S^Zbwakl6a
z@Ho@A@ObY`U@9eb;&IF?mYC}!A8L}lU|oyuZ|0C{A{zl7vSehoJ^HE$7yKl_MLgOJ(@;z
z7K3k}2#%p0X7GXtenBtIpwflGW2t5r0q)!-&Q-+so7oayPKH!Gr9AtsX&`NS2!JwN
zeTx83u;E`ZoNGC7wK*<)y~=c@`w`~hoB*(oy@zVxUU#gHN&m!no#(LQm;?LRY5=g?
zR{&s|+RQ036aa3C)z8?aIvhVdbG8&ea!qwe4ZvbNvtS?ViZDmVaTfitkI6y+*vHOo
zCNL)*z^Q*ZB<|r0EV>S~1b`FiFo1PaCo+&7W@tl)kVUm<
zIW!(GUQJ>EJIvWoRJKRUK)Jh~0m+sF}iec5(42Bdfbjs}z&O=x8bf;*qc=%>>xgmtf8W1fU22Gf?5IpdX>anfAdV
z5-$l%V-7_TvFdG8wL?0X^py+M#eH5Ncox`WhuPt6I*rWL(#zxRNE2z6{=vy^~E}(#Gy_vJ!|(uC0Lmbtu9c8z(fGN-d;4bK!j%&CNP!WYOhVTx$Ui0>Bf+Xmn2Qtj|%KO!J#r_+~K&o-1~;^t+~|-}M$c
z#p?+Hs06_00I)si3;@$d?gUA!DI|kMsmh2+pKms)O;24kbt>Gy_TYT6iMhl4J&uRx
z99?UVWkbi0xvJo?mSD^(SO$!lu7(BTu?{hSr=0~LJ0}FxHEC7I&Z`}<0#OYqy)D@q
zS+e~C7QEcyqh&p~LninHzbWYzqW>F6bB_JiUXbT~jUPt%t1P%gF9}&+|
z%t@$CKgOtnSO4cThX(uv94ikd1ZBmr5UCW=iyg@V1~x)`@(P}GWYR5@O&2WyjgjCp
zOn^--fNvplXaWF#ijNIs9SF;G87OAVsRGax5iih+oNx)P$aK@kw7yqdfaF_5+jFO_-{n{N_LaS|DP@WwYY);SCp8WEoJ@ux|--Xo-j+%gltT
zYfah?l$$^RS^;p}vU`6)c(#AN=Ae7+eY4r+gMoPBtq>0NNm7%oB4>
z$)45OhBP1=()JqkE46i*K1l
zdqhak5LSTiE&6zhST#u{&j)aahp=pfcfY=64DSF~7#MM@mqQaU&4ocIj|v3f9YnN+
zVd5_LwMA2Ui-C@BIdgay2}kcag;u|P-=sJ4N<#ewpd$bUF*Ih@vo69N@(&B)Tfr>e
zL&QFmiaW?n2*Y&j4u^#KeX0O-Lc}Wo*g6-4luS>7!i@S>Qg9#c)hn4(7bKi+YmPCm
zTJ-nV=4fFRH32Xobrl17kTq$TDbyT`R={Y89qeibbO+!!taY<2QFvp`!;ImWN!Nf(
z0`NW}{Gf6?*vV&64_f^ra9G0}dLUs*1yk4!7M*`0kk@_SQ||YPgQr}JZ7)KXOZVme
zwT_?r0MA{XW)3{=SafG6a{wy?vS`X
z1)v`Q-7sWfP3>pV{@37hc{H$rS@cIr?f#}3wXx`(YYxc^76LE;fPI@y_i@ysMF6M+
z4jY-nKmf+ZncjM;MVA7AK~O3KN)7^`sL$lkfhpU`6nF{ln|Pm&)<{fr+=4yh;@
zZeck-LP9X|XCGbBq9Z7*f1F%5l>x3{NT`dYM4nIDG#`9)E^ye&fZ<3uU(nPQm;<3R
zFVV~R#S-=H&mwK3$^GN>o~hdyKN9%BT4v=Iwdhfd3dD;<27Cg*W(-$&jBnGrSet~t
zry30S6bYBSrq5|v$E3^4I;8dxDTN{im^w9pb!t1~M*+X2u{pe0+0Ye
zh5M$ZZD`ScJzcp0_OzQ>e1(WIV8QC}*rJ`VL8Ci_U_i-nNN`zd(9ohj
zXr+#Lr+@AD9I=U`!=RO2OF9bFX06=AY{w(D;h*M+q^(7xe@7q1YGEp!7|1QLmw^)j
zYzxu-v(N`0^Bx9ossKzvLjKa`s2*4C1gg!R`
zHdTtDck?)EKeLz$M89+sVN;uh089g*Bvvc~ve2r~Y%DbTzGDtaNH_wO=ic{+MWdk(
zSanJjfUl9zqLkUI#xUiU*we`=RmA)1@4+G}=db37r>R9_x;n*VgA>IByaEzf6+X1+
zEC8xOfCCJWkl;ZAt667@z6(HA2p|A{NN9pd0Xwcspe6XnXPwO%&kxLko8TFiOV}Tc
zePz-yJ)L6H1%C}R*al2I<1>iss|Q_m<4BA7h8#V)QU`yF~0Jhz0$|qGftGc|FE=lsWKdrey-wrx@{}MVGxV5UB#e1^mD0
zIu|glruP5OJk67aQgob-Q!_CVDk0~FoJG#(!~4oB3PlLG=PA@3q#;+yDB_b$z?nTK9L|d+mFzy^ec7owdLbZt?O;M~GL*gf99j
z6FOP=+{^vF1tX}bt!5)Zgk4@P!`Ujtd$Q12bATf!ImTsP-tQPm^y6(T{*K3U)NlO0
z`06T~aN8Qsp5?YB^SNye!$v3~;Y+q1rFfiWqvU3a1cAfI^KLI`Q0>Pn^L0v^M7?fj
z>UKU2keZ7>u;PeK!&7-{(?LllxczkKM$jfg0VA*6U+m>8*pr11WkIdx>%=`ne|p&!
z3wnl(pgyl1%}SPT?T}yF0;{O|lQHp;OcX~GKVhkoSfn|$n#=QSYJ8xV8}Nu4!YZ*<
z#Qh$36W}0+P$D|`#0Vu29%qYFbUZ3D8khV;7PUXIie5mMz7~s;_xiTS*OqrMl&*SHSwL8yOkYv0T2*$tuUYLEH%Hv%0_HW-#}9S*MGw%;GPi
z?46oFe<4e^P(VY(5vae$XzH=ra6TngkiYJBkC#~SIvG>1)s;7$
z#0v7$f3Ol`>DjrP9!V^d8ph`TgBE`!W9q=ppj#3x$SMC|CB{-O)``AJtRM&dgB5=*
zTk6SfV*_2RAiMvUmEkUprv7XO{g`M$cKI)jyZi=Y_AX!LME4YI^s@F~_lO%o%`L;B
zw9Euq$~n2VH`dF7*13DJ0?>GhtYLv;IGEO^OSGyFFFN%97Gi#i917SeWihQxIf9&h
zo}0ZILDe;hYuYp6z(6W=eOF+S+N=cr<5Y7^Sg|Rn8xuS|GLH&wiO#7*i)l`?Xfm}InPQHWLJ6&
zUcF32IH9N*8;kdIL;U+*=GRZ@G$7t66Y3ee)pw)oon8*Q#jWY0_V<(=mx+H6&pMw<
zx4H1Jm&v|))*~B1{ohupDgW7vJ_B;2QB}vv@zH3?3B}ALm(el13
z!>`d#F1qi1OfFk~X3OdLx2Vm1M(T2(RtZ}uN0GKzzdFuU(zi%G2xrq?cq-?LkY7PL
zjH%&abVFC-u}IZ>qkr3u$97{Vsi*Htw$U5qmCqAF^`2ymdW=-t
z+~vPU>hj03K;`n=jjCGwLq?B9bL7vCc??p_H*YoOJP}mii7}5uGjp>m=*vhI^r$;+
zH)OjAd+P=Bymq8oAN{ASA5Ubo@8iZIRphC@$RgxV#(FgQe)(-H@m427c!~C>-t0r9
zGIX{Ksyh2~LiMN|MT+*j%gwQ8AvLBx!kgDk^U8OKpf(==C1yFAYZ$RLyxfIU^E;QF
zFnX;8e`Q3ag(gNlNh)vjMsM^|Z&U<(#J0)k$uR4B=CPjH&@tzWkgtb9ftHH`=4L!>HU<{D6#VD|&eo
zw>l52yCqp54#Qdk@pV2$#!pu%Pp@nB4}yb8te
z+U`b&N5lJJ^8{{B@)~zrodZkbUW-@y
zy=>1ou2sr#nYtcDU$zU>!3Mc~lP4
z;$^kd+}b1BYoH1{QoI|{_Z#IFN@Kn3&U&GOn=-;p5LVN;)vX2j@8xcHYm_W>M{y=)
z)ZD$hm*cO{vb{VpLJt&&YGarx%>q;H#b5b6wflbRnHLQx$OKg9#)LQg7v}**e>=#Y>TV7M|yciBX`ndgnLk2
zUW0c=weGo@faF&V%~t$~Ox#OC1qzv%>17@!w7+G9z9{zD>L!Wxy5_B{4|v)2Ojl{+Md**B!{M&EBHb4?XVX6DQxjth
zfN>Qr*E1F5es^io$@#EM3`EiFBvVK9XI
zvRc|#hEbK3Wq}ILnlwKpH)J-X>-+L`4UBJYGSlu
z(8?JGtK^mEjCOxBos*XjnB-}(0)@k&jrmU2ZY<9#CB3QGnG3wccq8DQZWDm=oY5N5
z)5+Z9!JGm`+UEXjwQI_Yk?Ic;Z!j>EsOd0Dy1K+!S%Gh0-
z3C^2hScMsNER-;>|CEyGhSxYJZx77)Nf{l7=DQkh==cVy`|OHqXuR`8VAk&caBHHJ
zLmnX^ECsSQpCi*=vL0<7UQ(}7JXNd+)`|4u>Su<)lAX7s-0+fyrzIkk#w3Jw!n1y&
z(U#_?eGmO{>&4vqQ*c{7O8j_cSGx69eOzsP8g(8~Qom6m!Xe{PtpCYf?J<9~FQd$e
zl7^?mPeUkr48livLKaqfk2vR*za@gLV-z@K0*rRnI$4X@?Xlk&eTk{|@iPz(o(Q36
zwlmcdDVM!$t%(4OQwrYEz8CJO^+MEoWJytLf2qOK{H0c&q)q)o1w8#ggWY;^#zo5<-zD+~Ns0@@r5K5+h87FvCxGFZYBK
zqj3-}jh~l=ry$(w38#8OU3v}GZZUm66HUs)hYu97f;9eAJj?quk&ArZ<9yz{X3@S6
z>MgCTtUm+e6wheq8JnUr9#r{^Fb%?9o>1%wi=%-e#M5QrSqN`43D*O+uO~dpFHy;Y
z5uQWwho7Z4fS|g0A=(L{OZ);B4tbu0u#CBd%~=f)H{$t5$djDsEApxp*sAw*Zp|`f
zQC{p02^cnL;W
z!c>JJ=M0SA$3!N62@@@5k#Nngu4M;#3PWB4((RAs;yYsI^Nc5!Wj5*evD>4*D#(Ac
z>(kcw{6qVQXOX4oWg@-5cgBN!{|-0Zy+({w1p``Q+*ja!HP@9e$b;Ccx;T&SP<(gpcv7gxGhM6I9;i4|%-59$@ZbeRxXKsnApp+DU7u9&uIr$Ets
zQg0=TJR0OK)_~mm@K?oTq>mLSS^#6;&t2~k}yRCu%;fr1su#2Wk?n8AoFr
z-_G9h+hxfWIBOvm>~rco`Y#O#HNEWY-Um#Kpl7z%yY5(Vte2l|bqB+XWI^OPMyQM8&h4&p$9VaM
zBgAjYf)?R}e{_T=yv%wFMyL;=-+D)wD48K)oHy~7OsK1F`MG0s^zwdh!U#nWI%OT<
z8ZU3~7M96ELkM@XNvq0v$jkE}Fdt5gprP`YiH?xjdy)$r>lpFdGNEbA2Q2aRJOsJ@
z#8isTE-^wA6qOC!oIA*>t?m8B_#Ii$_UVIkg0c|gNk^sZ-2@|O!N9f4-jx5F2O4(`!1&J+ahR?N)%|J>B_<0cmMKoDL!mRy8Wp-R>`JP?qq9L
zQ_vtMwoU1cySGHJB}vAphePS7*OkoDKf{+My)+tnszo0Y~m3dZYfPwN^%uH4^M%%?KZ0mbyO?v;S0
zUY74~?};QvI2uAYn+$R#_rq@Ob`kQ6DNQ0%Hyi_X&y&Xil!kFTF~+eFDi3zURFL-`
zno2#6iO*!>I9+Uylf0Mc(zPrX!IsTY#l({0viNglIDrg}_*zE~=NVqs=6PpRDlx)|
z(agRj4O_&k#qJ2J`z`JhkiK5Fh0Q~gLkKc$k>Z!pb4L79#M2Q#X|x25szGLG
zZFJAujmhRn{FQC-pAF^3$*$1^nW7%`w%k{`YG(*z*SWqW$mw*$S7YL95zc|IrJ=je
zAXDvA_IjxibQbdxy^~54WPK|77!DeCiedfmH?q|QO3`obVnKFiT+yql(P5MI+^-k!
ze0VjyPlLRLDKztnB_ib0P!nTb0JG60&YS<09Ng2n#^1`+g%E18pCES!c?$D_&R8(Q
zMJSqhw>Qc7z!4Nm?+bn>6Bok>`};v2LQS`!C5vD!*_fBWyvt8Qg8Yq(>AkM+W$IE0
z|JQE42ust8_PviE5QWR#M-3
zSL}!A=P9(xY7Bg|%0)k@=rox`0Xr+XWQXf^f~-cjulKt)iEu3z^EeOCyfnzOj!oGI
zYDV}M>FwA~(d6M=FF!ra&YR+#Ec_eAmeJ9mCrb9M)|EQCK(#*M2|cPmO5YXzaa~;N
z2=XX;d*0tpqA#Cd&RtfX(x*4^Pon<^`U&h{sUr{arsGnni}K1VMX-A_<_$2qGi0j`
z1o;dV$lNggS*E(7_?huYrhU2)ClRm#L}9INZy&-V?veLT?CVH8So-+k2N>=RIMJJ7C;X)e#1Hxz^vuA2M+#gr~N-Ld^8?
zO796H+y!Ay&JmV)`I)=V_)l5rqsjUw-#EtCUS8}yVT`*WELiLaZ+rQX_rzA2xCcW0
zIgU``WnV`~jBqc6PBR^$i4;R2_4@Jj(mh2)fVgYG>h9FAv5-Mkcn)L)<5s|
zaH{gWXfJ0bgp78uj4_;)JhEz8e6E*gHFWRkJs=Y!P|RXCOr1}VljzY7W62Atu=jAcM9@I3`xLc|K#cMl`yy2_5d{0TQRnM#Z>22&ShzH*Ffz1+o40=?oW
zJ2M(JoT{EV(=nF@W@5~TVWitRCk5HrZoNDnC{tr06xVbMnX;NJRyFulGRB-mxr{N6
zgo0n(>sLXJU|{C0zluRBmZo9~6g>iS=uvL(ImqgcnHV90aNl@0Qwj1d2=e@3S%~#q
z{vU^;i7_69@DwwC)q0RIBPg@I?IPqCV{3?Pjfau)l^Eo&wEUA;1&L6_{V1V=O&)_=
z7;%=Q>o-yWnXvxUjGfXA9xBtiQ3Z-7kUE7UsOrcUylixk%_QCm%*2=zVcg6kQ;2-P
z%j57a6UTwWREQME6nt)=`mMN`P}Ds5Dr3-ToPmo)tuA;J?VZrJXoKeu{$KZNtSrV*Zm(9b_d
zL1t)7%o5{~vM^PqDU|N^buWKvM4wWI31d8k=3TlYF+TQkX(J=Vqh#V~6je7kLM<;R
zI6`8CXCS=sg(EEc*2!CG09+z|P!^^^7RfSJv1saV2A9bUHRxGRIbLw8_hIzO`zEm4_g-dI3c^Py64u
z0a5T*tbZocmZ-sZl_UQFe4;?R7tw@6v%lZ@i!PmB(zJH`FpR@zzzC;i$3{b^mz32>
zOfeHhI4w)x6)l}!a$xOvtg^g>CLEJJ6742S?L>+LW}ygYVAn^-yue#@!^WYo)~x_1
zka_<-1KZ=>y#>ak$9zYDDz)-BxKAw8%VY}YVtc;EXWV?9hYWi}SJES@z~Qfu>FqoA
za0jI1B|;7L1NKLNW)qJmvH3z$E@Tyx`iHD}1oOsrj%!_nn08*$
zt(og>gFKxzw7%kbOoU2uEaF`)nF{haHaN6$Hp1&De)RMEATz^lzapL>3yUFas=)#v
z+BluSwY~9p^bE$XU-P)Q4cj?xkeiNhud_@>$uCyXuDa*j
z?{WuuT~qDCn~M{X?98R!evTC6@kT0-r$h(8q^0N-5w+SH%8pANm8t*Xk7p--Qh7DbCcHD3ubrh|oqSpu)EL61
zZ;&%1&CvL+Uj$t$e9RDJ41trp_!$WH)4f+PVNityxydZpPYfnTl5z)A_rE1W?`qFd
z2Qf|8stW7T+ufKIKQi4k-+;RAyp$DZ%{moET4|+0H-R^@8kpp!+hF`HK3S7i4dLcn2_A%b7tQ3{UNz=5_pe
z7!P_i0^zesP}e
za0Wy>hMM}9BL+F{UiV?B5$a><6pF0M3$hM`axBc01)W0uKu;iF|Ii&5D)1EbE@z8T
ztYfG=reUhCARjxx?VIu->&?iQWT_$Q(Y`D}Ud*M`jT)g5gkyYJf_#|j(A~_^mFx@I
z9rOLu;AM{cE+Z#@?7So*+P6uEc}qdA!V=4kc(yDxh24|3s+$S&Gs>(Qq^5YV662{8
zsp)G}GJ4|GlKqd1U&coBVssb!N(?fG4OiQ>^3;5c*c@V+ug0=y^Q$HG_(618Xo2Ed
zUyVUd_0>p!rS2;c(Y{(LWa0o4rudh`gPhMpr8Tq>4utS4^{kvhKG7}DK4gAP7E}x8)^Kep$N}f&
z*~_LPG}F`fHplwuV?qFV-{YaF;PrT}4C(oMg4$7w3UV7giC#93PIyg2*}Ul5*ED6H
zC;lPuI&yL@evo&OU*pSs5e`MM@l3bQe(o$MTMo(N7@QXZL^uq>XY_9BS$_92ztCRz
zj~9s02E|B@y3{`gx%_4hxSL|a7;VvPrMk4`VQ
ze(34tXAqd`C(&i|R7r0{_s*k|=Dh*mzIRZ-MiTyeY}<6Y$jh?-*iSL<6QEiD0|GZ^@uWna7*u
zMRVqtG(0G=Bn>DF&v9N4@{S&Q-A8|_oMqAW`6UM)lq{144Hq}GbdP_K-Eg&fR7Zb1
zI%`2m%Y%})Mc3#szJaUyARk*976r2YPV^{rCNzr{@c+WZ?~<#zI{It=bT5J2=jCDR
z{ysNVpJ+t&_w#OYN9bF<9P*VrY($l($9TCcs(8;s(5BS}FK>mYE9p-bA!hbzZax2v
z&SW<)kA``KBbvz-GN}`VYqq)P^cFAYP{@P-CMHHyZN0Wh5
zrtD?sozMMhTGG?YG$w7TK$=QvOpOHxQ8?A-C0_pfe;pLRFO$U(uG{FE*j-+}%(d0s
z>rWPq*qq4zrF6z>=>KB#PV%K{db!^Hsgx^aQdtk?s8EIM;AL$b)bej)V#E>>s_;;$
zneXT2dJ3r?P=B)c0Tykqy~N$#N^A*TPUs
z{?5xmFy%QTsJ_qiij`ho?+Ee7x|ZsBI0_BM`!%@A1>S<8TC_jSY@-^lmlU;o=AX%O
z&?hQ`Ja!|C6!nlnZlWKs{_WD&b$6de_q<-xGLw8NzWV>NUtNm}^6){nIExbz)c+r}
zuj^5ReC#84s%?bBA#C)ce2{B;$zSE$L`YR!OW{6~t@cp=^;n)&bJTQkNzDTjr8m(7
zGpdK{Sc05PTsNaT`#idEaY-2u{(Xzt$$5!)1~^lXS4WR8E@{?!{%TC*%f|}X=d9PC
z>AI01=g>*fK=`fn9~PIiIxtzIiyn*d_Ze!`bN=Y%7`kaSTD;7u`mt}6G`nuz7my2h
zdNeUoxb%1kl|I9lUf$v}qEeBS|4WrdJB))TxjTNz%k9_N8A1}w7q+Zr9
z)x!CE@w%Wd_InROe$BFgh4UH_{=bchPuRIf8E;VTd~#3flW!XnLEg~9y^Ugo6ol|U
ziUoNFgeG}sazKuWrd42sJ&hp4-i98ML@e90Xocr=!65(SV=`V2;`sc7H`)1|&R$1z
z_&B5b?=Bd%)9xq?zx~%!)O(JL_*6JolIM&Na#VM^x5+i7+`jaCDS*TXa$*Ub9
z&dGw>z`mQ^FXgq|%!f|(Nce+7XvfwV8b9X0=C)to^YULU-7(jXGO-VYjXND-yO%Yw
zz;4PS5wyFR7*j1LZ0ZCVHg|rKsTz!%dFvi^V{sc!EA;pudA$3HDPt(1OP+IKkn>7Y
z+#rYTGNC7~69c2NRGz;7F^#1P8odqLk_VtWE
ztDZH7d5iAjym`*VA2K2D9{7?|9PZ^sCi#&95w!5$^96ugIM|dY{uDvubNwfsg%U5%
zFj<}$p%{X_A)xjbUJku-w;NJ5Ufh%XaLhO(U&A9e_Lm>5>yALC1qV>@eO5p$Z@bYJq{Fs0&
z9E@VmR|snSV9w@O1;o%yYtMHGmfO1o9IcC>)%hy^vp`*AotM>|w-X~Aiek@~3_j%O
zN)@_W77l~3=Q{>RFSL0xzZW1z8yI`OWUy?OGg0v%1kKL&e97R37mSb?L9?@4rYeDX
zhL=Pd-azaEb}6UNY_?7I>-=>Fzqj*fy)`Qm;uq1oB*Wo`j<*g;Ny
z`y6H{6U~9q$`vII>h*2rj```D7Y!s&syId8745=i
z=CvmFvcPtw+k$$KjB4)I{u0-B2e}uc)xqR5f+lRwG7xIG4RT9!_k#d~W#L2=Q*LvF
zAiHq0+EFn=M+jHmdYNNf>0bhT5e9>9JVYkSVSM5D9)i5u6*n=0=620*cAFGImO$W8
zp3nM!evc2&52w(rbh9bub;$i2C$w$(>u#5em}jX7tRppR3Gz!Ep?xMJoQ-1mE755y
zOZF{n%ru5wz<5-2^-6rw6&IDk8zn=U^(}1ZI*=fTGNDlM9~9wS2tTo`*1RmpU#UZ8
z91BHAwUAql+C{b~zT6t*Q@nV{Hr@^qsx;0cn2ATr?s?@nKoJ1i}Pgpdc4GLSlqVAq>l~u26}F
zc{z%PM+L`YW#TdvJDD)0{~sNvALKn;{`3q+#o$z(mJ>%Zdky5nZ+DwHbVPY__KETibwtB
zV+HJoLVoa<4|47+cKJ$U>R~WuS8Pr5mk;v!Y3xXI{}tn9REL`i*iVMsz}?IJLEg)3
zMU`%Z|DgEFmoLcejE=ac;xU=xPGSY@H$-m1l-v_!{tj0@W7gtZ_r$2(pw#kL4RX-e
zcGZdrGD`D^6|f&1`PJV_kkena&GkxSs-VWa37c1AQ;rNW^}8$RMA@v!EAJR9WWPc3
z8soG!y592grcQQTu||Zt$Dim<)PwAPa(?O-iZm%YIkI0yi#iU?WkRmscbeOW4e}9wq(mK?5#*dFCc0nL2=WlOj}$*43u>`n4sgT3
z_g+@kbpxvrG{-shJlC|lNNNsOz)XdHD}GWYbc6rt;X2PC8-J{QRy-DcQc~P0o+_U1
zb3+ricq(Y>WbISkuiqL$YrqDR@~!(1vdwMo=>Mr``zP9_c}kYF)Z5Kgz8n)|_7V5$
z@Y5n_r8o6EM+hjsKMNSFn?S(1*md1o<6zRqa0Zmqs!p^(gZ>6Yj
z&RB|8nAMoqDzhbY``J;{A52po4PYynmCA@EZ2Q(RNJuO}&8PiBXTuWut#%AOUeC&g
zZm!x^yZfvX(trNRGG4DTm|Cr;!E1KTl{58TMD&X4bIPoyQ~PM!FU%?-%?Cj?V4GIW
zrcr169qycI_a@SYyw5AI+Ds{DPkeHdRYJNp5p8fi{~$mARHzm+?Ij(5mgh_z5z#=I
zRc1Apv#~}(Rtf22iRfz<)2gZ5^qfs%bEdtF$kF7TuDsetxQN9w+l*Nyq&pGOEW@;F
zKA9VL*|l<}zLW^2;}?`!jb{s0#f&hkg!DH=G(4MDy9T{m*)L_}OnV&>{k+VJ%B%J?
zjIDGgB3UJ*?fYC*6fjdngfI_+{nQw!m`@?FV48N=QFSq?Ej-
z)t0~|o9%!tXWCPVP^|bB<+bCNNj3DGlxLNa9!W%fovE9V`J@N!lr?ACwnXIWSCzRK
z?KK=(Gb_w0A$<-JJx!+7HbCo#U86Q_BO+?lbCkCQ+Jm5%GM;9Yl5RkZFMSeIm!R$f
zpCjj-X-_1gNAWdfE=9X#jon683F&W$97kT$wnW=@q;0n5Oj}HZLy>r{@*aS;%1N$m
zDIt^owcIr>Qy+-Be4e|o64Gsm=)&`qxeV=UdPzNsN=Uy$MAc;4gV6T9SjXw*In#D0
zLPf;$mA4hz@sxxXkX1r@D3Nv=&rN+W>J_BW=5wZ=PejxH1@d
zv?r!L1nrxQw{^&C+LwvwQCz6JTHn|FOY6__tP;|@XnOiJ0@G%+x}QDU)Hzc>Nko|!
zDYI5fIjpJYQbKw)5%pZA)w2HP3+#|3XWA=?Xf|A_yjs(rw%SfCvPwuFNrbUFF|8K#
zvl{R!H>Pu@eu6_&OvkS)vsUx_GPLc-g_V%r+1M;4rqxn@Ckt`qHEoWF@-9|ht>eGn
zV!q5OA^iyv^&h6yBEDp%>jM_6|4r{tOntx`%G?2Ut77X=vr0%85urm(OnWric}4aE
zT{+V}M}+PrUZT8O>UU#Rt~ypi`Vt~M<%wx^P;>o-?rAdZg+%Z_U%e~yaj5S++geOk
z3F+&J=!r0`7W{Y4a@FLtL{v?0Dz8@i`+aGT;&dex5Z04uO0D!q(u~mxyZV
zEh3r1j;J#o-4mgd^esf>Ekz2o#=rh>YXCXZUPwfqUMB5HXy-6%r^#oPkbZ`U`X1As
zj5exeb(k~lAR^3qQhi}WEQ~8EN9w;h1Q57#&<}=a$K%MK3m5~07i0;_5XQ5rsNJp2LGwnx2sEBxl@@iH80xzbj
zc_<-0aX9l2wGUHwM*VevS5Kz?fQahpJ!RHXf4Q3^WR;Mf=Ozh>X|>K@ywkj!Gwna!A)kBkbaDa
z+NEi=jQtl$kFa*>D+JbhE^%F7VsawrX^B&Rtf3Xh)~c)irCkQM$_tj
zExk$&j2P`rB0M#_11ye`@}^)xWl>6c2oV*<)LO20TW8vwY0o7hXMCc}TB4sp
zLYllvNVg%P-8<82O@1lILbxht+7pT3s@;m@S778*8M!tCZxgLyZ>%UK{UH&ag++?k
z4H@rW&`v6{w?uPBJBSERV?`pGX6XtPYAJs59R0p#c~%MORzzqxm4TKR?K)`v9@aid
zc_wGP8;Mc&-Nq}x+-EY^72dN9qG~=$NRK8$^GQs5J=#LX3_01fo9kQGv|ABdpMNeR
z|A97Wx(>+7vr0(cOoXB@QY1ga#S-J)0Pot4&Yba1BSM|+GM?PNTIRYzyPH~)+m(>M
zmWbSL+8fa}-rseXrmaGR&SRG%=Jqu*auc*)_R|VEQ=U~yIw8g*w@{J%j+irEcX($o
z2c@^k8LbTw`sQ6mQ<=VyxgOBAA7wAQWtEWrng}D>LPheSB}Tg$TD`rjz2=Ozvnu27
zF_;Uy_)D3)1>UK2Tx85DA$OAiqV=Ap-buabugzu
z;hoT~LZkMtgmfn&wEu-hlQpB=1+6XXG@4{4XS@c)DBW)3Ny<jQW+1%fiJQtAZx#{t7=ai6AajGDb>8O8
zDIwjSh|DQs=8QHN+Ivi+)FzDfG7+^2qjls~H_6%%XvZ9*IS&3;O1e2QW=IPaF>}Tn
z3Xexx{aemx8;`bi$F4w&b22v!+Jcrg++~%Jo=SwSbD<*En;C65w3fS7>*YC1sJ@%|
zkCsie_=+DrH&&o<1gVQ&a6`Q%JWZsfqb@S4I&Ra9B=uPOYMMdL5?T<^8c|k%l2tAk
zD^NI!)G;~tELcLH3^6V1WX)0@BvQ!WrY_eKHZ#O%{RLy-&$3p5H3bSsllm;fCG)1N
zOCX|E57ecBnwVw`sTZ_zwQLE;5Yfs^50U)g(KhqVTEc5Yv`mxPKut{Z
zFsa*WyS)%g_=<=YII{Xnu&O}eSW^GG+;+FJmavXUTjn2THgFSDje~nT`>67PC0tG<
z_~6%IR)NAtp!Q`uLeHQjTt`IC*W}I;A|f|7aLZy#=t4xsR4cz_WF#hb^C)-6zmbr>
zkO&K-g^J|s$y1>4QD}WeSYMqp+BHOY2I4K!jz@b8%VLf5me8Dt*7Nvcp%ToHX&xi>
z1oC1zlT%8%B{9aNU4a+>E|U}BJ?XBIRYH0gk(2o{R}q`O8f_x9@6jBLyNvc8kxp1M
z8W)NGkhMwBI&ot>n^`5K4<=%tlAHQ*)LYMTTMw47mWWo^lt2Dc1}4kET(?rQgoBA_
zp$sFjgegSkla2m3X9-Ud(Yh1HkgY0o6^toR_ynm9=Q93N4OvUc$C@5SxXS`JG0l^t
zp2;*v%WX?&OGK?#Mf*$ErV?4h+Y#Diu!J{=9O4*>B|Jss>)F;&a+dHmky79O;%zeW
zG?b&8*dUv=gkmBsoRNh}$X8P^%`>E4NpH_!n6reFiD=zViFO5Qyj@nOk$N&KZQZ9O
zv?Ai~^X=_we>PN@>RGsbFLOPV64KWZ(e}>{S$Yob5A-eCmsCRf10oC%iD{olI~;fO
z@?g#qZX=@o6Sec5%AY0Dg+$g1SxY$TEc!of8bEnlDb}V;HJ#L9jP^J*X9>NC$f2^f
zOV(Z>vW3~D+PfvJCZhJv+9a`r7m55h*R8)TVHJ^9juG#cjT!R)V)C)kmbH{;iKx-Z
zh$YMh)Vy7qPL+7iwp;u(o0%z|j`;4mS)2%;@P1ynXZHa!lgXD3X7E(>t$H$F~Hm;r-byoL^|-i`V@!d<
zSD-cF31fMXRYJOehz2F2$(jP0!dKx9eAL~b@p=%^kQsP!f0>&D?FBSE16d`c#}Q$i
zUZ_aE%o*)9Xjgh`MmyPC3$%EEtj&ei?g8F1%wSF_=^|p904ZYTj5iP77g$g;FxnC#
zY$)ssw0NM*&4+f^KzFZ7NMB7v_o|3lGui@Z^KlkUCTFxKM`mmpw=3}CK{B@x-sNbt
zv#W%3IT4opixn|*Mq31J=Kc0Or{ks^+FZ;bLh0q2ZFO64Eyj!6nO$#`B*T?G0#cF~BZQ
z&S(vY&_jXm3HgitA|pn=@V~V(Msy%gQp;Z{l*bH6^6=$rx=d
zG40!E*Fjg`XWF-jXm@Lb^1g%i3`+7)PTU>mQYC~bDZP$A1$uNGX)AiBz5Khb4b<_Vj?Fx>KlQYnC2r=&tpl8
z?{b#VfrzIL7$d7%&nZy23hFNyqN1~wkPN_^CpoiA0yi<$$E1$&W-X!5Kg>QPv)Xl0
zpzsr@?P-oOYYB~r1gn9XnC4TcSEa32%~?XG6S3o62}V9FvmG(3Kw&~^Nq_4fvX-zn
zk&ceKB2W|4d`9X7?^;XfPsDq6zK|xQTIB!)=Zw#(oqCzJPj-FYLICGN1a7(moQ*Dp2?(+_t{7meAlI
zg$>liG;5(Y!I^3YmQX;%w}bwXtkSwhT0kyVpm3eq|2m!uRh^~0MZ{O#8zIw*kSQ_B
zS6t{ZY?CKv3H^xpJOg6P{{8<{F*#n@dzDT))LMlatetVo|7g2U#7%3-=P1}
zU&s=c{^LTA>Ov_P_NNOerEopb=lq2%VT`|!uJixpNgg%-e@mtsJPBH%i<90$uc;Nf
zBKgJG9xvP9L2LZGTLW7{6(X9?U^}sd?}@Z#fYWek3AKrYS<_>(u>rXvfP1}nI=bF0|(FNm4
zGV&9&0tN()XG%zKVm3y#FH}Ud#ArW5dz!YEAth(Lk;JqF3cUDnncEES1~k-0Rtf2I
zh@5~qMa-Phet|Zat7^10+ATyFb#?_>JXzL$g?1AD$W%-t3$X6B6d8@%N>
zN~fX5n?*!3i$IH~$lMla&BwZ_fD+Qxh%gmMO#3_9gU7k~glYFDayWi^LV5o{d(Al3
zKN>ldl0Fr+4r5IHC+det(9VZx?oR7hQWB83cfdE>d1nt^E_AVM=3I}NSb+3!g@g5UnffP+rMCv2S7zOHIk%#=qJ(rA5w#W5
z7Ao%;)8nS>;u4oZ(t}AeE5*4-p>7MT)rd2jlGnuNxWF?HKP|B0L)LbXlp6
zb_xmhCXcKPA#x-oFl`OA2?hnRY+4$B>{oyJ?#f(K9wfdG|-V4Nqw{tc3Je
zM5utow6)RxlyPl1XX=k#8;)lxa~;(AB&d}tA-w_9YNV#Ei}otC8Zu1Vi3l|xzofkN
z(6%Lkc9>N{x&aZ|VPe|)XnRu<<;|G-3S!DUOPL#>-Z9Wsm=e<85K#}RNWPv=qv>O~
z_Z=-~v{xK$SD?kSWvwB!H!+WgvPwu#CxV9(MT#1AKs~SAefV(9NlyOfX#Gd_SUy)sTI?
zyr8%CWcbr^?KQEaUH_V6ZT0ek+g*`tm%&(tu&#N^vATIV=(c~@nyZrU4ePBA^w~1H
zvV)^P(o?kZ#2EXaSjNhs%0FcLn{88O;5^+0BXq1l)9Ns4Zr0h#-?H^x+J55?Rw7oj
z8bo?=Z@O}j543Nc$5X05&Ftrkw=a?Q)zzf_mSfm@H(!XBj2lyWKv}#%k%lBs2F$XjdaWCaJd44Ust#w`}K8ux~_0WtIzRrUbNl4
z2OH$;+(dgbC&#=lb8_d}U)?*qK@NbX#ew-RFcTblRw?9?OyE4_9=uK1I
zcSkp6e4wAZSYiy#=npv0b#p;(z{!kJ@p73sR3>`4(gfM8)HW0n5t?c<;RJu*LFVWv
zwGhzLAFq%Zc5kCg(xnaZlc;C9l=sx%Q>u0s@^5yHX+JOHsdjvp7~yaThp@Y?=lldO
zyFp;^ir<$7JtOnjDN%EL*UR_41tX{!P1xsx(+us#Ghlf6C||E#MG-%W
z*0_8-$k!rXCH@KUdY|DskBpbgOY>8w>vAXTRa#W|HXrveNX8#W*Y2fe^Rcp(ldWr!
zE8QJl4nmed{6vJ4AhbVPlO8^w^>R^(y^fa{;basSqROe0iw5;lWeK;zprfN$d+Gk@8?{Lj3$VoT4EpzMejddDk{{4k(ra|5XOHYhg13$x7
zzS^n+_WfA9M($AzvXWoZVG@@Z;S5e$@;Xt+YWG3Td`A7Leh*?nHTmjunK}!?zv($d
z46@aI_Ll3w)w-w#UFf|8mD5MmP_}
z#q>4mOoQzDq8+%$U&_MyD8BgBeUcXBANTSeU<yvX{j^Jrv#gG``A_yP(u6?zahtyA{e!#*yS-2QN%R+kb{Rwm+AlZiB
zfVdG8(Y<@?LHa71xVN61uax03^bP5q_2dLupWgXcEPfrWg_udc7V!!Q1I}=LP>`>E
zWc_^njR;plsLEtbeQJ;o@UfqESB-ELilcp@gZz{)_*EI}W#MWF>*_iG1X;T#u55(~
zV_XBHe6wp*K@KW+KZg6QOk4}0FRe_|i_u;Vrr)NiBu4lbiYi!8Qwg#^JteF0f!|@l
znxz8v-wKRl4pc`~uYlxp?_0$tQLXBF62BL*E9u9yaVCS@AI~sFPKX7lYD4|TMw#e_<`tYP?+3Z}
zA@&wdVuTw}97Qu#MF-gq0^4r<1jW|sY7sxkQg;Z=i`?x7Id`R-Pj8BjsUhMf5pRaL
z`$E?ug3M!F2gN^%&=W#EhGSJqkUx138le}8-`csh
z6=d%t^HMRdEd7Lq{5D)u0sB7zZrJMPy%iZBn9(9X^)BZL93Q@k8e*A1?}h|mYc!u{N>2l)Wiq4|{&?uPKM2i;eiLH61&m0HN#
z0K3G<@5l}PDqHtJ-Cp07H^?LU+qVUY5$=U>fFD$XY{0sOnaIH3WJ;a30`}hp{PL%}
zYLIoWw|+4(!hM*^oQN%TtrvUwY(snBCf*_w_oLYFR@d7#@voeUlKgAn>zS%r&IJCz
zsimi3t1JzM@J>TFa0huip)~OptxP}-0
z+$N@~O9A`o|IB>8XVW9Lj8EAlZ@Al@>ZL}l!0SdGj^_L=?qRID%gLyZF$Zp!QLTj)
zu>TBT!kzB94sz{H&ZaT7STp8GY~Gpq$xUk}dU*2FkshQahEp4O2U1YBY*pxQ5m&LB7CKc>t|AF~%4)E$JK7u!8&s
z2J7y4S9H(*Hq9KrzXr?Q(R>$OvwvyJDsjb_sGxRf$=;O}V|u(ol8bAWx?)XgSqS7RICajM;>$C&*`PxynJmE{XFKV?GAss~21|
z`l^->ynd~_iT=^L+NI4dTF_rvv|Lx9Xd+oA)p0%!vNy9kb=U($m;_Qtm&hCtyy!+~0>W)c1W>#VLY($UAd{
zOlW1kuXkpU%XYd#7(re5mp`~d1X;)5#zAM+@5M)DcR2Gb&jhsV$A%e`jXbj;O-KgD}TmBgk9)Z9F0i#kz*K5ab$f!3fPEoN29!_U8f&Uf!9JOtgS;x4%Y^
zXZ!mwLJ5R}e7g&B8tqP=h-IM^!ngFVs?Z>}zT|vqgqA4o>@e9e?)36XHhN^@QJFXZ
zMrU7$AWsN|SYE_9Ghux6Q&DN7l4QL2`S8zTGN{Gd)n4{Oa(Wj(7PW4uDfnX|s+0SK
z9$lS4kYBLuVa*j!h;D(%e#xkY8ptL_Gl*uAiOQ#bsXDVAJ^4jmcI{>p!FZAg>X+_o
z=0?atUasiuRwPDHzx3MkZj}(^VNAW$Lq9GHDpENknTY3k`Da%abK2Y%V^LFeOgnBZ
zNVVx%km?FzupH#|DoC{l-M|E}5)|B4km>+>fXQGLsCIin>NwC7Oa%$3-n$^x5%dPr
zz#35Nj)GJt(C3bVOe#xY9jJ3>LF#nS7t8?bL4&&rQfGq-FdJ+Djr$a&x_|*-4#ku`|$wi1}1=&pn$v`Ko2k(tOC_4@Bruurh)`ie*h1F-e4M718Vid1E5caf3jd5
zsM8-0fWBY`SPvQuzyqKH%my1kp6+!FT|S0F_`1C>w$Y
zz-X`pWVZ58>!El6j0MZU4$x*89sn^|4)TWM0niOh04qVk2s{9KfXQGLs5TN0fSzC~
zNI>;bcmVVU)4&=~>p?v5AobseKo+b6bw=X>&=<@A>p_DtcmPy@*zjUn>>lt_ej0MZU4$x*M9sn^|4)R{Y1E3q2
z09Jy6S$F{S0F%KgP;E9I06oD}kbvqh;{nhcOap5`t;{Pl0?-F!!8%aqRXhOtf*D{v
zXfOv4fC?}hYygd4!vkOdm;-X4crG3QL%@8n8I;b$17HNG1Y1Dad^`X$qxokE*a})N
zzyn|`SO#{0HVg3ph{1A@w+Ih_ZeRjf2?{Fl0O$cGgH@o~>v#b41XDo*sxQU^pf{KX
z)-0y}YrTO7Kp&6=>p-0)cmVVTGr)S#U@0B|6<{{l02;rE2fzR@2joEUTX+Br0rSCT
zP`V5cfDxb)Yyo9&q#0ni&v18Xw;Q)?w20DV9ftOIpEzyqK!m;u&<1|Q-9
zPyuFx4PbJkbgJ=3cmWImb3hIhufhvp2$&BxgVK-j0vG`*!4{Ay`vftx9iYv(cmc#Z17I{*0=9zIzu*BN
zmySg)13N&QU-1Hn!E%uI8(sk2zyz=o6l}o@pa+-?R)K22;|0(YOa;mB)PMCqa0BQK
zrhzq})}MF*^Z{A04%FF-7eHSy1FQ!P{=y5O0?Y;*K;vzA0So|hKn@gd#|vNxm=89C
z(j9nV2lYRKKqc4$%68%fFd8fYTS4nxcma$B%fJrMW;Y%HF<1`xrk`!nbgCPe09Jy6
z{B)`V=m92!RiIibojMN4qdk#RS*E2DP`!#v?1=0Qrhzq}RzW(|3G@M3unyEor&Fhc
zzF-De4;oZWr_KfyU^dtQ8t;`(bpZpw9FPOW)zYa8!4NPXWH$3pX<<5bDHs7N!4^=q
zcRFb26Tj-WS~2G)RD`{4!92V}uIP-lO<0Q!O%U_EG18!vzgFdJ+D
zjq5o3U62F79FPOWb@4>q46YzBA8ZDt_3#830V=^3P*xvLfYD$H*a})Vz!P9BSO#{0
zHbr;>#9%qdYltU6H!uOL1O?n^2haoPrdMV7r&?oN0eXU|AOY2z;04edOap5`t)_SZ
z^Z{A04%BIe7eHSy1FQ!Pitz%d0JFgc(6~8m0So|hKn@hQp#Cr9pCMpA*bGWb@BkPA
zD!~>|R*DC}Xs`rq1+81+0WcOU13N&Q1MmQd!E%syARYkSzyz=o6qMls&;v{^qyASB
zsCEz@06oD}kbvr~@Bruyrhzq}*1>oH^Z{A04%BIl2S8si1FQ!P4#5MU0?Y;*K;uL4
z02lz~fE*}3>|7ebVR!({2b)1@8$19;fJ(3hl(oeJU^G|)wu08}@BkPKmH`&esU1-pkh+h4)_iT8V
zaK`^6u$Qq!33~5TjPf;#Rmrc$9)D)D52y3W2UNOby>#M(`|+xv3#uruE0|KYk6nmN
z8(oMzJ6(t@qhFVy79vjdL6~!V5bAOt)Q?VlrLbKQUHXc)w~e*83ogtro1M
z@{Cv_YqeCK$vy~Swgqd7u*?S`tnonzn=M#hMH{eG7V1|cHMU?qrDiieOmd|K>&V(F
zAH>=QAB3>Of{m1K^qaEQNYnzwr!7Ua%Di6Ytw`Ad{ObOu$
z7H%oV3?D4S#XbljQ83-owYCk4SZiy^MiH%T$NwbI+D0#fq1S}-#CVQBJg>9@7p_#e
zuZ5#=hm_u(S@!nd%H$v6t^WuQdFO9VtZ+)FBeNB%rrmsJdV}IpY*oNYi7tLwR}v}>!gyp$5*US4$O>IpjH7kwd<|F
z-TdKurC(-G7-=tUaBr39Nv~ZqYI;~{y(7Nj5M256_jNB97XO_tZYxgdr#)gRr-8o|7>9`5`pO}NoSmo7J6e5y+?Cft&}Rb9FiJXgoPx}iP6_3R1iPP9+>
zF0bk-tRuZk_(WOZ|5w?YfY*3+?c?Vh^AH)tToF7mkFC;Ko@!NFiq4}}=T=*)efxHN
z&(Vgaq-~JesRlui5<~?lL1;=41Zf+Zwn0h|goZ{N{O)`2wG(~6@BdxbKd#5Uo^`Li
z_O$l!4Cfp^Fvu}K*X8%{mUtW7R4365$9o5j+#Pb;
ze;BzJ?Hdxo8uKV>VtRqRc4v
ziM})B+a;LTMIXQ?o1Eo|kmhG#zA1XPU{0*=YV}G#r)|_c6qpG(Os+$OQsxM#h3IC%
z8CdVDW&NDqk=u^z{evFvn=k@dZpLiAM*0H^#p+@|=W731JfdN%;;;5aJfUMrFcZ~n
zj~9@Q?5kUbNr9!pU!8e~c#-R#l?QmxH#~UCv;vAyXO+C&fMV6{i1EvU;b^!D5lv}n~B-A9^n
zj;D@V=J>x`g${69M4gphXR-P{U_yoi=4~0t9HocUFqr!P-O|vE#!DeJ!EzQ8087ih
z?ub%SW>ftI*OWQ^{U_MS95GvNGB%8u94S~cY;xoa>DG+_O;07U)xqKqc{o-}2*Xt=miCj6qza25@ht+}ScSu|XQ
z=4#>91{TI5IlKXodW=rrk%Sga!XnLmErDUtaPh~HE_C?;ArW?YwIM9B
zC#y7*fcF%b1nAcSsq~ZdHujt;wMBRsQ~mKW2@_c)Q>K}A8%^OXGSZXJ(%l2-I;OEmra&|G
z=NiT$nb<#cnsM?mvIoZ148tr<4Hb<=7NJ}-Z)KReS|pQF0e7K;B#lLML!l
zPZAaxyzzgMnRF-mlA$W06dyW={WV!q+01rIo7hEm+fvE9rK@8M8~Y
zNQ;CEWO}@1(pn^wtC^<}DO_+YlBv?nC-6qbBAN8R=&c;fGvm%8zDWt=g;cX&(o9Ac
zVa0Tbf0GQudnT-#MIrM@hU4bDZY_%#Y3(s?s+i$Cj5M~SMK*p0neY!zkU=;b!GnDz
zla=aZ+ojePnJnod9X9^f?EEZ}$=A$G>{~e(StJv4iD{N(n?hJ5lc|}1`6Z)&36d()
z)YeRskwr4em+9_9$;cv^0?pL<+GJ#rOl)8p(zFo&7Rh9RQMW}q&Hd%tW{mi?NSl-i
zOdY^RlU1`wrcg6w7^`ewi)7+HWSZUBaHAISFk%>}2qPxdX
z0cK>8Op#_j#WKsnStJu5!!*4)Wmac_}Km1h2w!D*39dIG&QMVoMJS~SatroNQHX_1K%zd)kU88SF6
zVsPe>2s`<_)zm0Vz8q*4xot^OaMvtb#+OAH
zah&CAavyq=?O;*Jm_!z&gUoss=`&L^-(V8e%a;+53RIb<{*B(CM~h^Vzoff3v>zFZ
zWC}F15f{`fyhT)3eDUuY7RdzWGtFl*
zaxCgWm7%E&^bV&!i%e8XCVUWXVR5@_5qAaJBivKKHA_LbGh!@4)ZwnlZ8CXT6f$N3
zy?%>p!-k$Ac;{9NTodUwQS-J%E^TPh2CR+7EM`FQt9tQ>_=EmK}-=yR!Ayz0;VFi
zj73aEabGjdEZnf6Ka03!*TZYbP_gMZ;BUE(G1hbQX=zv^4l^<{yJ{vOO$9
zWH=OS_Ib>wBrOV=u#`!+ePc+ALgs4nml8u-6tYT_fpvxi$^|=UWSYK=em@Yu7A0wk
zCOe230L^NE-U#VZ&NP-)Cr|LAvQ-iy%r#gte2P
zu*eQhTESveRGB0er5O2|oGNi4$o|)LkNJj4ACjaNh0N4sUrB0F$TE`QNq?D6wulZ#
za3rrJ*9BLy>_&?wI10%5!@J7#Xwi_d8BDknt-{E$Xt*rR9hV8vqT$Llmx0NE=`0#9
zWfgrkK_`(ji2B#}M53gHn#`4pMT;h3+-fGAhk~*e77drJxr4I0TQnhFL9W>ni8hOH
z!C*^WL(-oj(PmNDBC=sA*n@B$wa7k>&m{8_Ryo$fBAFaA&Gt(dS`>$sn*3P0&?2vJ
zS(vnMne;;pH_j#&C26rHJBnY6LME(b$_p44OleWbTun|v1CX>RWEDw&vsg3$S&PD^
zuLB#}RwBnDYFVO*oA7xIrm_ekF^efm4w>$?2qI4t+vT2&MG%4Y)9{h0&>tj{EP}~c
zPhUUdemEQ1B8XB=d?jIG5k%4krce?i7D41|;sFT|iy&e)B85L}I~tvRY>_>gNvhfX
z(peUzWy&;pm-LlIA(OM2a=EmrMIj3`xlj5^qZ$7?24Xic>08oS7A0wxCP#~3i$a!@
z41GW@S}dX-DVs^$fg4P0MvEW{HSq%8h0?tUK_i&B@0et@bcsa}*_!asU*|vJ#4zT
z7qSSE+o7~ABnKl{dB@8lNlZD#Alo!*mVC@?qb!m|(uA$*m3y5w-dq)XFJ5n|+DcCY
zB^!(4DLt2@D~>D*S)$2v_*^jyXi><-ZA=*_q(zb_LY^k;{DPEhCoPb|AK1WlCVd|)
zuZ%4UnW4#FBh1q%izHF;Qcb#&(xQ+_JLon{NQ*j7k!ikWzgurIwJ1qr@|bjsG=oKw
zNSUe0UeXm7C1sf=yGTllLMHE=hVRsCwow|%qWCQU+o;)6NorBZ*j*$CW6!7iUr@*_
zO|C^7@rj~El4zrHO*WO{HuZ}jY|3u>^+~1{g)G$Mmr`7dB+(3U`Aiur&1+G}Y)w{5
zuURCC2YYUXX0Ju2>~M=@L8k7Z-(lj{qL4+J%tMKo(jrNeD1I+fc9WDAh0M|9?)7-E
z#-zKYI#P{5S;qscj9G^(}_It~4~
znpi6>U{O+4X|lbfvPcpoP5*%|Pe{vJ6tYBeqz#<((D#Vq6IQE86nMXQOHtFW}>yZ%Wgx}Ej@VWr@IUbAHM&xTE
z2-k%qEFwkBex~>V+cLhvU=g!trY4r*4FjgI7>Sz|nkbXczZ*pU(b!WSpf5*!S;W4$
zfJ9i^BIC=VzLfi!Oe<7}xmaW?WN8L3@alqDBF(0v&`fGk$oSuwbO4q;k`{%`AsKcZK9I{EvPhqmWLoB6>1X3v^mr%S*}+rX
zgPX@<8Y~Sloof*W^Zu>*V$HX}gDyI-814OV-s_rAIH;yP;zVGjTl@$SG1>M8aE!=G)m#W2Z<4^jXZr;GIlx{2r;KUPfV77;R3%o)^FCM`_
zNT7qEV~#NSu3)o^ut@%1ojgiSx&
z&c`j%{58^E7HizZmoa?->cRH1Si|RNegHa*TX>7r9sZ14sSSRWidsxShj?^{rJZC3
zZ%I8ZG6TaGYrcil(_#&ua7s;i0v(onihkmxq84lNsnUFeRMcXPpY+pAzfme`v4$_v
zd@Fg^#$vVqL2hC>9n?K*w&xaW9OP;KZd8N=(_#%DIHSglLPavp(9ddVFpJe0xuu#P
zA`ND-#!u2&ray*?Ffc6E@cEiQj>(Ec)FMZ)9)f;1<_|jPEM01`+Q7}!e1R0uVnT?g
zwi&}^ntw$KXfd&xPp+W&=%5rCuq7>82UyZH-%ncF
zVhvxejz5W^k#e4H(xe_1W2(zpsQFu@9u{ld#Qn+izez(|tl_gY-^XJb+G6!o`HWkk
z4e(;3sfQrzX(r3mN@fs{dRk-#hA+~5d#R_z8b1DlI`I@DD(3?ID5L&b62N4(=G_Y9XAXoFxpd#cg*6>&mM?39C
zr&rNU$E%E)#h5;JaVgP!3oI^-3yb7UU?yH<`lWa`l_SF<_>P9hO8Kz8p`FkZeEY`GH?1xPM2Re?B`q>LV;-2IUWT17V~Uy#^p{T2
z?y*s2@iquzAKJm}QgnWtxwYZK1^)KKV0d@S_yKV$DXmKz;H8+9&pNwfF_ZLYCTwQ*XICMr}&GYWO0w
zg{td|=-Rlc^c5Ov;@4u0uWa=a%)Axq6M!#unp(`n2rr(-Pgloznxh)LglMarPG5gw
zE6#dZ>Ns^{ljvZHw6DdQ_RZ3K`8e}Bum;=p2!K_LyA*(R~r#FM`r&qCv7gA6atG8;IHAn)?H
z*_zvr7v{Jfx5$LJY9`lgw>*a5?H573rp_Vz^m&ugB1wG2t4Nc_@G>ghT5RGe%*AgW
z$0svohgF8OND`mt%hBZa<%YCKvUWRMIjj1wI#)fMHkTeAOf^OpX*AH|7He`EUX)|r
z7D?h=>V$brc|=~_v`F$Y-V{*p!@IwVHqB2QHx4W|rVUy%eLmR#5rrU1G|?H?`wV)E
zAQDrUVg@EL5*BeEAWsvqcD+78XbI(XRx&WUM;7CBCSlr-8U-JHX8@=OKDmck5;Lz
zn`yK;TI)rOQR@v+yr>mjX&1C$+A@0ScF}n0rY5}Zv1fADNwC=3va6CQXSM1t{HCz-6K?b5nEN7-HR3_YtWsUE^Io5pw;5i
zpqoo&O(>P{v|U5-ZxPNSc?8re+F36=Gt
zX+oR8H?(;cyHwg3#LwZ-CO{L~WN4eVSWalu_GjEFuM6+mD4I4jD&=X%O2i$FEJBRr
z5x{@^^Li0?Nh_IWiWJ|Vn?a?F6Q3R@`P%FSDZW9s6iUCfqdyGM47zJ1#WUz8QTY#d
zncA$Dw7x-iHNf}actYIdRWz9;HJGKmZ#qpMF3`$h
z(v%jNE}H2uC&jKtvUY>AOha1qjH^$JL;AwigDD_Oo6W*oOl-XkQfC%Bx-)+_M#
z5|tJq+ETeGYv}!3JdC7;#p1KU-t79GNr4fbKrMPNq*?1X#~Cw=S2~Kzq@%ZG>NKco
z{mW_EJDW^Ud-;%~LHAcGy@>t_oo>0@;RLEt<1u7Xzh$Dr5oXL8bhD}aNBN*mVKFeZlJf>uJLBjT}>sMC67WgZxb`bHS&2)=9r+7`1XUqMzU|q1Yj}iM7xprq<)4b
zlm9kBXEE!t|33V7NU0`Q;N34B+JcuZUw-H!CW~yiZJvfV3P@Vy2l+2z?ECXc2lZWO
zN^H=L*~Fr3m~E66L%IjyXUW`5l0jdh|7yXZLAOk&cnw)oX;BKFyqWICqX;A|>cV3O
zRiN3%cms{BMJavkcSzcNHdZZ?7H#RX$TmAK4}dI6Im%11s^s;Td3s}R;cMGu{|bh(EUrZXo0xznYgh|gA$vx%cx}9Y|TbM%LX^-90NuB4L1$nud=DQ_gojFPGeJDeh7CN-%%z8Dj5!KH
z%f!QuBuof!kw-0DM4LhsYomXm#W-sibSG@1ivV`X6oMWS)}Xt}vbma#`P;O%LHC$t
zt28?w*>n4yorKq0(I&IDi|cd>L7fo5+%y_=_h=SnFVSp6S&Iz16L&Bz*hC6J-y+VL
z)}Z^XW>M%o&AupmW`pj@9n<;a>7WC7OgvobWYC>UCD{zkwi4E$`-5dmHT$X5X|d_%
z$vc@AU6w>4=nhG1&|R-tc+OXW_YeeUt#!;UTK@`PjHtii3-jbcD(NdzvjxJQG3)`$
zmT7iB?zeD^7<50|EoqY}1ikmS=@*0UCd~#oa5a7FHD(COKKxHjjVfeAz
zvIKFT$(>K83GM*|@DB=)=Qc!smbM>`X@K_63ex^3GC_T4A7kuu_+>uYeju)~X@8d>
z?WgSd-~6Xg8+6|VV?RoxyNycyW%ZivhCMUAU8~W(XxX^EO#3&V=zn@^J
z*CS?vDOYXOBb(ahAD=LurU@b$cXICUWSLakQJf`EXt4nC>gR0s4FqDfVWu_A)rRfms=tSz&J}Z>TW0O5v|aHc
zlko|`YP;m0B;#}nE!vJVSCw7Id57!vt2QjKh9wk&?#6V)P#LAsolpp6H~1pFo1PI{~;Wbs0sQN&pznutVVYYl@00!
zXyjvH=Tp(?Uh(#X8XUmw#F(Ge3jwFc^_f4@#hN1HVuwa|pH}+dB7liX*0M4xqe^60
zoK|@O+%--vg0i_!!n?5`vqsOmo5^}
zZV)81kWAeeCWywwYa<3R*Y|`(^;tSO9y*DUEA3XTJ(*2y
zi>8;2wu>OMX5HNq(s~ZkcB2r~Y_69CtL+kt#V(aX*CSV(3y>pf@rMZ2A}WJ=R+(*%
zLHD9gf`*7c2(}J9B52$co-o5zR>;xxS7%v5x6o>jS)r0bvr-AUDnS&CojmIhz42|g
zG)gTtNzeUYdl2=k&wA!-L)aB-yQ$I}U)f%
z2t%{zTvKS`MRpk!f?CKsMF!n`Dr+W_Qf>1amI`hX47wj3g-wIn0chY9p8QGF1RX3h
z`w)ZfdMdfq%Gd1caKZ`OpnJx$F{Mn~0L9=-mvd}S
zM5kN3XkK2kBTf
z1WJE7l@Z~%V&mKwA?U%kSUrUfuM_Es=4FFA0W04GY>HDTwe{<3ReXG2koM8FJ&|la
zlwv!L@RQrHFVyzmU~@$OTLr7_BTa%!I)RS~;>wV=YbLI2B-`ctZq!`;D0TRYU#Gj)%HG22efY_
z_+R#?YV4~hw#=w7(}P7M<>b#D(Y!M-;8k2Yg|>s_14H)+(()<(hd-3$Cu?1zt^c|A
zG*(X;p!Jlq;yaN-mwvKwix%{HeCog_PK)3?O;>L}XH0h5hXnpWlFl`IP1Sd@(=IrJ
z+Ah_bOZ6m7gG;q(gsi_^)#sBjG?FTqexq!*^3(<>L-MKYQnQg%r)U~;j@pTcG|o9w
zRof|ut4u1}{%`xpr##*OzfTSo{EbFzyfSV5&+VjX2U>fL_4M;%olK$g|FXUGM2C2j
zwY6G5#c3Q|pbe|Hn5uA!6WKKOPp0O6vU)qIil*Q$ewHSx_m8T794c3?mDQU?^&FHT
zDV20lvolm*#G!EusqON}IkU7rtG)^C1Bh+PPj;I^>VG&e&%$*o2S2g=Je@VMee_!{Y<+A
zWu01xjey6U%MQMo=>HIj$X^t
z+Umtu{Rq?GKp+98HH&SXcmz&{R{mp+Rqua}z$qosWjK0)1K>IJOgw6oGy%%4*WpS
ztARhCCj^~UM)NKo;prqx^N|`k6{Si3h{_~fuThz#CQw-(TbtsaO-UsH}q(2uin*CtO+&YXQAqP9Z1(Ly8NAK{sV0mH5s-{r#K<-9oK=8%@Ps
zi9t7R5-q19V5v0dW>Xp32iNS4`Ej_LwV>}qS7>IzK^ee;C{OCgND_9T;Bj3a`pCoJBZ*$HX{t_KQCa>Q5MOpyiQl7AzI8h=iO20-ut+U#eh{
zB>E;plU<|^7D=KHN;UbE)WRZ3?6i_TOW?-=r{I-DMx;gf%>N7-hJMuA?CC7RU(Dwq
znr}l4v+fq@kGW-%ZSgrC)zpH*qF$#1Xty#=-zT(2Q~#$t9`ecYNZzy!o;EVMMP1Ya
zP5n@1?p#~cRP0oy`u3EeEHYC!OH=i5^O&RFqNd6<)y*GaEH=c3_%h7ZiVKe(J16Bl557n3QHd1H4>xm3tH^>!wh|ka|7qSo>w_;cwJ>w-n1F{ORt{
zx3aa9zg{sT&SIptz<yh9
zfwA1t7^+{Ay$SVZ^I8-#k7VmFF*|Y$SiI7CU?weY$8{4|D2v+`oev6p*(fR=HO|l$
zWkGkFfmDX5(F2OD@<|)rD5cuuYfOX;42#GHXQpM7zG5~l(KfVbsoX@Tsgtjj;ix*5
z7DGMd)sYdCq~1<+B0FX#G5Ld-8)#{9YM=s^xP{Rk16Hj|#Evt077W`c+$^D?#cjzt
zlxllMV}>fAN$d3sjETi7YaBb97Hx2$LMIks;SGG!XkyYqj8#^vwEV+WZ<3=;yc5HbACs
z2D#ARFfX&p7PGMcYU0ZfLs$fnl!_EVkDwVDj|Sa*Dnol<+;iYs#7;5hYY<_6OvN0g
z)tHW3&zYox8X*lUWY8_6GIR`X;g!RMG%FW*o+4!BrJl6Tgnt8
zB!xu~*&v2A3Y{%UEP|=fN%%e@eOUyNx{SW+N_j1UC{n{_;iZ!J<>Y@y$1=%jObu2Hgy;#BWjP<}>J)QrWbC0jcP(FE_q`!K{%C|V%cPsYVI6-h9XOCoa1zFTCTZo
zi%m;wR1=b&R!vjBWuk$oBJXuRO;mdpfhg3(9O;`NEVA|0Ql#+3tySZaoo@J;!dJ=g
zT(Oo8W@87*)?Y$Yd*;9{b)Bk!T}aV78Z^c4BXAmNdQ$b8i|f|-ENTl8ORPnqdStHC
z_Qo8o3`0vWye*D^9MamuqBn}FAHa=$&+7QeA&X5B}qv|`?X?1fhiKsasSTKv}k$CZ9=Iy>&
z4eLfk)&D3OR44+ZN_(j-1+MK?Q|F;>=^N?dj1=vR%0hdY>^iX*&u4e
zKYraMk+Sm$1|{3-SJh-bDwRiNR5Ci8(`T}}VLmDq*hJ+LC>fed)C*9y%FxOQVrj8F
zx~_VCz7rl?O4^CQAms!ap*GR`tj$#7`jXXcSguY{nNMZZEmh_nh&xs56f{}PceHGd
z+q7&)jcQq{(+Z!{c`n82fzRno0|Y1M(5yOY)y@>BZI1#fo6o<^WNndg>WRdBNLHF4
z_IuTQfzusBsA*rMjHL5f7o!0)itssI%
zWSDj|=%!IAxnygDyT!nuTYQB<1?A?0#lRxV%K|6l*0@Tg+}ssQjr`_#{nr6A;}xbw
zDsG|E13z`B)-FWxbG9L6)T5{y+g_uJNWn<1B;mY+eJ5k(9o2r3)ApIP?aXIbg{jm2
zO9mMyEKwd=sa&zPyns+6yB0Gz(OL#})L14^cDgS#8Fg24kx_82Rt~^az@hP|+DT;<
zl~F(8Yx=q0|4Cg~gxmV*c~o|mpFZh)Qgu#s+FoBmZS(uOo2lC(Qt5)m>;zjoSzom$
zQjU5{;`J@{K`J6Uk4mTLlo^>tYB7BF3hbit(3>XqEaIjWrU@MNJ@|ngxUbDYl(9>l
zOLe*hm+Ca#P!G;b!&I-YowmND-D=p^PPgmxcf(=x19&r%m9z*^7qXnJv}Q~`vz{jR
z9j2*txCzPB${*x@!w*N*im#m(Dsr(?C%8;gLrU)h>?7J?8}Ig&O={2XoBL5{{mASuur&d|wnfi{PTMHfiov$SCpkqpGd>4iKJCMZuKv~=|wecJ0
z?uUwxF~tZ3k`~Yxf#cMm@#3|Bwn)|js-3O{R54o%90(NZu*}%Z2Tm|0T*RMHqgOhe
z@66T;bW*++;Imi@NLX%d17%EvUQVK*T_LA5p7hjhg8#lydsaG~FppQR#7_xjVGqL|
zLy&v+V}_)xdr*E5i;M1YbwdXBHu)#%lih$)mGUH%vpA<^`X+(VA9o6b7ZpgPpv_U{
zbgjkRfGn+L2cVP%T8mps#aG%^YAxy)bDFO34r~$ywJ*a7Z=FpslqJ-YDUiO9Ly7XP
z!ngUxma8_aoQ}cE%3)WxR$a`&aFDNtt#Vd}$DW0*SIz`o-j%Ja=Rbz!yhuk<Au(Vd&u0>i)u$Ee(tW9Y!g|pe)4FI9XJB_G#R%%YuKb~JY&R0*ozyHEDY;@
z<~Jj=ezwuuLADLg-)T#cPT+RXcEoLvZD-sL*>*^K$j26&PGQ^hen4IN)@dGPJ7zF+
zwsW}ou^q(WH&R`{*6H2Kb_;(C&2~)($k(a~Yn_gLZEx_$p=^%~glv0-KbvNIhCk|b
zQ2nqLvtp5|T0h39H44peyYn{X*HYYwk-M*94f7wY)B
z)8MZKYxq`%)s$!rd(3L8sBziXU%YH$
zU9H^Uw2b@}-z7mUdx*C2hjlb>;uzo2$ge}5jT{)lci~7-8<0a#
zqWqlCy-*!chc-IBgT~`ABCX!Uv0p`G!+F1J%y@nps*M<)4D07{96_d(U&P^g!N@N`
zd{2!>GXEWy%pJgSP&MpoTI3$cL-F1{Tl*Oty@wn9Sjcup7zTMc9vN_m-G^huwPvPx
z0P;>`PyItUPOdOEk3qh8wb4HddBb&De#rASP6iG!)5x8W`{R>J9C~--=!uLeKZm2$
zetx#Y<9QWF=Rb`+9&#Qt(ZZFUCqNtq5^1AZhY8hWg&nN)bO=H*fdV)rox|1eV*s}bMf2~7niVzvx3)nmj>
zOklEmur5=$nFDtN-a%n<8eJJ${b$vLnGHbN3`sT}ie;sePU=KM*&ge~7j&J4A*h9H
zj~l&+Qj-99{g-fe9c+Q{fDzjIeW5bTToKPhyW>1kn{j00Lnd&q`s6rmJEqJ4AH=zb
zQOMrnMuf^b+-%{bSdGhZI)ub4=)SMsfF-DTInD-uaU!&Ob!2SagdNmB6g^P#FX@ZU
zvDzi`#EVxLGPG8@AxEFEnP5_=PdKApVgCvsL~-Ec#)l`-equG|oRPx?}V6)s^{jf1wrlV7l3-V7koE8e0&=NiG$(
z=A}IG-VWE=j1A7ec5t0UGpT)&K4I;9B8+&_zRyqVC`7w67OG8y4Ypm3wQ)@LF#AY&CtW
z(>**e2ik^AM@O7`6XvYhyVZ&H4o;Z|iUTH{LNoHTV?Fps8Dz1x>VjUJ533+X3@sUS
z={KL&xtP-rw)i_v&d*u8YvA;!4+JQ3FZ=x3Ma3V(TN0Dj%qR<(`%sJ;{
zW>@)=nP@qO;S)KY>u@c@sYWkgIDZ3m;yD+3W`997&vVAM|4)cfKMXb9+=EinjB5+c
zrmBT`sN=uX7Q)@?NS@Q%*WkQrvD4|*{zX)YJuyNOkH5ZV@@5&=HEg;Lj0ShY6d2^LNx1OW==c8bL^*mvYnvstz+rfzaz_pxN
zUahu3jko!ZF>~SKl`Q{Dt@N}eDb;D)sHwbb&{Nq>RqyfhR`yKOS%1V{E
z#~Bn{M#7H%K!!TL2iG{M8CNK?GSpRjor$d~S3wzEh~y}TmtFtfvlzMwtJT`QPC~D=
zHK4s%)H1d3IF)bGVe?P%glOF=9iERASP%W<=Uua!))#=7_9^OL(H)@7TRgs@T7Y
zs|51)Qbjg6e$PD}%qqC{dV^y8_V8)Bx>wJ2n*}Uw=WeMAN0xn00*NXlL
zREXy}uNr)BfU&cio_Q~sTV=0cT-D_dB!7<$2NOid?&kqG;7GcrrCIY|>1(hf5>nsn
zg!;SC>C`G(25KBKrvBzogFDp?`<;RQCs8g2qilLSP8YK&8VdK+*M&JpQP~X!FXOU_
z!MX@r4Tk+$T%!?(%H`E_xFgQ)LA?
z9dNF#?ccQ%(c`;3(Bu9>saCWv+T)(oVtgHl7qwVXA7b1t7}O42*BEk=mXjw1Lr&B3
zkkT;7Sz7Mdw>jh;lR>ph)tHdCM7zpFvVZa7=4&yKlyV|wE3Tb|X6e&*hdTl(O1n?9}opXmdx>W9<)
zYg+y0T=x32e}EOnFFvtH0k;l{weT#iQ7HIIu|NfWq!faY5>KgC@+7WHT8zFG-^7!y
z#lVb2d}d3F2hU+Ivqy{DXWr;>4^pgM(&&Dyxo5Oo`qq#Z9(RKOClVnG3Q9fhRLXUd
z$KTTuV{@)PiGDZak4N#6BTwoEwxs;$Q-mW&A_j%lXYla_ku4&+ZXq&#UL2P(D^KiErGoD_g8yqlm0?+G1=D4SP*DfP-
zYO4o5$1~-xbIl4$S?w`0)4aIDj4aA~elVko^3?CNY%a
z@}R9o=DZNR+f4kFHz8w&IOSnK8ks}+wH$M&k@CI!%=H1~HF-wna6MV*H}6>UeY`v;@f~Ww<%#lmB%sU@-wqk`jtJ#sG%4?hP(HKY$ZP?H
z3h?d-b3ki5HfJ8ECK$*+nWz-;26xpo7T!kwHh1>!Sjc*{Uni1&+uMpbMgm?Dt$mA9oU>ANrCqoBm^T
zL1&R4L>F`sISVa8ISj{b=;N*;4@4Jq6FCAEpv*3L7ZvC(@@uF-50M+de^0)fO{1Ss
zgPuV4TLQYEm&j94gWe)vkBlj^KgXg;`-uE+bU|N{SE36jvu|soMY%tud?W0y5jh1d
zge6DopY9dio6ajN|`};|8D$L(jYmRhAy~G}7d$L-I4ZzfQPg{v
zule^GXK7tuTFjbGmCkstKkefgA&p92@I~SlZIzk%*!-jMRFC8MUEDRf{kWMmD{%hl
za%0mP`bleyycY5$+pH>tiya-k*Sz28Yeb
zKFCj@iJ9@6IPQT=qXRg4l^L1e3GyOt>C&b*1JX4zSX2=EU#(os!>kk>5*CCHax3Zw`L+*_hVj-qL9>rE?K&Id%PD-~L
za)B6SLoSh$Zi8G8&CbL}DW@8l6VJ=@jm)99M`D7(cNF^t+P{lq&|)Kh3He4;nEK5)
z{Lx6j(*1y=mh@RM(^U>^?lAr+co8sswa*Kl-HNCup^_KE!|$kU{$Icg5eM06f<2WegZUpQ$bK~xXHl8kn_H7`qlY(}Gyh)Mynfk#`SK~uZp`2<5I-I;1jq5~SPN*xdAp|s7v%;L;Q`3kpzoQMgsJBw(jPPLn{xsR7y&m#O)}G^^Cy)MVN;eSl@?2xYUC^%xI+mJ~
z(lR|>@dyX(@MEF;b6EIw9vm8f=!~|!$1u_&jLLQI8
zG3_=SuOKif@8SC2(=>%=FXUme+u?GYh3S??l$S`8aw``2%@l|;<7%mWG33o?UfLhT
zaYoW|N3>81#~E{u*ifE<0N3;XDVz*Oe=~6fjwdlxD03>!lu~mlohTC&<@zWPZ8)vI
zbI#aPK6$W4j*$t6)9o!Xf+#N%Kfzw~f5}Qy8&1P_qp9hXPdGBA?>P;>n`ZQs!)2!9
zMq`4^4V<3aV%)O}xa)lpOD+9ysy@2e=(|8ZEk3z13D(_*_U9bFT}s3`yt@o1?pSuB
z8R?Ys`3@9Y9zvv4CiSu(gD@||!*r(6~HNi8Wi
zk(~@z$d@p5+3bTLzb1Xm)pD=2C|Ap?WO5q;`F2cD^vRXepNmMgC!;X=a5)|YVu+0C
zCn5I{M_f`{p~lRNOX?X3BFayq0`$Y3@n9MEl&46Gz7F{vWX!abyP6pC>ytKz4e+%{#6#O4L;(8w>mtC_V50eq3
z_y7IS-*l7$VzUt32fQP-Tm<=M83mN*NXC>yWkI6cUAl(X4yz>SD0h%ivl8;t8%9}UeNXwQ1RXaNPe_f|LH-ltoAw(Zr^^uC2zjj7WJA7I=85khD`ZBW
z+;sd~_8D6tua=g8?7#fFG+i!;Jc)eDuSw8R?k^J&Hzti_p6AA-ojA&e{DTa+J&+HG
z4L2;G$(+v(%exY=2Oy8q6BNcDcQ2PDVksX+5V4j=A@`9fmU0}XVFnTJXr{=t%^l8T
z7?#viJ}Gh^$ghgbORe|OTJ*`y&Uv(!m1p>679-IT5ldnqj$5$GQRXfvO$tGI5$1pD
z2jeJ1Eh%$%+!RUSIECh({WoiQpWekkdMd(D7R>rWK4eo`MhMn3(&JNQn+37T_N|A5%DGV
zAD(eC`6R=rABGca#w*rb>Eq>)=SWMehx~%fV%*>595D8jizMwv$V25)oVQrEqD9#f
z-1kk#EJ^=+aHO*k$X}m$r0KW^T$W~x3Lwvt3@8tmoe%ee6A=@1L^%XoD=Uwd1!+I@
zZ-@=|j1^0a4fl+V=FR(VV&`7>9878O?wZp$CWXoU9*j#Y0DZ
z7X5otpLsZ&QVz>H^AcLWij%MO37+2#xD{U+u|%ImKADWW)ovC~YCpGgr;9fNwH))4
zf?TBK{IF*rmuWc=(+l!N%C%mujP|&no}n1~9mH8$jQ$+rGK#)OAZ^f+18EnfZ=G_E
zwlyF1is~Qci=Uy^MMwSOuluNZ_VxBv=J|22Up%@8BL;bNL%({DdF~bTg26}d#bov2
z=HuGtIoSY&CZ859>u>P;8_ct@diW_k>c_rl9!E_AHpBUicbMmKnKL{bCqBQ7Z|ovbJ@UoRo99sXrJ3sfc*wk(@pF6gwCyxL9n>6l3!&%pgt~ZBi0Aj-
zX`W5xK#w@|dluer9?A7OXEK`7(PZ>6p1U_if+!?x2yFj{anKs=$Pv;O`Y_f@;GYAbn0OjE0iSJNO
znsvr|0qoZf9ZWsy0bApI=4`Jo0(c9~TLC}BIkSsGM}-5AqraK$MIeh83O(^|ApO25
zI1$(t=M#~yZg2&G4Oo1WnP^`@L$gQwUNpEHdh*x6F)flgj$
z0Rdi4w?$*;*AjYYy!+Vw7$igBW$QoCz{CKD&^nvn5j=HB9hs*xXv{6Z)jlJ_e3#R
zpX}caU%#(wiJ>u>1qiFB~|*$d`Vez|_I1;5_Qbh`kK~LmUD<-6g8gNbXPn%QV-Ej~n$KY2`^x
zOfe$SYjigf-OFSvFu}3dgASPN&NejqRcIz={~NMrcUXXG8~F?mVXQSh4tHldeSZn$N>_8IHr9b-k4&6T3luL
zv1lgR9Y#UeU%S*JL!vJG(;lwvVPNw`KSx)tTY!Sny?czhqy;<(bLXeiR+u(~4nvvM_>
zeEq)1pMV27NIs6E=ra@m$Kyqkc^L}zc`b9Chs`u_R~$uuLpKMyD-QZ9szSPzNXKJW
z&GC5qT6mPpe`@g1sCWGZM{E0dBzLLfE9p_kH}pnxWcI4#<6+PLzc9H?9UqS?d9<;Q
zht>TzPrc;p_f=i0U44)kk%q$jm(#*sh=N*md~f1N--FPtT}K}+>(=q{pnYOvbL0U%
zL8d^F1+{%Q1=sP7!*LyKUco`6eJzx*mPc(E7IikLE|v`*Q5kpW;b%Fu~zt
zgS+mDin!L&&Oyx0a!kT^^6(glkT(^!d4+k+HM}f#ZFT~+Rnz}e@svL;>-+YKLqVR7BOb@K*LZv*ukrdm
z?(3;O?!rtp950UEtB#J0dM_dv6JRH_FR=ngWss41Ubo4bz`apEum9v{USAoG&`H>L
z;5a
pO%474{r~J_NchIDUriAda4p4}Ad{2W|d@t_+7-==Ckd@dWNUjlyyEg4g#4
zjw9&wqc|=i-6b4I!d>}BkNabm3Td>)dwF3D|vn)JIIH1}O#(ut8c-
zR6t5Fz=<@2fCdB&N>f^p7JBFbB7FbZb8m7L-|u^W-{bGe-kqIov$L}^v$KbpXQr_+
z$ELBoCnBsTBYNk58PSdNv%gs9+zD)cz7bWVL}Y(H-kr*W+VCJxkeOv2i0R0W@35-J
zPWfla+Suudhv$!Wn<4H7CVQi6xPl4^=^n$Qn{%z9aSQA2yE4jOs&=j_oHu2>+uhL1
z!!(!q$7j~v@%`B>v1~f8sc9b7%&}I-*W$(dtbOrK_#gZ7Llcr!w%DpyeGo?YQT5^6
zwRYt+cQY2`4K>$Wk+oW~uk*Xq`h)SZRaQ;!5|(3a^QQ4bhb-2)8ULunDm>njPb{&b
zo=&o+r9`n}>#LNzSgy4{r8z%*-ioOGe)KOZvfL(@lL%Km8ixP5HM4eI_I>__+IKP6
zR1hTH>+*0jdrI9A%4e59o>D)-dc94Y{{7{T-KK6G?$=cRiGF&m!>=S+dcAA>XqI)m
zUQ2!SxbY4`g7rxKL^jbHQojxRGC#Ne8^LU)Rjqjr`^EaY`Dbi`^>B;o_3}U$FZZaa
zyi7@9lwMX+Ld=6@pt0c$kC~&b>=p?bS#zfmD)WSx6PIPWP0e3Cp{n5PuTW3-7i3oz
z+**reiS!q&aC)VtzWs5e+l==Y6jT+Q{2etahp={vRlnu)Y_>I_W%H=@$H%!%x4VgX
z`?m5%c#tWq?JaBa9mgfXjaF^>~he$2OqHL7!azIUhf
zUFZJ+rGuHK?>`92)R)8?>kw}O5wE(XcX^VXu=;ng*xLLHU4}Bu$r^C~1hz8&^!>vb
zO(4D7c(x?}YB!W&9?zw{8}svuJLs`7|9skN#zy5YdLV-H+voFFKQxW=KQ>vt9~qA!
z9C##6pGZSUk&E-qLE59Nz;ZvGVlC;_ip{YO<8Of#+q(|?%4**`h5cyt?_HbC${*2t
zG-HL9{#YKrH7~#5v09Ap7;F9hxR*_~Za!Y$Q%pOVxf>*T+Igc)9@f@na&IbSB!M?s
zU7pD9n|%Z8<8}z&&J*f=LYB=VXtW<{cw;>8`%0EA8sj!Y{UxC&A!b}nP~Y;klaPs4
zqbHZJk=E}|J{bSSN)W}jDXEVh660&DL!ZR>tmS_%YbMJkSws7bh|eUhuWU8c+r(F<
zJoQ9;<}ol#z&KjTvhmhyPxVOnwFILHs~TftS$3$zDt;;cw&$T
zM|^?-$2c9-^%!1{XD~XUMrKp9KCwol^+@Ovf>!el$YkEWp8xqurdz|Gt{MF$ZNtg|
zQ#&)PrB63t%dEXmr$k@gf%T}QIh8K6!k=jneVgEw>qPKLR*z@uu|wAD&$ML1`uv$X
z{MsIC%QJP^8tXhjBdwJF4?@nr+P@D3=MVksvCY=m{w;B^c?LYlZx>jv3>ePO6IzHD%H@@?OR=_RG3as|ecEv<9pMA0VA!^mfQFnW?nYS%E
ztmg^#XyyisKlc_d|JM5NbG6w?Yvyyq^qf85fhLtguV{}HXZ@Zxahl9}J}LToImQy<
zXEqw+G;7QAb=XJNx#v5vkFBOJ)PW>??1fouxmEVU)BIGK)$_#!9R4r9*cgg{^`aL?
z&dL|-uwB+4FAjv{YyVOmw!<3u()2{1ly6CjjqmtbdmdxwmN_Igl&7OFKGcMG95f`>
z>hkjOn3Yr-;ts5^#JXol3W>=Sdveo~+?)r(dQW+ckUhL=Jy~nT}3*T?f7FrkHZ>6ukgrXShLd!U-^Z%aH
zm!3oWk#f?7^fOi}b9?D$yhN(=L3j48HSB|XS*F;~FDce4{Gb`jw8}nc#V>y(`))tH
z75mzHb$FUyM%CI|=r+2x3rYqQxY5BWjk<;hL9H#ejt=jbQ1XRe#mn@mc%zjR!~8n`
z?hj`x%(C{3SkAt)#*N&F!?Wk8UJ5UomOpoFHN{ghbDFz>64X{%@HiUF+ZmLe5@rUO
znXJO)w7z3pZw#aM$FGAjQ$KzfzmI%emyZ3=@zwc+UDkkB)vf!-?_jH~xCzB~Ck@8d96qTAKe`V%b@Mk&dV;kMP#wYi7(7%#U+-be<$1^xY+A=F<)%a_&#@lN
zelW81OChxvZ-|GPW%KjD$WCOncUq_(6x258L{~Jn-Ir5JC|ABcDBV16S%)VNk2(80
z2*}(o_;g8+V(uSny*Z_R%mpgoRRuv*uw+V8e)V_jz?7Hx7ZXX{=9~R&rdee(HEV`7
zu$mzCw5my}7do{mn`zxW^<{nIBrtPqC3o$s+%DpEovGH9sV#WLF*q_Uy+!^EsRMGGlvXpaZxOxk^xdu?>;I8;97xX}7N#k1pnIm{bs2u&NB2*bQ
zBcRF%>!s-rB>t0LM`mN0RU+Hg>GeUjK_b}-7uB$*JR&Y+|Jdnnw4YaK?U?S7rA<8l
zY+)p|Kr1Ox3pZ4vCR-;(wL8$yT!jsy`Oz1AAAsH`=T7u#YsmWXqi1gnK;N8G?T+dE
zu?JFGPNyu#Kb=-H?!>!J`V^yS518S@Yg7bw^}sI9?u{vD?*RS&In}~E3`R2THh(QE
z^3yv9y?eC}Pm&}nkR-$Z1^vWY(zN_@%=M!ef7%4gaO=~`WhnG7!@V<~#ipJ&vlaij
zQpV?(&72;tf3g`$!W?VD!bFa_eYNmOzUO!A_QG!ZzN2H@US^rg+cdkJ4JA6%o#VaY=7Obx-WaM!|uSwWkWv)LgX-i2!zb_4`96{%9&-M9me^}
ztEM_Ku!d~ohM!DCxH~kji4k?6Eg_AC&FBif3Dzb7W^aau}86E=B;kjya*YZEm6Sb?d)mgi8dp;y$KHr
z_jWMPohMW%5xldfna4vDtytZ$jZcV{d(+)7rUd(sN$TR?OCI%fKuP{&<)M_ncdc`4G*$WR_TVOY>XBE{SeP>(r;Z-
z*CvHH)DvZ{S`E1rAywXRZe94TerOZ-d$I(5?f5YGv)*+|;;w3tC*cldt&$
z9p^oN=%J502k2cuiVwk>%4w~&*8cDaoNQq~%1L$n@evpV6Mt;VmRM_lJWOQRvAvG9
zvFPo%tu)qp{;_fo#{^N_b4z`Gsmyw3%T+>tb!!7kW^AoV$+ugFQ8MqR7^}li^{ak{
zShF^UTH}9;aX`&kk@f3O*ZGw)Yx}m=Y^>!e?ql`ao}{k?3z!jBi(g}`N!v|A-m(4D
zxU35pNTv8~2zM%pUu(qA^+AkjqMJ9smQ#w$^aX^b
zT06G2fLTkuNBx=vHM8-rWiX6t{MLr81vK6o`ddGJF^wr$j_J^EzA-s{t*1t`f9>lB
zSv_jkRb$k?QEqsSt&&|etVX*%81X~9hwI<`fkm#eWN0K$Itzh%kI#N1FcHeF@cl48OO|-?l;nPZ%+L*}e`uR&F=a
z!-LG|Zfa-Ps?>hxgLUip_nw5v{TBr?2UYIBNNfjd+`)F#*T#b#{e6|z{aarVp6WEF
z>W69pFTdlVrx-8!%n~J^P_@*<%PHA+csM2f{`|_x)z5EuBuN2Tx*ktstE@@Ko3q2x
zRd<41bxBsliD15OzLjv|9^gE2;#Ge14{?F9X8J86VX`#IX6=)xo&Tp$sxi;0@!~mOjU;RFxi|fteQBkg--lXt%Bb!m
zWogDIhcWgIWW6@xza5qi`QH!AK5Je0w-#nt?BY?>J${i|h`N+!j6Yh{!Y@Z<3%ib@
zg$Jn9@XP**THjy3j2iZpG`4oi)fRk|gF5c0B-n_nX(-%!HI0`YvpzYVC@s^O(orp5
z=g{|GyjBee?_EnXK0Ai~Hp&sh4b!5c6+d@Owox`ywYrxRTEFr%{Ri?elT)@4|Hzrw
zM4DGEEA@JgL(3#<|Mhq2=aV<;u^;n?-)P5Bch$`_zUSzaistN7*?6vOJU_GIyvna!
z5pOWon&pVE7)xWTMHyqw`A)|L_6J;G$u!v0u%ShNg*9Sx#5je$%Yt668o{3Jn
zotVP5aW=s|UWa8dzB@;}UXP7qmJK#K8i|ED)P|+;{rhmq?r*dMfIZ-)ev
z4et#%zZT`4S)C}KML#es%w_7{NPwG|tPcBHJZiE&U{LY3$vPqZ>Ws-aKX^f$yN`8>
z$)1PV6fvL8SW1e*9NWB~O<^q8-r1FLl57(%3kST5RwzTZz;z
zd6*4lY>arQ7i$TA&gjKjU;@i~fwR7&)4v&UmhPDNQozBe2YcD?_hw&l{nUA|abr69
z?Ry?)`*~O?3`y9z5a>t83ifSmJMl~(X7V#*#oRvZ9)5PLIM9dHqf~o}-B0O*PqD^G
z?f0HS&mi{KeOX(6W~6P>3okn3@l~T`WU$311#VsZv?V@RiOG2cBk4njfAR*qN#LO2=oy
zpsH#z+y_3KBicX1QafcGL7S?-O*x1?M=&=1Z=f%C0#t>C`LyBpLx)&r&wGZ|WHEHC
zR&^p<`BD7VpEcmSuZUayL8?FJ*o_B(AN6wbzEwF4XYR$iB-(j{*iFu+ifpdN*oDtA
zPpF_3JVe5tYE^CM6{dZZHkoBLzGSHIE^pGXy$Ot9VWeU?Ge?vMYmZfvdae!!Zu
zJhbjLrQg-A;6q}1p@;>|fgGpd>
zfDKylbxsee%Zxy7gR=mNlPjs|bRms;#mrnN8-<}uJo)X3k3`hq$msa$BpkT8P~J$G
zV<|~COUH{H<5&yNtlym*@DyceiV91v+Jng^OZTeq2Zs$ai+;CLK4!I*j0tC9kkp~G
zDJ+ztGdjIta_ITM3Qjlk{#N);P>9RH+F6Aa>;siofeasL*15`#bRVej+|&X7&NUzG
zXce0317)4Btf}}wi>pA~2P&=t!MQHSdilIanZ#;C%66N?Jg(v;j_lCPsrDO_Af$}q
z#eg(p!W_@~+VR=!PtLEGh?`SbCpJoSu-FqwCR?mo;$FJoHQbg)=Nd~gnMaWxz18A(
zi*;u|3GY-koZr}I=T2qY6nq&n3)4?oSjKC*h?Im
z-pHA3BHdEhQFGaRPCx88h+Q)iG{fJ|lD3v((s}BXVfiv((XUXH3|$lpEvJ!=4RA?o$$l
zN*MO63%MzD04QPDvxNESHJOupmYcsdbjFKKE9sO2k*b7Y(^7}icS}_4S)$Ad(PS}e
zqI`2~w8$I79wVA}%MgwPkfc=11C9g$De0KnNIgk`Ome*tZeipgkV#gI!L>AU5XhAK
z$`Ng#FNuRcW=c#5WGdV3NGb^AVB{t_{Xrm8*%ujCq=1D~{>AAG5}68qb~^J37XVGSb>
zIPLhP?-J2uDXUu}?@x&+*G@5K{VH=ld0Ql2TgsB%YgS1hmMIKv9at_tUCLS}iR*Iu
zp!!7wpJ3d~WkTmOx33fK!I3fI#!~F&OmSgtY@Fz|3^rk=7`u$sOq8w<=6)Zd!joPn
z$mJ$s8VX;8ShtKl7*InM$G=Hz2tZVR;@_$@|64VCOdgxhn#*PtvxKLo81DB?scn)E
z#lw?Lnd-?Yx=EKOxV8X>xbQuD)|;i^lHUhZ%q*qye!z_@a)OKf->|-94;Ep
zvTu3m36b^_^D4NgBx#i>YJC_IA}0L=Ghw7y_7kh?raM1uF33TknK&hn{=`z*2|Iin
zo5rG6ACZU73QCHC%#(lG`P-R^C^K>8XV|xecET@gAFH;Xod3AD4GQu^nmpY+xX#vg
zkd1V8B6OE38)=6B4q2KOhMoNzi`UqEd%<4zlNynW3s>K{rM$X;Tf
ziIRhCctkcZ$P!Y5u)DLw&_i%#oX8PRm9R_ytljr8`-WK}>`zulUw#m${6x{_Pu7&J
z5ik77y8jb<^`ER!hE%{+s7sJ=q2^A~)YLE>EE;lB16IRuu;|E16;`E#g)pTZqSC>l
zAh!q6pr{9e?)e7is~Uzgh3ZH{K&3M!1UaeDt8}>|7eIw{6VD%E)eWj{?mH?zIs!#z
zqWJlUbUcX2qj0EzmD(I-2$~ry`Wxuk|EaM1RKOY
z5g(sm$&Dn%3=kon=2f^B2eSck#nZf`b>ZY7xwA#_31{S&PdGD4E_L#qN?C?)&>aE?
zFQ+GGf^mn6!6zAAd5NVbo$9+zvQd=pcZzlJk2qJ3*j|5%g|dIFu)X^K7z|&vM1^Yfv`ViFw!9
z`$(#nv!%hgP&-T=~%dNKg+bJh}$YXOSAif>sGB-^_Pz<>ZMk^?ufZ}b_M3TaLu-W$W3LhI=ZLhz9
zRGdc5)6J4Oc0~fu4CRI8;=|g!c1Ur#AV1A%CRi#@5}RuC<@c5njJwZDWjt#xhAabb
z9s{K^{kdGql~e0fmP5HRyW~yXU?9O{o83!@c|(z~6S_
ziwXQvXKwKuv+Rn_yt+c=N%!;iD6pTrpQ{R(*nXxv|6Rp++C0Pu>HKQ8$a$3ijB&p7
z7+=C8W(l8fM8z`Yh?FOJ+t_u8<@G4ZDZw6tjF#jS3ak*XKFRCQrkMI9pF(MBAAUEr
z|6Cs)!+7ZyIQe)YkH{V2;|#i-&ard*@!bem9xL8`hEJn3vcHoe?-4)r=Wi46o&mgF
zLiS~c*x*Cebf=p+$P>efk^#KByXymBJaO}ZBs_oF;+(m%>|Q|iN+U-0^I
zQ~pt=NOe`L`+_%O)9sUA@P~Lr;kW>PDG>K9;O!b#GNmG!(#N4XZicy87?IwF=R|mP
zOb%oCVgY}JmLX;#@2L`PP#nE0p$DSak{}0-sL=b
z`f>+`-|Y$3e>tBELjSd#Z^QC#{f2jpsO=M$Bq|Xp!P3pL*_GxrH?slA5)C)-ddcN@
z^C6Xq*MV3HB$6Wh17tGop&R&E+7kAq@A-9=|Mi1??nfRR#-@pDzwt3_gFSpFe~_^s
z?Jc|b^PFuEP51ISP|>#UYFU}W9=Xh$^HuCc6WSkwlA|5`+
z7wZct7<<*P!uBJMQ20>
z3iV@W&~=EoP{w1#BNurHKYvE_yU0Ielf~~BdHc{=AK^Trk*VU13Lb4YzrLV~;UiOa!_H8SV*}wqNE(yKaB=<
zeAPVok1RCvTEve4Ex(L;&3@n5Fgye)V7Rq=CErFD#to=I6X$RzJYq9
z#?l;OMCJ_)aF+P&2Krhl&Ryq;uF7d_7%MJZ#WHOfE9b#~8Y}L*$UM~dP*(~LvFda>o1V(Xp0_E0_*+W`4
z)K$$~Bu@Or7wNaJqOY1&T{(zi2W2-Qu(!;_DHTwV(fs3QkCA}6Sk4($5g}M
z4__x^8_`Ttnv)zKtSPmV&f0LQ)~6lm$O;C14#%2VbV1C~6a>)P-)YL%bifad
zR?gx)ckb9;iB&dmedks9l4sd>B`80{=($C>TADSziPF-K)V+z)JBrP*XWXlN2|aFW
zfY){JR`GCW#ezxuXJ^GsBE<@4DZSYYm#+7X{svY$|)8I}UCV
zi@tPHcSY)59n!%0HPaL$Zq_kefJV4|p{K9}Jn|R^LQ3@|RK}{xNH3J8oo1%We%Nv`
zJx!5z-|;l~UC~oW59M9QbkfY`!c4ntxKE{Vm?Y19;|wh_5y-`DoQFc(2KJ%ZlBSH*jLfvF{Lwv{s)gKJz?W)
zeN6d^N9WD*=>@s;7H9!9g90)7NyY3LFlWsy0Ng(mj8d!YiRV$Z%+Pc*$ZSkcA~drY
zxSP-_%?VfF1X?Sq^-)p+SD}Qel>!s1@IW7>4lkM|UhkuHk2&KXPeBMU3PaM(%d^B!
zeUzQ8%l##@NVA=fx|y>psj2w2A6nuCgplVnzVj)iPQ-*j#L+_UtE6?x34~=MBekQ&
zA*edbpW92h5{zNKA41-tQUDf+ReX0TUS=bLNM2-H+6bQV${>pG9O$R8eko-7MsV~9?;-o4Rop|e(Li_q4e-}N?Jyy-?EpB4H=UKfNvWq<{1F812HZGkQ0bO
zVktKeBNH=T5Qsq}C<+8j1)$JZB^g^rF(&|05ZDMwvpjo8x^h&-xL3V~jju15PT?Q+
z{ck9jRamQczoWcR16JN#OX&oY|B$6@4ki)%}C&Cvg^Qd;-BIQ|m^%%2Md7c~l
zrvT^jdK)-TQ0t;>V04Uq%2vKsFo@}ElqGOPY_Yq3r^Nlu&mn_U(hHSrvY&5mP{vn-
zW%|)h|?xK}ptXKyr1khg?wJR`{jk;?QNK8UO3Ja9>fz
z5$R@Lk!pm!{fhFtjGI|-L;0q%xz{R`eX5>4N3KFIrgoyKAUvm!Ms-gb>Llk7RB;Gy
zz^5LB2v+y8P4-(M>idkB?GWcfRSzVx+oi_w?YBg#OMOT$+X3@pfq37gc4K*BgG)`O
zqJu8ADO(`I!_*f5eJe~&sh+h5`h?*=2hG+|YjD*K3mMM69XrMHF!f%lcOgvu0QFuC
zSMRDl`&YE=4yb^OaHC03B<}8UkxKULuVQn!`aD8$8b+uo@NxHyP!su%o%YKSD(5hV
zhq~31nBo&r>IGMgJe>t+)DAKW+0YcN4$1l8blg`5rSjmEQ1M(1wchkx22XBlx)r@r74y%`a7m{a&>bFR!p`q^RjE
zWoOq`$1t|mj;pIa#6mW%grcN@ydq;o;=^p7SY1zT&5G=@dg>y^Pwx`5
zQq|;;!l~evAXu4V$Ew5{;y|h@c*SV(P6M?cH1Qn`)RVkotXSJnZG-IXg2X38S|hbP
zFI*`mH&WNaiJaY7jTNu1PK*>cnyaB=U1K#gG+P7BvLT1Rrs6oY5cmAzLUT2miXVf#
zh-#wRoPBF=X{L@MPjt_gYS)mtJ24R#4LG+j(IeKhRKI7L_J^%75)9Y=<1V$If@!pF
zrw(OX?0N0fm2w(SwO3oI(=U#3iw*77c=2U>H3AZJWqY+TyhnewS3lBE6NR{#+#Vy%
z+>#igU!nIID}~iTZAgTeXJTaAwj}mtTSVh~)XuoUdgUIq3vzk)s3vk3om}gVGWXw(
z>XRt@xuZ%!-SCB?r(eYT_o{6jXiZUkulf`fHS8pd`gBrLX$B)YsWrK;!$daAUIS}}
z5qY&5^%$_;Nb_UGi$!T_G@EO$Pg6H?USf&#
z-xHJhUQ6UYsD6R9>GqHsPH~!#_f%(x?4mVMX-y98ORR3kJ&ZL6bqfw8z6R>19!yNL
z$M;fe@c4q2KvV)YGiit*!FP|UE!g_6E<6hV@9t@0_mRZzcKavQ^$P6#>wVR;a2KBF
z2U=%~hEJ>eF%jb#)xs_0%xBccIlp#6To{1)Vv$}Ps4k~8agaIyR{y($)c!ajcMnou
ztxh6}3|w4Nd3ckcuoOMn3+>YpT!RLyUawEoR9RdQO^&3I4qFJC&jzb6kbnw#R^7>F
z+XtUjhlXOJ54@r-M^f>M`ey3@2Ns@UKt2@)`rArL>IVXHAo=#r7N5VWHg4>9x+z5z
zK2tz0I66;uQf`wkr<72xc(%Cts(KGU3YYwAY9qdXws`S1bsTxuFTAF9;ZRUpzpiHT
z(%Hg(UA?zfCZxUZeBn4o@TnLy64WMWnu@f1wjJ{Zb}qj>QatpgS`1lV<1KZ#0*}kg
zcht7QdD(DLMd9}Sp|{mM@$rAvE#$p_;$3X8npbB{!1}x<%lg=
z>VIgTbRMTpLTVR}Q?E&lV9R7xm50_Z(=m6H^b_iPGMZ>tj+!8UoS&iYQgM`Qo&%}2
zz`i*Lcb0VAzjiSq&;Ir+wN(VWXrEoFs;)+Ok&c@KyuyZzhj4Eabd=HCCHw%%@iz-_
z3i4_agq!sZYH|ifhmYMFv=qkyegcMmd-5U%?0utJKY^Yta0S5}-c~y8eVu$X$
zYIoYGJ|Ok1@tf6BsWSa%i@Fd8saIFlA}FrCIyEy
z{hUfaoxxc2(;37R`swsNnSMsv1NT6T;rP0=S3SZ)qrWnPO>V*yE=vAT@BK%%>AnO4&c>0fv*x0B(@yWj05VEft8lF
zjENJG2i0MLK-v7tga6XJ`_P?0d*~r`R3tB-CCbjJ2^>~)=y`P|sPxB$pQXo
z!-DV<0bGmn8|T%9fpF}|AhUdyy|PRl0pT)Bw7aO*r-lY!RNGW9uWA5#CbrfSCZniN
z*;M{T^@|4oz%cP%|4CU~*)#M_Lbpd;QlANimU-|d?wxl{7oF!NB-<@2)GRe3E6{Tlqr*4X}hT7c`-$AvKud+#{{#LYYqkElpQ?zTM#fDd$hWj$GUJdcz
zCfbkChB`IXIx_TXf7ndRRN>#h-3lTO?@aE}ps!-WXIpDG6ka;lK6$sMz+RbapKGtZ
z%AtoKH12*R6V}8g&^?^JuA?@X8)ekh<@M59wEK6~riNi87a!6xYn9Wu{e$2sn$KQ=
z*rZc@tUbA>_Bg&5B91?-wS$sT>k$a)`J(e9S{+Y0ZV2zJO5tZz6_(EEM=*^~#rKbB
z_hVs=URtlj!}zukT<5rO^aLZNSmqMkf?T*NUhk!~rAL^p)Aw?U6;DTp8*^{CdHR&)2GTFpcsa#hEb*x`?9
zwSnE@F|8LrxiuA8myj)>7=tV7ay8o|m;QNHSi=
z%+W>e%b?p@aqMNS4(zMYA=-Fw!t^0pTNgzFYNpF;n3-bl5N$57?f23(*Bw^8u!d@f
z|M}9y9yLr`rGiQD^_Fs0{6c?X#}00fm#8{&btwY&M56ZV+5wJi(`
z^T<0|GAVh(-qBh`PCesuJPe~npDmWYqfO-7w%Hy3t2M;6GD=B?!a-x0Q^kMY)BXaN
zzV*KLWL=-C>cr6~COy~+xSNG54Lq<`oO@qOt>O1YyW1*Ii4bz*(@h^jW`$_=KWzw<
z^4b5>KBrN2`2ZVhZ@&2A1Fc)ljTfgnHv;9*qWliewn`@XY>M+m$Z+jU^}t>|N&_V|
zB?=!a!68erJy9QOeIf(2YHCFM_d{(gE3j{Us12h~z@eGi>j*m9pQ-hR3~V|An+wDl
z0U@?^idZm0D}eACJW^Xgi{}~zTArU}cN?X(r8{krG1>vF9<4p7;)cY0DmcbY8mHY1
z!R5S&x3mx4Wq77!wv#X0bh@&&zp%7L?#MFC%|}}}hA)Z%^R+mB`J(;Kd`SCXYFCE6ew*RCa6ZAxz}(P~hdxK#Uy()~-d
zWU+3k14M>i9*ekT+RU&Vii{zfK`Ccq@)9k^-mwg%0HGq6Yga=_@io?Jzpy>_p0!%B
z0v7g%3;!+n7wy^M8|aWJu)Q0#*eG4P%Vyd4?9}dZkqhX}Lt2yA#h0eK-KTIv>vvHw
z6Psm$#JRUPS0S
zIeD$g&A;eHjl89#2z?>T#L_aYtLS?|iwUib4+@Y$gZ6(puGQ1!g^wI7n!kQScM%Gflqyi|*B^|A8;2Ul~<^VTd)28B)>Qm*~UH>xj
z!J!ElG(F(MDquIfC_3KMVnoJuP2MR^yRP95F-J7Ip|$##+Cyk%{k7!uUb?9jL1$=Q
zp*8sz{5s;33azuORH2PPYCry$_A56EPeF8DUca)5zJ~VmNv^-fqXTbsVF!Xy*4RTe
zJscYJY5RR$ZwsSDWVUoafRpVnLuWj4XO%BI$lO*c!bA0q)Y-40dIL)T3f0L37cE}#
z)DmC0^klyi8P}~oCvvzr@6y-CuD}_I@2_}knR&in9&@o>7^Y{}BYFF1D}4YuU*1YT
z#`0`&m;MNgEZbIjml0%^ZWC8p>y7Ss7Sybb{$zAY0Z^sa1U_8~87d6S<94xVoA|1S
z{_Xz-o;~sby(>2$Gs$fvGCoz~?4u9qi1Yrx=ws)@`hfonJp1uS^lM54o#0rbX>Rw%
z7+0x?DASvZagXWYbeQ%m(-U>)_!Yy-bg#YlF})wFR+3v88YSmhnrq+xgnr4x{Lcu>>a7d&y-k9bK7w5rNNDZNA_#zlYlB!+hRl&K9Ru
z=nXPfltYp`)d#uESyj1#h?PLdfv;?U%e;iQWyn#t{atvYRhfInWv-UFu%|Ipm><46
z{Sa$iHI2S5^OVy94fQFPd6IG&PJ?}1=J={vJ?S#Pb;e4aJ>fEQWLenb=x7GwX`DKb
zA>u>QCG1fIC07l#x690`YO9yaTvgT9BQA5Q)7HZ-(?(l8eU0^WnKzxr9>TAxaX#oW
zCsj4}0A^H0g&r<*ZB^50pq?18N{`PNCD%2qD@azgZuf)!PD3>P`>-~6|Lh}F7neDv
zsuR;?{#rGg&MtFB)w*?ZnUkw9?{%5GoF1snjxKXPm4T#`y~kyatU~SJGC!}%;brm&
znWI<9`I4kzc#(Y7X$%jK=@Yz{tD0^DnpCZ1YYMNaWJtVJz9Gx#mGZXCOm~uRR(PhI
z4TmLicwQEUXUgYyiix(Kl(A$ULBpCN=mwS+%JJ4KtclCqw?pReiYu%!;==b*Ku%aA
z#P7-X$Mp0ptf9-Cc}#Kul07YZsDW7$kB}PG%Z^`qeWbo5{Bn#n`J|?dNjDt>6yy#fOyAtS&woEn6m$r
z6vE@-ZQ_A%^;-PYaxwHmVc|q
z-M7(sC616Ng6_OYgQcJ-dIJ2ZTpo&wT6;py5sDYRyZhvV#DcCz&;1tJp{wY
zmBG32xoJU)2-;y{`B4AC(E>=Nsr7(9-MrOO*D$;b0BBq4%SF(G5ftBRzFPuf-0F
z(3|=Tlzro--k;LqoB9BLxg08j-lfi8t3Gn$?cp~FLqn2%Tqd9WK?#n=>fOV+xlv57
z(ECRgF875ilSj5_nc#ovmopAg#G2x)E50l4y!k}v8)@Fh4uKykAChjR*g^S_lpdM>
zK<|ANT5HnZ^GUT-wggn%+9cN(`*@7&Ud@}tG}
zkz2Y>M&7C0dL4eac$(XIiselkV$@-?r%f^*X_5tYH|ZV|(_(L=1SOlQsdoQfq!(bYKPE$QbTnrKXwKk(d0VLhCuEHdIvdVygR;1bx3L41UK
zRKc1>kR2S6R;0
zRl{aRz|~a&Deje5SDPqTd39Axx#HPkWHUovT@^Mn=<3Qo+sqims#IEXljVrPcNvd_
z#zTSD;ZOu`ufEIpKs0Y@NZq+xOQU|EDUDBTiV>}i-|UB58QQ-!CBIsOCS+*gL#^+q
zZKHM?g??;pXSCMX+Ew=&s}yG2wrOzbBv{kM7$0}@C)|WZ`=bgu;vi)}|1RswSc92LQ;7Qej-di1v}r&Vd?JQ>0wX-!gc105nYYggj#J8WR7Ya
zK?`m83PRhaqbBZraqANz%DNg!J*wk^3D=%x9<_B3WWtp&k!yF@iV~5f~Nw;
zKh{4RfNitIW8IAUes@q?1tEf)~QRyU)AyV!q|SwI8A
z&1H}7MlA)lofz8PNOIAPQ((^prx+r;yU`po*wEdmUNZo0=CYyP;O2z84C4eBLQTHe
z-DqZDmW3QMZ;@uSuU4b2BAF2rEF*HlYcxE~sE((ApQahLAm~>kSF@Z(L$9*3h=K(?
z9oM6XTLh)BRMo`gG^1HGx`xX?hwe;^b-J+fMv68)jG8qf0Kw9Dcq*T-q`2MtHD6Q<
zL_vheeG(}Dr(qw}(^Hbfq8=EKgZa0Km-R4e^$4YQFY}Zz_d$QVbRp?77wuZK>wQmT
zvNsz27s8dUzUhVEBbDbW_z-1~*99M47%-zfp%`pRr0Dg4QLmm32*+;1v&^^2Nt6Nt
z6=F^vq(u;AUL7o2L(F=>NMXA8?g68TTX&a$O`5U&N{UiBrA~KDDMWr!Bb8?vo=6%-
zwlUP?Z%eZ5rKnI^s07?S0_IB5MgIqln*TQGFCH|!al}C-cuMVbd^xb=J82o%v50@j
zsDlyzDMwrpSTW;{5eMU3qb-xlGx00{ccXX!C9@R_wy;
zR^04q)Xg9ouh7%I$w(Jq--CUk@H3a=ikN562r7(}VDK7v38y9=hL(2yq3<9M@8a7sH4XQJTt+HqlcgYoF2aRZ{Sx
zkDr5+)6F{0*RqwjK_7?9gT=OojRqOI`-J=^305POFF-{Ih*S>8@=@HW(-LKAh}goY
zr1z5mIRI1(0C)^pl!=A%RFgjue}u?|V|bnd9uGRfF)!p#axYk;fl|o5UP_C(*W<~+
zkG*A~(!U?(i4yA`
zH4=P#FY%78L~q$Wgcbgn;q_tNx#0|YCgN(y
zEWdaWgp;vAOY}l-R6DbrhHdYPvj>+!u(yAbdQ!`%B1b
z={HY_r19M;86%-Wbtj331teLghU0&g!tUP(2Q5cwAJp3?5TQhT_oR{BLr(c9K56Jl
zqA(~aP0r}p@JZw=#PmsW6d@v%Mu(H#y4sA
zY$ysQp#qO+dLN@*U5dXUmt>V@6kMe$OEF8RKlU*iCK-}9=PNrh~KAkt|zc&9_sCWuD$PkPLG%4pT;1ozj64{2=K>}Ikan38}uM==rxf#!HO10tIiP%^6PHU%ljJ*1M|oF8;yg*Y4fO_aFI9wo1~`bI>4Bo
zkto@R$T|VL(}Pz+Q(?53IYzoC){{uq92hAGV~LT{m3hJ&;7eAd5-bLaTn)?~Kjr%(
zOvfOR2<4lYvH+O~PrTGiC|?M?cr0rrpy){n@m=qt8)uorvG?I?!ckq-$KI+wiqUMC
zzmEx!YjM6dG0-^Kg2)?aq{8|P6Z-~2;WR|WKqEN=+%lRbmkV(=TpIG
z`tSkAj>$m|lcU>uNFG(rmmZr*UIY`#VaX}t)JciyLRwv@l)N;oOky;=8c8AcCO!I+
zRHia0$c{7+N=si#y4{t*%QdGJMl3gyKIvnnG}5K|9aJizlLf&u*Z)4sqR
zq7pT;ojwQ8|5nL;x-#~KLHD`xbU>E+lE)_Lx8*bzdOEZK!*HH4_*y4hlrS=3U;!tS
zINL0s8}h4~KVSwU@C1w32OIP?;g1I!HELn{seC8oInK#a1G^Pj(20^_OilbS*r=0M
zinjoeg`^rY^EfiTPYZ#YDi5txRQE<=E5LAdVk${5<~TEXXWw2>#f3vkVkn6!xK|lk
zf8tr}g@An4v)Bs(`L)j)12aI50BYc)A;Dy~pl^Iri(XYi^xyz;mfNRe=aQvd2A2g<
zD;Ic*n+PYBE&R>o__w~0UIwi_ch>bO{_bSN@C0EslEL4Is1=_*XEfCf`WF{evHLk#
z@S)=BbB3oqgi4d33QlXJJBPZCLCIev=%hL6QQbExmjMcvfv+;*dttqvHxhBKhyHyM
zp^~vHC!vUspEqjOuNtURp2K_tRg=95G@S(q7{D-pdETg=!IJ$`^dSKNQ<0*9s{p~o
z$MpXPlfB6>wRAHBAqUJ`4LVgdPjWY!kxD_kI5XgP_rR&FU^zVoRu&Z%59DGjRujDO
zHokZtMnB%cIf_7~^cRfqjHL7wc{0R0q9Rrv4Yc*5=rll-DcA{3(19U^5VCHx=$}3O
zRPf-Km$fs(6GNvtQ@_r%l}V62(yPz)EJr5q}*zFEV#6ofElRD#z6
zlyie(XcamlmK%Ob@D+TnIHg&Abck8d=>zd4XGDGv>4^jfdurZ`S>SXOi(WMD?qQ%E
zOk^m$q!HTzU2a5sIU^mTVeTG=9mbk#UM1
z@hFv&#vhp{b#l5>b~)~nQlkgDgPTuo`bgRIPPkHrx@-C+2j-7iK)A+u!hDNDVjW}0
zzb^v4!6^)amha0^QAUs-1*!tg~dcdXKYxyC&uj|@<~l5f*K-K(m9@B
zX5rtMjgVD+z#zH9JgSpKI3tK~a?+<%i~#v~)Bw@xLn1-)JoD)OjI^y)g5xyt&!MKr
z*DQI-E2rga6mPZwqho`X3@ZxlD2|#bz!$o4o>3Ld(ANQPAbxlmZj5mHSD0dOmY2P3
zG^`_Y!Jcr3GCr*W)2~Bkw)-l__6isG4l$CGak4f?K|3li6Nb6hGwcs3=HkTLLyQ)^
zWmC|5r6k1NgQFhMB7&wQJqR>kib7~oaqcd(z?t;*Zlw5bhveUK>3twKpH~%EhZs#V
zv}9}+$M>PYK|?~*#Gb~djeJIcuMl|nXT_u^5P0NBA$6&$P!B8&8zyaAsH51qx}z|Y
zGuh+JGL^UUO)<^{6+Qk-Hc4H%(;WpAGF?u@nL3?w(8;hQXvOWuY5)%M9Z5H8efq;rac#K*&oM*k0I?*SiG(e@4JoFu!apR=2gY)Ar|
zrS~41ltqe47o=Dyf`T-~j)jC?1PKZZoq$n5DWMK3B_K!@0w{vCC{+asQl)zvId
za^LUsyzeJJGdpvpUUN;mYH|BwiFEE$+g!3cqD^{yH7lt)+Noo;?Xp3y8G=%(`KZ*%Np55>s!M1q`NB_
z8;ho`WsxJ?mCRfy(}mmJHH|kK;g+Hj{@oiCqjiR-o5~iZyJOhUF>d(SV1vfG|9GTQ
z-glh4MU*gxfBBJHCOPM|lig*Z)WuIsc0UCH=y|OC6!-h23iSCDaL&@%gDLJ>BIrJ~
zGTi%!YbZ>0w>C!{33=7RrVzaI6Mt^1yD9wTo?-Joaj%9Ajt-x?lf^$K@>f4~uNOQk
zNi{CAfJY{S6_GmKeNdRor5WzNA_T&+6Fl*3^i15ISF-Gx?l&ZxkOS9HywWVU0=-|B
zI?w$PrexbZH@sYAvAgr_=(q)LSPIT!R~75D+Tb80Qlk7c>1ifC%!
zVs}TExbGU@yxiT?EsSOl*Sg;lM)SAUxz9z%j)w5IbzOtBpbmB?&McTN;6uK3*V2$>
z(N1@1TeD@?9(S!MC+`S7#cgXDAg_wN<^lKIsG@SGkno(xzd7vw91@W8SntzrA7IMg
zKke@CPAYg}LWUo2uHbl_522sP^6@|{mg5};9iU5+bx@`f#x@o<#9Wh4Dm+Py0Qbzc|J73%@5&{CZ*BNAcT*@py{g
zD~!h>UNe|pSYa{BFuE`vOY!lA@feCv;w1ypVL`}X*Gfr)0rB3Yr7xqaW+EG%X*l&D
zJ|2$1hBe~2ab$;+5MRRUSCmRbr{xq)+z^gH9>gh>8%{!eS2zMA)Xw(|NDYP?9UqQB
zX2kiP7cPSM(QpJ(B=+~}!TfL<5-x@#8vNeby-p9;iTLf`xvRR=Jkt1gv%Fg^slAYP
z+j)RG!Z7#35y--%->|0)UtqdA&Y(C2xKgBZ?L;{q@!Z)8vawN}Fl*Hsmg^)RSrpTJBm1hTs~EasI)oHIPb)`D<9`22xdaqOPR!$ql4(dfF~1
z(1cM1_?R1xLly~>3-_fYnK3?`>>xl!IPM^2DBH|%vXd<{9CvUtlx;~k+3n=Z<_(^Z
zo)J77!XKRU9G>|kb_2-&muxL9fNJZO)>56w|IEgs+e-ty|2Y;uqP_GEHaGT92dOE7
z1|6j>W`2nJfz2MseVs{9OY5L?z4>V=XxhS8(InLph1c91W^|Hj(<$4$lSICq*yc`B
zDM;1+)=BCCWdjangDNgzGJmy))Jbq{J`G`!N~}k3X&;-`OWH21=G}YagLvx(U-6=}
z-vwjOAHE{h6;IsY>tDepA^vrfml-HQ4{05LZ;wT*-0np-V+YAq$OgE5|XetDQw
zCfap&)O2e=GE5iG^UJ~`j|(H
z?kTx{@DIjI;~^gQ2Xjr5UKR8HSp3c;31&b4;KL>ZQn(47{#d#r=KsNtPm$&VHQvpT
zhDT#=ZcUf|5RCK?CgBbSYBaG!OlD5#v
zx$rY-0%fn9CGDpAZe>aJ?D}eaA=zq<&we3QD|!anM=J`>`9i8(G~GTMp~ZOY3u(3j
z{rxvrNu?q0;#}0S*Gj*Mm!|V+>!imCE=Si(8IXl}g=6{(Z*F{pbThIrx^AP?+ZK)S
zZI)KYotlETu2q0sM0mE95ArrkrNygLm}?6l^k52autn-F38VP5ozf%`GF>Tqq`Jal
z)_spuOW4LI?*U5Vetvha)W+5!d2+urMLfNo{jwjVwHw=+_kdIrx??R4NVVc-?m&xS
zcZQ%8IW2yCKq`$ZJSnY=`*J24bQcT}91c7W@omv5G`N=S
zJY^5z?kQ=MAYRzd)AOV)MS$1pHwo&9+gaOl81Oze=A2Z^_1CAUP++|dO9|}52hwBg
zz!j-E>v~=q7n&G;^So3)3aXyf|3tUX&SkIs2{1mK%jf+m4U81?KZUxilvL23z(?`mb=;Im_52feVgQ_d
z`pR$Jls+fJ5Pbe^XS^R
z+g4`V#mKh)(62GFZ8aWYA|DVd|0akx*0OeSaw=5+#>C0Bg!TOEI5{&)Sk0bIl
zpCrn)#NVg#or&@d0V)%ll99ZD-$<4NB8(CG{D^L39sJ03j`1;m`Q<3_0%Mm;!~XsQ
z#>$kDYrw~Te5Ae)V*;CsHIi3~4oz?^15*;eps_@rE!eHxUXkK4E&
zGZ|ZeVqt^2O`6Gx%->9w;|^1Xa)lWz5M@vkGL#nYujel{lT+Z3Z5|un97DN0kL5JS
zP^R!}&E>M9cy$d+Yz455V^6oD4)L+Ago!-2qkKI|{A~gIz6&Zlw}3t9g39Lb
z8eK6g7(DIP9TU2oWpu}cLY{C(ciANhW7xjuFv{68p2y#9y!P`lbeS)}pj;2Rrg(om
zdljMZ1z*%d)*bY2rLh}(pTVg_R}#M$2JX>|0IO_@^yJer1^+>M&AByxrHbm{gcml
zSAHovY8>p#IzDf#H{B*1veV;c6^6@AfatFbmlM-|JZTH$;zH?45rd1?7l`ggBlhuM
z1){qYyG4#gY|&lHx!w@~#+{O4dFKj5cPTa|*N)+mL$N(y7l`gsEaNVT_2MjdECTY#
zG?4^;IhhuBmtda65@%UOcY!u>W*0SZJYPm|^Lw5v{W`LOFm#0kwxl4;y>?vED@hy0iWLx|fo4DP~>
zlmzek6@_8HM?g5{Um!&6-ymG}CJ%7+z=uqvom|;cX7RVexE!%j7m`l3$DBfwEp_c$gmz%F!~J^V$khrO!aarq9R
zWUZ5OeLN%vpOj$=YA%2G6m|`Sr_RU~X#0`Q$&(@xpMFt>6Hy19*_6LEiI4nK9v^+^hV_zh!vP%x3Rj!{LZo4=Sbj
zi0g6|87nJ$6ZV{yM-bJ+8Q@K6@^oX|CW-y^|kkp{P(V+{I6{#9Uy)UwJ~Q
z%4$8K)IlHdyEZAB&uN@GOJ
zcT(yR!TZHd$_r>ar;}1SreJwOH6B%60_V}k9(Dq9_}JzyN>4Vnvr??!_ubA)^@86Y
zJ)&zZW3r(N&pHkgHWUCCT|+k$>Dxc})=Z@`v4(Uw{?-m*=_7ur
z&wp8HCVYS$ItV#*zbi$uIf)9koX)SHtHLTK3{>jE6nFiBN-go!EIwqQ@(N)mhFOV4
z4pz=%S^)o&xQB7&RR!xYi>-W3`59ZrxS`5>v`2_<%=+yQun2Ig`A{hl=?x2+M))f`8!9U|LRs+W;lqC#
zp%}KYyK|$I91sq^7_Bq}x8b+ZN^fC3Yd!|CjVyf(#*@P~j8SR{^Vp>^${=AH>poUF
zfI(Flr&L4GeH=QBU3r|cwv~@$1S~pYiDvHes||$+e+hpK$!+;v4FwV*0w8rwj27{|
znjqrcU$g3=l@v2cQdE!@ZalYsm_HhYqC9jQ|$
zD|k0zBbcJRPvOW6C^*IKK8nk-8~oq!d2e4{zWziBh=-4|lb&k!61KQ5_
zpHkz05hGbD%p~|pln~b*eYz5e`-i>?MZemXBJnMO^_Z^6;{7A+mFY@V-%r$^BGo|T
z&Z%Z4suA;!^1}%2+xV>+itYL2W2RKZzVJ0uD&hD#$COp}=b5vd&(&rtsrdZzY}mqD
z!j{ZNUo-j9*-A_70=(p0<)}m_{6ASrQWRcgaZ9mYGFawPWh8=9Un-dhwtuPg@Mq8y
zRI-lC{wS?AX#c?_;8BCw5u^`6tcGS^VKrp%rC%w_MAXz}g)-M2`o{lRp-d-E70<|4
z1`}uL!CK`LJm03S!?uLEn73Z(3X?AU#y85kNIbv3+N^Z2?~M1iD%0Y|(}N1e?Fw#W
z1|$an*u~qF(pU%kwqazb!`QBDr7(8~wkTKj30&3dvg$jOxhCWVGL!+<2{-t;?qC+r
z+NtzGPA2VEE+EL?t?Wc!HteC(YAc_%PdO;Cy19ynzrCNT+{f$XDsU(4V;c_AIm3KM
zlw|h!A*DavYL_2E57%u+56cwx&~(TST@Ndz#gp6FhljC7d@OgLQjG6EOsm<)4jfim
zv&}y#-JJ@@P+bd-ICGZzv(hA#pCP}(rvHp1z{kfPMLIrwf6P8muN+ez2O*@^FCYns
zH@@W0oKmu65f0DpUQph~dx^jPhms`1qSTa2$~qKlaoJuaZ(UZZqOz>ZfP!n<8NjJJ
zD{)ns&%e5Y;X$D);i}R+5)u`>=XK>1aQgV)H9i`{=
z#%e_jefj{k74kH_s}x(eQ~k(|a-Ad<>vSeN0GhkU=hv-Sq>Dh8a@sLiilWj
z#|9~Cy?7tyHr+G8`qBtxg9r|75y+Ute^yjHaI$zaU6sMT;cX1{3dJAus1*ggQu}$;
zYIu%v2Ih-B)5X-sQTx4OY8iAT5T}l!R16`ReHX9Vw2m4+^_RGxXJF#P^d`8Hb5JuS
zL4_LR4Av?^#XEKedmkaNeno=XJTyu!6;}^OB61>G&4#tn$tBczXsJ?3^%;Wj2*28k
zfc9HJ#Rb>#&1W)DPJCYHBy4
z6E>}`&JpOx$r@@6n?z!zsx~R3PO6#_zn+K$VFJQ7qLh*KeRd*MEfc?sK9(;)XM7Bq
z?Z?L?HP4I1pXVJAy!I_`QdeybCs1s3L-ibj#*I`T8~K>}t@Aq*JI(1vYAy~Up4~*X
z4#?uhA2ejO=gqQilkNFA*MzZ8a!%xwrZ!rt8>?YD?gVzmM8Y
z5U*@x@AXyB)BWhhmsFUOV$)ty$0101S)C+=uH>Dcw-VSdFROKtKrbj>wV(O~QMcad
zubxEOGq0!@L78hjK;4emszyd4FELO}wy$gS;6iqb^&g}@008?9Rwp>syzd0X`M4o!
zq!nz3MsOaE$_W*4N4#L9sn#?Y}p=Q%y=fA1GiIB&>r9x%4$YX@*Cv}VWe^>1g
zx^{-%C^v?wmGBzkDZ|x`xc^dkTI8F?si&zpVjuDHlhi?x$@gHf4^me~u;kES9G%}F
zN5;@$5Q}qwubQF?p&M@neqx&X)qgE8KCYkuvlgj|e9KJL1c&QWo;yoz7kZObU?~1M
zzdT=c{V#QJ)KSqc{@7<~gK+DaOVoA$wPbFwg4X%#%hZXmYdV)b$Ohcd|Lj$2AOx5S
z{N8HSZJRHR-k_Evs8-#eE|1&)5!OeAg7pzTqOWdHOUK=zZ$)+l(jP-Pd`nbvivd(<
zJ2;wk_!<~LkB$CXok5K!eWNB&<8{9w%+LLF@$=uR_M+{v#aXnYwy2A!{c<@fxsl?1
za#TEPLQe_ns9H;8@9$Lq0N&)C#v&ZPOTCTXSM%_D%5JsYzjIdESD3T<4~02{`yb7D
zI=3+AZwCu=UOV(?&Z$2Y=A3q#sq2YBXc|^
zN+STi7OmL;9O2RkfcBgA^TTp7ui@6>2!RSyrmByoTs5^SAtWoqn#XFmHn2BhwYCVh
z#%g0h=y;--b{yhErQ@`Ma%oBOXTS`3db
zsWqcD7EWPEsl$)^wMvxy{tm4bYZuT;75x6%`Mv6OtuBu(rHvG!iO#U
z_nHlzg&nGErD4_jt?F7WJbjl^u#ugtuGPlF(OW~y2$6{@^8>Xs+_y4$%Tx`7jAeXC
zT|foRJyl=pBm}1Xi7VF6=V1Flf;IY}yJbKE0|gdhbw7UCZ-gxkv@d8?_iLzuI5(Lk
zHwNIAu&#}@A01|%4ymkKO|;I`BxbA%j!-gUGS|CJ8N#O3T(DW33;MKcpbE!hmT@v{D=d=}}0VVU*J*cT^
z%-vff5>S)gT9o~$Xz`-9AujzZKr5_a4{25B<}manKx+o8*9Y+Vo)7M$)hQ@al6}`#
zn;G}bm;dK-xopNuS}pOxmwfL_+H>Siv0Oi`;2bEyrVZ4(;y!z6pyn3!{E>KxL_=XE
zzFFjWHgt&QXZ|5tD=~j0du52$#(_{}W@GuoA=)aOUL2SM#(d!+cn8F~neSE&>9kc)~IDv;Aci^ZRm}{YmC+K
zcAv}VjMI7u?o9COtZfQgbykgKu@kffPODY;uM@O?M0be2#(tfot*8Bb)?{sc+@zI&
zOqhZmK2Kiy7!aMwCVmWvX7eo{Ym)?F7Hg9M+*r=WW}s^av-qYA?NiJw2O?OJ;FY{M
zM>`|9PVGjGCD{E1+C;WIORK|_`P#3-a(-#P20GCF-Ta>g+Eu}I@MAFhlG)YyS^}3B
zX=aq``+sniY{=3UYY7Y~y*T?q194~@-?voz40|_!ZMl|GfcR|lO6{66zA`NPaU+T!
z&(>az1f*K8)xIQ1*}F?%y#^Nq{KNIy$1a50CT$RXvx#ksHfv9XstZ*Sw-ty$XJYH`
zv|dqk*I);ysPXLk@3gtNZuQx&b;SY6Hg4B8qY`Wt{hgo{tMt8=LiJt)X>aWJ07DiV
zuv2S*;PahYj?-dgwuK0`3wCMXQQlh2M(xp30poA>XtkY{27(X2xkvk!C@!n_5vcj4
zecD63i`b?8*l|uw;bjhJKj50mt{v3si@$!&svgp6yRLiA#Gb|P)S6j
zJBzkyp|asew3WgpUiD`UvJKy|X-Bc8ZsOk^)mp=t2seMxwo|h|9tT3rWsOg09nrOn
z6WTk3De_6pzP>kAjR5O#QhSvWx1Q8&7Tn#F+IBmBN}Cq9_Km3Y~_3TL2@i*;x2aG+ren$IH5c6lUNoN74DeSwmT9&=g
z&S~E{g%eoI^V%tNx6B1xMGIG0Np^gYVX?XwwLMO11zzcQ&1+9QP0+AOhQY`Dp*5uC
ziE}`}wPz&T{+C9y&VqAcsndp!W#?<2|3mP~LxS++d5!bP`+g-2Bf4#2FwljWsQ>#J+mfh5PM}dxKF#VP`k%$9~u{ZtXVyKNXCRkZjQQTk^xT7JsVkFyGSdON$vf5qrWMDa3=@Wko!@V#%m-XFma@%nSVtkomL(ZmbG99;LL=oIFA
z0-z=M^!CV<;nROc&_98yWIrV6QxWtmu1`U5rMO*Bw?w@+V&5d{pbzt(6ZLmR@wX+c
zV+p&lD76jz6$nHa2^TBs^YL+EMIEGCmRU(}Bm0&&OUHjX
zS^P#NeS#qFT*uO@=v_2tx+fLKbc0k-MQ;~M$#PO2tE!I;p~OZ?f!6N^guyI!q^jN)
zO_r{vARjTVP(AFE(b)x$puCBj?pneVg83a>m=)WQuT2pUcq#v7WVuC}hU4kS(
z+P_^(e?F8l8oJF$=~X-2{#;5y`!h-FIg8z?ZMXkK9X*lS@0+SWi8fZG>Rl1I>e|!M
zzpmZ}v2}Iz*ASGgXT$fSdOFqy|Dv9LoL+fTUxxj#*^TuRf-sf6(^RjEJI~6d`f4{F
zgiq)Xa2aL2+v>!9n%EZTKa=lktApXSnmuR-ocxB@Yp=s$5;n)R9ds6VkT%W|1@Af-
z|97Z4y*uh<#J|7i<2&jg5-s2xI_b+r@%$I8Ul+7EhA-%%U!^*ycGK&Lc?)1vMlZ(m
zy6LY)quq`@^cVy$_0Yc-7P68(^*M-r+tZ%oXM5>yU~Kz)>93G)?fSj-x@xo!$c(xBhAN%1g
z)Zk<0+d5G%TfeQZM)2TmeJg^Tcl447^4`(Ea)IYDFa>>lSML%<5imgbePCmoDVRnu-N#PRPol$pGxW)t%@Kko%y_yU+|STeF`Sf1NwJ?|->c*Bji9H6
zjXp?L&OsUd(dCCgNse3Zi~jB+wO@p!<7#J427BqFY5GSPW&Ct~h|5zT7yvdl4G^jc
zGv)wp=*O(Ndaa})+-y0_&5q2GXukNxxwXXoD|Ajj4!0G(+&#)UK`BoMdW;Sc_x&8If=lUl;K;)&(`b=bcuvxDbQ#fuF
z5?;fc;b51-iJjGb>d;JA|?5%@UcysHHk%?(8r;<
z*G}to*wzzzAe4h!C-oQU1h=n`$e!zDI)fw-&63&BJUtJ;iI3_qK^q2)B$n|jHZD80
zIJbV&!M99j70&6IcscAjhfiQ-RKK7<28!R{3;M$_m1xFgeZ3&wy~NsG!Qz_8$6V3D
zGabnu=3{kF?~R+j>9x*7+a%n|-~F
zzoQ?=rJY~9t5*@QDW%-k`-I?7f8AySyDunfWsRp9T-*}_8oytj5orYRv)~R?mMx4l
zJ}<~OE6Ny)->AGeYY=S=$8Y-vcQD#mh~ICxjFrx38!%DqaksG(394ie*V}IDh-3^5
zJ(!_$dtG7~jRDeEWTP~45KWSSCaJJBvT?l7;F|_@mYHm>V%VOt5>=z%E^5pFZS0Iv
zf2oFT16|jQiL}CJX~xHK_ecV?Y{8b=5)z>HU9^#Ilr}%7?KbQHwk4}kQK`lj=*AH$
z{f=RTEJ#~yp=N;jVi<46?IgK3Z$Wh=P-<8Afr<8CFbz5G1Z7An%n%~KAwy~L?l*jq
zX*4C617^wVJx06;#PY=$Ra|637+1WiaYj6QF~+C>#_FUPqYQ%8F~$=l`*JVF7?1bu
zm{_AX$N(2&0sD>o1^*xJis!OjC=nz^&NPqGjg5jN@kr)8h_$9
za3Yz%;5YDed1PsfLV}G>w*$rv{4Q)}{JUxlZ&2C*!rMf)7`C&F(H&WUDT|$)RhXhF
z%KCEf)Q2}vbOoa>1X3DRFveovVB0GpI+@+8WRQqw^~%P)XpC@w4W#a4u9`+I@xnCL
zuqL(2@oL^DAoGLT__~g#)-mQ$6sa52VVCL~ZCx8-3K!1c?1ZNq7?t2pkfk*+YT{vq0zNYX5>>&BxBZU?r&cz>SFvsxI~
ze!t^4TN)5Iw4vXVt$)HGVurIKpEOn#`CW<`PZ^m=uxHYilkW!f6=L3TW|
zt#OSu8Z=h|u0f2B#3zY<+AwXt$e^x90;|^9cqeYcSrDhfYmgR{LT^PZ=+c;~UppIj
z#b3^Vv0_ZJuSqDFMXQs(^VDtzWH|ToM$a0PLy#)L8r{-l_V;r}OXhk5DBS6J<5lNj
z4Fjuu&+|r(*y_)0&=_0=YcvJ3v?LD7c){Dn23RvfE+>)dhlRo0#ZtwivG{frwmq*=
zxv>0t3X~&^mF0YtEjuoyBn~?`5853UP;rg%VI4kv_^`>)*04bcK}R3^yq8gi9qMJI
zumhsvsh~qdV{u9eyz4+qsuHoJ7$|blwrBwqUsEhR)#zPEZq{qOYs6_wV*DjS>a03z
z+uKM=gVl@tV!?alnvPC-fNgt(P1!4hJ51D@(E7J|o$zQ3UV~!2hw-4uZQ=l6oS^m$
z3(g1^Rucm;K^dyF_(7U~Gf*Fjz#!kifDb=MlX(RcPbZX42TI4Xi@lA;1*@mYi$-ny
zHu@U2W!j7vjW=9nU~T{tCHFB>p=(vx7i(|<8{Nmc^f3lFb+jo`ykcLYbiC0Z0utx`
zzaX=SJ=Tv`Y7_#Kztk6plL3Zc3GBR%ey`5fzGOU|hH2Okm*V$F(c3qC|A@k5)WLPg
zZ_uSE1y<_(zVlkI=q8VDhy9ivnioqTM6|`N7wa_E|
zWrG4NZD3w8=g_u%Gk{iF`lG5uR3oPl5h-vuMM{{C{kZ_r$pRhBI8=$46dg=^!(Z%m
z+>DU`qZDspR58JbR+Q4>b(kdH*UxAM6Dwwaqr4E!>h(9ul|bW|G}4j}78>w^3#WTl
zz@@v{hy9JRD3R6QsDt2_{zd@7{r*NP1T9`MDk6C86$7M80F4O1(XI3-cHtGHSq!Wv
zJUUKRet^+j)FRm@FB@l}qu3ohNhh!|Iud>_im`(~t0khK02w;_Hx4j%if4B4F$0at
zvlV`ne^bv(!Od5`A&rIT|wv`~B
zo5Uujq1h39YZ?wCRG2uz*>+IjNxPbnh9278Y^tG~|1t*qA$I=v$DyXn^ZD9wMlmAE
z@0nl}Cz#mBALRYM#&7J2N$mPZMr-;VqD9pMxqbB{<2I6r9L_)^_pqNP8_#1G_hcAl
zc-N1y8#{#^ekseFVpMm&zJ=8@F2i`Ipzg$}#%2&Yf0=4LhMPN&|I~Pb!d}yi_n;Nc
zj?XZrorCh=f;m6F`I;K7)8ojurS^Z>d0g-yn#NIck|+G^u8a<5!fY8V0KjC7YR!PJ=4t{g$GLP^7DGM7
zqRkdABABHptXH&o1FNEo%j_-Qp25C$nKdKt&X~@E3w_0zB-TR=O_nL8RzG3bwPMlYw5<{J^nvf5<
zUoyCJ8}?M^^3kTb1Elm&?%p2r6Y>0y?1l#ux`UPUf>MEVslL~I9C
zxg(M?eh_aSgc#&{_9?KI-n33epJ-NEMFMr9R|T;tL@_WL%-%q{8eOVu+Pssp^~~S!qLb^JFHude
z)i>X#E#z{2bD+6>O~~v%xd_^@hAqr3QH*`iz?7xg98U%#IFU_mV9r7D3JuMk!UZ;=
zp;=S-gKcbRl1scZ4b27QIB)c0<|;%}8<{mJI-rr60x_UTjm+{=`j>c|M8O;R-eT%xRJM!m`H*!Ipsg`L@Ex30=%P6>lRS23Dto!+P
zM`F}q91_pxvloLfJb8aVn;kTZCr>&6y(SS_ytey^jen(646G6zdFQc%K{F9=@5>a7
zU~)4vv2w_J1zy>dWv^2J`t%xNUpbXQ^@)0O7g`X_%o!liUT9_xrm6k-anlE9+-8YT
zDdFqZMR&F876+$YQ}CDKXq*9!gx&*j$qShX0`(Sb@+C{+oL;>({jAzTJThJBa0
zAw;ghj((KR-U0S&2eUhh^2a)wunMw)b$=RT&SB@DHXG9tEZYgYu#3!OS>SFqG|RK}&Oqx6
zY)xmg7O?w5XY7R=nfZ)^-PzBWYYK8VY-CoE9XIXpXvtn{gc?{JmfFJ{LG^sy!yJV%HtK1%k34v0
zG%c-KZ2tCWneFXqCaROM4uNtoMKOk5?P-3PcAnb_z{oAX*|-U8By?JE*^a?1go4;^
z$B7DLROwsppElk?kM9d|xM3TGpar_$$%TUhZi|AMK_?rAtxq>{#*m5^U_I
z#FzQ+FPgnQ4z?z<{%-=W)0zHU@c{e!O|u1x&v?sx#`%m@@fM~&oy~d6>`N63Zv#F5
z7d+?1-Z9Nc?9gT2Gjnmf({3g7V4dGLJL2~@^gHSUbGCR7m;VpUI#D339!WQY;@_Ml
zj4#0z6J~s2-P}1Oi00{a>Z!#b)w#;j5UMunIG|0p|4ei;>G!xv1VSRaG1Y5(Of3D
zLz>krVUk%%+&7-LoMgHM;R0_r*#x_PH)}V=goMRUY|Iqg7Iw4uGjKpnWUDez_AKi$
z)m$cC8^yIx%xGD@d>Wr2?KA~~6Tx%m*zq}L
z8M?cQbIn0z;$Uuoh#|{0vLgngX&^iV|HsaZ-v9f_b=Y=Fn|3IKF6dK9ck7|ng*XGW5>#I+M3307l9m$)Q;
zG|5^mx<+RrC7!JcSU%Q1#Tv;g`YpHMnoG&W?PQK*Xqs}iEMOhQ?ucL^FI(EWMgE{m
zRJJyV_vY}Um8}DUnF}!%(jWy-${pM_m#?pCHAfSq3|pJ(n$2d{u+CwS>9#{Z28Vv7
z`IVa16N32ndRD)-RZYz2Y(Q=6W1#jwwXL$WI8*9a^|5ZAt7G*5etuKON&O9};?<2T
zIn{artK|JuC;78fWX@(MQ;~T-d#tXL+P5xJx3c+lt!{W`-wLIctcTRmtb0A=UBfoi
zbModPRXmi#66>Sl*=$&SCwXCgWZub+*GJ|VtU&`O^~DBAUBl)!uzHC1x3PPn{#3|Id
z2?~A0K5k<57S9}HSDRQ}#cLbcQ%#-EuQ$c#scczO)cGmHZH818SluA%Jh_Rz9t0dF
zv)mwN7Rh&!48^WG&5(H=o7T)p-O`LYu$kRyhEbf}#JWF@b&A}h9!KtJY}4b&J(eXk
zcXGFCj?~3$M00fW#~k)+b0@Rf0?FyTMGI><(KxoZbaG#5iR4^fp_SD|5U*}yBc1@7
z{KnbpC#+uL%`6uCBrG039LtVAiNSx*q^GRvAidOo%9R&KDL)VK!}l5ZHF$7U_B{-tpEh#tvhU6JFAy4
zft6}+v4o|MVhd45?Pd?Rx9Z2Q|Ig2Mqoq2a-`}wpI#@3P`h05#D~}|27Ig(_HHVk$
zhFb&{-z(2ruK>=6pS4y9;`I@1)bmzD@$3km^SotH0?&QHS_^$#Hm0Xl+jZswNau;n
z^SCF0AMI)BqImHHztG#7h2=giya_>OXBwZ<$GYv3zRm_r0>SBg$^h$iksd#hgRQm*
z`O||fkb*MVYqMhF`N3CVrx6j@=PS;#hFZ@R3O?bP1=H2UtMBnJpXm8
zQ)Ef9Wq|#ZUT-k?Y};|HOnWRDdyfpxB~SrhE(zUngL`RRF9o(NgdoJ^|`PRIwD
za5x5u(xsnSB~1_=cdxdf#+Jj<)&qghU*oIRTU$lfg73h-C~I4f;2(c&fv`D`U;M_J
zBnbKJt&LVnQ^?1Fk&h>;#mH+5k+4$~`_<~0w}yT#$!y6aNMj)*oO|K8gp+Rkwgy-y
z*&iFNfEy7X95}Fpx3pyD+hiqy{!nj|)l<;f%uUt=+gYj_&{+A+_@uKIo2@yrRwcsr
z(W0^6H(OO*-yOo+c0NnqV*Q95e{ZomB50mtRl;fWT8>q#JYAi@JHCMKRue8o2HZMX
ziUH+CO~IiHk{NX?8t3@l91Dyt9~-;PT7)TQ6~D7C3CCFcb^vxIAGh7oA?wSg@4$NB
z&%fURL@bbiAoR5@D5f)Wr}b1qSX(s-7X*N&a;o?kQ5xp4*LPa0lr@opm{tm&M+BC>
z(Q3p7?6R!q`U6%3#Z+++x<|gc3cz(SVt{o@!BY(VUnYQmkmiqy##e}VEDd5&r0cFVx_>;Omf3$oL|aL1-ih*Y5a>iOqE__PebGGIh5?mCmHBR5qJ8++$ra3ig=~`^HMF?J=t%
z(0Rf!Ye=+f%o?C;bsJr0wDA<<$G#9uG5Ef=jd;{6fq
z&EKuTsPebpt$ieC&9X0BB?D{MpzvYPoUpKM5xhVovCQP4%a0$np`ELjt;v)>{V$8?
zJN%o!tj7f-r*Kz{2yS3%zE!_u5%2F3-4!9B@5iX3$km$^+|K*t16zf!c!$3&@U_2V
zf8Da`I>zJNKn{=DGw2I*}cI_XlPFxUHF(nj)UY9-fgcXT^coWDt7k3uo3HlK)iOG
zf7;pO6~#NpSf$RMnz&Q`(gmNt;zn1`Yjm|p>y9>01EFRo#PGd{M2Y9uvZx*?d}}SQ
z-vg5dUepUcJugA{bq(L%8%>H=_VaFiJTND*h4<|1>0`?dY=7C46L%bN1K}mClI_B?
z27sR3&r@2=TgP(xd1{Jh*YWH9JUi$q{mm<$m+<)T5AYPn|F@us!d1p7kEF0)ox?AjAE5ZCgLH58v^uq;03_
zyPop#MfaxM@ZL0z4Sv@XPnpKP>sg6=YU(i0P}-uF4)Y8aM)UahJfB93Kab$c(mW+Z
zw7GYL=Ynf9ZB;(!yx%|4vqQlYPn+z~47|z~&cSSKTjk)!hy7v{{&0?Gnn2=X)8~1n
zBIM=gd+^{#LZ=0G0z1FJQw^8YxJ=Irl>AnvXC{T_LOZ$1LOYqyU+5`m7u&MPspsj>
zJZId%p46{A*ML10mwA?x)YH)wFgkQ-85{Ay8^cSj^z_35XCJNhyc@6&cPU6mqe@9za}Xji;WA)?UxAJ*(r!kzK)3g~&i+=lE9c8&7H6;@f-!JY2`q
zzwsp5vpoM>PbO_WO*eXGM}Zr8JjYW9Na5M)SuOoZszoWmU2N}G&kMMiSJ>t`DG0}z
zd%I_paDsod-LnV};zg|5_gH_US>Nx0W@|okpopDR;hVnqT(hxs^={8c)WIfuJoB-g
zoZ5q3C6ilwJ@rL_@uz<9Y(!@=nRLLj98aV52Rt^zdPc6tmTUVq*E23|H$0caAy^p4
zL;G8|gPzjKf6}*xkA8#LhAWsC!JhrmlPoOd{2KYHu~Yx0ktpnz-ifuB5n;Bo#>p0$L)UmWqY#279g@eIUQ-uFSoBp-!$|S!GS=)m_K1&`hxW0p9xuFJuyaqT*ZJe!4>fS9V~CB$*&;|x2NDHcH*X|(Z6b#Q8_k>x4GpRFA1CZPY*Gx
zAVRz=cJ%|8$kHshVzV%>f+}F}TX4iXDiegNogSo@Z}J-n6)^
z@6e&J+BzL9_SUgg_9m1lD06`F6gjj)o@iI(`I&UbVV(~YD!H!E?^?`$IBYKF9Uv67`H6cR
zy_|T1c<*>{!a1IS}Ygv))1#ncg~l=0q=Vx*nhE$p
z^`^TVT%Z#xtemLza-vflE>=&6Fn@cYO_=ZD#2zeUF?GF9r!8_=Ze1NN*qL#5o-Ph+
z;1lQDGY++WtP|_(aHv*0e7H_dHzwY&yZ^Kk``h8Ob#!8f96nnICzj{%+1fijoLSH5
zVmpT^IN9MOv~@l{%(J-+ZJZdf88PnGPArEd)$_KfG}hrRv~Uu?a@ZWroytx)?2pHt
z{(i(J)$`UhuR3nw8#@DB#17T-_D=p~x83_xhugQ?iGfd<2$1e(9qM~Kl{n`lR#k%u
zNIW;)W)@a)VrTZVt@XXdt-WLeQPe7`!C2(E`6v6czPEhxcwF_73b_mksX0zPWz=8{
zV(VD-2Hr&RkzkQKkS;~8%&H7n-JED
zCGntsr|N4LZDx<|jQjdkd+xz5BIfI7
zYwfDRG9nJ>r)+LRZ(^Elc3LE-lf=xq>#!_gHk|lAA$BG>MBg5uqBKBq(0;&)p}WK(
zT6~2X395*J)j5sqM2f`!BvaPw?CGOH(;=Prg`E?9!TR;vUdVasW8PDc2I9pUdryhd
zt`*n`s|RQByG^|r$v7{F_{2Wz;az}h`{O;mhYE;6ta@*6`GhSn`c^hAlALDecn1d5
z(jfg1fdUtt-riWa^c&mT`{YwKh{CPXwvWI2B5c1tHJvmZ4@leYq}6iL=*b)U3Brv(
zl#^c7HCV!n-a3YcB6$%YD^!7;jK-dS5j(TS&b{dMv)49BO0^o~dIaE$fH(xi`y@CD
zwxu|6{V$y&+ftlxRZi9tYoex+tZ^T2qAQ)0^J=hz%$LC0pO1-7Q|a-y3vD6KE)Aw3
z;OPr)^~n2cCa5jJOnCrUG{fi@x~2v%!pzKc@*#)@F5()MN!`rU20$x%SWrlbvPlGL
z1ymWR)eK^&56v!6_f!3V&MvAi9S(%hl8nJGwu<|MYBgcJ8Hl1h6(ZBklp6R7qD3zC
zh{_c944#eco*~SXF$$+|2Uw%N-uPP7tpfR3YVOhA{Y>(+1&Z4sQK+yUmsct#z6mn!
z0)|n6T)5)^?Z!popCTp%6e;*9qL&*~nK`SdVWa#ccLFoQ(
zAhgwbY!GJ3ML@VnCdXhT1|~L(VjF+(B(x+#5!hm-;am&>dEkM7n1edB?gCNPd=h^R
zjgFo(=)!0-1q!C%!kG{gB3x#9vh0gSrd*BPmt}t?YzJr_-cPjvaLDgO07;-a5!NDu
zl?TB^&%i(=yx?8l=xLQmnb!&_aN$U;LBN_EfXV(i2=UtUTx1Et5hpD{cabINLg^^<
zDLm=U5|o`KxHurOO|dcmG2{Pr0eZ0j<1i>-IpnD!5$&wcf~6OW&mfTE!~gpVObdbW
z|Njcin3?`By_8T#t@N;2xzOHHfMme2=UNV)_>@qnMSHP9Lk6|z?-DC-hi^`{j3
zUl!@VF%072|Jx$1_cYC1D-x1!kdlm*Y0#`X|L}5wY%)CeMn^!=BND_!Z1A+`t!*;l
zIL@MaDak~4G=ovlHw-pJ4MK{ab(|MdvT|r2CQWH0)zj@&CwQ(ha1)K
z=J?3UX2Q{!x{$C-2?Xn>B(sBqy-7k6`wIa)yTZ|kO~i-05I9b=SmF?G@ic=}hyYC^JrKGQ%*1Ku{3H$S*8(L#^#;OY
zFragX^5v83K3DKw1a&=7Dpj216zI^}>nts_&QfUn
z5wzb2i@{HJ1T3q-u>_ozQ&NgS53W#z?s0;eVxR)#z_=8eIrxDkVa`B2YOy{E-9plp
z0DRw$hukn!)8Tm4fW}!GHfmr66|?fKfa&Oc7(ggGEXX7x;en%vYFBm@ov$WLQ{dwp
z{0s8q-Aae5M{p*2Y`0}J;IxenA3_b1ROx^zM`so5ZRy0_u)MGq_aIzVp(%>ls=hQhI~VFOFIO3~PIHOGHB<_KpVdAbag
z_Ct38)(7xU8$eqq{nGIZjnKSf==Lswi;TZCIz>LF0m9|YawCQ}I~$R{H~{yTOEmO-
znca7L&eL>f014=ILNIC|%B7T{>ZE}w6_Fy@Db)ZJGqNepRnuRlB@O`7pD>^@@FL;~
zl%vML(12vF$6pPzfjOs?N^q%?QWIr#^5&k`0j7JM40Buj*VOzR$sV8Jw6_p&ozuudxs#NtC7ljgzrYAha#
z&7_76@?xldk_pjsz(odJ2;O=NzXf>X_7Ahix&v{YAdZ>E1`C5lL_sAapw4Z;u9xOG
z-E@J;hm;(o6eNIy3zQBgsQ%(u-SBcxJ6$@3@aMjk!Z=utrx4EDmm##D79N^i8}#^L
zUBf_dJ^*V;Yicw>95UkRv@NEkV+V_+yDbeCdzU?>28L-*KKX{izw%-Rwgj}Jphz|r
zxw8{FyQkN>4lQ?pN!RI)0H8W6*8!;scex$_j^GM0e8PI{LvY?oi9^*iGS`8{hT;xb
zV^Q~y=u?8gYY7-NM|HwnfLL=zqb
zaRM>P)BXd-|9Wx(#`g$Zfbne#0psfgHri^?ZF^%ONec(WV1d)nZEXamGO#ujs*@tw@f4?1*iw60iUQMd&LC^4z!d2P9$T2g&>C22VcWL)ofgC2Vx!|O6SK5
z(!kbhwhp)wv9|107eSep;N&NOIH-;dkdAJ&K#QeZuzD~u7~wCAB+^+((^}}jUgun<
za}bk-HAK<{ID+VTfbH7vN~Dnz(nR~9;6iX=!t&RMEqSzO
z3<#LkVbnqUwT-<%UYi7L`Az3JfDaczIma1e-=on#JlXAcVvT}VI8d09@+5e=*J%Eo
zr#S5wHRvgh9^xfVJw&eb|KTN09Rogwx@P?-I;SfT*y(zaK3+Qyf_j+b3=DRLV~r8g
z{-cn6l;I5!yibL~Y%uJb1h8H99N5spU8p<~i&cqu2Ftra#4}E$iyi6WM7r9Mu1=(z
z9qC39T#ZO$L-%Va-XUnBcn5zm_R27CGVO>zf?DLqQ_HeW;dT-9!vigm-^*B|A~CO)
zePKe^ZiKHC7_xn%0?L%={`pxu5tFZgexI-
zBqX5FZImaRD2+O2+PgU}r2*SBF;SRkdh#)e8OR>m?ST#jw?3rhBZhWp*SmzDXo`Mb
z#80$FKdm)?`;*g+F-2hH95)p
z2nhWDG4~zdQB+<3nc2;zC!3wwnORcUgx*6(iZV(S6uV+YeNhY+u%ajl9VF7>gdTeD
zWoSxJL`s0rBy>cSE&)*x5dOb&XEzC!@B3a~-~Z#oL+;MZojd2=d+urX+;eJ3SM9Oo
z*O+YD^IssU7sd(>!6;?s;|?gosp4Ba6~|&(quPWo@TBK6Z2>rf2O~O`;K{Y_l&A51
zjM?SDbLPd2L|I%4S_T~FXNTg7`f#H7=egideE%0nuEG?A`4hJVaJj`lHx>WPLERD_
zIRX#&1VET&mmxo)#2`6XOn|En2ALC;X7g1xB$%KiQwr1!62YvP;3$ILR(BEeBS
z+^+;BTh*`lGkjbD;WGdW2bhQd7IL1mmSZmB_ONOuh&f&4PK^NXf?mAD-5jhJgC(8^
z^cgi6e#cgA;TIRddR&CiX;|~iMdN><(_{pc8YQUIOyJW{W(L{C9nIjL=HQ#Ql);b=
zg$LL%bg+grE$}j!d<@RtqlrZ^ods{k&m6QScI%?df};1uAfgS7O`wfK+=j3b{y`fB
z?hI~(Ex8;@9m6dcr2B;Ue4lu{>I}e+-}cgH%G3oFCOl5J9xUcB1|
z@jV=X43ry%Jc#O`&f{VKVmUV3!`gMYS%;g+_N2QnKnWDI?e<~8gfk(+^ryPq-%C*I
zT<%YJh4N4i7C)lX-5)zRe|Z2;P%+Pew_N4^w-+#Cg)wxam!~4E70|w7qgmmOPUfjN
zcKZr;9~TSo1g~BGD%e9Iy3P1;j!PbVEdNtXOsW89W%}
z5HtrhAqCl`*P5V(X|5e!ZQD@eeT1hb;BhA@c`pHKH|
z?lV$bZYwW3V5P(F0&uVdnBaI53YUAJ-9VoaYCBwP5vK|AEPoH?R!F=N?knzSM`+-S
zbfAwXL5imbqkKNr6tOBG6vHCIfhlL6s64E3hj?06af~2Z+IB
z@S)a#!5@(l%YF36v|yk$G=zC)E))yW<8}WKhVP<_P`GS8FbEKn=gRTg(
z)?fR4;h%jTObvAL&w78c=<`^Z|H0=k{mJK9^>w>HX(Ib|EcUGqe+6i$MOrNH!g1!`
zz%!tdz_B6>6v&YqNt^q5s%NOe$vr}7=>Z*Cah%7~gSY)iBaU!c-guOcs?``-M5YUWR-u5_e%u
zJcZIM6o!;jSP_Cu&Vt+uVi{E|j^>CvnI-OIi#t~%i|PmkCKBR2PthVgtX=N0HW@CQ_%dKlsQ`o6Nt{^i6?A6Pn>{YlZO00}%3|opAkASo$;#W90JYOphBX%yH8x;Tra=QdNl(SH
z?IHvVfw@h|7C?bOhOzk5v7Zm~2}qN$Q+0-s3ba7TC^s;QJFt6Ui>G0r516~RZJXEl
zD0)51lbDexR-1d7v#1*M1b8T5g5fD20{U;;NbqJWyKeza!JpJHqyRv(87QAi2t0eZ
z6UQ*%%;ECUQ=7`35W@|S&`4V}%w}Xkl8(XDrSjci)3*8OMq~(|2NY2{!8e)B;AJ6_
zV4yJ+p>8W82rq=?)aE!XiQ&PA`8k+^c?@Wzx_<%XujD9f`OAQRxl0PYye@L
zcrOqf-_@MOSTmmkT8dy10gNjM<6umFINt>-0?qb>rNlz8gL6TcAjRwx-~=gj`Xe}S
zsmpI~Re}Ep|KX9=8jrxW8%d`?V%Qbq1ee&fJ|@;0Bs@F@Cp^QqnXbp1`$Agq-jmXa(6a@kGql98+X7@@vn)=$
z?%skCMRwS5enASnz$ZP#3!ov8d|{=q&$HgXS3Ox!N1N;t+PBWrC1u4
zhg)_@tW8u!e*D>
zZ?9cuIs(z%v-qqI1_*8oMQ3#|JPDZ@AWLjXq8qkUGZ~x1l^_h5k&-xTp4>%6}(VjKCEaXI(lh42%
zw=aBhvHzADZibtfh6Kr(bm_^1BU=D_%e&B2^`
zurwGY=OFxVoC7D00U0!DxF-cvVf}DVDKQ`ah7ZSobv=~&ht0(=3o#d^O3uY!bqx7~
z_0WJekMzWs(!oTVb_UZ*DTQB+(^LIemRE6ZwGUk$3AL)kS=cD^gnU~9YWP&*Mu}6g
zaJs$BU}>VlCuX%0?XUs(C1Q}wi_Fa_;Q!}5U2UD
zgzY23Hr;q!*`6SLg_H_7G|7G;e3c<#+X}ZC3BVs
zc6WQi-TPhW06Sbkqyd9z;-g`uc&*%0)VkEx*>LxF>S?t
zh6pZ0*B3*{s&SZk!oe6pOUP40ilF1%f$Bfh}Ey
z2sr12lx(YTLlLcwW#LpvmC(YRZfr>7xKF4Mi2tKbuq^#I;sbX=t;s-+1aR2Y=6S=_
zP&oJ&a=&uW6F%R)6cdXedjHTKG-jA{0P|Ib8R7G=EI@16?F(xK>N*U)0G-Qap{ZK&
z1_qYje>5F`^YY~BFftCYHV1VEz%$?ydrgDbXesf~l;Q=&5XvT&smL6F=w2bIym`Y&
zhjqraCxm}yGJ*e(Aod3_bJFHeLEF#*i^*v
zzfci}O-1Ctxrc`*c}i7Ya|K02c(qr<{HKOPV?mrSzg{h*s_77(Bew(p
ze<6Y3C^iWcsbh`_N2>oGOM&N4Q{btC)nbHqs+t-R#y@p`1Vv5ulo0}E;6#7}eMRg_
z;VEvP2>7jUa6}FtZNPbe$UpQBFQKdaHM#xYQJuhaO9?gY%uHx@#Ui=HTYt#%2G8uK
zz{B5o2;c6R4Hn6s*{sU5R*eMQxBLFHVpQjM7#d<@XvqJBp$|s;Ll{H`)vjKr2R9M^{f@l;nufxrJ$U5l$TAb3hX|KK4cR4iG!HnHRJ)B1ZTL8Yp=#egLu2ndhQ$iF;3#RBl(y&m#sd(_sD6#*J6M|_F^u7DwC
zDmzAdFq?1^6EUZ7#dgKAj0M@~6h~}ppZ1(!**`6gphfM5B
zcdrm+YaU=1#0`&9fXCQVb)~>M4^=QzfN>q72a^Z;sZggV8+HA
zMwro^d9AS1vcrGdPBm
zT?C}~5hiINTiaXF-$I7ZXCz`Mxs=QTaasj1)|2yGe!tjE--I_8MEFaY+h7;~;Mifo
znFzQTFzS#r%cEEPL$dy7RK0k9|88Xw(zsg2EC?(EXdCWjVi_E=mv8{&e-=&xW+SBl
zOZb=z2o}RL!?`}NYT3Ntfn16WAQz2Y;Hg;M4Xo6#dCy`FDnsuDR-`%tTc0?f6Cz2%
zBkB}}z(s`%JmteMuQ(RvqNIhM#?{1>{l&@pyQ%Zdg`V1RR~e0MBai^O>7sualpq99=35$Oc$^KsAmlafsTEV}KZXU{VkT
zqRhblO9r17VAo#Zq8{AGVvgoP}|&hOjB-MWl*@65?VLfM*vW
z2YM7ZK`-+O8dmrtX>wVQnt?6`Hh_Zh6yP7u(G}uEpa3Q*wuQiss&Gb}2j!3e&Va9*
zjD;BGPZaYOlz&8gR~UL8@;mQ_0Ug425z=$*fN>9ob%eQ|<%#~cbn5@&${n`Sn2=FfWJ1FDdO!$#I3ES5bR(YaB|M5(zSf0F1I7#@wZ=U?!
zgU9<0Nb{D7j3MCrLM{WvL!c2`HiK$S53KPdJ)lEGOJFsU_YI*mnx{kDH{C%oTS#&L
zzR*=H4E)X9^0&eO;`BhA_m{SPScckIZZ68;n7NMNxr7ywkT%8P2%+f1YsxmcfnE+p
zC;?76#FTv&J2!!PUzKoRjoq2-l;6w5Z8IkfjvI^kHb}VXd{aaSDR43?1ORAikCTJZ
zK)}Tri44|AISz+7`exH(&~CWcK#zel)EHrahKo%ET!ooK*mX1EEL9gOsbDX3_Bf_tYJI$I;epywkH>`ES8PTLH{OZ+X-bxa!D0V-jZQotm^>B3TyAP5>9
z%mH)2GisR2uxA-g!$0%XtT2F&_K4%_lER0CS@PbVx(;zCR(Z%x9_51)-Qar&E{
zU+3{=M2a&^K^ufP2OWH
z?7<_t)tq8R1xV~2q4A0YbP1S7zgBFM@Y^8!*TZ)eVI4Zd(u55oJ{1D{(Mony1Odg3
z6y*Ho)y1*F#c4E19evvM2B*d2gY`Y4Wl>6V`pnElF
zm=Lb&|HTbDY$5#yJg2?2>-?X#cCiecfA7}rZ{$=qSq1HG*l1uT0MX3B=l<9XN4qz9
z5>iry7Fhsd#P>q|)ldP44S<=I90lQ^Ms)0~QvSF8yg0b<$ARF&i{pUxine{K1Hl;o
z%%_^U?`?4ykvJO%qr@r0e_j~4PdTJpM8H=R>o)*sFru4S(ZY06ykyPIMPcA*?iNpE
zJdRHW>%%G?73~zBzol5bygf!^(8+)@&$-ojD9*o+%2-#>xLl9*_*PF|ynHUtDtFk^
z-XYDiGLCt|!(2JnVYR7^4>&bC^i6EMwd|y4rWBJodXzf_G>3~wdghF$k~QqK=UFF$
z_7hS6)qsM?^o7S7d*Qq#MiEI5tnP|MHKSR;a2uV&uIs0_xi~*2eqwgS3JvP
zdDmq6_7_hR|E=93s460u9$A`+2&Q~g;L2_hFclH64j#3dUh{-G<)7wL>znx6NXoqF
z*^KDq_uj(CHZM1E684__Y+$U1
z70yWw%q?G|j;FXzrRa*qby}g$CPdTzbp_q1tCUA5_A>R9c~K8w0NFh0F!Sj^J*9Hw
z6ZnapEc5Hx6kA_OXx9PP@M<-)5U&~eyNUo{H`gKnI-wHB1t+c)U0lf@UAnkZ7gQ>w
zZuJ%aW8Lr*e!u4R*@&4PQCvS62}+9^%>3ToF#sn7_w5dIJF;MTg|w8WEAE#w%sD~g(8-zC&u
z2{1mRg^iSC9Trxn`#5T{^D}?i;6I6Us*#cu|6AB}+PO2S68G3_tazn8^mt?C5&6e_
zn%`JyCtu!7HybNy^3~0h)&$3}Z)~PcO_Ve&`7uqDnyGL~9pFGOLf~e6aD*F>oh!}Wq)`jY&2Dh;Fs)c#Jpg;^zA$mWZZEexc2cmAe@Pw3d4$|FF(x_2oR
zVo~`j?};OBj(P49rQfBL=X*sU6kei3
zLEP+mCaGflZ}G}QKkrGZEAQP!pEp%1P?Nn$EvSa6w5KokCdJbjQ@N8OzD}xC`M2-k
zXk8fSQ)*}mQX%0GX6w`_XlEfvTTSj3um
zV*nek9ll#>L8JC2DU`4;DZ;wz9;K`#7p|a=&6Ed`qt9xlJcI;|kvAr8igVN>0i_;j(d?4mMW?a*~FZS}3)|I~8d{3neYGs{KyI6esO#p`^G&
z8I+*;0}*g3yrpP_&0lm+OXX1=JF!mpU@Tp$OGDJSKVa<5Vec@dkSkgu!d|o%%!|GI1=xq*tSZ!jNdLMSPpo3
zx=JihULysfO~y&Vsd%jq=Y<_rgnev6V@C~);9iUZEW-RJ+CIw$euVRhF9kRs{7s|S
zhY(OA#6!`t%AqxFm8wYDe5$QdMLxQVy!R=!q@`A?`;-L^P8W0a0cC-V{3tUYRyey(
z(j!W;1G(QCKdyMi^M{{QHp=q3URJ_0%5It8D11(75GL=MNtIt#>PbIwuA)Q>NnGNQ
zXU7`yigG;+X{!cxRhmoko?h0jHm$fi``U1gjUD+H2wPO5_R^czU(rj)YIyr-Ov
z<*NJoDb5)A`fBU&2*r|IKTih}6e7k{tI24kmIKWrNPB-tnr>YktMroOpC;2M6VaWX
z*23}1AqklovnFDGW>~osm2FN&-yu^KkAaOkZDQ+b(y9_
zXoEMPLQ$wPwvUKRE~rpe&fZ{W$H#P%6tfO$oHhv
zx>d^XqN@3!c=?&t3a3Sq)+qNBJ;-$7!2@fQe)!(LHA<*MzV@|s
z$jr2JgHm5wZAE;kjCVm~oUv8efTDAr@+FFz+mwF6R9K&GQ+^H?2PU2F52)++%G1>2
zAj0EU-=RFq!#a3}G7qmb%2(QAY6s*ijU?C31rVky&;#4z4H~mW$#er5qlF9=Ee>Ff
zr>?t{L{uNVOW7>Bb|b-p=_#&GKkZSTN1KoDRqhLbVa?gAxJA;OwO=bDN%^RK0^%nN
zl-1#x$drWGOPQ3oA8R?2?%uCVM(Zc`D_0%JnREEC(jGZ;>K{?2OAww99|dB3PVvW-
z+H&C(s|89)T5NsrjiLZMY48c92QnGOe5cfO;_|N3N(PE`rDKR*@dQx91knmW}=(kLt4slLmZtMJOY@cC2l&|{9V
zW`(IU;~-N%psQUZlyRwk>zAyq
z;`hzcx7VK*9;lCl+v59uN{bK7DXk6?!&JYVTDc^^aGI8}pAgAT)8d5>%c)fQYvDA@
zo`Z5JGanbME$!kGy3(FeR;}{t2q~Dfhw>_@^B9;9D&F4q_+Z=FcH7pvis}I=;>YSg3TGy2D<1r6{7N+S&fCTUj;nf{c?~oocJ4F-`+&tC5m)#u`ya
zeMpjzAoFQmHCqk~gW?Zht*i#>cT57_Y^eS$yS`$fb0WRoMD50_=TsAQ3YS@Ts%4Sx
zcJZBRZR8&N?oRcfEZ;gwpPOnuX^!=asXirgIX%`)9j0AAh1FF=nxbKH70G>mZKjr$
zZ=SNsG*{b8^6pbsj~41xe3;&ArQU%&-HThP<>H8QsBoN9(@4?Hujsp0Y6Ij~N^Y%|
zOTIP{IXzS!)DRrjjxz7+YL;t}swSXfh`E}cZ>{DssAcX|1s!d5ubRZyFW;+*{8H81
zs0kF;#=eOQk-(`_8_bD(W1IDBTeUT3WNLZ8YEeWxwSo471DmltNX;9Doaf&M$@3w_H2>L*CQ`^1B4eQokej9Tqtat*Tzlk4dZqRabMT3a7f?{z^H*Pc+{0V~OR
zLj5IrSubodh6X3+fXkQyB@x
z8JU6Vy{J~@$|8R)@=CspMfcsm)Mh9ucT~&8pXxW-U5!JeNoJK4hk2qOb?B&aW@Rf2
zr6e6D>m{`sDA{)}sXs_TQn$GA9)P=#)4bqDy-
zW8Fbirdfl#t7)=aaG7#HQb$%RxI9wCy~Bx50cPay@J#$!>0o2#r>N2_bXq={5YImf2p>vbe)%AnD5W76G*#}xtiGY%_ITSZa9BY+XMM6YJI-H|ctsaaV$x$g`
z2P}>&d4sQl$7UVS7G*Pk0@a_aHkMCJpm!##RpdhxtSOT*otXXnDJt>+PoSGq)H?E>
z-c)C*x`WHmX=*K$)*aJSLzZy}_2C(6JNeWEnl?i%Q}#A63qpSb<^Y!f^Xv?@PI6JW
zOSs>a04txb09KQkYE8Ywv1l2au<|k&9v+%7v+qn*LL#_+v((zIV}1I&&9a=Z3yGa)
zsjqjc7N}>#
z5^qgKM#;%)luuwAY^qSYcoM_RedDeAE7V2~%-CD2)K80MOk%uqn9FXTF`>hjoH3YC
z>>1m6xWtTU*z6Zta3CC~Jddx6W=v?v7@GB~?HRkZ+McnhYt#pL#y(mrX3UzqR_(*o
zJnb`e8kc+5sgt8ozLy7dLpgW5`b#L2#Tk3lPqfP$fw92|62j)m0z0+d3u-^qdVR0@
zI_D!ie5gbxu`m7CJDK#?Iyw4?npU!tKOI${bHb+i=m~W`X2EitQtuHoxz%ZPxYlPK
z0`f){6XO{03UsRQG&(iR@}5x(Kx!A$h4bKDgRJsDsFNJvLK`ot1DPK`cuAclWzM{I
z33N2uy8e?oKNQq;_I0%-M(^TvHCZsa_?sBL;hZ@=c+d=^7x>V!oA}UtYyVAk3y=J?
zC~q}6r9_@K#BB>VH-@yt-;*INyCQIZhzYE_6yt3s#R;tg?uMXU&Nt2R)I82xnQ^ph
zoc92i_qn}{+*Vk;_n=%7yRmU9($ig5yiIS*TNijhExg`|C0__;0{_|ToyRRt)4a=h
z6+ESTllb}r-TNw+r*y!NFY1POA5yE2w+emU7GnK0NbuI>8gC?cw{qDy(c6G4yqoAn
zYEs-doa{A9mq=QNPa^y{|95Xwpf3u5)zaR%w*ib}QX-GtPQN#qTe;}>{;PIzE3im#
z*eUP<#pVMmVf@BOL7e0!+Dey-UC_g>3wB
zJ+IIs$2ah1LwB;dXuHPVr?4F7HukP`iHwd{=z;BVUaLnFFA|U2Z&K5y-sj|fJFLk~
zy$*-#`z1i5I7;(F>%Fps_TA&HA@>d>qupOhPo{pKCcJ9RZ|1!k9<>lj)?k2Sw^=gB
zDao|`KJRJy=kwO8`@Ncm_VfSc{TWR6tB&5?QrHDfwfkK@O@7(i(fcj@KHbyQC{KW%
zndQyz^C+Q{_k^@$^ef&pB!8`mLjG|X!ESz>nx}hfC1++^^w1!Maz;2sCR(rO4)uZGGKHQSMb~(Zuk0nzSJ_
zoEmrb8dZj^fSBh%XeDG@lg$ED!xf?p@zvP^afL)~E?i|%MrUtA#;518N#Q`KD>Do?
zKRzL@A~-pmY<~5vxI!YgNL-z;uMqZ#pXJ+ENaW_k)uB(sXCZm=hYs3Ta1i5~hbM?<
z*aqR6SFQ>mAcu0z`Ei_?P-Jprn7%$q6<_t1_iR{BZrdY(Vk-fcd8f&4l)JqKF-VGxJ_GErB|^4{%$%Gr5{_et!S
zY#f5)R%@-3L%dIf#^nzT=I8__Z5l}TkMd5v|5(YJ1w|L^`V#`Xl37@E3)iQLE^H|g
z&vT1z;rU9sYqYmi_$CY6DM;C8JwMu8)`4}o{>f5F)amWG>i8Y=RH;;2^>$oy?tG)M
z-i7GS*|A=|6ci4}a&NWR;X_8bMTA-=*u$(lmwV$JSS9zb^j2xQ{s<5&gcX%l_Aj8y
zV*8gs6Nfp^z5<$X#>$!Y6^4+nb{?TsE4_`Q0|GXjxgc9htGq~m+>a(bE~Qe=DsO^(
zz8`(M3ge6NyH(!iTvlD}t;*$-tG(yYhl^`5rJ0nq2H0?=KRMQVYsokIQ~kBxRkuA&
zupa!>J2X6Q$vJ>+rz`~C@20{pykaIfx?_fb)feCSwpE-r=o_Y`Oz?cpneb3zimT?+#@rx>>C5
zCMtBi7|Z-W16-ax)+)OOuXxE709Xfby$o
z>=ExY={VhW)Z0})u-RI2)O!)=nMc!)dr3ZdrO=h4
z>|Z$N!Ce{to%aspQP2C%TT}WS6v3{jC%vuZ!lm@wN$}!B_ymj!JH=Sj@7QcFLisvnE
z0kiP@i(BB_quUFguxNmvSpL$w?SdS_)sX!9gb
zpnKr56XVz-niosCG1^LsjnG`y^(gI_>raR2l1F<5tm$b*Ya^c;N3#{Jgz~>v^%6MI-IC|=^MTZwW|DCEjf1;{hXvtllN`0#wTk@5{&JuOKUTe
zGKVu-i3oYvl@Avmun&ZWn3>eW5AZT+hFxs;Yx}W%Va-m_Hh?hSpf}2Bi7=RDmC>p?
zkc@p*dF>E1rKL3ZM=jA>T1opng0J4It=)q@uBoj(C&`<}QDR-KA>V(ZuC|&>wVpPP
z%f{j0k}dBHq@S{
zR(EKNrFFMGmk*q_?rW&caX1EC88v!^rjh$jt*5K+8oZ^E|EyM;o|&SB(`R>T_sd%k
zS+RF%$&!3*993_s=|TLawoSEO^3`lBzo|A(Jbmr%5>E%+tsQb68#l^L_nBG>W!$s3l?
z^_JQL7_PRhG$1^#&$iY&fd@Tuuhtm1huy2q5x1MP)!xMIMQydFC@!|unqUmAwEMJG
z*kW2rXYbec;caU}JIxR`jy#~%QaXgTpy-I)?MEvsx`W|Abc7wf``xRitre
zwNGhJdrU66+x1Cpi@fOyHF-+=jC=L#Q`%&C`}bDP(^|9)!Z80??atc>11t!FF#P(Q
z#)P4N2MxMj4n@D9bp`JBctNvVOL~K%)nITGy8lJ3zg#e$_PwZ$K=sc5(thSSrJG6`|KsjYCzKb)X7?`r*B{XRn{D$^(L
zYoC#;8HTGdT!F7Nto^!r+W*r*4*uMLr0XTL)=gN9Ae4655*YgaQDwqD`R&SsgF
z!He8D=G~ztvLu-K!JAxT(|BunchDRVNqX@k?E>I=xQ8|daMAe;t%<7-XI!bs?JIOR
zQ)^2@GPSy{`Fz(e?pnBOI$>lf-B&A*p10_$rMX;RaF2b(J+`I|(o$r3%TnvBPc-N6
zseG;%u;eeLWXti6DdES#*h2)o4hw}seag0(-umD>|{Y8BmRN^PcURj_CH-c;cH80*qh
zEn1R)>^XWmX!lwgGhOQ@7kp|dGZ1DDBT02akkWVKE&?v_^Xz%#jM$Xe~h>8?4pJ!IbgrT5X@}Pw+5D9vAdWr`mcb}S-_%~jwRQcbRy9ICaLK9^uD1(goB9WFdT)%(1iBTkcV|QF
zhaUY2z_}MmrrCfSsp!ez5S0|YJ)HY`D|$I8t|xrVDusm94}p?tUfWAg4GT-4U8;UD
z@!~3In^lV~XYA#`t`1`FSsE5r)|ImwH6*GsEG*@Z+9?%l&RmC}w7RGB>7oE5eJ0JUN>2wg*z5nV~nm?k`-*F~}W3B7t);AO&@6rD=9*EN^{>b2Kf
z$lGpGkM?>OY&&0Quix)t^P_tR&6}y$qIaIulM_Ghk704Te?U;sprFnBFxg4SyiE(A
z)HPS;&5tqR)KxJQI`ZUSYa^T6(Cjw0J*B?@){^$L{)YThUz+^1?o)>bj}$`Jgqxy<
zt%aNP{6t-+@1E9c!pfjOqnAce{~0~0(U}!c`X%#A_^HBFHXJ@YHKciRYG}+7DS8nk
z4u`o2(N5Vc<1+J?iMqXy+jVIO8i>1YH?Z%hu(b`RHP7e`uwC{1C|#jSzvzCd@T?vO
z17iJW^>%Pq`QaD64y}1skCaByk(GL7DtuONERCaT&*>k^fOGD1fNeBweNKN`BC6Oy
zzqdtpU@jddqC*8h`8*#ZEt$Oz*-19dlMV{c*%t?ZzeN0Uc8
z=t+^&$8Z)nXmTug9*}stHc-e
z5}F#?JZ7vEJ>H%a5kZhA<-j;hN+{mx^djihSQ?C?LI4u+4oT5tY*32c
z$^8ys^H&exJK_DtVKC6b=zrm2?tc4%%KuA$67$&eUt&5N|E$+?gt|vi*Pr#Ol+aO6
z#gyLJQB3LJj{2RL&21g^m$2(y=Ow+C>!*I09v9_Y(G#fuOL}#e?JCDNblSx=O5R9$
zS+5n5$5`#a+MzC2bOR{x&dd74F2+muINJWQo?Myn&OJ_y##$GT#xN;*gyAKUV2@_~bx?Vo4
zkNrAT>x>RfB6nxKF2!}$Bk2Cl`peQ}TG3f=h~lDMRDD%kz5FVoCQqROb}{=^{Y!|?
zZ@i{gmZs9!*Yt*Er;0-?4l}1TL*jC8=0#d4zzTDZ5W{nWF20WC7xlVc72Ry`I(j>j
zK7L*Afc~9*U2l!qs{4lCU*5TgK6?Xo2GHRydPP#YfVNGeK3%{S#jP$(@!ECOE6bzZ
zH2Dq8-SDpZTWIoPC8IXo@upbVt>4ruq1}s1u$a3{(w#K-j>Xz!J-w!N!rJH^?W^P_L
z-RBxFW|Ow!H|6xuBk6~3`fF(D>GwrLiUTo1F-^ikY-SsK}WO;YJEmw7XXamyVL%o$WgL;0bzk*jV
z?scg2M|vzeyJLx-6q=ui^pPG9kcWJvw@a~MEkazeNCqU^m|KvW)b7zAF_z{=1lMVZd2=<_p(ir1QtkTmA^#5C&&kjetMYFU
zX;=`!m2QZZ-iv;pTgpW4Zkw#~yIhA(;f-+dhJ>r=Ok9a)e3h1|&y;4<&YnQ_ITYGc
z-&%Ziv!`xJ%VaCA~Mpv_sUZ&Jsx7<`lsv|x<2Y1a!
z5Fi_l*x;}0==DDOeev_$91;@ln_{3mpq6=k^f&OO#(niNC_41j@4)BB_63VuM63Hk
zhC7*0hx+QZffVk3x{p6xrJp_V9s21#@ZqEVfS85LsZoDGIiG&)uV3`fcc+FnLVg~6
zDmx+FABF@^X~IR+%=f07NB5C)Af&OuR-J)5+*{^SgO5S3=h9Oj>n3FB2_J)&ETByv
z>kmPYNf@L*k!Vwmyd*HLNI~5;dErQIV#3`sXviQv9dBP7BpOM|5{*2UB^voS3rl+h
z&B_9HE+o$<`Y)&%Jy@U0tbFlc{V`mI4*@}&OO=M`_n_))Lv&r8fezRLQZzopK%?W?
zsK6tBc!++NlI2d};|Do{thwBa+~zW>F;s6VEv7C*A+ar{;Y0PkXtvWZ4DDRX8U`9S
zkB)FLo6?3uke^NMhKunYI9yLiSj$$k5O*O&8~e*_0p%}a2G$K1Gw|bZy%x_vg%M)v
ztu7;2G+aQ#N9x-FIV769vFxUe(&r=#E_r1_UqsbPfMy1yKx0^nLR4u!dNlSSKOU`j
zl83md@I$>4C6S)WO`D|qK+c~cy^`a=#lCd&8ff1n(yKW}U+PQG-vO!cIIbWGyR$LC
zOQ=z{{*VW|5DGS}+fKWRCat}f7X?SW0B(+hQv&G1r!liO+$m^fUrmO{)5FgyBTkr1=;
z7GaI$()Wv?dw{`3(u3V}B^2Eb^0y|7^+v8)HaG9F81jSQhLva}7$NOjtdF(7TZ1Mq
z!BP~AHt^l%fAYzXmfFnp_uo3b^!9HhPAZxVJ_c>`3VTxEPE*WA2Q1TDCodO#Z=S*&
z^zrESTz#k#;Ct@nbYU5&+zN_aE{bZ)^-KV}U^&Ey5p-ZVkZA=bew?e;Lo+M`0ONOO)q)*mE4j5?ZbMQ(~aZ4ok8a>Pbk&;E?DwMraPe4aa?$>$0V0{%ug5b$UB`ZWRQBsMY#>yuWJozXe|;NLD}!
zr_z@GtPLuy{m(&t{WEu&5v(^a!8|MC70G1d)@8^2ka+a90@gAPD`=cKa
zvWh1tt4Jo>UECJ+`y7%^;P;%*!NDxr{<+>CuGRwksIUy%Bh{u&dr
z-e2kOs7s4*4`*mht^o2Dy-9x_O~1ZLFD2jVLmzJf7sd-aHtDTh+xY}v1f^|+H0_>(6Q_`eIEwm#x}uq@7gXf_?7J#
zfX`{gcKsX=z_cBJeF?ep(LP}AT8IVQE+3@z$KLctK6;KiNAmT~(ZW#-;so$^6+OCB
zzZcR<&Q555%V_^j{q87!Zf3Gq)G8{!3zmo=YhAMopI=QkcVUgKrlj4XXlxfx?FN5c
zLm%zd6J48^!(V6(P2%6*&4He}hSrz({q1fD{(>LYpnLaVq()NbJs6^rJYoADH2yiA
z-=jb4vNcC~a4$L*1ZDhQfU=g3?nSk=bki=9zScjGKBerhp}Bo(MeNgaBz#9O(1HS+
zfd(047=`bL;xLW${jj)X(Hg_Bl
z6X7{1CZh2{{W0c9BM;ir>gI!xktR^~A<*Niw(39+92WHW$RP;UtH^a&e*w;sdKA5N
zSg(${Qx1bBF$UwfP35_ug;N?4WOeY@gBEZ}f+uZMu)^jlc!k9v8K6<)!Y&@!cGna~vI6
zN9R!?As6{$ZE=u!7SKa*>qSDhzvo-DkV6^Y>NUBiOTN|blaGxc>4csj7mT3f6MC62
zfj5^&P^%OAz=XhYKv@3*ZDK#%US>q+PU!t&Phnti=M<#Q5chh@{7zpVw1ZqcfhG3r
z_j*OhcKyHC-{D!m`8}keHP(|S^(daymreu$RDG*tBb
z-SqBhy^1h(;B@_k)4Dh8z#i1YS!$qGZgQQ`Yb3Evg%1WGo1NLil>D{tULW#E
zYG^`AqaTc6m8kIW>Zkb6BpVgbm`6Fn|
zS;#R^Ty~t*dvMwAoc;utv(M>m*md;hb09RhZgk$hKkB@F|M+?I1-*~`0j8Q!RObi1
zBP74EKj;m8Kd|H=m5^PL+z??)fWzm1(96h|`jO`X&=HiNIs7zEN{%oVRyS9R3n$Kh7e3{4Fwmr&29`ntS#Qr
zS9KRzS3oq-g}{vlFy-XFscU!^Mvq<9YesBiOPgH6(l-05{wh8ZbbR9f5pca(7@u$*qAu5-oj339L
zLMR5lxq%XJ--krQQ9p^dWfGN>P@*^{Y5yLGb8kA0$C%%5aT?`}FN*|&
zfN3*u)Sk9GpzQycc$^k|sDn+P4ljN`0Sa~C3Eo-nhR
zB)~R#1N&eF{b*sh(J^uGMNC*ki5a$S#Hf9;w^$W4B;0UOn+U_GQB1Roc5HAGAC|I6
z_wCy#{4o%g;SoknrX62K7}bijw5Uj9JpkPnY4E{OXO!`Xw24iURjF^3QOcgcQCLQ>
zOID%hqm72z+7-6LD-NpO2>8A-4la&1?w9tDjxk!xU-f5WfkMy57!Bov{b+2A(fLn$
z2179SEG5>c9Ukltt%owxoY%qT;FO5nAg&v%Q(^@OnuJ8uxA)nuOexG_iW4ebB=;=egKQO$nwDndDib_fGMti9jy%%rzD-^4xP+mFI?9kH8%Gq!v!ldAc
zq!8;qi#P6-zNZL}(Lg#wccYNc@4*hRaaT~6o>Q3b9p=3*pWQjNzDJ*KtSFy(?#l;t%ZlEzy{ap{nD
zP>RoZ2A3H=!y|{^WDX5v9qq$PTa;i_flhKL!Dx;xvD8GP7w>4!NHjXI008Z|GSy5r
zRD7XDvQZk;;H6}P?bEZ90YnZRO*Tr&`>)gWWCL7KfZGZ0jVX;vBk53Sqqek-+4GmKvX}NVkD!u#>EcurHZ0Ks<93YTun8qF#Ra)25EY@
zjF_~S%K&5ZX+;@S6mwTq0reJA)FzE>hy{?PMiIotO4OvRp)tvMw5*XXAHY&9Yutx#
z#g#K^fnQknlrtQX%icnzjPiy-Rm&MM5ggZ`fcG%zgYu|&0}k5djW&>9ua(D4o}}6p
zaP8nlPvIR>IiLKoA?FYHATL
zEQ!vx{TAV4G-g`8D;aF&hv}V1u})=!Pu;_e#dIC^a-mB13^yuLbQL2Sv*4>@FgNX2
zMWDi0Rg7Oz|I4bVKMKbt46!BgYBl3s!2WGDV>@79Qr&3gI?a43mSWS4H2GQ|s$Rn=
z1B2#^Y3RW+>R-c1kq`EvIW;h|C;QOR8b&2JS4X57b>;nisA-y!iZ6@GvnV^wcsDxp
zq>weQ!tAk%O4T%4$vfS&FAcKF(3(cn?T=^Tv9!}FtZB@YymlnkK0YiagkH=|!0lm9cyw~es{pHi6b+L%{T8P_Yh`sTBbuqA`=}KKA-8Dkk%IW;uutJTh
zCx|z;3+vIrdd3|X50JvTks!0S*>n_Pucj2eUf*bhL7h|Icp69@)&L}Sy_78OG4TsFv7R2j+h|`(*u*vz5oEyu
z+42W@Y~$U=B55zZcaQM_96c`GW2Bb_XMUy=nAc_+*~!?4Gs~~OVoZ|cvr{OhGcfGh
z6suWhBTsT&KL}fcLS0`mUbNnS&3IadzuDJs7^h`vjFtN)kbu^`X@pu)?-;urEN~y~
zX7q%r_Wb*XPpFpf1B+L$pbpcGxz@cOVEOU^*FN2i4{&;nZuKzqaNE=!BU52%)u4JA
z#^eP17JND4%+g)WNTixUXdgbW>dvN18AcDh+BMT)v-+S+qr1$f-8=u2-kD5$0(qPSaeIn@edYXbO`NRw!td>^7Bv4>X`oSEdhP-(by)hB{bSaaK
zB)MRd1?Qii;6>BRsz25EnccLiPe(&Dtmmdf-gBLwF6hd!9HX-Q>vS4B6D{P-U^rV6
zn#K0RWTUP1?kwXY+0}az>Qncu3)7#o@aa~eQBl6H{9Zo
za-F4EqQz&%URiE@%09CdRvL#xrR7%WXU4-SFOw&C8Hw=88)bFdV`MqQPi@0zQp_){
z=>5iG$#L#hZ^}JjbZoip8=SXrMr%ye53a5az^Ni<
zv|>M7ds19^qD&RfrqQbhjfb%(lzY%fs5111XcH%#;_>W<>^}S+PB?KZ-^>DwYAtN0^){SwD+
zKs~J(bnJGDEl=Tm_IZRaRbFa`O(C4Q``8*3=}Qg+qn#7)tBFJQhvI#UrFqt1kFR@#
zd~gRvCHiIrK4xu9^i7Hc-Fc~k?;-4@(5i~Q`tpH=_PU9wTKhUUQWLw{(Qm%ETt*?G0xHboavF>=>H%N3~;}gDV
z+U|k)KvC8Sxc%8G>iP(eFY*DL9?C~3x$g(k)b_p_@~wgP61~vgH`QOltq%kfA%8J%
ze5MAmU2XrqB~SX6iSKlJ8k2}Y$$HwCseQZr55HHfS5g`6XMPq_j0zk_81La(0^(m{qrDee0yDR)gn!%|oS`R*!%AR=cE;)*YRFWuxVSpRDKJ^Buy`oS&>F
z-F*`zFq?xP`qJbB(<$a7^yBPwyC0uFXBeycs4(
zSYrqHu1FaW_bbYEQk@*CD#S#-2?GVPvG`g3oL}D&j?#%z1V=#CM7Ru~XqFn$0&n0~
zcnZgf9wsB&G|JNm{uax(8z7V&3dD-!A}uA_(bUg5ZV<>ABO++kxI>?t8
zE2`L?2uF&IlY@Yt_#nq7g>@mzXVi@64mm?Hr+}Ba5DJdSBE$GdML|RCV~foL5ZLqS-u4M+8k?KmhW(kw1wUu>#HZ<9A_;a>(gW`j7{Tljbk>r
z&arYP_ea!KW9A9l|IkuL4Pg>7R_DyoD-!Fv!MVz}9d@R9hu@KCk7m|C4
zudIL8I-CgPA(A^Ish-39Hl$602(X8x;&>b8_D7cZ-b2$kmZ#AJ3lU`N`z5~0xDmh9
zmkivrYA*F*OY;lrwhRkqGHqYxt0Qf)9Ls&PA)D-_1uJ}|VB+1p!nanw(TDn5zV@)A
zePj8)XDwm#N?%=E-dgD!&&+nhD&JbDIZv+kH8tk4e8#~K14r_b1=y&xWVP>Qv>0E_
z+0J@pjjxj&zKW}cyF1d(8qP$Tx7HVF-TKt`h>WHl$@MjZm^voccM34~sO_w4S?hh}
z7&Pn1=RSlZolFfj`qHJ%*3^wybwS;{BR$v9S=EURe{8{R@@X`qp_9$>%QpF{prK=%
zeEot};bEJ6DrIi=MYw)q=fUMPWV5dlcYXC{UuP_!@>_g&K}{>%f@&XZ@l~s^nGJFr
zaxsu*3iuYg7OaZBXmPeU)Xa>5J`hg%jh!j>izb?>wbj=&*i2Rv1d7{CbMkyih~?*e
zj%h^8JU|xsO#eJzs{Q;Oew4CJJi2e2uLAe`-EF=kIcNy)m@21m7z%OhQ59o6-deE3
zhcgkFpSAhEM)HXh^h-XvgzK_9eRp8Kp55u20*j@z%a?$ePuk^s6SHRx+vO8l>iFG0
zKPKqY-M-3+MYCAqD=~qYv>bbUUGSOlG^nNXPFlLxHxo1V)Yrkcb(+)4sYsKonP2T-DB-P=sUt3d*xx@8*m1wc*NIH
z)>bloEFN;K0l{ph9s?`<%-Va*cT1MP9%X&~tuNFG)#=bl-&vTXH=P3i9ZHd>eFr0Q
zu@&tPbMKF(xTl@XsrDIQ{b$w(XM7LK@|BCUp;Z7f8&M6~Gb=p~5G5M6M!
zM2i+dLU_Nm&pBgA{@;7wdvCsvb@$%Cv-a9;t-bfzd!0siTpz-EEV<)?XQZjr;;!rC
ztb529{(KJ^9_9w{F8)^I@?GT0?RDh7t0&VpADHxq2QJ~d2s9tvZXWd>C(i!E)y%n>
z9{u6cLGnCw4TP3Rf9S%yJY)Mq7cLr3rpk|8N6=i6k6lYaHa>=ycs9aV^~AMY3LV17
zpy3abE~G^%H2kSc4SirhOQ4p>^Ho%C+WOQrgxkE@Gq9dPC5J@S&FO(dknM1l$RFhO
z?xfg%pl|k4y?n)3sLlA0tNvPY6S9c|6>Mrt!x~t~MN)~6{i-oyo
zO29G02`b(p~9)D|?^Fm$H8!>8|!#4i6*UW2EaN
zX-E!t+p1f|*`Vm9@Jo){3ouR~js$$b(5{dFgW|+dcrJI2u&yW~yWO?Q)+Oh^%GNHI
z+ePzpx?7tCOr~N{?hr0z*(i5sF6FE!cR7^uWR$!5|4AuhLrWr0xF>$uP+`@IV?rvO;
zPh&+nQsUjYX=kiEp<-5dvo^9{>u#dE@K!rJT;&$X@}j(jR>r$aQ#^
z%fI?_p5mj
z3gx2Vvb$ZjlDQN&ZcnW>(l={g1{eVzv^M5yEHd;
zLfu{9uEH3Y-|a6Aoz=XBo1Kio4MRN;cwm@-UCh}n-S9Z`x3RUQd#h6)Hd|bnN{IB1
zw0`CAjaZRO6CViy|N0$-8k??v_=HjFRZx{r%%yzg~zKg6PS7S-+SmRbFG
z>FiFC?#-gRweRB2mwyRzjb4?T
z-JXcm5UMbj_fLKr`!7s#cu$k@Ll<{aAg+{AW)F8grfN_3PQGGxxu;tXDmMl`3UK=^
zF*oM8U@#SzU58mwac0zbL_OPUqTmIJQ4jYT<@>r@2TPZ()6{`(FPGumK$PLpKFT@B
z-6iZPito+oy=l;wgWPZvxsFy3au<$D^JQH&gFE5b#_d7wQ<1otw`+v^GM2&zY2j%1
zBxaUA#+{em8{m)mZNJFj$lsxf$q8~4_5(R$%*eBVd9J=Hii$BoGhDyznK
zZfrJ9HM)NXTUBKHbqJ$m7e*Ya_VTG`*irc2^_HzEd*Q8R1a1G`T}HZiogRPh{)jns
zp6@OIPBd=5yD^6^&39ktJD3L+xG_D$k24G1opVatACcE^FO{x7qI-sWO7P_~$Wo*G
zTimVb=6ZMK;IGaiLN_D6*?@>S9N{t}3U5TjCXOgzM(p2+B3}QUu55I_3wJJ5ev`X}
zbonez-sGjkj=HY<4fk9qt#K-Sx34Tz`vuHr~tL
zX0n0CY<16;PM@P<+uT0|KRE}Uc`0UxyB4;Mz1!XUof#Cl1CK4^PzTz)10t^c#g@61
z(eg)k4GDV4x655WVmz)R!U0o+7Yg_BE_VWSp=-B$b4=PH#D{tV`z40-@E80#0ujIvF6ZNyqOyTy^gsnV5PnAnEO}E>JHNQ#n7EZn-h=q`PfoY?cp=_poaI1y=R+>IJthy8}yRyVi@3I@ov3r%ISg
zqAGlF$CmAiIzHZF$iJcQ^Iha-H=xTiR~tKSxUt2Fh!VHZy)swRkX!D;d^dC1E%(>x
z>Sb@c;cVa+qyKGpPUxW3#_&7t+5ysz^O&ESI_UO&4Ahh7jf4m8lwc)oP1Z=w<$c1w
zKk#uW)H8frtu}f+ahDY;W&SgFYQd4XPS0v6Yf~o5JMHzTWbY!X|BriG;tI4cvyw`p
z3h>c^^>!FKj6X22eJ+JOcke=j9DMHn09&~EU$~Du*BgJoaL*B!kY_sOx251wm(Wlt
zR5V16pAsndlMZb%1_a7ZzRNr?SY9UiFHv%cj8ibHX<>-moM}L)+&g&aWjv?ybG_-e
zQ29?zxDX~2_HGA+%io6aCFt1P@}JmJ4U3Zx2O!-z6E9Z{at=4#np{f4^0R?Qb~|Sp
zT|IK%!0dOgLteTL*vlX6;$0(rYAsvahOp~^75U_7=l}5BJ?ptA|MA@Y{Bo4g6H-C>
zSKjlzQAqBFh}Bf2xU3m{i^!+=j&9vz@WPfp+)#Utv;hZDBLyC76UA->0nF(lOz-u(INxMJT8
zu>SSs;&780v+;j@Nvg~FTum27^-aN5Qp(EvJY
z%KU#V#vrnA%G2f+4!faJG@pOVXL)P_5yxryTa11kz7WI%bpdG99lt)WXrqRKk@;BJje*YsmPV|mm
zedWUJ6=ymq48iWcauF`<1CGYdmG+65To)AY@cMs(WJJ&9Xuu9z^lyt6=x4?^vFMi;
zU2M^l7M1#&xZ)OVVbKv5U1iY=77ZU@;>%jJy+y~F)Yv#c9^pi>yaQ#}cNAy!AbA)I
zSYfa{7Pe;3V0j#t_{~1UP)F=HpUKCVw*FksVCM~gelBC4wwjW^Kp1q}7xGw8qv4nG
zwgA*h&LMbw7|k4l9yE+HhsfKcV^3+qP-rOR6*A0xy5ca@G?IsznFJ0;O|PN!;bw3r
zg3Pt6Hx;%v48daJhuV)5-
z&S2bhN}egNlm2;1{+aS;9N%h|d^LFXGj!iX?!Fyp@oYIaCv2Y$W&h(dTqrStzxxn|
zm!p|;dP{rThlX`d-Et4vdT6%XtG*udmCphz_qG0b#}*_&}m?|MoC@^6h^mr(r@YmVj*M
zp8P9$O!nkq2)xRCs*zFn<>TZ3l|C=~<1>uei{!oRN}%f!xf)hSKP-{wNjrX`#!E3Z
zz4#MNUMdffF8@UF%g{x>qAknh61c_u`!awRP|4+T8g9(3TrPhvUD-n)tbo$HyN5=u
zz_ZWx(18{59O?NU>a|iXj!l`FEAbGKf2BOfIgN&_68Y_3h5WYfr5>yC%>BKzY_;42
zlWzAKd5Uy>6Rlb!e+2gvp0yZx**}*g{A$8I=Q=}f@QTAte3Vi++>F~Gr?Usp`5WYs
zphn4!a^+Wmu(|B2l)l^~f5%EcX*09|Xrs;Y5ao~gcqf$i#b{Yho3_f!L3(V%^UIB~+vE@6
z%+Pmlb9&*=Qhr+H39Cz_7Xcz+JNbCnZ<@exKNzJO9u!nv7ula<0K*_X*m
z6LjvQ$%>4rQz$86VnoYZH8nn1nYQut8osJE0NeBm9b&&<@6
zr!Eh^)ucqBZtlIMbPl!sp-^TOC5}p0RZ7vJDoRdTT}9~_`Xk?AfU}gBiJ>D;APOh6
zI7-~UHmV9qPgTW8*-C}IN}_X{(ZH+VyQ1huA68QmxWuW|lx8?cc(R&OG5;S!?b@>T
zL7|52?MZwW3rhw*168KF@;OSqs=AUGZu(OQ;_oFjHWW8KsjlRNuZ55rq7DkyP`qf7
zZZ(v8;x=SWB@Z2}p}d1eRNqFOWl*QLm0>JR>INmA0%|I;!6R9Q5Zo$5p;KxqgR%Fx
zq^8o$j?Wwnpr@Ad4y%o=ClL6imJ)*=)BAHUUsX$43enouR`NrN5w(@lF{Xr~`Ox&i
zxRH0Dw!)XGqv}AA$<(%vQUmgQUkAM!g4`R7QY6$>8VBIT32i%};9zMzrG#|wBwebf
z6sIrhDX!q*{Ej9k?Wn6H(3X18IhTKanJS$T(EMh~36WAu^8Z7H>njsbn$7i!WK&cFAf*LAapa4E%&=e0g@$RJM5a
z7UDgP{S|%y`U7V-J2X)m#Mo!M_~YUHetI+QXab=oO?g+0>5A`SNUou2?<#!!`0Tq-
z@);EOo+Y&*`+rQvhY|6%kbRO^
zk_52JUwp!pYY4A*p*8W+bNmO&Kv9htOt{>eO=#GCv|Hm3(b38`hb}h-a^VI=Hc)v!
zD+N(?ThLeTk1-C^)6Gi}tltMsN=Rr!?od1;;*AaB{%-xwW!3tmHBF
z^u8SbMX$?kyB*)nuCV6fKaPtNTg2WG;#*(dA+)}$QU(s{?{rn}IMMabcUM&B*YvbI
zy6N`Ec<+tbS`Xz->EU{6-b4AR@=m^xF%oCDJd&J%Z#|W8e#{#jc(GJg8t4teIW0zT
z3M)MPkFRN63|YW_w9}|%Po+w{co`nZwH19j#8!|_Y?rU<2}w{X*FRF&WlF_f=*sCd
zx0f=8>!4(BrL%c)co)s+t;qHz;w`-u55H@^*jo`dUjFK>@ZFTukCp1IGcxO=w$FZy
z=EoVCK1yr*j5EewJU#5A^tOglJblp@*{9R$zECVP>1ki38Q;?#KNxzX>nBPX(*?;c
z`tB1Y$#zF_@Ds(&9%xeP!#q3tDgBZ0r~O19TiFlBB%RK4XeQ(nYT6$bY!}Vz
zuaw8G%lZC_3k%T4{gty^>|>uotq6URhl&nVJUH0iV4z^Rb0CJrBzis&V+EfK9i)i6
z;i6-1n+B%-dFL}FM%pK;A`Tu$l|cIZGbnMw
znijJje30ogrPfQd=X0e{0EQObhb9XX?F6^z`wi&FR_v?m&=*QC2vhk>XuokZ_Df8P
z(&^HdO6}kme26-P?xiY8dD%$fWg-4g!fl`^`lpaqs+qf2R^JGy^=z|Qp({7ebuj&)!M$p?2Crcu
zcT%5W$`?7z-(ns!HIR-BQR0o*;V`YiB!fpOmE$upi-Q|9o(HT5WDSKmgv9rx4v$i5+BNs(D8)nHexpRt+utZ9
z?Qp7C8qG5F{6-10?+6RX({Gdtb~TEK7NeCvQH}46QR?C9!Hh8&KP%br1X9dF
zB{wN!l_Z?RuR0cmpG@7yD#`ZnLR21FHdcwW$Jq9<^$dpwxt)Ky~Xe)`M7RCK(OnB_0WYC~v~
zP_`hbq)_Y&M`L`oU4cP3Y&^REIJyGD7a6KgKpT#tvFV1A(l^GsF@z9Dyb
zj4aoP&>~SDJ2kDbg}ZL{Y>x0&_R27&J;>Jil9E{EmorK+bl?oqa#4TC&aDA
zn5fRDhChgYJK_f=ANPtSKVaTIna=&7jJFpUeU>20$+TpNg8KquN~5^FR~_APek;(s
z9mYjmcoj=LoL{F#OO?3DK<{28{LTlZq^zb1qWw#h82WT6bog^I(KC}4FI5t436Cws
zoai@p9~VeRmtmH&cM00&gJp_~S8a!uVNO17nUWig_uVojna_(LuBWs+8qVL*JMJw}
z0s~e*8BQ@9G2dysN(rY%%akzqynrU0wOlF4%?)I3Gws@Pr4a32jtBFtz?hs&-W5ut
z7;#M^5PSV7A5K+9AEP-dl*Yl6hNCng)NVWAIaey45ZedAoRv!MP_x7sUyWB{nrhPg
z;U2uJG;K3wL!q9ptweV*QSsD!xsrpruTqNQatEyv?Qvw4Qb76{PQd8W1%PVDM7L&Ec`k0i%+fC0ij7JT72BvpvaUzfyR)Z+2jF_yzhZMruSeclkB(Di
zJGw&2%}Q7n2Si6u%}q+>;3>Zg4YG0xis0LXX~|65v&k$L=NbGQpFTsELV8|`+N|WG
z!JCyhtXju!R#JfdaWfQ~<;?5JW+c;JSE(}bM8=%Lg1fw
zpy=ro^dr{hKT?w)l~b4l6yB-y;88_8l`iIPK`?(-7`p}GyOhbVYz!>grL^Q(O7L#w
zube*nqf5MdIF)kzq})TVd;XK+HRmkA$qZ4qyVapCS(`J#9M%qmzdwA5{9=Rtph%slXwnZ~)egVya?v
zKZJ>zG(Vcsk6_=RWu{Uz*z4yB1hqJ#d=Pw=ebj|fnF#|BK8g*Mhxe(&M*FgI
zP^ru@r92-#>wQf5S)5ek`GfCg=(`R-W8`n3vd5J+(ygIH$C1I^p_F+XwrD=@DaDcd
zgi@HbIHqD4>nEUuP%Bt(7GiIHuxH6e%t@sbALh$(8r9J9loF1&HJwi>l@qZZFijzE
zVBtG9kTeqB46IJ(VMma=PKoUvJetpy$#CIs6?2Mg{XCuOp2m#8nw5#xnR!~t5p2#P
zXvJxBd_b+o)78^T9!UQ5GzxH-N}f@?>@wt&GnndL=Q&o0X+Tz-QE~yc^Ndn501H4?
z{&^|KS&W1`PpRcu<>%1t{23F;yV)pv4pUupl;OW%lki9S^%rG`HPeRArSoDV=g4{G
zGp?fg7qGMNBlW(Z3}wi}3rapb=(-5K3&;5vQPNd3^CBh^hw1V~r5a0Cz>kfAHAYvz
zvRiCMl(>x9%Ma{~C6Y2PqaXIUtTcC?r306hxy~PqURSW4B(097i&xMGXI(|p9DHu<
zzp7jdfOp&#*OfW}#g}1V;WILo9Ogd$idTcHUJb4>3f@*O22{BHJ&LP)4}e53%^e|&
zwH1GW4Xs77G&`ILZ&3#h=o^>vV_rG~8f5QofA3
zmp$_OWaHW+Wqqi0d_T>8fgW>#e%c-F*Rx)+M|htczztt~{w-USZ|zYP=d_=|A+@C9
zrq~AKOrUx)2d~~&=2FGogA=*b$sFc;1~e{K&20r_0~ZV682I8?sv4)_8?4kXPEC>a
zk27xw;ImTU#=$t%9RQ}k#jCfN`OmVtKOknS`5_+EATOI8Sn4m+)x!S~T|80EJH?}F
zG1jMh_zBoJhj%NL@Tg6)M+ceF6FusO7#?>$ss~$9(Fy7$gh29!gz>30K;6SLQu?Tonx6N?I&_lVj+gHu*zq0`TQ*ViNqY?Xs)xq2^p4j!@
zPdLXLX-(7=r~PeI<6U(fKiK;{b&c}l5)6-QcS+2%Q#hC&4m;AWi5eG?$#3X0p#_gE
zp*Bs`QqtKa#-yfd2o9)?pc(J0>C&0?RO|z_Y@R!>eML0Hyj?Zu14un@1AX;8SFEw>
z0~O!E{+8OcRLkM5}=Cd2|f8X`tF#-MFc!(Oy
zk7Ql9k@jw&v{o)36&b4bhIg26hpJJ|sN?AOSugYsuB8pbRITMV%fVUm-dtvm{%oIS
zyt@P`F7I&=zS{JP&X~w)N#5UoFrRuQZRjMV;TqONqttM7fREX+_#1fS+_vW3&E
z1-Ku6brlcN>{kodPEq?6eKg9&mQvjvmfM
zaPoJeJWh6Eh4K4Jq{EvHWRCBb>0)?gOBdrRTUfk~+fJJgs0(qipwmJ1mpUcU(Q&jl
zPxPS3!n{jto}(_0SeymQf!IPXV{rqBPXy&cZ1T(4+KAoj=uV>!sk0;A!AapKn7QCi
zGre(GO-TI!Klz;!(}!_AS1)?Y?{jA-oj3ntpj=z)ml
z?ffvH)tAv}_6b}qdQfVl@-ks0C*WO>D6RZ5KI^P&EEu_8#!qEDmcZU1#JgU``#9c*
z_z1+uzl>kV@e2{ph0QDC@lAA`GG0M*%E#sj;)9wisP7RqU#|5G<;6xJK=Uw^8f_0O
zPzHJ@S;}-Kgfczo#0DNJ1h9bUL0(4!7$i%-IC0|%bgwfZ7?-lJ;Y+SewHRHj5bMf=
zs))b|ImZBTOvvHY>+vxm`YcoJ9(>7({uDxYGu2wo(^T%LS~Jk^OmtA{Q8j-q93yf>
zUxpz3by#Mv(8i-`9p`Z)@R*7>b|F;dxT-iqj1P~ierK*ANEYNh=%|wz5t!-;!JP$a
zcv5Yd$nkaziXH(K$#=o<_Mxog$DJ(T&R2`gXWTsrwc_+prMF`XhUQCvspfYNdrzyY
zc%*-RMlI}2rkOQkONJIT6O-xU8FhH>Xx;-K7Y;Ggf`yp5qX*H(TCvH7@2uKb3JCE?
z^wrM1v8X@q-VpeOrKt1jlOkegIQj%GjV4Qx38IpL@SthZ_+!&)VWKTjB4l3Tf=beo
z6zY6X9YXgqVuI+_LA6%VV;F8{2$|GKR%y;L6e0k
z^dF-)8Rk1`euhvv9htnX|P}I_sX#P%dZv=w};iqMO5sPDu>zQX$&>K
zq$-8&aYXyE42*o4wdY@>is+K+3Aabm(?zrc>{7CZ(4q|NJ!>x+sQ)>ht}dd;%c>k~
z4kRjeS@pzZ4W1D!Lr`=Ns!~2SE<9`OtfDV2t4hJFF*A>0*=jQ08bR3xP&ivBDn#UPLk1wlwusN7WxuUAnWP4yzsy&LtIKe35-oT^C$BQq%j$*Y|9*ik;N-?I;
zr?SQrZbX{BmWOX*NLlF~b_zu|Hr3-39=%V%&~hV_#Sls&5Vs5c6=LJxI&ln{;j+A)
z8PdW%f#$#rv<4>npve*6SB7eW&G+#n7xgsWS7xzD41|&r^W|(K
zVjcCmrWUM(q>(&GMq-d`)>FnyR^=S^{5DE%BVc
z0_kp6ij`wtmo6_IJ&L&pir)5DH7@+PSW8LKH)E)6WsHZVzpASAM`m3H@{Lx10KR=QJ&g~7X7#Hm
z7y-B{zNse1yxtvi(WB#P8R^6^x^z=5NN=7{Yf{5oYIE9sLXDwm-q;knaZAlXVJA`H
z=sO9w)jUw`>vuf9knqI{G7a_HVOTWZ_43=%oOuBN?+ry8WY5&
z&oUQce0ylrUA2rekbG~)=AjdJp$N=jLVw>?M@5SFTbO$Ub&vjtKCT(7)3SSNnBZet
zmhhscAxKULq(}GEgyyFRrN$J;Ohrq$s-&!1kk}?FqvIDHa-l4>LIKV%;EOhv8wd+0X=`9
z)<{khuLEDUZ6Lqd6vlHrW&$2OHHK*r)vJSR=YMi1!Dyt9>%``z`G2TM(q9kgz#l5k
z$Wi1&HGgR4V;};7aMz7Z#QVe#AF72y&99XpvRB>MB-G!^hw5ACpGWJ))~51*stIA}
zkn7nz22uMzvGfe0s7KJOtN&DGpjXt3mEl<8)}N{up<<8J{Hd>xV3rwLAvl{>;4yew
z@Z`>p+A)ppMAjz}S>-jowu<-Mrme;kFxarlSRbJ+@e=bROPyj1)LO4E
zC5+2Y)H!0+GW?l31(#x#f7JNge3rj#jJ6FIB_73S2Xf6?Ep{&OrmYX0>cCsZOylQV
z+CwSo`kZXL3;sEDGEVbEoz5PaImZafqp8lEBl+;Xy%CXSl+3H0lcJtvhu)iGxZPTV
zKxyY0>ZWPpveF1mdqX;WhBj&1C6vLKrE7TtmA$w+AwEXSR&y5z%zlj--c2JjNlOXv
zWFk9pJ3XKW+yd~cV*7S_04{cTeUwm8JBXF@oq}3OPW}k$<1$(!@jbDg6}1dHUtX)m
z=T#FbXz8K%P<(8PU7%^@wD#n#s3q`m&^Ie;#h3zr-1F5eytFV#Jv4maure+?beZPuU63ND>
zDq01nbk|S2s%n#6D>mWqUEb_hg79?!;dP{s7wq?MGUj`=Z=iEl8>OmiyWlW{9@o^W
zMD9faia8t!@!ox_sah?qmpB?(OZyZ@BMaBo`Ul`FzptM5t83$PWKfhdu#Oc);(kmt
z%RjrmRzbS)oX*zQHoI;!OnhPyg|D=XerVagPwHIwBWI<#zv9SQOhSi
zF{oNctsstackie*m+&R(-5s?WphiR|tzck;cqI#&B^*6?vzyl38M=h8I!9vo%<8VW
zoz9WQh8|ih(?dNq>~kU9TdV5CCd=O+Yq`a+jP9q64--lv*qes;fOn$fsj53R+&Iur
zTgb?M1GJt&xUPQbGp$4jZw}=es!fc*TaWL@Y58%ryfZk+sk~%Qi
zLZd*WKkw0sbS)pg*m8n{Pj=8V4j!k3@c^Y$EfDGX9O^ZmOFoAtkJm1vu0EZB=qq2-
ztO)>5q@4)DWylRCt0-zB$Tw65M0#+aKAZ^5G5U&Q{j`9|R64|DEd9Zl7k+Y20w#^B
zOhW8->NH8)0w3r}laYA-B-Nd))s--jpTZO~^UI*ftyl)YW9VHbM1w#iEFOFr5cI||
z`Uw=ETb#OzVy1x1qjFO;f@DV!k}iBhkEWn7yXg#Rot<#FGF5BI+r~4eYH6?s+BAT0
zsOjx#V6>IGBj~(BxC0dTW3T6Jl#XT&(=PGv$>i+U~bYtANtuEI_5JIopZGH(wR)!GzTN^
zVkX_4qqTD`GaAp;Hak($ob$AvJUjY)p4Je91XPr5b-~=k!vf&86oc(l7fd|1vzWo$4;rHnV`C%eBsVhVq+hCwyjD3nS|e
z&#>hh&l(mihdNkK+6t`|O8of>MB-%e3hjhl@Y_{T@R=(`!Htz>!6Q~_ALE+X=c}|B
zs5Nl|m;!lLkGsDv!@EPh%1q>ncM7fIQ##gt6VmH!`T^T
zI9L+o4djoB0GSZB%|=k80lH3gGPOE6CvP%ctzmD&>77TuOsxUSem4`HVXHCis8%Ha
zhicCp7cKStxc0U5%SHP7g!YAn@=$@3nz%+)?WA_rsx(}%o^e*oBQ994Ij7AO7p&jC
zpxGC!4b<0ZDSpJTDPmsWmHM&m$L<>+oqIl%s$SOOCHNt0aak*&Ji%vydG=tTu#?g~
z(M0|=6yHYIYNB7GXRia{zZSbNVENBbO1z@IDg8Z^+Fb$br$gzpD_Tp;O;26H
zz#UGZSGDT6hEV@1jL%T&&!IHha8*lker24$ir&PfeR56XYhy)!)pQOv_*E;NIGnE_
zhi9|Ecs&PSMQEnnTqHKd-~EaqFps3=`1xLQ$qlMu{lrsvUf
z1R#E)OW$dYDg)}Isi=azvM#;|Szgg!pbm@*s{WDiQ?@Wc{}B&iM7n5LqJBEUR7Ch@
zB(}lsSJ5?Nc#@tk0CsM10iEq!CX;FOh{$3IRNk_96(Qa)-$(%3T=b8>$Gx
zcCKAvV2;rUj`hEeUYA#dpKn;oV$2#ldZ
zMWP@q+TQ<$E-YfZGI~nnG*&HG0HQRBbUzwZlTI7T=)ySWE{l#flHM+>vvJ&7R%hdQ
zudL3-v1K`(jpP1ux-gD^mDAZcHYpFxO&V5SXXALGyqTDzf-a2Xo(j4!j!!G-Fpku!
zqP{`8no04M^yfH4nYXf@Bwfm+%9ZtIn5j>ytQW*oaZ_deF58$TZ|dD_8)}K_FZHgXf11tK@PV^JRdr!szo@Fi%+Q^x
zdL8tO@?JC-Y+M^J>V7T_^XhC4Pk41UhY{6uHiy-#q0Td?8C(IUIL@HYdwOAMSLT$ax`(#Br$@n$`=R%Ae71q3iZP-@
zQ(c!%WzsuM^{TjmGl8RVsegM@o&ELPY^oQnXnAePY9n~l$?mVmbf6ND-Q$i2O4q~A
zPTpyzzk%)^(;SsS&Ggt@Szc0rU`q)JHaF8#ZA_#0^=SNV`#vga3ypjqrP@lT-q)Ad
z0Qx{5Wd7!1RJOTZ8n_<*@t?y;cr7&;ny+
z3$6HG$lPZkWG>e7zhzeD3z-FG`FtVs_W43)?L#5+M;{8AO(v~`%=pbRA8aLLcD5EW
z3xK*vuT1H!^;X;zDO=w%)0|iv5t*(S_5ucfbyQk=Jx+`?;87!Pb
z5K0-R$GuEz$Z6|NTWP&tNgFdxcZY7Z)0U0X^SpfSAg5jBH1_It=asa`bfo=mrxj1n
z{#<=ddwR-xu2;IA_a)D<>AD;`o#kYnD_%)Ef;8OYNN%rdFCT~*uglVl)6}EAuA=ax
z+CvSDq$R*0bbv!Y(H%3CqXR;jRI-CeYlM*J-bv&X9^EM+Ao2~KmYBQf*ru2G@Qrs1
zO#mwNsfD61CuEm(<^;(4_i=EDfUMhJNxO(NEXvDw%r0xfL_|)c!H9%#(>m&j(v@Sh
zrK6Di9D-0Q&&@=&6B4hUByT5?*apGKbu31t5Vi<8ztUsDWDXKSk8?s4WEeLoyL?M0
zLB5@w7LBybSJLhx4Y}3soL#<@$*9Xmr)hL&k?Eq&kZ1`V1O_?X;?M-j*+qnkbwLQD
z)eQAU$a9ukJ4}d|l&vnY?dZLS{bu+^0xEO_w|h>=Gbv)zG9LWDl+6$yf
zmrqgiuDbfN;k{E(f()A8RmiiUE98N@cQf=9A?YX_Sa%Zzd9xdWcaBjzGnk5Cq;Udy
z!Eqs4UZ$f%MP=?lLg;K35DB!5vI}^KxMSJ^-g_l2YAOW0eTuqu&n{qN#9{`pxVsQ=
zUw6oG{W$#&3@R?72jm?=#e0av1_()K@YSUrA~797&k>d*mk=;ETXhC;b;7ON7CH)4
z=r%@i({G)YUE1^0Ani0RBC2}I^z2FcbO?L)I1TEVUD$Srg<;>=Q^<3sC!`xf&&-hg
z5kgbwt&c=nD-JEBG&3|Cp~&gn@(==b89{SKh`PLlgwSD}5C`cFf2-$3sh9VGgSHpn
z>S3YVe@03?Qle*wl>40WWiL#5p8QOsdSw@_&kP8B=4U#9Sn#;f3;BWD?*kROn^Cb4=fX@O&bCh=PSh+2
zI+QmXgOK8xg{i?7LS?vU`^S1RexoW};R{2-Y9ZdkY@*0MLMGISD;zuDl6O6#w^Kti
z8a3hy-^@E7wY&7=P
zwI&Dh__C)}@#%>)tFImpaZy(;%Iu3ezA=Z+^%b>=8goS~4C5l;z1+h&_?)naN8O3Yc$7%qrTYHrL&)23e%s`{d8aGDLy-hy#QKvSYsEY
z^sJxm;T5&kUw@0kt@|U4Wpi48y;>n_+1%;n^7nsOHPbCTh9yYk0A1YZE;sdO#GyPWlGDYLZ*%bJDe9Uoz$4Pp{ioBSbkSCOIrJUfGDG3xp<-Ysi;ox3`Ut{K
z_8u>w<2dvatu#ZK2zfG1Z4TGBk)^ZsNVbfAVHtBx0LrqTjQVf;`5sWx(~~rCg2;9X
z2md}!8_nQp1YsqgP7qSKCn7YF-kc~xA0ou&F>RuFauz2pr5$FXA0f|G*2Z}2gUpL&
z6H#Ij@!EI`B~E&2H5rvl|825tBuz@MJ(?6B|wz
z(4GjwCZ?N-%Q!TFj+&u+2zj=ca~IAoUpArG3i4amEpWO4lw~U!_20I#A5gFllp%yz
z$D#3b#tc0{i0z>=MTk^!3KBD@)f55!oI~?zwi()rkmu+z)KiS8Cme6GRbVDA`3XNp
zTc!{|cNi2aKsf4TgK*Xca>HrS7ODo6Z8cf$EUP&T3D_D~KqCL2I5eGZn4!?A5EYiO
z=u{zUT?AngKbk6_V>l5eafO+9m=j0PeKRq78iYE-Y8kE_kX`O3me#kQI@4-
z)PGyb$v{Cz7}G?yCpk2Y{xU=H(;)=RVdd#U!WIa^f_yPuKxc4h2yHV%7dSM9oZpHx
z&$q~WAq$vO2pE}7Kz7yZ59V6CGYdNRGgv&^NV1e!MzZ8=q;5efHnaAaY<~q?D5|g-
zB9r79kPa5H{tO{qcMif7jx&Qx5QHf_Vq))bXamK}6i>nzYNBT`4`Q=XamFvJQQ7$U
zh4Fjtg8-vv>S4B>WbyvnPF9@*=pjI{e{ssp;}P<}j+<*1md9HN)gvrpG)$%
zzrVvS*riPJ%)_?L3K}>MhF~U5pNGwZOu9eMT%LNq7t7PS-~V4PPs66n*Tes7S^fWY
ziE3bp>UE+RZ5HblK&CC$WzRkA0*K9G$@|dynHcyi+~L6SLmc?Ti<-rHo)9TBgKn=x
zU4Z6Vr59jYaTWHhGBb>3t6=eR9nHYLH+GdPI-DN1NC1plqnFJVj|MU0kFL?U@lGgo;VK}zOIjQx!e?1jxCCyr=C%(pz4>0mmCy-SARS9!iNm-If
z-srw?Y{cizvI)!);;^l=FGr6v^t^$g_OFmoZ&tL6n*zxS((LhU2^(hPS95Atnl+=R-_VQ4-Pi*#tvLM8jr_f~@}%LG+v8*K=aG1g5Mt
zH|V2tW$wp7u}*N{0F8I=e&fmpJr-UrjmKND?G`$Veckez9p^kPcLKy@6cP-z-+5ykf*(Kjj;!xJz#!E!#&q|=k<)1@Uq+E_MCL`jds}sUtT^Mta?=ELSw4x+4?4{vJYB%rbUQ1VTEOKJ@GtHw-Sd_H5-D^PD!Grh%^?g;@~P(Ie*9A)P$?LTx7Ejk|HLlP81IjIX50dUos8$#~u*A@7X%
zlEqU}ikB?bvD1L!r7EPP6e&}-Q>pgt%at=??Hc#q7rew^hWf#8D3d;DAR)OAV$c;reUhZN|DaX8jrcL<5h
zET&SM+DZFegWxaF&bxywi6=PRh^o~|Dv_f%k|Pl0I&V&2)=7FJ-hG?-IUF`!5hqFc
z9C-h_uTGMZdUZbXj_VdK35-p>xgAnLhu60Ni5_O(*t#BmGK@Q2FUeEGzm#z~fxC+?&O+F3BF
zi~&(dxM+0|o7Vf8rStliQRVtcdMDp<2BJqg?tv2*+orc#I;7$N3P3I-+y{W`&!%Bk
zRx6l8EE4?4iaUT!n;dgU82jPNZ~N;fxl%K0JEaPUUCEE-0lbLY?8G)*gvUyt
zsH+%|7l2cDO};i=-@ze4iTDK|9)QAkOv!9o4Ot*#-)iRI0^qP>asW*gA}S-CQeChR
z2saS3|8hv&NH!h4$RXtii#1^3k&$rkk(q~0hlV+&HxcjjzqkfQjB_anpR+2#rpaa{
z_|`H&bvWp8gQWOWzX)o;{{-225ZLt3D2J4UjMp(UuuVe+Bd?zU
zI;)m|2WAmKQw6f=05hvV&dj%waon?N|7PNBS|1s*L_#J%!&I)zP-xJ|eBHRXaj
z^KD`fs}(;!>c(K3x(7QXG)Qge6|bLBF0QO!)|-`P)4~{s0QfdDfJei;w@i_2IuqeY
zaKJUvchJzy%Qz($@LL$q1NA0uJ#zLo&Hs)=iUVE%xCegw%w%ZOC==k@${bk#x4vTn
zZ2DItlY;EnJpR*>wrtM9D
zZ##2fwX$fH39xB>i-Q0}18^BWz=Isn8|Dv(qp?*ED&IKCljz^UoLH^wwR)LNE7`cg
z0{0{1as$`vo~h|Um(1G0VsfkR&1KtO>p&5=cY;dQvFB4NG%0i
zKJ*$M3pU+gX>J}c$2hk)kXWR#Q)0bi(+e1K1(2Hnlt#j1d|7~%u}ud~=ee%e_cL=S
zgM<}U=}n4?`yMQK$O%MQAj)CV&w|*r-`k=@J}TXe_04hSR37QEF-~FWHH&upg^dIB
z$N=C{{U;b#5xADryQ@tL
zbu}BC2EU)=k^ELmHX~Tye-@7k130;_Q}mnfEIOx;X(icopJNb%sb(VSopz0XY+XErpdylStzlnVVIbd
z?;dVWOK(`T!3fboe3ux&*7G3r4C*VF{ln2I(J98oCA!u;Nw4d>%t*GMO@DVtJWKCp
z(V~@|QWcb)0iHM{{PeduNw#T;=A4)Kb3$}#*dnQ*Vx46wSE$YZH)vW-q1b^f+JGf25@&`M40q-NZ
zB~}z~|KpIbiueD@40--}cC%?!ZCV#QJg|kn-x$EQ@W6I+f@srG0O;T#0Bj2vpeF}p
zqwgmOfbTkU@F3wm^kN2pnk$ODCc-ZeYzu>|JZ##{%)@tsS+Fh4jXujPY+Be7LIBtn
zZbp}7fK4CkrVuxo1KYyd=&b>;{niiI1{2{I2)2d8&{LU(O%031EoM;w38&Fp8DP_L
zRxt#CZQ*%zR0i1eh*gZ+%z~GOx7KpbT?lOpra>1IW+_h+&1!7&nAQu~&+1Oa&ZkxK;`Z03>k1((A9wS&k
zw8SJ(AS~M71o#DjZOeCmm_2`)MH^Tg?lT8gVQE;iF^8=d&2MoK0Irb_&!hi|JkFcQ
zf99J)JYW`_a1YxUOoO(X^n?lU3jiz8nXAo&g%(|EarlEdu<9ITm2a{|M_ED$Ko0JI
z&oSOOk1(sC?W}SmQ7%z+i=HB2oTkYJVb
zhERWw=aJWWzIySo39xC@a&rbN0NhQdLq2X>o2CGe58Z}xv`zA8
zSO4N%f;X9?&Zb+*A=P0_JzymN-}F)2Bt6OZlr!aC-TJv{X-8YMG9<5!&ia&=w@r$#
z_%-fOv;&O)c+5P*a|zss&sX8KBl?%uq&wrBQcFhR4?zDja<)r~kJj;g7c*8oPlejC
z82?bcc4B7Hqg|4#(1&hGDADIE6jx(^uW93Kda0VQ3_extd95Vg3VU?Hj@XZuo^3V3Y;$0>De3A(UYFiiJ&~eA4v%vxfVt-kWF2i#}0`{
zU~h}2f))c%06a)|hCafwoJSvFTCJ6sy!*ak4s7WnYMBYDMFXol#SWDKu%+vcUdg5Z
z6qLu=yyw9VsXW)h?v6Y+AI+TDpp|p5Wk7d!niTCXHDO#xC$6nARH##uUfnmA@ob#_
zLg`sw+cdn0B<3^%z{Y8|-vronDi-RPocL(8U^tGJc1nt`djuDAcm$?1UtW|o+Gq|x
zo2Fy<@m{C^6a}DqxH&COv#4i^Q>@1+s&kS@_Kydn;>Cgc6b6gOnN1JjdyGkPs&4}4
z#Otu*=({}eT#LTTwDVo3SVRZ_ufs|q2Ob5zE&2minmieu$Q)9TklRXVY|+?ng?04{
z0I$$e&~B{fb5%EK3HeP)RKg^dl9y-fiHoJQ>H2dH%+Ro<9=LsAvs{g0A_J#7p}D#`
zcvMZIgf2<(p#!nM(}-)*+a<}>@jAvn>eWA)rQ+3CE0l{_yoGWxEiv0E-a2G3pd2E;
zum){w09aZ42oAi^XF!Sah&VXYY&x6Hqpe+%6sM2QA;YlG0q4cm6#9pyiNNa;-bWy{
zD-wMcFQO`4SxrvmOnJdo5&eosemlvemqEEqQ|T)JQ+?AISOtIr0CG=l007h7pu9&U
z096sO0X>yD`7Mq=N0{SpI&<(MVJb!q4=Z>%60|y&nb=VF3qUm_hh3vIO^6j=?mCjj2gcH)Ih>>2*E7!LprAGFz{NT7G^Mqw-Iq=y}87*
zY3*1^?8pm1O#m`stk^ya%ALeQ_-IkLBu_{GOy*P@@n0?EN&8E>W{hd}W-*`+5{4Qk
zz@`?!F93A`xD8=heZ)dork{cqO1aD6WL=&VuUTkVx=MN^GGQ}+Ycg*5l0N(w>+)lA+H7xbHV+p`}05n1#T#Nz8
zgXxJ`PN_MBn8zHNBBBQX`9UWFkUtea{(|cG5-bFw86tj0ci>i@jPAhn!B}%G_&u|D
z9}xk0O*I>B(KXma!CP0q0DJ(zdE~*Z9f+!D`WNPFZ$XIp%%M3V@
zy!XNDKW5PZF?V1qcsl1q2W1+8KGF};3P49hd{V|N##a`ddDPl)`uZd8_)BQfN225J
z6Tzi)3&AAcGUne6k9YmV)PpuXh)D?B=Ver^7i`z_s}6~)Z#m<80AC7o37#2Lw&TH(tAO7{tfok0AE2X!9KJ<5;5`c(NQGg($=kBNphlp
zC1>_AcoafoaudB{HB}*OstI1ny}4|wsBUjA%xcDe0^G0Y!`%HBpbs-GQo|t?MsBNV
zOmDm`sRP5KBN~5^4#|<0mMh*-mEU-$L4-BTq(72!_cnLtZ0h`$8xil|7%%{UGx^O%
z!yp!o_ZtQR?|!ajz^4GLN-_6RZCd4^m=;L>wUqy3ek;0;k%NFd^_E##Pc3>7=9ggJ8J(HZq5=kPw4LVVzwb>|
zoy|9|0x$#-8d`yk*hPybq+oi4*P^~n%wi}~=Ape9(aAzA!UT&up8yO)LTM}EPm3-t
zYwpZ!W)8yv_#DOO0rb?OYXi-#76BN61PM=aSJ=svHDGZZVrUN`wtz+PuMy#zYL4Ps
z7VSpG`f@L1fNLZG@sCW!m*1iRH<=|mZTG%Om|kvW+&94G$B+ojM(=)V9)+MKz@_@P
zF>Vaf$DvJGW}7NS%(oQV888+Jk1#D~fK4rcUjW7-q1h!9U{jmJ4(5;!zzgiZ2W6ub
z3!v(;E9Mu7@klvK6dk$NL~Y+q9Dwor_|y(c8luSW*65
zMYL6_1sKiM`ClLyh+=FLk(~c
z0M5KP&e-rgvYJJAp(fB)zJ1I=v@%@S^iRr}IvjP#XB$KU*WLBJJoCi%O}t4}u!es2
z?dK=C4}ROw)KJST`WPHwp!@>BD~F_q=I~#RR^WDwMc&x>^&MaitYLcKt$HHpxZkjs
z#6P5tPTp0%DMEqVo2
z$Qv4mm;?92pRm?Wu<1>hWwNMY(MA@F!_0!`Z%}$dA(paePSgZi
z+AjdCDML{cJVl(0nqYbivvJBdScWm(fpP%h7YL4sOEQ-$DHh#|
z4qFl|GMNRdv8jd4QMt&XN#>~Z3jjCd4s=wmiK`Y}^uOqO7dWk^_Wyt8ZtgTn4IMP;
zFg24(Q4vCq8gf3LkB>7?2ubLn+|y~AG@Vp-RO|>LD%}W0I-^vCB5Z_^3O7RdzdzSn
z>(RIW>o+eS*IMiPtZT1*?R8#z-%(-Ns)^UiLUh)hr43FQj!|kyVo9o3bJx2jxWdar
z$)l$Dj`C>0tU<-os=tSqk5NmG#lkz$>^p6cS#u{FKfQFE%PvhZ}NcC^>zgHbAJ6+~
z#x;V9|49=!s%-W0h`aJqU20$=TL0hDlG9dvgr$7#Q7d4**e=MTb7AIn82(LJ6ZkK};Yc2aH{z&Zb0n=r)Q565}3
zs6MhbRZ63OklC}`g!fAk)J5LHB~!b|o2jL|aGDrFUE~E2WFg2C-*eNjuVg`8CizC!QNMHDLbp$FkC*G~=9#m<
z6+xZh7k;PVFE2-QOxZYU1a*dAb9kUe6y&AMYP1#coh-Aa(7uVGrq16?GB
zJqM$zR}5R4K~DS9)`f$&h?$R31zNR*IdZyNnNIL>k1?(rLB(P!ul!xhQox;CmUg7h
zq_t~eGwBV@+-Jm1?&}-E9u{LZ`UGK
zUG}@m_B9q8lSi8v^9VE@eP?nAQf=;&zP2YcnyW=cQtD+QjPDZYpo+#
zoH42%zEO`twZ}F`t&J3Q*)2vL@{5eZj1?&EhGq&cu&AIhQUyKy4%^yTEW)08jobcs
zkcnl{7VO==W6K~_;=QN)BL6DuG1j9a2A7u9PZU3(JB+DRTXkUzqTYj51$OA_JxI9e
zPB$%o2&v)m{u^!6ZHWkK0f{lo(R5-w*4S|(QWftEwzcSzhW#ccRWnwg_!u-tdYk8Z
zn-_YU##BF;G3v2U^E`EbPi^9;Lw}c1wG0JX_kQcDdGcT*mk<2|Lah?bkoi;lRzjQOiFETh&h$C14(7f8d)Ng;FFpZ!2TwoD(|h_0
z?1W(TPyyTY-O8?*);+B}vo*|2bH;9?%EJome-f10p863|w^%cjA+LfMf2xds%J9iV
z{$Ni--Q3?u)v!Bx9?X{(i(oyrF;78L$`jKpqB~Nz(=OwvL;ez#b{;EGd@8D|{qvaQ
zpU0j4(TPx_LIp^Sdm7wMo_o6I9^tsd{+892SXIFGpMUY*D)G$g{&4PEYSd#eYSc3@
zI?7YO_3oPHn8PYYVRR@)6(~Lv<_x@~`=5!_{eN@4wXh||b0si9z1TsDNjy2CEFJ=?WB|3*$cJ{dd_If{3alFn^3=g@B0MBItx_jJV$g9mMhDB>D*`++5P0LnN
z{($n(ZH!q=5RytR)>=Etm)3Ws3i7B6b&l))@+Z?32c)j!=o#QWA`?q
zM@Tsc<>5!TSW=~%xRUInUUV*e%F8FP%Yt+82naP*_&in6XzlRQlJ5t5m84yZ6a%
zKj^iYF{)Q-?B!37**q*U!tEkF$IHSU5azIdrx9hLmmeT`h=~!j
z#e6g^K_x?GEU@xfH&31*6aCRlVDhP^8sr)5R@1qz5WzY@WBwP$?>D>R?s1EgC++7>
zekRJ)T@c#u<@SJ%_VTkPW-2kl-D(17q0t==_A(C>`pD*fnHYfLmtWmPFSWzZ{03-8
zfdf_%212+5h3@zUFCz$YYLpo%S4Yvbk@k#VA6Z&__Bf7(RCys!A|2@uRL--#yyj&0WFCk*RBL1YK@o>SI1%ruvIV*Ko$lnu2*Xfp
zFL#gd@MD~On#0jOxzuD?sDyCG!ESU5@)Wj>I#K4y(QGc2iKmEI1+nZu&Ly3^?D~v*
z)*=*+fN(YaBi&$53F&PmdKTr?Q;FC#K!L)0p><#gVYQxBLi&srI$17X8EA>oMnW6#
zm8@mTbH&%DNtAquh^z%WWSvhHj5O@KcNrWvoe#p8)@!K{X@Z;-l+8O>?5Je|>xz_5p?K=A}rVg5YBQ=g$E
zFc$}AV$6v!zuL?7^Isr!pNHZa2G}8wVv5PJ|HG}ZQVzVIgfQbCxGirF%d@^|bk-Pl
z?h>O!P>SfL)oWbD)0dG4%_;$HCTS`uN$5oEPRDN9BvQise1adyQ!U+AFe>?iW}Jw7
zzyrkFGJBCP8uoXO2VbNz+PEmeGUOfv^NQ4HA5*a(B=-DPSEk;+OtW?=lgm+HWg=6e
zOvRJQG~N4r+CM&*BgZ@r!B!ax95@ALjcuF~sz|qEU!GiIm5hM9=N>%G|Jhfr_EXUm
zXI-Y&NM)*wW{xdwe0=151K`L
zj^jO_c!n4;DYyFB;UJu*8S^92apOu`?w5#g;4~7t_?5>MNcDrwqCw+Io9q`qDGQH~
zaJ08@lef@6dI1X;Bu01?LV+h-u5WKril^eKp5u
z`1pZq$z$h7y(X5n?2?GkYAy*U|LCfHkjF8^HYVNvSpD7i9L#(=k2qQ|v9!3?A#CGx
z_G;C?MEc_ANx7WKygIEQSKscYymKL>Dzt)7sQ3jaD_?e{39=ms?D`zK|$Tya4j2oM@_x2(paD4NKC>7iEA?OJfC!UxjgkUp(F5F1nCcG2G57UKa$}Vc#5>qDDv?W?%F{P
zuxsl=+NJRHwW8ZqET8TzzmIAO7V@Jr9w=>EXWRl$i5HBDW~IvZJ^03h?e)54rl`PK
z3fPwb-}Q5o=ij|-?q0-;qlFJp7Z&YshrX|hu2H-zhgoWpLDoGwl{y8Vye5LWzpae8
zy7M3h^hnu>p$PW
zLLpu(!rlf$cVVNxGSmmZo
zj!-43J-h3?31g@iE&9+g+IxA1H?dqMiXn8I=?Eu#+0|Pxg0{M^VPjOw=Yd|fg3u68
ztdIo_Z5yXLLg$}-ppIiC#%Ky-Eo*tz?I2fnPo?Or;%M>|wje&>Z66(=zY#5%LXTVj
z@RYqc@TTl*@fHr1f;(r=g)4TcZQccU7xa61Z&C0EQ1-&x`N!x
zwVpV7ekx~Pt5zw41_%Z0Smx~;U8M}N<6)_k<_T|$&<2Zpv%{pCb%K|Z*|Bej1tYXY
z@x;^4LXge6rcxv6)K~j!#R}M$!^Jy$yQOi{o1E;1EW_67=+=jL3gZq;*$aqkl%YMQ
z!?9P8iw|%w`5K`EgyZQ=)bR!Rafg(h(XW+-15s?h&5gN1wrk~1`HgT8g#68}%S7&F
zrk>j7KVxY4u$uQfveXG;=VC_;a>gN6BgVZ8!FCE2Xr)QQ&V8K64)z{fipLHn-Tvf!
zF^iRQ0$Z9|zy*18=ahE1#;lV~2K2bSLiU|;xu*tsl%uW=)MQ=s>BD;A-}~@AWjqwy
zRny$-3lDjjMY6Y*7~wDo1HI)SS9eOeox$>$4t&}^&gJjRURUVPOvtlWikEu14|l7#
zMn4eY2ngYTFUV<>loN@?BIK8FWn&%*bNAD`^SH8JW?U~*M?qNFKH3nM);;*tgHox7
z$!UsiD8eyTkVEKe=^ut{&^2mdA}XGyPF(Tgqsg$Pmup&0u6MFN&6Pc%IJ(4mABxul
zUN`om)R~{=WoLNQ_BfhqypO~?27Pzliq>d#l9z2W?w~jN6kdvV4DuhbEu)r?;KOiA
z_OWswPp${&y8Xtdy)3vT&)y69M1&I{+{bp9>UEG$&??JufDuka@n&OJ@q=tc#qUBn
zKb3`(AcS3VW@U*nbW+jQSLh&%A{(j)Cz!Ab9c@?P&!j@pZ!WfET$?Ch
zrxLSbSB44q<5%cbh@JOPkV5nxD&p&K;&c
z-NOyOQ|V+h2ydoy;m~-g7}kp#^-MGcKe>wq*^T~QwxSh}svUnJlV`*GgeNQ42HBn2
zDxGchCRwC<#tPV*@B3ZgMvx%4Pz&iA6C<39iG-<&TpMH~#_Q9t@TDxAhhl4Ecde}%
zAE-&84#k8q&WG{8opQf({ugtCPE_QtR2o%e1?(006aC~M$R%9tSWFq=LQMU?w#&c9
z!WH%o{QqO4JQ1V#VoVHS%v7zOG9#l-~3gIP~znWC%6~Qjc^r;+*oHK$U~{h
z=TL~~(8pDIw#d*m@b)^>Rk9#!QDNnv??t#4LX*kvzJvUY@%P`{w-K&`(2jj7b@@Ra
z)h(4ek5jCzSg`w6z>Yt!DkVE}~po^*uIoXq5n^3MEC
zCUoF(3g-ZF<`rH(KruDDHo`q9GTxaZy=>*3`MWIWfkq!W3y*l&$XhT%1%w6j9bv`%
zj0=pXm|P?NLna2ns6W>!ntR#J5fUQ|hH&jOj?l-;FS!j_*eMG`AgpEWq1*V@%c~qA
zF~U$3mmEU>ui{z&Ik$^@h44?A7)HV_XKA+T{f0KK*|iZWAw1%Ds)D?jwLv!u@s}(N
zhY-%#f;@$(5M9zT5%McAl`(1+)LXxCJxY*0>71aB`dd^whFF2(5iqxwx(_h+D06c3
ze(nnmBisvNHREwTLkd+Pn!Q>6{RmPO}j5!uY|Cimv9^&O|_?anLJVd6(LFn7R(#uiK_$-IM~wQ%LR0sgkyPGG1`X;{nE)EQ~)lK7-0s4OHOhv)uh(zk8mGa
zZ;g?OCrRjeft&pWc?%QCmY5h5HJ&Y(jTP}}h^uG14+-{s#>s;|u^uTg!b}LywbRBe
z2d-WY-_0iM1%vLMG=m|{&_1edSP~Hqo(8UgONXo
zVk>a4PC3Gv*SXQ=*`;mjCsG_T8$~$l+COSFr?hQ@c)S#IP=tf5?RP|%%~2N9JclNn
zU=4~UktGvPAj|$b#Rvyi|Npn+Qgpl}T#VBNv8|FRz{%l1r&+tbB{yuMnDnUkDNv(`
z{owLgK|ALN$5^cn^`|d-W3D+XF(c>e_~IQipmwdmAun*LaJKbP)bF{{w)NxtQM7px
zML5&?ESmLPY10%RyRUk#v}=YRi{*MRK?vtayS?)EjtHqD^+QH|8EQCD+UvW+LUyhZ&YObtFTE`j
zVK4O(_D4Wnh!R`AN=j=MHaw{R$)pQEJaDk%S_cyCoh>asrI-b{KACw?S6hhswST#M
zLGH!6RO{}^BD@aaRX?o{GBeUPJ9z!$_AFmMOqc>OpXU@SP`sENSDxku>D#=_ui>`I
zrbG+L(LJ6j;!=pO{OXn*L0;M4PB%7*&~X0VuB`@n8gqcUTqbJ%yf#H1ih4ZHrpWZ@
zisw1(NFFAO{RGnT8}seQp@ZzwTpQ=+tT@(%<)p8R94W}UM&dmR5!6*a^Pu}mZjZ@M
zen^~G<>F}~XdQJiTQv=k)h286Ha`Kqay2cg`9f)nw(%n{o2#GR*2t~vgPg>W(T?;y
z8b|U-blMBNwAPL^28TGh>4nnbGx`&20fEYvJgN(--|VrMTU`bDcN^EYO&3AK|2Fzw
zEnI^<<`~(^;CxLFETz4A;uY6@;G$+
zsi)(AaL+2p_l>TzbbcaRq%=0aW(vNmT;@q?#v<7n&(Pc4a1`WBcd2-W2)YB_)NhoM
zlfx_f6WBRW-#H`t{zbfX`2cOJQPREU>F0FF&Z9}5RCYPyLFPfq9ptp*+^-x*XUtQ@
zc`CYf9^1q3;2u?{rBQHn-NvrLXhp@scKSz&cfca*lVHlp7R-4#WTtE^YZ^kls)4qU(*GS7vz%tgFQKY
z7Uo(ugm&&+M+^Y>;turWg8Ung
zlw-mOjZwTpqn7J~9LqD8>*rvhq!`6_tlZ@KASY91-7~FBZySnd-`w2o%W`+d2g)go
zy1VDZ*bjbdUzQ;6p)9%`5lXb}^YwgR$G!Z3`)*4*f9#9Ib7iO*ktW_ykgs8g#XurL
ziT%{#B~(=1)g@8UE2RyOinG$F;r~us(~}Ca(LWEW0t@1`EJ5GhKhf^d6|a;w%EZr0
z(Go>{{|JMe^}k10V%?4rTR}XMy))g1)+
zNH6zu=0=dmOZIZrCdi?@0-<*`U&2BQHM|Eo5m&8vFq$^Mv_Y#x3i||S6rPtN$hYVs
zs3&ps!F+WQFGs)4*IU@Hh_8uc`yGxS0ji;3W$<
zAAF~ly`Ii*T@PGf8~Ia}+7k!ui;)GL=TD
z{9wY{WD!{4Dyw=Zk#uQs%iPHu3rsu0|a{Ic#$mV;8Cl
z6?)oBjuGUJtL&AHA&W7gqsLeQ8}7F6+L$TU@sWBz!lOlgq#V5D1n
zeCFljZ`-^e5ut?!iMDK2*X$Vr$W^Q5M*Y#r#>-?#TOFT0<(hbqe{$m7mHQLH&QzC0
z$1f~x-7a1(%Nh>GobH}vkhk}i@ARkMdwDcrVQHIo@e0w^^L1_QM%f^H@wBvHar70@
zCbRyA=<3aHk6p`R64sdqC8c^e2leM;Dz@yQMnn
z+SfaiM|im>%tMXmOv0?)QXO`u-`(MQcP~d!$oBsb?W?=S}gu#ag8jhV_!$RQbqpU
z%jY4g1G77lNipps%^UEi+Cq@m!EA-8sCscJhoNi5+aLWhUQJLJ6lBfItaZiF+wf9+
zS-*X;^?5~0N{bKPKs8q}?5_uxdsk)-AbQ26?i1awyu6?|l~T#0OUPGvn2T{PJ#YzQ
z#U!F~x&5i%cn>qJ)KlLb-df9(hrtVM7V%_nPOxV>!2KgNgRoz+i!xEMI{iy!Z
z($<;y1Mxe<`|Ss}Q557Mm9|((jG%?q?+izBSdf$7b*IAXW#JGM>v*Y2o(gi~t!f12
zi$q9OTtVTC+6Bs$tUL6Wq8FEz)@u`QfYD5q`|{qd6A1EF;<_2#*@o!*rKN34;}3Pc
z!%2Mg0oOX~vUpL29Kz{4$JEJ(QP*Xx+=hMxF<(Abz)sr_JK6ODLC&L(;hFOb)6bHs~ksRDrUT?t4nw}?s^ezHIRs8Up-EI$lb(3FPmNIdi#$d6i^@yKMI%qi=5-U
zrCDCy>@A|w?eUP4{EwppmzTEKbI2!T%Ew=EbpCQZt52f-q+c6j
ztOjR>A3$Jui4$2k7RCSTkS~dLa;eOa&q5Itur?^I9m9}aU+j|#~*(#3nxHW$E_FcBF`IbdrFBBPDC;9@H{&eS`t0Dg5N-i
zzfjJTQ1t8Uz7orLc`b_@y~$;SlOdGxVAU^l_A=tZ>d9}yLd#Q7ge~D9?_r*xsk#1o
zU`fOlrxJOZ<*aJ?(~Y7wZCiL)&
z5<#wF{>^H0`q$Bw&og0poo#NS$=BlN!ynN<-Xu#4dU+3f1uZf0
ztxU)h=ki9in%A{nR#6fbUWpMjrvBjvggT6ABHmJx(n=uyP8KvRTG6$h(`@jvqhln-
zP?h+UUBpJnU%XsSA?!_(x^FRd9o*Vt$rc&ZjOXP)-R9#;FOz1rzcBoJ2-?1j6|i5O
z2s2s0T`@wQA{|sI`)N{LB47(pFheLd!h1diBU!<~dnimf|)!Sr$W$tRdr@
zjf`+nB-G-
znXu^)t2hzD+4e;i=fiaoG$yAWb%eTJb}`AJu?Qs)3Ny^d)l-Jk?+bZT5pO5PXo+I?
z)9(ZA6udm%Arq}2?0)z?%MQW$kX?jQ2)iGC&-mT$GyYA4Oc{*bPrui0F-BqxO`Uc>
z{oeM4Gx57j>OwOsX2%1y9
z#y{MqYrN&`&nn9Vhk2E{kmwnBlPw1&c~evBWU{dzXSHnRL8vFVP|5HOla2i
zVyPP?mw36LQHseVXTc(9`t@^Lw;JilsgD}<3me=)^Q>=K1Omroo31hg1l^{Q4IMEzzNYrMn&ZvfpcqD{_x4NnBMP4=<
zn`g&*BG@GDuI8>=4f3=W?xdT}dH66$sS>uJV})$GHuL~Di3svu1`W1T7K>oZWCb$z
z`@mf;a6NjE>#@vvWn3v+M^m;5jD;G6gIwCueTQy@?kM{9b%Y>W;wW{{!)2idgaiFc
zi$PXEVA#zh#wdsJtly6avZE_+TqP5l{gmH{$lIIU(+_eW@29DKh!u|!j~HUL=RGpI@?G|_e<4b#
zhE<1)Dque#u@rNfVFdXs=D5S9fteWd6c|5jaTgA97QWEN(B7A7xkU%nu}Vw_d^M9`fi#{3V=i!dek1i6LzD(*sdW
z6(XoAB*wfR#>f6vf}HfC-AUCHF;!3n>}}GM{BsQQdnSo0D1?lA70swOVsttqzLqM_
zd3jV%J4u*66(e6}%;^^FG8N0eWarhU(H
z6}Y9B??KQoW&{=Oa6eQA+4Mbok3Eh$eW+KKAC)1MaC0-azgWQ3Li-7uI5^S0P8UI|
z!5RpnDB6x}WtWlZ(U{S_O{b
z1XYY6Z@Ax`96gSS!hKN8WBEq2$SNWIRbfiMGqK4c=TM8#J^?)-hA@@
z^o=pHmT=BCF_^E?9urGZg_y_mRz@u0%P*Y~65<&$qAJmX>8%Q22{(P@7;0UXpt5gZ
zq{A;cC8Spn(eeM2%C8Dh<0U(f$|@oKV}^BuUgI>iszZHhB6}SpG+O%4h@ULyy9h-LBr|ypDDzEk#p4nmBuvsOfrx4M0mua=h
z@MvALoHOlxM7ro74b3XEwivoU>MpE=^g%>)Vbf}V;qE5(BOf`_-aw>`LO-v(+FNM5
z-44sLN=WZZM6KGi+E}<{FEx|$oM|s3q6yjy%A09{`j|2HhFDf9>CQytYEze>y?mH!
zPNqGbh?>)j%G?rdhw-jfDHfYOV(HgTn`?AK{bSGlEu&LXkUPd>`ZRC`Yex3*qB7Q}gwQn$hp9$kPI@0ZTzb8YL_6JHEmW2=|_nuFOf{`
zLM?$7U`;)jsb>;V&-I!zYk%OTbL==KtAzBWL^Rbktu_Z5e`E(4In(Yzgz-0CsJz-4
z7~0q@XO)n?jR=+#(`r|sZiYc`Z!T==KbyK^u-BDYdjfy5kf&H#C8W0yQD)O>LttLc
zoS8H26GZfk7b&lH14`$(K0pcSeTb+JFs*h14k)qyG?y`TF)=#Sc(F1cfx5cbet9aZ
zg!J7+=w1@jYImU51@0M}_5vb$#!Hk}=Qr1$>spf%(&rLUYcg#&w0E%8trlV0YfsI%
zCb3kR|0VTY_nefFzLtodlWDc||C~NlBb#a05>ZoJro3ABkD?y4kDpaSdH@kUCe!vn
zdu9*UB23$zh`hBtqs--~4?Dy?r{&7rnuuJjNZ~PPM=))s&gV?qpNKl271ADyHY&1*
zm{mf$k_eN@#I!xp=F{w%PvuOzeW?5V@(txZjt#QBb<6`$XO)s(OhmuaYwF|4eB>oI
zHO-l}9T8QoH9dJ2UrJ1SBHC*yr+Pipo=Zf%-doCh657G-HG*Wy
zvr0+dN=(n#)F-2Ug@>*?HtkbHbjK@|`4qIzGt}`QvPwu#C&GhBOsnPmv%JQsKESk7
ziKu?8Qr^?hem=xKCMBdZYl-POnOY0^m2R4lGxaNOnh?LO%v!~d|I@siRYH0&5&V&u
z_Drp{VEa7aZP(J+8vY=ZO*ix51`OQ3NfdIbX_8%9WZ5vpCN<#_#K
zdv!Cbg!J!p*R;kJifGxK80{)($IjDfX?f0Q2N0p+RIT^4Sb@T;p$#6dU6Asu64KWb
zq3Fwv#zhkaGKJT`dy-L9)yH^aiSR7q4YG1A+7I{D60SU}g!Jn~6)3z3+SfI$zGjt>ewzq8v&#Z4G1|@0
z_NCv-kTGYxzxJ@-8Lj*zm{Xwe7I=@Kq55Z)kRCxq<^nA-S|4a1qmebEE%DYq4b~JW
zycJsi`1U5XOYEs4;jys3yiSEJnl?cZor
z|Bcp@2=%}6bJ1i?fxziOrfx=05>bcojqUfTbjGWM*Zg;9
z&Uk4eGWTsTr$FIwXm2x*Qk_sj`gtO%6GrQSHKSEQJMu`kOg36eB217fzYEp~;QtZu
zXwtUWmMZgr8WXXptzhM&*Ku
zX-1QJ6(=sZCua#K5z(?v#@!f~XYbAWrymT)8yEzM*#P!rSKN9v+{4yx8`)a+WZMh*og2`a`g)K;cAEd*0xFVl-IkY9t+nC1~u)8ywaa+a`#d5jt()Ui86&A^Qn
zDtwgWLs4tJZ3#_@_}ZIT!gLtR=h+@a&Jvy_a-g#o|0Wxc5&1l86(ws4ZxB&M;d+TB
zJWk|@W;Vv=l#nSbA=b)KDt}jzR238`e1gFpT|vzBly5iR_&wk%K+)65|CSeDwl
zQA;?0h`-Tsf5@sv3Iz(EggW3N*FpVZ`#)C`)BcaCpF;f#9SyrQIVGeYA)@`Coyz<)
z+L5@McMG$Y(3gldP*l$?VJ49?No+}eOE{c}HVt5m`%@($5Gzpl45^vh8SS;HV=0#s
zkwaz85}qZpbRV0N=PY3+5!Lrv82L*^W)b=RWw-vegmpx;5d|Z$gxN&KlZ_3poF&{v
zM0HL!GJng+9AbHq)!M8je9gAMx=b0dgy)Ey!M9w}!pwn5XVG&opOpzuX#eI9T(sD$(dMA}lgOyDKPn+NX!RN4zOUIh_G
z>dGO(oC1X}K|8^lQ$qS6Z!XXhqrD8R*&x^aj8;HIGoYct+R%*5y#j9q7E}+UO3xy~
zj>59QOB5)a5ACX9?p}>{5)s|&uwYJs!UfQtzzdpqDj_|Z2ouj`ftDEURcOa$GPaMA
zGhPQGtoiS)4CWLld=1`|d#piZm5?4nga)z9XgzokMq3DNFt?|6W3=muKpQn&)>PFL
zD104S3IiM(WR;Lfe>2dwDOLquV!TE0Zia_Ta!N>_M+BElsFFE4fLMXT#n75!fNh?v
z64Et@&_%B^nrg4nmOy(54Fi78Xv2uG#2Pz7{?}k1kC3^g@J8TNHE$)P`w;2Gy(*Hg
zI&8FM(AH8ORXU^1Bcca!ucyTd6fTE$A{U}HXO)odNJJftX;+|~Bmb+enR*ft>RLQf
zncqPD7JPMnN=QFPMB7`YeG~0L6>fXWv<-=H-<8!+^C@VoK;c`^nq!rMW|fdm6QQ8X
z0xiiHZza6DQE6|{cvlgjD#W8?Wfj`4%%OB2me7KTI#&{wDIs5;G0oegZsEbp(U$No
z5kC*89PO#G0)?xg7BK&@%|A=|jx~#>Kr$P+iD}l5dNFsXNth*+6VW6LYUP+yW@@=E3>L-3KYIeYAe>XI4^4n
zg+x4crBNAR64R`Mx)DP>>6|4jCUT6kT0KryH5MsQ_#XS8c~^O}mU7ZAW{sPU@Sah7VAV94^(5CFVjV=zQhpws
zpX%Yb6UWOe+*pCa4W!=WU26&F{^Q#B0yQzshfpu|p0$Ky|MBd^3Bf7>)*m0i{n3|J
zs=^Jsls0e^(`PxYkT^TbNY
z6n=qzmcNiv3h&?LLY657!~S$3qiiBN(O<|C2JCX72Xvub^TbLi{E|F7=qI&8S3>$z
zBK#}!<;xVyFTu9)zJm9C&Mks1VI2`oXt;^^L0S5m$i9qe8ZIs2FXj_r+GGivVVr)j
z>(4CVP$GwV8U5^~GRF!O
zen;vF^mI)6vX*cV5kKi$7O077wm?0MJ7+gBM*{PoX2f*PH&vFtM}0f@rqf9!q%S4X
z87C={FDDsoE3|FQz|_wg?R_FV_R5DmEmoj#8?nFs4A^
zPq1Db#5kk#P^F|F!fBdV1YTmi9K1Rc+)Th|KaIDUKs-%Wen#79qMJ`BA^j)RLvWL6
ze?faPT8$f~?My_6F^?$kuV^y^N#R*!m6E=O2#+E$^$yfi#<&Zcb_fxsAn~Kh{2SUM
zxmF|cDk0sH2-Drfw7;W$9%E>8rkz3rZRK>R`8~R=Y2wX8NTv{ab48{C8T>3p%!dZ#C}9YADq02vzUSA
zj4_7@GthX3^4CE7I*qL!`DbKgUq$FxvN=R=Q!~O@2D>1e1x&ybO%3|vJ
zL{wRxR%X4x^dk=kZB_~C4Mfl;rd4Nr1J_b}H0@bLXph5ZLbX|h0)@4qokKo#I!Z_%
znIT5oNK9P^bu|gh`EsVdjmTk?;2CA!3+-YOm@j6Pkba7Y>WXRWD(^Tqn>6ihL^PXx
zR(b28y@&*rUJ2~qnRXwv
zU*aiEhE2PY2oE5ht-SToZr}lGMy-VOLLyjBOxqxz`o9Hhs->oWmxyZV9A$1u<`dy-
z$WTJMBN3i_V%kP%o0331%$c?Z5$a+5obv9A_Hs(1yh=zPPegf5+ZgTF!+2iQ|D37c
zCZ-N_F5G-IPz4J07URp}VMPh)X^yrm&=R9Hfi?&GcqnJINks5aoFz~3erT)9-FFeS
zk8$$wBlM54mv3q&M#IQ!8+mQ1)5cNLR9Q*K<$b^!SuKLrjD1mgAjtFXcDuUn;?bQ2W&5Z2
zp^1mdm5E;^SL^z)-zd@n$v^V${yKHwuSEaL&#RzBB9}BEx*n&h;6Z-dK9ne}707EU
zN<;KXKgSDlEnh!sA!bEMUK@KEyhg+t)OAG;a_|7R4d;ryQ0Df9*6R=VRZ5Vrb7x&-
z&HQCFIr+^W-8;5#M$^(|4bF&PSN>uYbGNxKAc8!fcNy9!yBp?LR^0By^K9;#ESJ6g
zq21lbT`|bhf45_XIGVi&4JKJ3UR!vNvYD!q1$hQP?NNr=H$+fPnZwz#8fuXH-D(ru
z#0Z*Y*Za+V91-M$eAsgl7T&}{tM(8^vFWQz1es~UcRLr7V1IfzqT1JQ=S_DXA_Y0v
zT`Yb}1Wnj~J=1k#L2ky=j1-9x4uWvht*$gdwkWgxf_NnsnrpA19*$|E;wT`wg`Q9|
zeXD{iqs_If53E&K#_uYu(si|M@J?@6mp=D$(sVl+i=&f`_qKRy2-R$KsLoCCa&_QU
z8*jCEhoN6~wEM>RTQ3iScLYw0-iDV-)S?W2Hmwd?Om*$=052D|$kQho
z@jD`DNB@S|+Dp&NyTi*H_<;J~_{Rv^)F`>Y{SNp6UUr|5ryrJ%-<5@bLCEjqE>XwJ
zn=XuI?OB%YnHb|}7{{ID`jj)hoLZKjIuR4=WTHEShMe>5jXc832l;-7k&@qw*s*6>
zi)-WeL@bAJXtDFc@m^klk?LEb{QX#J2zhxKBefYH-Fb~Pl(6Je=s
zsiv$FD_@mY0sDuCYwYJ*XpjqOO3d5h4I-SxD;s$YsTHbnLC$%?8XG?rabBIW79HXb
z#XJSVA#^Ar26@0hd$Dzu2sS2v7{ztCzczeHeKMDb7S<^%&U{(l{Zj%T$;|0wKedbA
zlKDq|^0sRX8$~z+MNAveIQXoWefjJ~lRP7wiDDQXwg}_Bob;PJ+xS=(&O*_Kjz`@n
z@^A}WGs}+A6Jwl>=Gh`h_g~8^WIcc7mM2_k?;TUqpXcnKp@dImCUoyGjz|
z1?BE{U^hkos#n(doD1oP%Q5_=_*cST$1_xI2yzG=AFJn8B3Pv|=2bARW4@z@8st-}
z?UBXNjC$&wzKY(gXT8(67X3P^Q)JN&MP6
zp^m?kg_}^UIoCa%AoCbsn1UxpxEaEL=6e6^A35hH_M2hETV&!EG!svECW3sI2P_jt
z=mX*F|F|J4$gPeLe=iHSLRfOJs{ujwJ1j3XiI;{%p!ema1Tp)^^11`y8`(D)lOxH$TA7@4=;ih+f%8&-+IixElj|`(8T8#;gq(>{p3k
z71o#oU@rRAJ+>elUTd9D{F6)#gm3_+)Q^>W`D7D&V_ik5(?Na
z%b&m5)tDetcNlY(F}091=Dpa=Ji#|%YN&Jh98B^*|FxDp>USBf!0QUwkInxp=UUSb
ze>xf2s;GRGF}2<_W;Hg3-{Bf&ke}Y*Z2lpeS`;Z@|G4l85mt1km?U-Bv0{*~ZG7}P=_$Yxlo7h4Cn!JX#6#;z#ElWP!g)(B6tMCZJhJf)Y&4}&S?Qit72~{PqBk08cBHud`uje<`wy~SPxE9vNQ4Kl
zye~CecM)VUl}Me35gtVG6l1=gYmiUaa*{K4U6Q93D`5Yi?#gFf{aab&1Fv7h?JifoG}u2k-P6sM*g
zA;_1h?(E?tMo2;U`WI*3q&k@$RC5^daG6j|TKt`31bHa?88Tr6_0PZ6cGn2<7GH=e
zSy1W5dqR+>IYMFtxv5@(vycgaajuy02$|4;Tj&cBWPM)Iz{5`?9f?zmFgn`1=?o6I$05`}+v8oxcwwXf(Vw>#h;xwE4~x
zqh&#hqMEZDA;=$|bA=c^PQTos>FUxa93#lRpK&I}$b=ep8-9UDPd3Q8xQflx#0c7*
zc(SG=1i2npL5Ro7f`*02*W4fw
zsF*5*5t>3+!E3-`-HO9+Qrt)l(AKwjRzyfcl_3dS}58bLPm_YuiNDTD^T-UT_H
zdM8g9p$x(r`d2;BAitpA$(NI4p*4ytyT*=jrI#nL86y+M*dIn$Ux*;kCpKles9N69%VUrnrzJ*EpV*RhKwacBV;`UCCT8Dyc}*|-
zAT*g8-3u`jKO~}htLaa>g+`D~n8~TrHG(d^596DNt-Sp88h4iuW8t;t)?3|DklFz{
z^eISP0VaSIU?=EwYeDK7FbS*#d3_5~UHWDUQoRUF0c$|PZ3U^Wpf`xYI#8=$L8=?*
z1Ez!Zpw8_DsUDyom;p9|qB{ywJwbmk6C|L)odu~A!2mEDYyyq@7o<)F6(F18f15$m
z{}!aq1Vh0*umzOdRggLtRDti~8x)h8BOTZ4$VIUp=6Tk|v
z6Lh)<4}eKvCCKCEGhO5XFa@ju1r>Mz^ae3l2Wkz%1E3F>4%UM@gYf|92WEhcplApl
z0R6#Ckbnk5@cU#
z^a0bsdQj&{JOKKE8DJwQdI}GK{$M6ZK!d0802lyfgH53EOgxa`e-$7LHiM?m-~lic
z%mZ6M$+LI>RDt63APK2S6_{1*`!Duj2vG8^mB8sI>?WfIeV4
zSP$wf#si=qm;p9|q9u3$^anFR0vasE17HA{4K{(snPpT0Pyw=FGibUT4}hUy9@qj(
zR^S0p1?Ge8pzIAi0II=4kOOVs!~G*a(W&;sMYf%mfK&@D3gT1Hf$1
zYrk|RwTVFEcku$K09mjZG+l=mz)&y`Yyl+u4Z09JsVpwkAP|1}$^2w)}1`w%aHUSJAX0}4LE3!pcM!8%ZDBVGV~z;v)4
z)cF`MfPP>G*a(U~!3&^2m!FEvg1zrHvU?IqXwwv$*7z>tw9iYRPcmYfRD?nx^|LgP>UI3H8N|5(8
zUI4wo6tD&qY{m`UNk5K43an
z59<7i7eGHS18f9EJMaSN4`zY{G|2ph8^8cC8*BoNf5!`;0%XBv(DVm=CstvOnPrsF_Zk2nK-JU=wJ(M>=&Xr~p~888oexPMrycf_Y#IC@D;*&IMIqKG+V*_DrWP
z1l3?6$bq)C)2U0rSg-`_;KPPghdSxh6<`8b0d|5;d*KN%39JNpb@2q~1*U*Cpr9UJ
z0P;p}WDM4UT19vP^a0bsdQfL?ya4)v8DJwQ+6OOW_+Ni86C|KPeY^k$fZ1RZ&;=Sd
zz!RVXWWi?8v>~1VL%}?-1(Y2aCCH;NU4RNT1*`!DP4NWi4Pvkk)M|zoKp!w2tOs?P;|0(U%m5pk
z^Zbii;0Dki%mfK&P=Xi005BVD0*zbZ0Z;+5U^8gi3J-vxU>?{4N=oqnr~>oBc2HJ^
z2S7Dg2y&opYdp|8LuDYa1ndAE_QwNY0$2fdf=+Gl0GI?;g1okP0Q3S=z#34{4iA9d
zAO`C|tpo4?=mVyM^`K6BJOKKE8DL|E{}pw>1E4>c2@=rYKs*2jfZ1RZXnYVJ02Lq$
zHiM=e@c$iOY%j_C8c>S_`=<stD%!zkk=Z^XId~N!a^T}uuj2r3PNq(qf^9w
zV@`Vsyz{ps^M6v<_^5sjid`N``EbK2({vM7lb>(2O&SHpl+mmz9kka*BT!r*Cq>=
z>Hc^4AopLrNVdw9Ew*4?5wbp5kL|w|7T%{8pLlNcLC$Tq`yeKUE|!VCMVRD+5N7%y
zgoPF?QhC<-Am!QYgAjIFu$~BGm&jN>-R5+H8HHiYvt)j10`K}Dsfh&}DK+PVq*g7J
zi3YMZ#RsuA+Xo>mvEY8nw%!Lx-D1HOO08HX3oW>CrhI~hn<{yR1)C{2--3tdk#CI)
zmJ-}#!PX+|@WDb{yj<2=>o#Hq)2&@?o2`h|w$`q;E%`qQRJQfYTR_qKym_KL#UGv*
ztH6b;qSDS~Lo#FE_$MjeCA@H#@S0u1iNYxzpzKtrmYvF&@!4+@mtvs;RyegY64ooh
zUv`VJXfQi0P>+@}^etti%=6wF7aeqb>4BLkD@iG`a>@jMuFNZxu1jn*LIDcxk?Zeo
z{_y@P2p8$5)#`_Ac5jL(ys54`7m{wuOYTJnf_cwzcrH8on|rGZ+dnlqnN2-y^6Bq{
zhuynzn`73?8
z&SZY2h-tR^&+k&UcaxWoa2Z;n7>e6a@V*8AGEZ4Fs7qO+%u))ev<~2XA3lii?cBf8
z4+Vug9p^B@Fa6}wyAeL*CwJJVM;rbXJ)Lws$svP$q9cnBzjYk#`@hUhDZMrL#=VxD
z$!kMoeBuWjj9mT6whQ&`6zb2+qASVsu!IMGp9
zrv-g|UhVtW``P8y4*!4pW*kWy=>vU+c7#`PVP!rLyywFm|G&z<1gy%cYx|rdm=iLH
z^Mv}4mS)&won{1xv(dN0{>KqLXr$nVT78xogDJmu^DJm5fCK@UwDHa+g8WtHT
zCF+0QYp*T(e*gbn*N@lZy`FWiz4o;B@C@f1$Rjj^Ga!#v`!gY@$Y
z4`9APAs9!4aa5uZl+VLlTW0P%$2;49j(Ptauh;$i@Fao3$;jG+X9ASj3BRCt%!!?I
z9g0nv#oM5I7H^?ODBJE9Jo2RtgC-Jh^iXDVePBAB>s{f0)I4ym*DG`ptk|_IW4*p{
z&a!@qSfrk<{o(-|3JH*>541AFax-$KO>Z3S@OBDicEYvr$wp<4&9#9Ul9#ETZB&#Z
zy=fu_d0i7<&=;uqY^FJ4kcs=?D1)9o!qBdg{CEMJ?F$Lp@Sh-y5HV@WW3G;M{Kfvp^9x)LLsrNUq2MATg6reEo^I_O)J;CUl!
zg(5^|>KhQ~{HAr!+Q;cAT#GS?X%-S8(&CJu=JvTo%Sx@!p7iB75W(gfC^5
zYxZ(0v-vk*;-{YB9)&j;C^Im(qga#~$oLHx%6G^J@3O}d9Mwe{PmI#9Gz?hg$eV2p
zag02ugO>wyG2)Dc;jHlh=2yy${4332)UV@q3_aF|gQhj=L78p0&YX5WXmf#8B|_C6
zw04QF(A*6=^et*aYtDu1P%z3IGTY1TGQ*%hf17>3G6C}XpRC*)@}&qH+B?Ks3-v!^
zt&C0m!a8-BfbCZ$i3KMGqq53Q&}1Yt0DC(|rOYq1Fw@i-Gyl~7kA=-7A;xywlZg^4!c`70~xkZDSU
zc@Y86G!B_2T*WlwSJ=WiWSTr-hG68e2*eNtCaM-{*>3C3A(@O_Qde~sR;0U9VS1}O
zhfI_FE7QD=ml5dSA(?zK@k_CklUW*40fQ9HS|emP%m5@EqJDfEW;YXd*=IS2mMar3
zTw^(hmP@H-x+A|?&Y|TBs(C#T|9h1s9a^$h5DQ@y}44#`xIi5-RK
z>Gb9>jBAM+CfU2x5`xHIBCtrPP_CsMvIO;HV(<9dmf9h_*J6mXb|Qllz?d*lF9vpX)eMOcgaSP
zOqDRbFms_(d=AN^*D}pktSn3uassImp)OjbrB);p{*#G%;EfR`a!4jmm=>#T8i$PZ
zYGIPlbxh-sOvZkC`zsU7rJzIrk}4JIWL@hVl1cuHiMlG|kVVKh3zG1jT@4Llo5t|tq16B
z4JJ#*y+cN7xiHV6-x#S5$xJxNG@oMvA>)utp)gy2vXSc07cmGQizXF|>mZI5pF^a{
zuBW%uKkfd{p^TOK$>xhoy!a5TQvadVHHZu@s}5;ZB+QFQPsSmcdSSLY3AhH!aF3BFU&2Pk3-*Z
zP&Go`fD1gvi9;sJ97%UuX4oP)BvU5La_p?xN)E}Sm^&{;y$VJ#>6x0JLmJcyb2&=S
z9&<=0>usia73(Jp;*d+IU84e99s@qgtQc-X}ej+7V%P+l7ARn#lm*Pi&<1URI-6&T%2wr
z9MWm-So79ZUWY){SaaRw_%uNJI9h(BJ?fB_CBnR@0pyTOcs$cQwZZnaLo#{7^ud)D
z3+0eZwJ_hkVtdq~^k~Key1P|pGKWl5D$G(8j#nTK$s|u?nwNF3I3$xV%-=d#9FnOK
z<~6OVMeYnhWxh*yGw0eGIV4jiOo6&{NG4?x(>$i_k3%wr!j$Src1Wg9m?>JJkP}E{PocXZ+CUD;
zR0wk^3dhmykWA`Sroqq5Nsl`uQzT5Wj$wyn>V>J|5N7*}03?-@Mt8|Nh8>crBojMQ
z$FM_;;k5UdWRs3zhfGo|%sGqg5OGMRL6}zxup+TS5Clo(zE5{UKegl0A@?R#WZKTt
z@#v5R$8bzHBv0%6+eWZ_Vzgj%7#(!t=6iLySTyQ_6DI3!ap
z%m|$l9FmzZooO~{$U7ucD9p(^QZ4d6Ca5~0F4v*zkWBW+bT>~6=a5Xr$ENgZOsc7w
zv{<8~&mk>}gn37c;*dyyy5%OY9Nj!dRhCgG{E}GP#Ce0%m
z|IWX*lO3XH2##vu+TbFXwR32LBV#es#p~?o&~l}V+5hqRXb#rVp(T^^nDBd@03BK`
zU$~Dk88Dqg%hd??&oaw7Z01WqK{J=o=VHCaBuf2l!ZINzqNJ>aL(8QsWx^8O1vs=J
zUO=wx`xicA-Nh>x6u&z=oPdxfNyy9K88Z$+YGFUnCUmGtGYaWLyqOY$OSYHl7lnc>OHwO+uOjv^yaYY~5zH-REDkRhPkwBoG
z?0g^yvT3mSsas>dz1i!0e)iY28o$SGj~uE|g^-c>5*FS(b4U`?P^w9|#p@C(`i2hh
z>%!vQq1x07*-yPYR5GWSDJ$_&WbVuyDp@JyD@r;fiNdF?Wy;zLq~x=75lG<=Y_YJ9
zZnqmqhe|fAH8r<*okO|b(nUWta!4aotxCwl@XtFn4%I{YI;LEJ=ZqvBl0?c9As1=bUZ|pXQN+xe$
z%H`egAvDweHZRVfZ%)0<>vCI-==W)M4w(qm%KVOrl9hC*9?OJ$L7UQ{k|}07yoU-(
z=^z2+WurNy4ZPP1IY=EiR5EKLQ!Yc+OzDs$y1HD*wwltRk`p#bP1#XgAeK`gg+F9k
zDD1ylC5KAZ2{~Cwha{0{_GbEh0TEAk4wbAB@+R~#gW4gVpfc0cGA5m^&EQaz7Mbn0
zdtCzcW#-WBh|rwxX?w0_<4_${3OQIEIaD(32c|qpNry@n3prie)1eljLCEH8Pd1qd
zr0|D|=We0jS!hS@EgX_W)v8Fw;-l&TY$$vVd9hSZq8?WaBpf0|i4cX@|3rdth?y(=
zBT|H7r{QTolQ>MktvE8VTl8b+4wJBX6CzrDIm8AuV=Gd`HN#M*D~G(2DHSFHwISn>
z^-SKz{Nq|^HV(NH$|n_fIqtqNkwezMMwnY-?L&5loQN|kkf!Y==tyoJ971GXWkOE-
z%kC^3f(wyN+0LXdqbFrkqR6BuOM#HX(OD!Nl0?c{AuCWjk`9&3`iX9*q820_Dp@XM
z&kzGScsLFP2676aK9j+uaVq&wNGr
z>`JDuZfygoI;299f7Hz9#Gpb9p1IJzd*QH=Pu)QW?{>AXOE^sOk70Tbw<&~w0W%+q
z?y!-s7k&e7j<5nDCrtA<3#9C34RUregGbbX!)g8wJDY~+iD|FA(?DCASY7jA|q@+F`E4PsD>=eBg<(DFUO=T`eRd
zWrvLgNUmo35hwuXZikHp$QS-FCaVw)C=eSns1bwdC;;^i8~MyV%%DPFU~<@)L7DJt
zQ9}+chmGk|_A-6j-3TfgKoD#+C@?+l!-u(R#cqPOq{GI1vVNnVZfr@oaoFglT=>=6
zkPaL92{laL?1-Jt93G2=P_R%8)@nr^rlF^N(v@|>->(&Q$P6r>y^k3Lw0;g7`3m7X
zqbE6f9J&JF<)75w=>Xr>BQGjX4uucd;rL<4n0}0}7QS8!=&+H`sH6X1;h!z&kp8c*8I+2_X6*upjSiCkri1R<1r8he
zd^7Sv45k_}Gg=Ra$;Zl>d4O(C(Rw&+bWMruX}7LpwY+?}mZ}Vz5~&
z>W~$+^KPy1Beb3lnStfA>X|_+t*66AzFhcz7)V#jSE#R<`&;FPG+h8!U01nmvrHJJUF=)Qut}G549i$$jgKJSkI&j#?7n%DX
z_PXC+FJ^U}t(n8*0d4`yX`q|x)^oGbD0A4@#-+mlgidCy95(XF
zqnLgu?hbK#;V}JroN&4b1@pyVjqES*I*}p_BJ7$_BfQb=g+uaodyx4yGx$lj7Y-Zw
zGU1P$VY|hlf@j-(KIy7$b_pr6~dKO8o;cTI-*@ENaLDDxxO
zwRIJA*xFb6E$#-Dk&NG5VE4)nL0*MbV>-DayfevbDThpmn=s_sX6hRi4%KO`kk=fL
zgpc`Dz^CI%zr|vd^)Zu{;r(N}b=cZhUX0(7E*COti6tG9#Ah!jWHRNFg_d+kvb!%>
zXfmJ09oIUs31(XxhqM`jTbMKG;7zh>kStRQ5YBD33;U#bEj;
zv`*+`Cp(8IYYB;{0f}}TwYNgBirSOHa4av0ht2iRd#8r-=Fs*(`bi{l|Ay%0KZ$ok}2WJ4Mf@=_YNSP0^qOc1uyxYbbbQI{JUDv+eH3X^ID9tdyf8i^8^&}M@5j{M2Ry$CenmIha05#$$3s0!&
zD9c>)Dk3#=8IxW1tW`QZK3N$px)0Sk)?w}b=#ho3e67PtzHk%fpOO!=sQtLSrFVHkqO#FZq1enqnIch0i@~3$^rL2fQ)so$?$LdM
z!6_@0%%%{v7)u)ODOd~^QmNJHk6I(HE5z=0j0xIV4AvcIXU$Sg%FS$2!T)Rm0|up=RdnyAF(wiXtHMgPOBQp}zST5lGE
zN2p{=q^*L`c9X8(f}Du*M?~F#FSsZc_CX!?4hw$kegnRhp+U&;hinu&EI1mv@ouc;
ztLZlbZ~rl}93DA(^wz^z&q=m3c*}&OLw-}kVYF)~y%2O%mrrboEe1=dZ2ReSt8^IC
zHwwQHhi@-oiYHJDeuCCwFptW%x1$JDI@H2f3wbw+K+>TU9y_m$FPZcp-a#bmP)lD*
zvfWs`w@1>UD}C};Ou7S)^tf4fsO8KTvKURqo^_~=T|+YJ*P-}4A1@t$#dRxoOYasz
z%jqnWLe#z3@R79`{8Cu7K$)=HB@I6LYttTZX;aoP?H_t2Y%w^ON_s93_8ts62J$^2
z5rRJn4cE0or+;WaAhaZd7@YDo6Qg`t6rxsuZHdI?VDTLuMO5NP4Ovo5k>z4juQt&i
ziOnJLfb1uHLl578<-}Z-aVv#j4z>JUM0~6jE(*nHq`H`Hv)M_dKM9XCg*V-dd$0gJ(D-!g5|J*XL_sBN-vVRvU2u=SSq;Ww(tW+Bd**kW+xI?XtZ
zLeyX#t`>t!99z83%y`Rd6WZex4PuU;Tar3j41QFi&T=V4P1IVZ^suMM)`zxG5Na?6*0J*c1kT
zBhwnU4#M~c1wMy@&`&O<{dw4v(Ed6_+J8(YY5?t%t$h)eb1^?Pgez^@pRGvy&7}T^
z{~Gaf(LdIHs9Ly$s=vKB2bT%
zO;ERc{x|FCO+`3`TVHTKS!P&3A%2%$U)L(a2q9nTjB3T#5xwSZo@;&jB9eRw@b%v2
zsrZ5=nHV5pxnE!}e+XuUl%jU<3K(RKU~revVCyKvch@WaQx#!|il%>0A27|RE^a{hXIDBgmnaLh3LgE~y5(7u(nTzf^y6|-D0&)F4;-MTq8
z;~x}{*;PBcda?Wc8GF6?ucFU)C7TTnGqz~PITWJK!Gy%YI#e*YmdaK!VHYR(@`+$2
zb*C)EAs_sq(Tc^U=SB7f#Q}oBsuOJ*#O4M(
zdZP15g2D73Pwjg^;&(a#wl)p~
zUktjH?vwE!CM)!Eu$sp0_RY7B4l0Tn*s&F0n>68#j|_^5FW?1LR_-mcJlX5^Kq-}R
zowXQ=iuC(CnWoIhmxQTkJMAXN*u)u%w3|t$d2+I!cI=FNDtb59nIX}fH`+@$uZGHY
z**abpA2)8;QWJjyE8mERtTK7h`yH*wWRr&UF=e8hKik%F*=+OPXs`RdDL=!s{gES9
z+vcc6lsGIAm_u4-7O^}1SzG$^6pz_eI=fm5rz|_eUP>%87mq;zW>rEN)uF*|7cBQCHk@iii?%-Ogb6!HkJ-U1G)Qq>fVi
z9POP#*Gs5pGlW_&eW&`(rqKQkZTc~atf$zOIJ*i8QTKPZ7hV>FM}&pD)L+1!G6Rp#
zxB{97-oZB>6;T&80v}C_fX-rY50$J>y|4k?^Wu)$Vlcf5EVfVyP+`pE@HDFbGP_f=
z7~DXm9}!SVA$|awk#q0`iV)Gn$-AcFlLgEtZ5OOM--Xts)nkfiEB!+@`846CtXOPM
zM@-PR4W4!qtu00li&P`F$i0C=R5h;ES!0XA^k1RWIVOZ+QIm^oS6B>=+e0JH$>zneUjOJ)p7$m*vtHy@AldPb+zP&pmfM^t8;QYs5Xc^S4r
zRNi6y4DVmp$(?bGX++kfx*BFwWBy-ux#dlqrVg1%vv
zyzBKC5&jzqy8hobe)swOVSe`&>ND
zGY(LBZ2xDTrLt6%$2NQBV=9vm((>5G&TODEUzEpoapo|TF*TxW+_IU|Qqi25^+@06
zQr**DXYNczbCyxr``FTI#;3w*%0E;dTSv`uDhsIWbMy~80^`jdCa)D`@K>w6#dMtP
z^>`@jUs!fMw!EHUm)ZXsX6tBVMw(NxDkGbqcd=OP;R%1Lz5cM-G{rkTI_C%!y~Egg@T)y=S8gi$qf(S;
zB#E^Bev>p6{gHN*rpMMo^E6DOi$!^C=`$Zu*&xbeYn&;eGB=!tDxOzs!}oe~Xez2$
zMdc~I@NkGTelOEE4Yf%(*QTLpB_qL|{J_Wd3gQ7XAq|BHkAkxIS-RMqWtOL5IOS2<
z=V?3=-_oaDV+f)nj4z6=Z2wEcJZKmis#}a}apBSKmJEmC3AzBBAB+)qyLblOL
zI3$Vkz$j`7vx+cd=kL8=`}Q8Q9W6*ky=zz&5f6)Nj@
zOF5*|VDe<9D#!4_9?0iVsC=PJ>@=@!s7B~4r5rP0-<72cT$230^w?fi`4dUC?2!Y;IY#rJgR3A9me|Z!^(;PO~7yy
zi=}_rvF>n2_<-M+f+^}047>N}BU4M|kdK#qK6H=V^%awKPud~oFaZXLQOTxGEM{u}
zIP4)7wjkN>Gn^{9>5wt+
z@ObCNV$lwlAY5x4uAOrrDm<-aVk)ZKAQmeTEcD?}8f9$Awo2{?%q9&pA0xvdvcZ{c
z*($MUg0`WB!{ZB*{vmxF#q>i9hq1oeV_{Mv7ESRM2MZ!-5;DB#P&k80gP19p)ZyfC
z9g^bqNe>KPv8d^2lR7-!qM8<+N8v^WeK=g3DKQw~yB||o#z)L18@DxR;&APpP-9z_
zib+#kY|z9ZTGJmM+j4Zq4vg#Ra1t59N`S{=X*_Atl~ZB9=%@WG`eM=JZ8>bED8^2N
zJr}l56j7SYk4elyf)JTP2t*_)BNH{?Y&$ht45nn#`~{@vqW^&+{1gZ=5y?s9DZ))H
ziT1a==aWs)BZ{0X5qMZWXMk=Ki?9gs7F8Dr#O5qPxo}hQo(NUb6{AQ^n8{>6>cvWx
zVpIaCLZPs1h@e414LKO*>jco(VvRF}iq@Gm!5t`4-dH#pRrkhVQ%$ZHEQ#8p($0w~M
z)xI^_m|i<79!r__2~+mP^uzBnzFo07m#5-SO!UY2(wGLC#O7jBXKoG)v0Ba{@kERz
z9D=AK5p^k=j2&k&n4V4R!z1jdcbJMrl0@9~n8!KB9OmMFZ}?L%p{NR^Vc{(X^Qeuz
z1-lp8I}GDarx1&FFulZltW^xFkOo$$u!Xb#1`Ajj^rnudw0&_1A<_RsZ
zLlCt>{JF|z;t)jE0{S|NX^6#g2%>y}Ig*KQyqd6(2B%^r%AtYWBM@qwry(WdWx?>Uf<~ad~#WpwqdhOD!AwxQJ#vL@y6pdrU1&A%+F~#
zPP?Ffm)SE5A4@O$+?F!5h@!*Ib1D=o4#|P%>9CS
zHODaJ9NOI6HV2pWRa7SarUU(NqIvF9ye^skC5h+;`vyWMlS>yRRC<2gY-OwZ&33rJ
zcK{xmW->f4{i`bs~<98|N$ocSpN
zIrD2KdQ5x&G4s@?cvZZN%Bb`6Y{Od&)={ZBr+h<$bJW0MFq=v>D4^VKpc*(t1zF)>
z#c`IkV%bkE1^F8Q^4Aj^i{eJ7S*GV)uV0|N*jzssMW3)1RAOJ$moeJcjGK$jD5TPR
z2pd=i*h6O7T(9e+byT(+RA=i`qyZ&Ch0>&b+n7Z*<#ttwPqMSo^F(J^e4J$kWv|^Q
zHX}ikk>ho2le&&XPfRLYcrGKF!dYI&=pqt{+4#)`US4FIN%JtK>P1x%#i9oOp#sub_#Zg_ulR&-~{9ZpZI@lazz;TP(`;FWQ(~Z=QrQ
z(6HW&&+$%ele>W_+M;CK<_;oC2?JFd%r7tqrGE#5cJqF-E*-*FdM}@>_Hzkk?@hhs
z;!rGgsN%u_UCzJ`m(sRS?_W6lAGUs99E|hCxGN?bc5YWQeLiYcEy{N=e{ndxW4@&_
zVbmiNe1}0s$Ji264USMi+
zy{@4mDr1A%n+`EMSCP;zucsXKCPo-1+!Vp!)E}X2u9Fm7IEM<){T9#@*t&CalnO{-
z1o&H6g}8KpZKc>=UVD=c
zLKq24hY)?fV+DDhxK4uz~
zC8CVdB{a%>4P_|29hPl+>vg?DKkudF3H2$il~^{1ehi6VwNOSc5FE;tR>n_scrR|s
zGL^%CFN~lj72o$uO9^r-kJiNUn*1}#jk%U|i0KQLX!*h&#U7hJ9b%S5|JDe%3a=-S
zbBJ`fKbTobpDDjt&LO%QwJQ@Y^rbp=m>}~+$_^%Mi)%qTbqFpA%b0MtX@(BLp@y~O
z+KyG94wK*mHO$&cvaKexi28@YqK@Ulo{j4pX6cZPgGx^Lg-IV$zYe*XKrIV}jK?y>
zlnxoNsAipzrC8xfI%KBk#_XyP{ob#eV2AXIidIyS9Hf~#B#D})?gAMX#1z8)=XHt?
zw#Fp_nW&HP6(L*}EC$2BLis74{YZDCO7fFGz82=*C0_fe*PiubCt+2i%-c)6Uafy=
zZPzboeOrGsc}wu{<59B((6V$leEEA}@a;AcOTBYC=T=j}k91LJwhl?`ue_C2o12$<
zSKpewhX&UpkVHUV6pMhyD;EK6Q7rAYTl_k8>S5XjG$f=Ger%9XOzemiY%e<2@vA??vKNU3*+Zy&5
zg4{P3^CM**OaHQ$jcnT$rgWLtDU`mCKG_Y(DVHa)oYpx{(>Dpso@FHbs6gd0bCkJW
zw0Lxr{5zd+qC+V&c*5V&%N8v?z_w7ds9%W)c&E3DLfcrTlKB(JT;U6m)Vdj7C-E!4e`@#5eaalzHzjo+4J@YS|>u&@WQU1UD{9kfiqfNTGX1RBIn`)|>
zvy=IU>i%12CN1~wy`lDR5PrO)JmCO@sAe$^qZD(*~+Y+Yvb;aT|4u~x+ZCxbC2oT{Zh!T?JtAu+P??nvbnYo
zB*VWm{-&n=DzDc-*Bkta
zI@cpZAiG}S&w{z0;ZNFqW=5>SY?)==UFBU6mp-sbrunAk*k6wyqfd7@3>I|GY`F!;5<
znzxs|w%WTP5X%22GSC+Wg+LY?uV+@IAXN)SGc)C=t`&`&vZzTOOB*(jvq={X-`jAS
z9MQ1HoTiAHAp81}Cv0dBVyBQk2d|uyaOVs0A*SQc??0!p6R*Ifl^OY^ch1mvVa0g8
z7spDhNtEx$aT&kW2lyzC`PbW!8V0$QUJhLcIRQ4z;XWMazWyOuQGwr7Vt
zh{JcC)jtEVDcXr6?;0En@zjFyjW~LoZS^-po^HZL_!+tlXIRxkP)!Xs4W)Aa`0;U*x^!Z=E>>ogB$N#S)ckLSK95
zcCMkWIX_dA1U(kl@Kh-LSol)S#;?72f42GcYp=&uYcQ>{R`f4JAXq7e*J3%Qa3P1`
zB)q`lLgV-_IX;9QF*khUwY$79jb%wfq8^y-dEzg?h!UaD7yN@l>o92fyX)#mY;^D-
z6WV42p|bmwEHnQbuS2kq>UQk1ls2*R{CGj4LA2V~6QGF6YKh;Z9Oa#TEU{RUh8F)rnAsU+)x8pJBPf+y7nE!xITpCP3>7sB+
z&cft%g2n0Bl$>DE7yFzOEVyA11-|ynmIO0k!wG4yCZ~SLYFUe5E>HZI!8L{EY;gW{
zhifs@+E>X5YkwL-M@(Hg(Kaa=$JYdXGc@A_4T2??3fGoJJmH%&Wuj#pqDQKD($cy8
z&kJv2SCU~0zV%M?FED$*^?C%#KQdA4yiS4okIX6SyuR_p(~+VDQ_uzH(TeHI9$V+7
z_*>S@U=|n<^)t+&60dX1%$Zd5K}Bbp9_zg}fy$ZYob}%5;8x5nY>bQP0)D0sv$NJ^
zdhW+<3G$rp$ZsBD`8$~>jzZJ09`DWX*lW6O@Vdng!zP2~50C}j#SGivb&ow?9~a%P
z>fPoIsQs%=#s=^78&(grQz2&@Cc5o%J8_++=Y==e^Zxz-{^3{cFe^!*foB=I>-m{#
zA8opPhcf*bWiI^A>vqFjeM5kABHfR^rg2{6`P#?rgva#J>SWSSDD8D?t8
z&ckBDxf55H!rpDVY=-|+&0U+l(}Ob+31nYZ_E*nt6lJ!8ufZrgH&xuNvFwwuu<@L?
zP5FEF@0k*~@n3Hm2+x{+WyrGjAMP%(2H{|VnwYzw_D?f!lzI0C{<^|!FY~&c`OypZ
zfhpHZTAhj+?lT0q8`h;fY0mMg^Iz3xrvLZeCDA#HX{DpOaIp!0@AZnVK3>_d*nIZA
zH@ai~5?Dr;!v%8iv+Mu!5Qb#g5_A0zURwW}rJ#A$R4>9^n9r6`gUf(a5qjgwN(9Vo
z*~{X#c!&M|>}4@Odh7kT5&qLwT>a&l$Zg&o&D~C;m3eHN*ZqcV7?3QW`?+zqZ#|;h
ziIZ@#%@O9d7a!vGC(pA#w|E)$?L7DCwZh}39BxBy4^ecU4Su(Qn0UV35q$Tp#o>2Z
z{%qCv$3BRS_})_%|8uUjb32L0AGf!tF2o?A-TT-*5WDG);$)=9s4y>Ac(E<&9<%Rg
z{kqpouJBIo7^@>~0y5*!xiHq^bW>X44e2-lWnnbA?S$9I?m)VlKHKq)GOuEJX8`WU
zRSvPMUa>sf*Pj2=#5}&;>(=Qf6pev*(m@-5{gE42{@9_#fb2rR*)-1&KwgBxkltGtK%Ryhc96$!fPG}r
zt8aG1$%k@s{SOsga57I$_D7zYa0kwo^Q>w3>33f2g|lzvEa9q8H=c~Mt#WqLhUUFa
z#o1mt+Y>AqaT?D4;n_(^4-Q}b^#GiX{f?*2-kN%Dm$Ps>ZP~(|-cf%j{5{kdD8W<_
zaP^%j!b4b@C$$qbTPiprdGu8`{};m$RB91<8N#kCX4v(
z2j}~OGblz>wY&*S+ai&xU%s-vFZe}BPGV-4Rr`V)DMw~Le{Ba0l^t@D^m@!+cld&T
z@}z0_>#VmXqBNtn(C&+8_K)d>&lB^c>A1UJz3f4hX{MY#G-vVBN({iy3s;72)d^A=N!tgUrF+KU06{{`8txo2;%GH06o&~w~TPAj$x8s$+N
zt;{qxuCs%Ka>4g@xuE>US0dYs*&ds0)YIAN$bb_)<3^?sm?pf*&Z~>xRyVlAa
zmN%981H9O!T!b3%6&A`lr~qXa{H*oA8`{+GSz~1`4sUI
z<^*sux`2y3_kqYFHhlh6f;Ru1{9(Z?q%(H(tBnN9yZx}dkp!EgP6J}S>aOHgJv
zoQ@1mQTbGK!Ko@YM+GReOCClA`l@_8D$q~mI&{Hle7BNDU!n%50oiXMbU}ZWUq%<4
zuCj-WDYHNCK$8wo`2lpnK$WMX3n;U1|3C%KQ2AuopQ-XBw9r{9Ul&3U@r3~jFHPwfx#+2iWWLgWk0&$e3c(T7f=o{
zcs@i1{2eyRJ5YlQRXz(fpv<5OpbIWi`BAjQ5S5cr0m=-pLUcir%0p3spvtY#1sALQ
z30jmggK)@q_-ODYdh#Z^;8K;pK@BJ~INKw^Wh#G#8eFdOCuma24AeAq!4)bGMg^`^
zc?epRasrOksK8K_FGUwzB{KTI32Jb)64^*VnUOsYH5jIHOSI56Drch$C^O>6Via7f
z@_ux|bt+dQ1Imp4ndpM+RUU>4+@SJ4RA4yr7o%v@U^rwBk;~!eMwLH67u=-s6Q}`Y
z4xLhT!Obd{qXM_6d^5U$GDp!O^zp4KPevEqrt(p=DDTkF#{a>xsK34WO#`@}RdC3A
z-XE$N^L|Xrsz(C}Xxaul^RfAd;W-({@$0w+F<`HqHS2Kx>OyPN3Hs-jS$R3+J-FY?
zVZ0DWkKe4^%a8B$M_TIt#+QfG2EQN!>YKrjZ=clEdR#zj_+K!Lid91A(-ptbJ{c{Jv9W_BUu
z)wx!_2eOA2W#OKKJPh4R`3=aOvV9{LOH8IV~xc|}XN6!K~{Dun!mcMEXcZQYLDvj6lw%-2S7+@zLZ0%w
zmAgXj^skjqhkU3I8xq#wT%7FCE=YphM-$u(`Je{Zy^y~_;h4dbkf*53K7JhbtiXRD
zpLfXmd=v7&*;ZzMKdr5s3i(%zAm*DH!by|{3rE3h9j^-^_r%-Ebh-@k4}aT?S3y1<
zt;M~<*N|V+f^yW{p+m70^3fcdK?TG^JFLv*z3Lw;b9H^ZGrj
ztq}uixz>QQ+kbl?Fv>x&2
z%hP5r1M#i~(T|V^Y7lLMd>8@9g6_aE11&+BfqA)(j#|i@t85D$fP5}S2P?yXU5}ZR
z_6*oQ+BKZShD(7k{usT}HImyw-i7{V0!|tmbtv|Pe2?~bZ^)18kUJG}lm;LtmX%td
z{*b>vVEr)KS7;&d?xAmucI6q6GebJeI10|!W?zq}ej0tx8f=8zQk#(@Vw5)B50Fc=
zYdAW7)(kjtty25nA-}0Zj-zISPDBSFM``PZ{>8}w9Zn}9SXQ7}m@$V_KMWPhyb(T5
zYs?w&7xXdpZ6Tkpt;JzBR%_oG@+Vr*Qz7SQw4Vlf9{e*_IQ(Lp;wPecat2QB*4l6?
zU8Zw5hvH(L>9}fDYk{tW+)4|2JLGG161x|2p4vPD`7(_sE<8`^^!zmBu4o}v;JKrr
zqep+#;CdOv7h5gCaeNon0vd7N?x4X%xo&~&*D&O6=zGQr$MFfDbjar;o@D-?fs>2TM9ge9l$19jAJz4%F6j{AU~I{a2Z
zzE$Pbkh2h5^v^k?>tS2aZy<-_k6U*C)F9$wbFl_y3FL9w9o!#0xYs7$0=Y
zDcy}oX0vlsc~~>!R_zDv8g9daIxHW9{EJR*&qID3t-{%p8=RDxI{&|nla9LBa26Y&
z5t{<}H;w#pkblImVS%RN_*NtQL&zZvR~Bd?jzd~Y&UC3d6jwvO8--)qwK#4?U{c=5
z^}mB{3g0HkgLJpU<#-OJTN+V*Uu(&&*odEP?Kv~fK+TwOC64*XnD#qxRA^f6h$d^{
zIAe}e8_FXQ;4=UJj+23fHgO%~t1wjPh*Rk-Ej6doXLW+2{1+a?(}vUPLw{R)%H=y6
zM%Dl#gm{xY2l0=LSyC
z%`ooS65RFPh^3bHoT@i{Y4v>|Z&jb%nEWgA0otE)_!2D<=kRtqoVa7jM>EnX=kuky
zq;l6gLnEJZ_V3n)^L(lX8s&W&`JD6bM)$Emob&TFZq9`KW(ZBk3^@PasWa5skW;m0
zTnp~nU~9wOQ69R3(M!1q8PF*=u0LrDQBJ}TrJn1>O_+$Be7437x9VwHh|teCX@wIS
z$uKvwAx;vy8G-9h;ho%JZ6ic!N1xHz4?vr)bP@xe8Pd%y_HbhP23
zHAc7jl%M(7>N|4&{|Gx78gT{tK*v2Vuin8??r~Ay;V4`a-@JEkr+*
zGj!_Zig%_iLR|5#(BV1&@*}8B497oLzkhUg<3{u)&0sL(v*3tMxhk>2lT;qn#_{
zO&Gdt_Ddk&u6@kaa;3H?SId)ha=RY#0G%1Ra)uTol9>TdhGOzz)7=C4OdZqrLhh`N
zxTHqtM8qYvLW79%Rp?jx;m&xVj(f^4Yl}Vy`5|P?f>MsRF%*)MroUL?1rQr`A$gq%
zv{kqq-=>=j$`R_6%kj%PEGhr2wWPdO7p*rT_r-j`j45XzV-}Pbpfl%$8c!Bz9E=CC
z3X>9zD930^P<~5?CD-+4a7vq0$g?y)CPV&AyMyw#8be&~>oEhe5Yr(K(h(&4|L*8-
zI?4huUkUC59@1LQf!s$&0p)R;G35pw1(e%q*YMgQQ-h9jl#ZIkkgr*7^WwhYX6QrA
zm=_rH(P!*?%9}LkxS_aGYrF#TPMz~tLHls6!V*ebgrch)JE@++8z*?+vFISDaAnLC`zcQ(r9
zDi47EHkEm)^(b14KDpW1f!1>J$dL9Qi6)3x5<_sDf~^>3?t+XKg7Os1|J3tV+!oZ5
zGIvM&(D!U=%170nJEdoJQldNzF+_XHC+n6!#4XgxdMo8R$lEkGY>Nfl_$k%A!KL@$}A)6VuTpz-oh2xg%I*l7%H-;DJ{Lhnd+JD@Nwclo&
z^i9a6+H~BOU7+Lp9mpH?0yG8k9v#zTA@9%(cmcXyM+&!V`*h1ixx0>tY1n`G?$pUA
zlf+V+F|Sw`YacI!JWg9;CFJ2ci*bLqaI3Ya{Eenv4f$NX6z46L&opE1``*GV$++2w
z<3kpL&Bl}8^df|Nz-c+ws0?z7WOEjwm0}3Q!)V3(}8}?@=4>8Gl~(
zbZp}M?x7!l$L+?GE}VQ{)onX2!MD`NY4Np>KJ~>soc<%HndPHzJ?mwhj{Sqi;mUJg
z`znaj4|v-2tdsv9(F$jCFcW9%cosP`tqxCbHUh#XGt4|yf!CQsBS`GV_3
zO!*4pb`g^%LHwO!;C4tyMDifL`zLgtd$k-^%Q~KscuriP^gVOKrHR`^TW+@xxn9M$
z#xw(cw8AGNq}wGvY2AyiwGX&{dc@-G(bn#^2>U>+Ct{F~g+k|8eE&TA5bUI-_W5wq
zWA?ezt-!XhYjuTv@Rt9+j{_p$n`IVWme@Nq`!>rzp#ERNI|?mefA@v9e6#W1EAwfI
z@^S=^IK%1>Eb@up;vPPEPIhri8x^x*$6ex8C}$LK>I?QE-!i<_%YitknYH^9^#vdD
zEkXsmgSaurHo%;n_9cm&9`-?7AwGZ84*V<7^HIVUJ?}NtK9+h9dWPdGH`#}ANAXo-
z%;>IOHly3l^T`9d14ze;?cZ~H?KJh6SK?{)SG
z(**eM0D5e_HP4@K{Y<*d=F{v^+d$@KpFF2K74F)=zUCU6?iHN#c~2qgMQo3AX8Rrd
zu>47Aa6VdjIo=*6H1seN_Qsn6?CHOd0qao?
z>;xP=-5=-%jP7Y4EIx&E7T^GSJ|4IN$bxJDvcNkPyPzk#`0!k0un71#Jn$44+=7f*
z;>UmtiA5LKIrd4kG`l>256u!Yp(h^!hs0g`&GlC%whOHqWJ7TqLW6eA@ks!NSOt29
zLm?d<(i`|7h5`$kcgXT6%&
za8gG;OdSn@HU9NlQ_BLac+ytj+z!^!LkKDI9nqx>nJhTs`5SN87X5gp?TJzpgrOEj
z!|-v=@OYcv-&(Kpf3zVTZeq*vPoif~5Z^=nrJ+XnUT#>IHagI)PB
z95BH+ybQvcP8@AxIbAET@lL6c?={3E3(|g&{g_>@w#<>+Y{rv!S^mcXrgmszx6q3y
zAX{V?Uauiuh=MZ|;_(s*^Gip5^f%;k8@o?_X7l|EF-rZS_7*dMOt%I-z|eUb4aM}Y
zXy`4~hTMhs?AXI2;-o=CzPHe_O!&?S+lAA07Q6dSdtQK9g$ZuB(pGQ*T96HJF1nV3
zuo4AfkMzeN=H&1VD$D}3MJN$FLQnsX|Kth>`r_1xiuot0rK(3rO-VD-SG!~T@QUd^gM3DxjP1-!fVh5Yka2W
zn#BE~s=iGEJiJa3a#U>!1Rlb9*2O3f4xayShqoDNaU5@xH4JHSI9(R=_-{Wp-%!BA
zO&E93$lDwUWW9?5;P5Im^WRXYNlol=05(&=9d~3Mg6?DJjyvcps{!d2BOQ+ojfdBx
z37+iopQ*Sm@%2!~?uY=7$NEGDa{5IE;?K87xPN4Thdcj&;bWa613ZrB(Z&HDPWRtD
z^_p)eP;?~X*g>L04hr*MPBS+k3YtU)F2M2XMucueq#TuzkpUjGzps@&(tyP{=&O2j
zL?9wMGB5~-4>o7vAkw}GO4!6_Zn`0Hf7BLysDVdJ9%gED+L5V)I~mt*Q1U*T-qdXJPu;ex}#GkUwawZ
zTmrf31h5>g!nE|`pXR0;6Mq<*8jK92;UK2t$WWQ*nK=AN(-cPpj@S?Vfg_j`?}?QT
zlJ|1B1jpbF?8-h3}4`q>ApZ&M_*tD
zrrPvgzCh{<;otf%YiUfol6ULVpPQHqd_!T?vj3kk`Kl83%24LbnCSs89TXbR1W%z-uHpw*TV~
z{Dfm80%8-6YNXqP14+2AU+oLNUtnTdE;F|#e$u&f{$G%z9s$ArAmW|*QY!Z>dvmpS
ZO`t{N$B4|rmlE3sit5eRFD2gme*h(3-l+fp


From 91b4729962ddec96d1ee60d742326da828dae94a Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Sun, 30 Apr 2023 12:14:02 -0700
Subject: [PATCH 018/725] fix compilation error on 32-bit LLVM-enabled compiler
 builds

Fixes a regression introduced in
9295355985202c267b4326b5a6e2ad5158b48e5d that caused 32-bit builds with
`-Denable-llvm` to fail to build from source due to that common
u64/usize casting issue.
---
 src/value.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/value.zig b/src/value.zig
index 16ccc0c642..503797abba 100644
--- a/src/value.zig
+++ b/src/value.zig
@@ -5390,7 +5390,7 @@ pub const Value = extern union {
     /// have the same value, return that byte value, otherwise null.
     pub fn hasRepeatedByteRepr(val: Value, ty: Type, mod: *Module, value_buffer: *Payload.U64) !?Value {
         const target = mod.getTarget();
-        const abi_size = ty.abiSize(target);
+        const abi_size = std.math.cast(usize, ty.abiSize(target)) orelse return null;
         assert(abi_size >= 1);
         const byte_buffer = try mod.gpa.alloc(u8, abi_size);
         defer mod.gpa.free(byte_buffer);

From 339cae7fd0af5730673fcba7d97568af24b5aa4f Mon Sep 17 00:00:00 2001
From: Eric Joldasov 
Date: Tue, 10 Jan 2023 13:20:40 +0600
Subject: [PATCH 019/725] cmake: install zig to 'build_dir/stage3' during
 building

This commit installs Zig to "build_dir/stage3" during building
so that distros' can easily find binary and use it for testing/etc.
This commit also splits "add_custom_target(stage3 ALL" and command that it invokes,
so that it won't retry it during installation,
as target will be considered not out-of-date.

See also https://www.github.com/ziglang/zig/issues/14240#issuecomment-1374642063
---
 CMakeLists.txt | 8 ++++++--
 build.zig      | 3 ---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7c63bff6e0..23222d73c7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -860,8 +860,12 @@ set(ZIG_BUILD_ARGS
 )
 
 add_custom_target(stage3 ALL
-  COMMAND zig2 build compile ${ZIG_BUILD_ARGS}
-  DEPENDS zig2
+  DEPENDS "${CMAKE_BINARY_DIR}/stage3/bin/zig"
+)
+
+add_custom_command(
+  OUTPUT "${CMAKE_BINARY_DIR}/stage3/bin/zig"
+  COMMAND zig2 build --prefix "${CMAKE_BINARY_DIR}/stage3" ${ZIG_BUILD_ARGS}
   COMMENT STATUS "Building stage3"
   WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
 )
diff --git a/build.zig b/build.zig
index 6cb3e35488..78345ac9e4 100644
--- a/build.zig
+++ b/build.zig
@@ -179,9 +179,6 @@ pub fn build(b: *std.Build) !void {
     exe.entitlements = entitlements;
     b.installArtifact(exe);
 
-    const compile_step = b.step("compile", "Build the self-hosted compiler");
-    compile_step.dependOn(&exe.step);
-
     test_step.dependOn(&exe.step);
 
     exe.single_threaded = single_threaded;

From 1b432072b5ae6a70a00464b48b7ebf8610293fc3 Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Fri, 10 Mar 2023 13:03:40 +1100
Subject: [PATCH 020/725] std.Build: detect and disallow top-level step name
 clashes

---
 lib/std/Build.zig | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/std/Build.zig b/lib/std/Build.zig
index f23b3ba5aa..a2c8a22e32 100644
--- a/lib/std/Build.zig
+++ b/lib/std/Build.zig
@@ -926,7 +926,12 @@ pub fn step(self: *Build, name: []const u8, description: []const u8) *Step {
         }),
         .description = self.dupe(description),
     };
-    self.top_level_steps.put(self.allocator, step_info.step.name, step_info) catch @panic("OOM");
+    const gop = self.top_level_steps.getOrPut(self.allocator, name) catch @panic("OOM");
+    if (gop.found_existing) std.debug.panic("A top-level step with name \"{s}\" already exists", .{name});
+
+    gop.key_ptr.* = step_info.step.name;
+    gop.value_ptr.* = step_info;
+
     return &step_info.step;
 }
 

From ec6ffaa1e47388a59035277dafbe3d9999a80fca Mon Sep 17 00:00:00 2001
From: kcbanner 
Date: Thu, 27 Apr 2023 21:42:32 -0400
Subject: [PATCH 021/725] sema: improve the error message when coercing to
 []anyopaque

---
 src/Sema.zig                                          |  3 ++-
 .../double_pointer_to_anyopaque_pointer.zig           |  7 +++++++
 .../compile_errors/pointer_to_anyopaque_slice.zig     | 11 +++++++++++
 3 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 test/cases/compile_errors/pointer_to_anyopaque_slice.zig

diff --git a/src/Sema.zig b/src/Sema.zig
index 8593edf6cc..71a1215dcd 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -25335,7 +25335,7 @@ fn coerceExtra(
 
             // cast from *T and [*]T to *anyopaque
             // but don't do it if the source type is a double pointer
-            if (dest_info.pointee_type.tag() == .anyopaque and inst_ty.zigTypeTag() == .Pointer) {
+            if (dest_info.pointee_type.tag() == .anyopaque and inst_ty.zigTypeTag() == .Pointer) to_anyopaque: {
                 if (!sema.checkPtrAttributes(dest_ty, inst_ty, &in_memory_result)) break :pointer;
                 const elem_ty = inst_ty.elemType2();
                 if (elem_ty.zigTypeTag() == .Pointer or elem_ty.isPtrLikeOptional()) {
@@ -25345,6 +25345,7 @@ fn coerceExtra(
                     } };
                     break :pointer;
                 }
+                if (dest_ty.isSlice()) break :to_anyopaque;
                 if (inst_ty.isSlice()) {
                     in_memory_result = .{ .slice_to_anyopaque = .{
                         .actual = inst_ty,
diff --git a/test/cases/compile_errors/double_pointer_to_anyopaque_pointer.zig b/test/cases/compile_errors/double_pointer_to_anyopaque_pointer.zig
index c7e54738d1..c695b7393d 100644
--- a/test/cases/compile_errors/double_pointer_to_anyopaque_pointer.zig
+++ b/test/cases/compile_errors/double_pointer_to_anyopaque_pointer.zig
@@ -15,6 +15,11 @@ pub export fn entry3() void {
     const ptr: *const anyopaque = x;
     _ = ptr;
 }
+export fn entry4() void {
+    var a: []*u32 = undefined;
+    var b: []anyopaque = undefined;
+    b = a;
+}
 
 // error
 // backend=stage2
@@ -27,3 +32,5 @@ pub export fn entry3() void {
 // :11:12: note: parameter type declared here
 // :15:35: error: expected type '*const anyopaque', found '*?*usize'
 // :15:35: note: cannot implicitly cast double pointer '*?*usize' to anyopaque pointer '*const anyopaque'
+// :21:9: error: expected type '[]anyopaque', found '[]*u32'
+// :21:9: note: cannot implicitly cast double pointer '[]*u32' to anyopaque pointer '[]anyopaque'
diff --git a/test/cases/compile_errors/pointer_to_anyopaque_slice.zig b/test/cases/compile_errors/pointer_to_anyopaque_slice.zig
new file mode 100644
index 0000000000..2c7334bedb
--- /dev/null
+++ b/test/cases/compile_errors/pointer_to_anyopaque_slice.zig
@@ -0,0 +1,11 @@
+export fn x() void {
+    var a: *u32 = undefined;
+    var b: []anyopaque = undefined;
+    b = a;
+}
+
+// error
+// backend=stage2
+// target=native
+//
+// :4:9: error: expected type '[]anyopaque', found '*u32'

From 94e30a756edc4c2182168dabd97d481b8aec0ff2 Mon Sep 17 00:00:00 2001
From: Linus Groh 
Date: Sun, 30 Apr 2023 18:02:08 +0100
Subject: [PATCH 022/725] std: fix a bunch of typos

The majority of these are in comments, some in doc comments which might
affect the generated documentation, and a few in parameter names -
nothing that should be breaking, however.
---
 lib/std/Build/Cache/DepTokenizer.zig       |  2 +-
 lib/std/Build/CheckObjectStep.zig          |  2 +-
 lib/std/Build/RunStep.zig                  |  2 +-
 lib/std/Build/Step.zig                     |  2 +-
 lib/std/Thread.zig                         |  4 ++--
 lib/std/Thread/Condition.zig               |  2 +-
 lib/std/Thread/Futex.zig                   |  2 +-
 lib/std/atomic/Atomic.zig                  |  2 +-
 lib/std/builtin.zig                        |  2 +-
 lib/std/c/darwin.zig                       |  8 ++++----
 lib/std/c/solaris.zig                      |  2 +-
 lib/std/child_process.zig                  |  2 +-
 lib/std/compress/deflate/huffman_code.zig  |  2 +-
 lib/std/crypto/25519/edwards25519.zig      |  4 ++--
 lib/std/crypto/25519/field.zig             |  2 +-
 lib/std/crypto/Certificate.zig             |  4 ++--
 lib/std/crypto/errors.zig                  |  2 +-
 lib/std/crypto/kyber_d00.zig               |  4 ++--
 lib/std/event/lock.zig                     |  4 ++--
 lib/std/event/loop.zig                     |  6 +++---
 lib/std/fmt/parse_float/convert_fast.zig   |  2 +-
 lib/std/fs/file.zig                        |  4 ++--
 lib/std/fs/path.zig                        |  4 ++--
 lib/std/hash_map.zig                       |  2 +-
 lib/std/heap.zig                           |  2 +-
 lib/std/heap/arena_allocator.zig           |  2 +-
 lib/std/heap/general_purpose_allocator.zig |  4 ++--
 lib/std/heap/memory_pool.zig               |  2 +-
 lib/std/http.zig                           |  2 +-
 lib/std/json.zig                           |  2 +-
 lib/std/linked_list.zig                    |  2 +-
 lib/std/macho.zig                          | 12 ++++++------
 lib/std/math/big/int.zig                   | 20 ++++++++++----------
 lib/std/math/complex/tan.zig               |  2 +-
 lib/std/math/log10.zig                     |  2 +-
 lib/std/math/sqrt.zig                      |  2 +-
 lib/std/mem.zig                            | 10 +++++-----
 lib/std/os.zig                             |  6 +++---
 lib/std/os/linux.zig                       |  6 +++---
 lib/std/os/linux/bpf/btf.zig               |  6 +++---
 lib/std/os/linux/seccomp.zig               |  2 +-
 lib/std/os/windows.zig                     |  2 +-
 lib/std/packed_int_array.zig               |  4 ++--
 lib/std/pdb.zig                            |  2 +-
 lib/std/simd.zig                           |  2 +-
 lib/std/target/csky.zig                    | 14 +++++++-------
 lib/std/time.zig                           |  2 +-
 lib/std/valgrind/memcheck.zig              |  4 ++--
 lib/std/zig/parser_test.zig                |  6 +++---
 lib/std/zig/render.zig                     |  2 +-
 50 files changed, 97 insertions(+), 97 deletions(-)

diff --git a/lib/std/Build/Cache/DepTokenizer.zig b/lib/std/Build/Cache/DepTokenizer.zig
index 8f9f2f81cd..c640fa4adc 100644
--- a/lib/std/Build/Cache/DepTokenizer.zig
+++ b/lib/std/Build/Cache/DepTokenizer.zig
@@ -829,7 +829,7 @@ test "error illegal char at position - bad target escape" {
     );
 }
 
-test "error illegal char at position - execting dollar_sign" {
+test "error illegal char at position - expecting dollar_sign" {
     try depTokenizer("$\t",
         \\ERROR: illegal char \x09 at position 1: expecting '$'
     );
diff --git a/lib/std/Build/CheckObjectStep.zig b/lib/std/Build/CheckObjectStep.zig
index fbeb87baee..e79ce9d3df 100644
--- a/lib/std/Build/CheckObjectStep.zig
+++ b/lib/std/Build/CheckObjectStep.zig
@@ -68,7 +68,7 @@ const SearchPhrase = struct {
     }
 };
 
-/// There two types of actions currently suported:
+/// There two types of actions currently supported:
 /// * `.match` - is the main building block of standard matchers with optional eat-all token `{*}`
 /// and extractors by name such as `{n_value}`. Please note this action is very simplistic in nature
 /// i.e., it won't really handle edge cases/nontrivial examples. But given that we do want to use
diff --git a/lib/std/Build/RunStep.zig b/lib/std/Build/RunStep.zig
index ce2dd0234a..5d530c7a25 100644
--- a/lib/std/Build/RunStep.zig
+++ b/lib/std/Build/RunStep.zig
@@ -90,7 +90,7 @@ pub const StdIo = union(enum) {
     /// certain conditions, and the step will succeed or fail based on these
     /// conditions.
     /// Note that an explicit check for exit code 0 needs to be added to this
-    /// list if such a check is desireable.
+    /// list if such a check is desirable.
     check: std.ArrayList(Check),
     /// This RunStep is running a zig unit test binary and will communicate
     /// extra metadata over the IPC protocol.
diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig
index 0c05a64b1c..bdb500d99c 100644
--- a/lib/std/Build/Step.zig
+++ b/lib/std/Build/Step.zig
@@ -37,7 +37,7 @@ result_duration_ns: ?u64,
 result_peak_rss: usize,
 test_results: TestResults,
 
-/// The return addresss associated with creation of this step that can be useful
+/// The return address associated with creation of this step that can be useful
 /// to print along with debugging messages.
 debug_stack_trace: [n_debug_stack_frames]usize,
 
diff --git a/lib/std/Thread.zig b/lib/std/Thread.zig
index 119328b2a8..d213b167dd 100644
--- a/lib/std/Thread.zig
+++ b/lib/std/Thread.zig
@@ -469,8 +469,8 @@ const UnsupportedImpl = struct {
         return unsupported(self);
     }
 
-    fn unsupported(unusued: anytype) noreturn {
-        _ = unusued;
+    fn unsupported(unused: anytype) noreturn {
+        _ = unused;
         @compileError("Unsupported operating system " ++ @tagName(target.os.tag));
     }
 };
diff --git a/lib/std/Thread/Condition.zig b/lib/std/Thread/Condition.zig
index bc579878da..2f8ab02d5e 100644
--- a/lib/std/Thread/Condition.zig
+++ b/lib/std/Thread/Condition.zig
@@ -261,7 +261,7 @@ const FutexImpl = struct {
             const signals = (state & signal_mask) / one_signal;
 
             // Reserves which waiters to wake up by incrementing the signals count.
-            // Therefor, the signals count is always less than or equal to the waiters count.
+            // Therefore, the signals count is always less than or equal to the waiters count.
             // We don't need to Futex.wake if there's nothing to wake up or if other wake() threads have reserved to wake up the current waiters.
             const wakeable = waiters - signals;
             if (wakeable == 0) {
diff --git a/lib/std/Thread/Futex.zig b/lib/std/Thread/Futex.zig
index 7efdc69d3b..7fbe49fea2 100644
--- a/lib/std/Thread/Futex.zig
+++ b/lib/std/Thread/Futex.zig
@@ -772,7 +772,7 @@ const PosixImpl = struct {
 
         waiter.event.wait(timeout) catch {
             // If we fail to cancel after a timeout, it means a wake() thread dequeued us and will wake us up.
-            // We must wait until the event is set as that's a signal that the wake() thread wont access the waiter memory anymore.
+            // We must wait until the event is set as that's a signal that the wake() thread won't access the waiter memory anymore.
             // If we return early without waiting, the waiter on the stack would be invalidated and the wake() thread risks a UAF.
             defer if (!cancelled) waiter.event.wait(null) catch unreachable;
 
diff --git a/lib/std/atomic/Atomic.zig b/lib/std/atomic/Atomic.zig
index 51e61ca628..5bb25e9bde 100644
--- a/lib/std/atomic/Atomic.zig
+++ b/lib/std/atomic/Atomic.zig
@@ -31,7 +31,7 @@ pub fn Atomic(comptime T: type) type {
         ///         // Release ensures code before unref() happens-before the count is decremented as dropFn could be called by then.
         ///         if (self.count.fetchSub(1, .Release)) {
         ///             // Acquire ensures count decrement and code before previous unrefs()s happens-before we call dropFn below.
-        ///             // NOTE: another alterative is to use .AcqRel on the fetchSub count decrement but it's extra barrier in possibly hot path.
+        ///             // NOTE: another alternative is to use .AcqRel on the fetchSub count decrement but it's extra barrier in possibly hot path.
         ///             self.count.fence(.Acquire);
         ///             (self.dropFn)(self);
         ///         }
diff --git a/lib/std/builtin.zig b/lib/std/builtin.zig
index 4a5e8a28d6..ea513ae87e 100644
--- a/lib/std/builtin.zig
+++ b/lib/std/builtin.zig
@@ -749,7 +749,7 @@ pub const PrefetchOptions = struct {
     /// 3 means high temporal locality. That is, the data should be kept in
     /// the cache as it is likely to be accessed again soon.
     locality: u2 = 3,
-    /// The cache that the prefetch should be preformed on.
+    /// The cache that the prefetch should be performed on.
     cache: Cache = .data,
 
     pub const Rw = enum(u1) {
diff --git a/lib/std/c/darwin.zig b/lib/std/c/darwin.zig
index cd1fbb9d9b..6e4ef59d38 100644
--- a/lib/std/c/darwin.zig
+++ b/lib/std/c/darwin.zig
@@ -178,13 +178,13 @@ pub extern "c" fn __getdirentries64(fd: c_int, buf_ptr: [*]u8, buf_len: usize, b
 
 const private = struct {
     extern "c" fn fstat(fd: fd_t, buf: *Stat) c_int;
-    /// On x86_64 Darwin, fstat has to be manully linked with $INODE64 suffix to
+    /// On x86_64 Darwin, fstat has to be manually linked with $INODE64 suffix to
     /// force 64bit version.
     /// Note that this is fixed on aarch64 and no longer necessary.
     extern "c" fn @"fstat$INODE64"(fd: fd_t, buf: *Stat) c_int;
 
     extern "c" fn fstatat(dirfd: fd_t, path: [*:0]const u8, stat_buf: *Stat, flags: u32) c_int;
-    /// On x86_64 Darwin, fstatat has to be manully linked with $INODE64 suffix to
+    /// On x86_64 Darwin, fstatat has to be manually linked with $INODE64 suffix to
     /// force 64bit version.
     /// Note that this is fixed on aarch64 and no longer necessary.
     extern "c" fn @"fstatat$INODE64"(dirfd: fd_t, path_name: [*:0]const u8, buf: *Stat, flags: u32) c_int;
@@ -2643,7 +2643,7 @@ pub const F = struct {
     pub const ADDSIGS = 59;
     /// add signature from same file (used by dyld for shared libs)
     pub const ADDFILESIGS = 61;
-    /// used in conjunction with F.NOCACHE to indicate that DIRECT, synchonous writes
+    /// used in conjunction with F.NOCACHE to indicate that DIRECT, synchronous writes
     /// should not be used (i.e. its ok to temporaily create cached pages)
     pub const NODIRECT = 62;
     ///Get the protection class of a file from the EA, returns int
@@ -3866,4 +3866,4 @@ pub const MIN = struct {
     pub const ANONYMOUS = 0x80;
 };
 
-pub extern "c" fn mincore(addr: *align(std.mem.page_size) const anyopaque, lengh: usize, vec: [*]u8) c_int;
+pub extern "c" fn mincore(addr: *align(std.mem.page_size) const anyopaque, length: usize, vec: [*]u8) c_int;
diff --git a/lib/std/c/solaris.zig b/lib/std/c/solaris.zig
index c2c38f46ce..b2225c6d00 100644
--- a/lib/std/c/solaris.zig
+++ b/lib/std/c/solaris.zig
@@ -1822,7 +1822,7 @@ pub const file_obj = extern struct {
     name: [*:0]u8,
 };
 
-// struct ifreq is marked obsolete, with struct lifreq prefered for interface requests.
+// struct ifreq is marked obsolete, with struct lifreq preferred for interface requests.
 // Here we alias lifreq to ifreq to avoid chainging existing code in os and x.os.IPv6.
 pub const SIOCGLIFINDEX = IOWR('i', 133, lifreq);
 pub const SIOCGIFINDEX = SIOCGLIFINDEX;
diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig
index f5ca72ed39..daaa1689bc 100644
--- a/lib/std/child_process.zig
+++ b/lib/std/child_process.zig
@@ -1174,7 +1174,7 @@ fn windowsCreateProcess(app_name: [*:0]u16, cmd_line: [*:0]u16, envp_ptr: ?[*]u1
     );
 }
 
-/// Case-insenstive UTF-16 lookup
+/// Case-insensitive UTF-16 lookup
 fn windowsCreateProcessSupportsExtension(ext: []const u16) bool {
     if (ext.len != 4) return false;
     const State = enum {
diff --git a/lib/std/compress/deflate/huffman_code.zig b/lib/std/compress/deflate/huffman_code.zig
index cf1dd71c75..4827feb245 100644
--- a/lib/std/compress/deflate/huffman_code.zig
+++ b/lib/std/compress/deflate/huffman_code.zig
@@ -421,7 +421,7 @@ test "generate a Huffman code from an array of frequencies" {
     try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
 }
 
-test "generate a Huffman code for the fixed litteral table specific to Deflate" {
+test "generate a Huffman code for the fixed literal table specific to Deflate" {
     var enc = try generateFixedLiteralEncoding(testing.allocator);
     defer enc.deinit();
 }
diff --git a/lib/std/crypto/25519/edwards25519.zig b/lib/std/crypto/25519/edwards25519.zig
index a8ca8e2fb6..b9ce14bc92 100644
--- a/lib/std/crypto/25519/edwards25519.zig
+++ b/lib/std/crypto/25519/edwards25519.zig
@@ -137,7 +137,7 @@ pub const Edwards25519 = struct {
         };
     }
 
-    /// Substract two Edwards25519 points.
+    /// Subtract two Edwards25519 points.
     pub fn sub(p: Edwards25519, q: Edwards25519) Edwards25519 {
         return p.add(q.neg());
     }
@@ -529,7 +529,7 @@ test "edwards25519 packing/unpacking" {
     }
 }
 
-test "edwards25519 point addition/substraction" {
+test "edwards25519 point addition/subtraction" {
     var s1: [32]u8 = undefined;
     var s2: [32]u8 = undefined;
     crypto.random.bytes(&s1);
diff --git a/lib/std/crypto/25519/field.zig b/lib/std/crypto/25519/field.zig
index 1885f9286e..f6265cba48 100644
--- a/lib/std/crypto/25519/field.zig
+++ b/lib/std/crypto/25519/field.zig
@@ -179,7 +179,7 @@ pub const Fe = struct {
         return fe;
     }
 
-    /// Substract a field element
+    /// Subtract a field element
     pub inline fn sub(a: Fe, b: Fe) Fe {
         var fe = b;
         comptime var i = 0;
diff --git a/lib/std/crypto/Certificate.zig b/lib/std/crypto/Certificate.zig
index ec4766322c..113d169cbc 100644
--- a/lib/std/crypto/Certificate.zig
+++ b/lib/std/crypto/Certificate.zig
@@ -1134,9 +1134,9 @@ pub const rsa = struct {
         return res;
     }
 
-    fn setBytes(r: *BigInt, bytes: []const u8, allcator: std.mem.Allocator) !void {
+    fn setBytes(r: *BigInt, bytes: []const u8, allocator: std.mem.Allocator) !void {
         try r.set(0);
-        var tmp = try BigInt.init(allcator);
+        var tmp = try BigInt.init(allocator);
         defer tmp.deinit();
         for (bytes) |b| {
             try r.shiftLeft(r, 8);
diff --git a/lib/std/crypto/errors.zig b/lib/std/crypto/errors.zig
index 4d79055919..bf37faaf29 100644
--- a/lib/std/crypto/errors.zig
+++ b/lib/std/crypto/errors.zig
@@ -10,7 +10,7 @@ pub const IdentityElementError = error{IdentityElement};
 /// Encoded input cannot be decoded
 pub const EncodingError = error{InvalidEncoding};
 
-/// The signature does't verify for the given message and public key
+/// The signature doesn't verify for the given message and public key
 pub const SignatureVerificationError = error{SignatureVerificationFailed};
 
 /// Both a public and secret key have been provided, but they are incompatible
diff --git a/lib/std/crypto/kyber_d00.zig b/lib/std/crypto/kyber_d00.zig
index b52f9f475d..dca4ab7ea7 100644
--- a/lib/std/crypto/kyber_d00.zig
+++ b/lib/std/crypto/kyber_d00.zig
@@ -80,7 +80,7 @@
 //! m = Compress(Decompress(c_2, d_v) - s^T Decompress(c_1, d_u), 1).
 //!
 //! It it not straight-forward to see that this formula is correct.  In
-//! fact, there is negligable but non-zero probability that a ciphertext
+//! fact, there is negligible but non-zero probability that a ciphertext
 //! does not decrypt correctly given by the DFP column in Table 4.  This
 //! failure probability can be computed by a careful automated analysis
 //! of the probabilities involved, see kyber_failure.py of [SecEst].
@@ -640,7 +640,7 @@ fn montReduce(x: i32) i16 {
     // we have int32(int64(a)*int64(b)) = int32(a*b) and so the result is ok.
     const m = @truncate(i16, @truncate(i32, x *% qInv));
 
-    // Note that x - m q is divisable by R; indeed modulo R we have
+    // Note that x - m q is divisible by R; indeed modulo R we have
     //
     //  x - m q ≡ x - x q' q ≡ x - x q⁻¹ q ≡ x - x = 0.
     //
diff --git a/lib/std/event/lock.zig b/lib/std/event/lock.zig
index 6e095b1890..9cda7f2aba 100644
--- a/lib/std/event/lock.zig
+++ b/lib/std/event/lock.zig
@@ -36,7 +36,7 @@ pub const Lock = struct {
 
         // self.head transitions from multiple stages depending on the value:
         // UNLOCKED -> LOCKED:
-        //   acquire Lock ownership when theres no waiters
+        //   acquire Lock ownership when there are no waiters
         // LOCKED -> :
         //   Lock is already owned, enqueue first Waiter
         //  -> :
@@ -87,7 +87,7 @@ pub const Lock = struct {
 
                 // self.head goes through the reverse transition from acquire():
                 //  -> :
-                //   pop a waiter from the queue to give Lock ownership when theres still others pending
+                //   pop a waiter from the queue to give Lock ownership when there are still others pending
                 //  -> LOCKED:
                 //   pop the laster waiter from the queue, while also giving it lock ownership when awaken
                 // LOCKED -> UNLOCKED:
diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig
index 34f74e10d2..eea1ec75be 100644
--- a/lib/std/event/loop.zig
+++ b/lib/std/event/loop.zig
@@ -903,7 +903,7 @@ pub const Loop = struct {
             }
         }
 
-        // TODO: use a tickless heirarchical timer wheel:
+        // TODO: use a tickless hierarchical timer wheel:
         // https://github.com/wahern/timeout/
         const Waiters = struct {
             entries: std.atomic.Queue(anyframe),
@@ -947,7 +947,7 @@ pub const Loop = struct {
                 // starting from the head
                 var head = self.entries.head orelse return null;
 
-                // traverse the list of waiting entires to
+                // traverse the list of waiting entries to
                 // find the Node with the smallest `expires` field
                 var min = head;
                 while (head.next) |node| {
@@ -1756,7 +1756,7 @@ test "std.event.Loop - runDetached" {
     try loop.runDetached(std.testing.allocator, testRunDetached, .{});
 
     // Now we can start the event loop. The function will return only
-    // after all tasks have been completed, allowing us to synchonize
+    // after all tasks have been completed, allowing us to synchronize
     // with the previous runDetached.
     loop.run();
 
diff --git a/lib/std/fmt/parse_float/convert_fast.zig b/lib/std/fmt/parse_float/convert_fast.zig
index 75875156fd..74beb745de 100644
--- a/lib/std/fmt/parse_float/convert_fast.zig
+++ b/lib/std/fmt/parse_float/convert_fast.zig
@@ -1,4 +1,4 @@
-//! Representation of a float as the signficant digits and exponent.
+//! Representation of a float as the significant digits and exponent.
 //! The fast path algorithm using machine-sized integers and floats.
 //!
 //! This only works if both the mantissa and the exponent can be exactly
diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig
index 3f8e7185d0..23021a26f5 100644
--- a/lib/std/fs/file.zig
+++ b/lib/std/fs/file.zig
@@ -92,7 +92,7 @@ pub const File = struct {
         /// processes from acquiring a exclusive lock, but does not prevent
         /// other process from getting their own shared locks.
         ///
-        /// The lock is advisory, except on Linux in very specific cirsumstances[1].
+        /// The lock is advisory, except on Linux in very specific circumstances[1].
         /// This means that a process that does not respect the locking API can still get access
         /// to the file, despite the lock.
         ///
@@ -156,7 +156,7 @@ pub const File = struct {
         /// processes from acquiring a exclusive lock, but does not prevent
         /// other process from getting their own shared locks.
         ///
-        /// The lock is advisory, except on Linux in very specific cirsumstances[1].
+        /// The lock is advisory, except on Linux in very specific circumstances[1].
         /// This means that a process that does not respect the locking API can still get access
         /// to the file, despite the lock.
         ///
diff --git a/lib/std/fs/path.zig b/lib/std/fs/path.zig
index 4f3e05cd59..4c320ae5cf 100644
--- a/lib/std/fs/path.zig
+++ b/lib/std/fs/path.zig
@@ -105,13 +105,13 @@ fn joinSepMaybeZ(allocator: Allocator, separator: u8, comptime sepPredicate: fn
     return buf;
 }
 
-/// Naively combines a series of paths with the native path seperator.
+/// Naively combines a series of paths with the native path separator.
 /// Allocates memory for the result, which must be freed by the caller.
 pub fn join(allocator: Allocator, paths: []const []const u8) ![]u8 {
     return joinSepMaybeZ(allocator, sep, isSep, paths, false);
 }
 
-/// Naively combines a series of paths with the native path seperator and null terminator.
+/// Naively combines a series of paths with the native path separator and null terminator.
 /// Allocates memory for the result, which must be freed by the caller.
 pub fn joinZ(allocator: Allocator, paths: []const []const u8) ![:0]u8 {
     const out = try joinSepMaybeZ(allocator, sep, isSep, paths, true);
diff --git a/lib/std/hash_map.zig b/lib/std/hash_map.zig
index 65b07b8e77..df3446a2c0 100644
--- a/lib/std/hash_map.zig
+++ b/lib/std/hash_map.zig
@@ -690,7 +690,7 @@ pub fn HashMap(
 /// It achieves good performance with quite high load factors (by default,
 /// grow is triggered at 80% full) and only one byte of overhead per element.
 /// The struct itself is only 16 bytes for a small footprint. This comes at
-/// the price of handling size with u32, which should be reasonnable enough
+/// the price of handling size with u32, which should be reasonable enough
 /// for almost all uses.
 /// Deletions are achieved with tombstones.
 pub fn HashMapUnmanaged(
diff --git a/lib/std/heap.zig b/lib/std/heap.zig
index 111314a1d9..7d2a66df1e 100644
--- a/lib/std/heap.zig
+++ b/lib/std/heap.zig
@@ -79,7 +79,7 @@ const CAllocator = struct {
         }
 
         // Thin wrapper around regular malloc, overallocate to account for
-        // alignment padding and store the orignal malloc()'ed pointer before
+        // alignment padding and store the original malloc()'ed pointer before
         // the aligned address.
         var unaligned_ptr = @ptrCast([*]u8, c.malloc(len + alignment - 1 + @sizeOf(usize)) orelse return null);
         const unaligned_addr = @ptrToInt(unaligned_ptr);
diff --git a/lib/std/heap/arena_allocator.zig b/lib/std/heap/arena_allocator.zig
index d07c45cb70..9489bbb449 100644
--- a/lib/std/heap/arena_allocator.zig
+++ b/lib/std/heap/arena_allocator.zig
@@ -136,7 +136,7 @@ pub const ArenaAllocator = struct {
             it = next_it;
         } else null;
         std.debug.assert(maybe_first_node == null or maybe_first_node.?.next == null);
-        // reset the state before we try resizing the buffers, so we definitly have reset the arena to 0.
+        // reset the state before we try resizing the buffers, so we definitely have reset the arena to 0.
         self.state.end_index = 0;
         if (maybe_first_node) |first_node| {
             // perfect, no need to invoke the child_allocator
diff --git a/lib/std/heap/general_purpose_allocator.zig b/lib/std/heap/general_purpose_allocator.zig
index de7bc7c423..ef88787fc6 100644
--- a/lib/std/heap/general_purpose_allocator.zig
+++ b/lib/std/heap/general_purpose_allocator.zig
@@ -130,7 +130,7 @@ pub const Config = struct {
     thread_safe: bool = !builtin.single_threaded,
 
     /// What type of mutex you'd like to use, for thread safety.
-    /// when specfied, the mutex type must have the same shape as `std.Thread.Mutex` and
+    /// when specified, the mutex type must have the same shape as `std.Thread.Mutex` and
     /// `DummyMutex`, and have no required fields. Specifying this field causes
     /// the `thread_safe` field to be ignored.
     ///
@@ -1241,7 +1241,7 @@ test "realloc large object to small object" {
     try std.testing.expect(slice[16] == 0x34);
 }
 
-test "overrideable mutexes" {
+test "overridable mutexes" {
     var gpa = GeneralPurposeAllocator(.{ .MutexType = std.Thread.Mutex }){
         .backing_allocator = std.testing.allocator,
         .mutex = std.Thread.Mutex{},
diff --git a/lib/std/heap/memory_pool.zig b/lib/std/heap/memory_pool.zig
index 6ae90a3894..ca6eb7f518 100644
--- a/lib/std/heap/memory_pool.zig
+++ b/lib/std/heap/memory_pool.zig
@@ -150,7 +150,7 @@ test "memory pool: basic" {
     pool.destroy(p2);
     const p4 = try pool.create();
 
-    // Assert memory resuse
+    // Assert memory reuse
     try std.testing.expect(p2 == p4);
 }
 
diff --git a/lib/std/http.zig b/lib/std/http.zig
index bb06bb1c19..744615d7d7 100644
--- a/lib/std/http.zig
+++ b/lib/std/http.zig
@@ -12,7 +12,7 @@ pub const Version = enum {
 };
 
 /// https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods
-/// https://datatracker.ietf.org/doc/html/rfc7231#section-4 Initial definiton
+/// https://datatracker.ietf.org/doc/html/rfc7231#section-4 Initial definition
 /// https://datatracker.ietf.org/doc/html/rfc5789#section-2 PATCH
 pub const Method = enum {
     GET,
diff --git a/lib/std/json.zig b/lib/std/json.zig
index 432f4e6911..af928bc564 100644
--- a/lib/std/json.zig
+++ b/lib/std/json.zig
@@ -1603,7 +1603,7 @@ fn parseInternal(
                                 if (fields_seen[i]) {
                                     switch (options.duplicate_field_behavior) {
                                         .UseFirst => {
-                                            // unconditonally ignore value. for comptime fields, this skips check against default_value
+                                            // unconditionally ignore value. for comptime fields, this skips check against default_value
                                             parseFree(field.type, try parse(field.type, tokens, child_options), child_options);
                                             found = true;
                                             break;
diff --git a/lib/std/linked_list.zig b/lib/std/linked_list.zig
index 577bae3d38..f9084a55c4 100644
--- a/lib/std/linked_list.zig
+++ b/lib/std/linked_list.zig
@@ -406,7 +406,7 @@ test "TailQueue concatenation" {
         }
     }
 
-    // Swap them back, this verifies that concating to an empty list works.
+    // Swap them back, this verifies that concatenating to an empty list works.
     list2.concatByMoving(&list1);
 
     // Traverse forwards.
diff --git a/lib/std/macho.zig b/lib/std/macho.zig
index a25ffca4fa..8bddd67023 100644
--- a/lib/std/macho.zig
+++ b/lib/std/macho.zig
@@ -356,7 +356,7 @@ pub const dysymtab_command = extern struct {
 
     // All the local relocation entries are grouped together (they are not
     // grouped by their module since they are only used if the object is moved
-    // from it staticly link edited address).
+    // from its statically link edited address).
 
     /// offset to local relocation entries
     locreloff: u32 = 0,
@@ -418,7 +418,7 @@ pub const dyld_info_command = extern struct {
     //    
     // The opcodes are a compressed way to encode the table by only
     // encoding when a column changes.  In addition simple patterns
-    // like for runs of pointers initialzed to the same value can be
+    // like for runs of pointers initialized to the same value can be
     // encoded in a few bytes.
 
     /// file offset to binding info
@@ -1141,7 +1141,7 @@ pub const MH_NOUNDEFS = 0x1;
 /// the object file is the output of an incremental link against a base file and can't be link edited again
 pub const MH_INCRLINK = 0x2;
 
-/// the object file is input for the dynamic linker and can't be staticly link edited again
+/// the object file is input for the dynamic linker and can't be statically link edited again
 pub const MH_DYLDLINK = 0x4;
 
 /// the object file's undefined references are bound by the dynamic linker when loaded.
@@ -1162,7 +1162,7 @@ pub const MH_TWOLEVEL = 0x80;
 /// the executable is forcing all images to use flat name space bindings
 pub const MH_FORCE_FLAT = 0x100;
 
-/// this umbrella guarantees no multiple defintions of symbols in its sub-images so the two-level namespace hints can always be used.
+/// this umbrella guarantees no multiple definitions of symbols in its sub-images so the two-level namespace hints can always be used.
 pub const MH_NOMULTIDEFS = 0x200;
 
 /// do not have dyld notify the prebinding agent about this executable
@@ -1658,7 +1658,7 @@ pub const EXPORT_SYMBOL_FLAGS_REEXPORT: u8 = 0x08;
 pub const EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER: u8 = 0x10;
 
 // An indirect symbol table entry is simply a 32bit index into the symbol table
-// to the symbol that the pointer or stub is refering to.  Unless it is for a
+// to the symbol that the pointer or stub is referring to.  Unless it is for a
 // non-lazy symbol pointer section for a defined symbol which strip(1) as
 // removed.  In which case it has the value INDIRECT_SYMBOL_LOCAL.  If the
 // symbol was also absolute INDIRECT_SYMBOL_ABS is or'ed with that.
@@ -1741,7 +1741,7 @@ pub const CS_LINKER_SIGNED: u32 = 0x20000;
 
 pub const CS_EXECSEG_MAIN_BINARY: u32 = 0x1;
 
-/// This CodeDirectory is tailored specfically at version 0x20400.
+/// This CodeDirectory is tailored specifically at version 0x20400.
 pub const CodeDirectory = extern struct {
     /// Magic number (CSMAGIC_CODEDIRECTORY)
     magic: u32,
diff --git a/lib/std/math/big/int.zig b/lib/std/math/big/int.zig
index 2406d669ec..b01d9b04ff 100644
--- a/lib/std/math/big/int.zig
+++ b/lib/std/math/big/int.zig
@@ -408,7 +408,7 @@ pub const Mutable = struct {
     }
 
     /// Base implementation for addition. Adds `max(a.limbs.len, b.limbs.len)` elements from a and b,
-    /// and returns whether any overflow occured.
+    /// and returns whether any overflow occurred.
     /// r, a and b may be aliases.
     ///
     /// Asserts r has enough elements to hold the result. The upper bound is `max(a.limbs.len, b.limbs.len)`.
@@ -467,7 +467,7 @@ pub const Mutable = struct {
         const req_limbs = calcTwosCompLimbCount(bit_count);
 
         // Slice of the upper bits if they exist, these will be ignored and allows us to use addCarry to determine
-        // if an overflow occured.
+        // if an overflow occurred.
         const x = Const{
             .positive = a.positive,
             .limbs = a.limbs[0..math.min(req_limbs, a.limbs.len)],
@@ -512,7 +512,7 @@ pub const Mutable = struct {
         const req_limbs = calcTwosCompLimbCount(bit_count);
 
         // Slice of the upper bits if they exist, these will be ignored and allows us to use addCarry to determine
-        // if an overflow occured.
+        // if an overflow occurred.
         const x = Const{
             .positive = a.positive,
             .limbs = a.limbs[0..math.min(req_limbs, a.limbs.len)],
@@ -544,7 +544,7 @@ pub const Mutable = struct {
     }
 
     /// Base implementation for subtraction. Subtracts `max(a.limbs.len, b.limbs.len)` elements from a and b,
-    /// and returns whether any overflow occured.
+    /// and returns whether any overflow occurred.
     /// r, a and b may be aliases.
     ///
     /// Asserts r has enough elements to hold the result. The upper bound is `max(a.limbs.len, b.limbs.len)`.
@@ -605,7 +605,7 @@ pub const Mutable = struct {
         r.add(a, b.negate());
     }
 
-    /// r = a - b with 2s-complement wrapping semantics. Returns whether any overflow occured.
+    /// r = a - b with 2s-complement wrapping semantics. Returns whether any overflow occurred.
     ///
     /// r, a and b may be aliases
     /// Asserts the result fits in `r`. An upper bound on the number of limbs needed by
@@ -1141,7 +1141,7 @@ pub const Mutable = struct {
             return;
         }
 
-        // Generate a mask with the bits to check in the most signficant limb. We'll need to check
+        // Generate a mask with the bits to check in the most significant limb. We'll need to check
         // all bits with equal or more significance than checkbit.
         // const msb = @truncate(Log2Limb, checkbit);
         // const checkmask = (@as(Limb, 1) << msb) -% 1;
@@ -2037,7 +2037,7 @@ pub const Const = struct {
                 add_res = ov[0];
                 carry = ov[1];
                 sum += @popCount(add_res);
-                remaining_bits -= limb_bits; // Asserted not to undeflow by fitsInTwosComp
+                remaining_bits -= limb_bits; // Asserted not to underflow by fitsInTwosComp
             }
 
             // The most significant limb may have fewer than @bitSizeOf(Limb) meaningful bits,
@@ -2813,7 +2813,7 @@ pub const Managed = struct {
         r.setMetadata(m.positive, m.len);
     }
 
-    /// r = a + b with 2s-complement wrapping semantics. Returns whether any overflow occured.
+    /// r = a + b with 2s-complement wrapping semantics. Returns whether any overflow occurred.
     ///
     /// r, a and b may be aliases.
     ///
@@ -2856,7 +2856,7 @@ pub const Managed = struct {
         r.setMetadata(m.positive, m.len);
     }
 
-    /// r = a - b with 2s-complement wrapping semantics. Returns whether any overflow occured.
+    /// r = a - b with 2s-complement wrapping semantics. Returns whether any overflow occurred.
     ///
     /// r, a and b may be aliases.
     ///
@@ -4010,7 +4010,7 @@ fn llsquareBasecase(r: []Limb, x: []const Limb) void {
     assert(r.len >= 2 * x_norm.len + 1);
 
     // Compute the square of a N-limb bigint with only (N^2 + N)/2
-    // multiplications by exploting the symmetry of the coefficients around the
+    // multiplications by exploiting the symmetry of the coefficients around the
     // diagonal:
     //
     //           a   b   c *
diff --git a/lib/std/math/complex/tan.zig b/lib/std/math/complex/tan.zig
index 9e1025f74f..a0ce5d18e0 100644
--- a/lib/std/math/complex/tan.zig
+++ b/lib/std/math/complex/tan.zig
@@ -4,7 +4,7 @@ const math = std.math;
 const cmath = math.complex;
 const Complex = cmath.Complex;
 
-/// Returns the tanget of z.
+/// Returns the tangent of z.
 pub fn tan(z: anytype) Complex(@TypeOf(z.re)) {
     const T = @TypeOf(z.re);
     const q = Complex(T).init(-z.im, z.re);
diff --git a/lib/std/math/log10.zig b/lib/std/math/log10.zig
index b1ecb9ad2b..6b5758763c 100644
--- a/lib/std/math/log10.zig
+++ b/lib/std/math/log10.zig
@@ -58,7 +58,7 @@ pub fn log10_int(x: anytype) Log2Int(@TypeOf(x)) {
     var log: u32 = 0;
 
     inline for (0..11) |i| {
-        // Unnecesary branches should be removed by the compiler
+        // Unnecessary branches should be removed by the compiler
         if (bit_size > (1 << (11 - i)) * 5 * @log2(10.0) and val >= pow10((1 << (11 - i)) * 5)) {
             const num_digits = (1 << (11 - i)) * 5;
             val /= pow10(num_digits);
diff --git a/lib/std/math/sqrt.zig b/lib/std/math/sqrt.zig
index e642f8a309..926582034e 100644
--- a/lib/std/math/sqrt.zig
+++ b/lib/std/math/sqrt.zig
@@ -11,7 +11,7 @@ const maxInt = std.math.maxInt;
 ///  - sqrt(+-0)   = +-0
 ///  - sqrt(x)     = nan if x < 0
 ///  - sqrt(nan)   = nan
-/// TODO Decide if all this logic should be implemented directly in the @sqrt bultin function.
+/// TODO Decide if all this logic should be implemented directly in the @sqrt builtin function.
 pub fn sqrt(x: anytype) Sqrt(@TypeOf(x)) {
     const T = @TypeOf(x);
     switch (@typeInfo(T)) {
diff --git a/lib/std/mem.zig b/lib/std/mem.zig
index 557e39cdfc..6cc60ddad2 100644
--- a/lib/std/mem.zig
+++ b/lib/std/mem.zig
@@ -114,7 +114,7 @@ pub fn validationWrap(allocator: anytype) ValidationAllocator(@TypeOf(allocator)
 
 /// An allocator helper function.  Adjusts an allocation length satisfy `len_align`.
 /// `full_len` should be the full capacity of the allocation which may be greater
-/// than the `len` that was requsted.  This function should only be used by allocators
+/// than the `len` that was requested.  This function should only be used by allocators
 /// that are unaffected by `len_align`.
 pub fn alignAllocLen(full_len: usize, alloc_len: usize, len_align: u29) usize {
     assert(alloc_len > 0);
@@ -427,7 +427,7 @@ pub fn zeroInit(comptime T: type, init: anytype) T {
                 .Struct => |init_info| {
                     if (init_info.is_tuple) {
                         if (init_info.fields.len > struct_info.fields.len) {
-                            @compileError("Tuple initializer has more elments than there are fields in `" ++ @typeName(T) ++ "`");
+                            @compileError("Tuple initializer has more elements than there are fields in `" ++ @typeName(T) ++ "`");
                         }
                     } else {
                         inline for (init_info.fields) |field| {
@@ -668,7 +668,7 @@ test "Span" {
 
 /// Takes a sentinel-terminated pointer and returns a slice, iterating over the
 /// memory to find the sentinel and determine the length.
-/// Ponter attributes such as const are preserved.
+/// Pointer attributes such as const are preserved.
 /// `[*c]` pointers are assumed to be non-null and 0-terminated.
 pub fn span(ptr: anytype) Span(@TypeOf(ptr)) {
     if (@typeInfo(@TypeOf(ptr)) == .Optional) {
@@ -1835,7 +1835,7 @@ test "writeIntBig and writeIntLittle" {
 }
 
 /// Swap the byte order of all the members of the fields of a struct
-/// (Changing their endianess)
+/// (Changing their endianness)
 pub fn byteSwapAllFields(comptime S: type, ptr: *S) void {
     if (@typeInfo(S) != .Struct) @compileError("byteSwapAllFields expects a struct as the first argument");
     inline for (std.meta.fields(S)) |f| {
@@ -3168,7 +3168,7 @@ test "replace" {
     try testing.expectEqualStrings(expected, output[0..expected.len]);
 }
 
-/// Replace all occurences of `needle` with `replacement`.
+/// Replace all occurrences of `needle` with `replacement`.
 pub fn replaceScalar(comptime T: type, slice: []T, needle: T, replacement: T) void {
     for (slice, 0..) |e, i| {
         if (e == needle) {
diff --git a/lib/std/os.zig b/lib/std/os.zig
index 32c73916d4..7759f1403f 100644
--- a/lib/std/os.zig
+++ b/lib/std/os.zig
@@ -659,7 +659,7 @@ pub fn exit(status: u8) noreturn {
         linux.exit_group(status);
     }
     if (builtin.os.tag == .uefi) {
-        // exit() is only avaliable if exitBootServices() has not been called yet.
+        // exit() is only available if exitBootServices() has not been called yet.
         // This call to exit should not fail, so we don't care about its return value.
         if (uefi.system_table.boot_services) |bs| {
             _ = bs.exit(uefi.handle, @intToEnum(uefi.Status, status), 0, null);
@@ -2978,7 +2978,7 @@ pub fn chdirZ(dir_path: [*:0]const u8) ChangeCurDirError!void {
     }
 }
 
-/// Windows-only. Same as `chdir` except the paramter is WTF16 encoded.
+/// Windows-only. Same as `chdir` except the parameter is WTF16 encoded.
 pub fn chdirW(dir_path: []const u16) ChangeCurDirError!void {
     windows.SetCurrentDirectory(dir_path) catch |err| switch (err) {
         error.NoDevice => return error.FileSystem,
@@ -6925,7 +6925,7 @@ pub const PrctlError = error{
     /// Can only occur with PR_SET_SPECULATION_CTRL, PR_MPX_ENABLE_MANAGEMENT,
     /// or PR_MPX_DISABLE_MANAGEMENT
     UnsupportedFeature,
-    /// Can only occur wih PR_SET_FP_MODE
+    /// Can only occur with PR_SET_FP_MODE
     OperationNotSupported,
     PermissionDenied,
 } || UnexpectedError;
diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig
index 901bb8dbb3..4ffaeb54a4 100644
--- a/lib/std/os/linux.zig
+++ b/lib/std/os/linux.zig
@@ -472,7 +472,7 @@ pub fn preadv(fd: i32, iov: [*]const iovec, count: usize, offset: i64) usize {
         @bitCast(usize, @as(isize, fd)),
         @ptrToInt(iov),
         count,
-        // Kernel expects the offset is splitted into largest natural word-size.
+        // Kernel expects the offset is split into largest natural word-size.
         // See following link for detail:
         // https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=601cc11d054ae4b5e9b5babec3d8e4667a2cb9b5
         @truncate(usize, offset_u),
@@ -3912,7 +3912,7 @@ pub const io_uring_cqe = extern struct {
 /// If set, the upper 16 bits are the buffer ID
 pub const IORING_CQE_F_BUFFER = 1 << 0;
 /// If set, parent SQE will generate more CQE entries.
-/// Avaiable since Linux 5.13.
+/// Available since Linux 5.13.
 pub const IORING_CQE_F_MORE = 1 << 1;
 /// If set, more data to read after socket recv
 pub const IORING_CQE_F_SOCK_NONEMPTY = 1 << 2;
@@ -4234,7 +4234,7 @@ pub const tcp_fastopen_client_fail = enum {
 pub const TCPI_OPT_TIMESTAMPS = 1;
 pub const TCPI_OPT_SACK = 2;
 pub const TCPI_OPT_WSCALE = 4;
-/// ECN was negociated at TCP session init
+/// ECN was negotiated at TCP session init
 pub const TCPI_OPT_ECN = 8;
 /// we received at least one packet with ECT
 pub const TCPI_OPT_ECN_SEEN = 16;
diff --git a/lib/std/os/linux/bpf/btf.zig b/lib/std/os/linux/bpf/btf.zig
index 7b85a618b3..39d25014da 100644
--- a/lib/std/os/linux/bpf/btf.zig
+++ b/lib/std/os/linux/bpf/btf.zig
@@ -109,7 +109,7 @@ pub const Enum64 = extern struct {
     val_hi32: i32,
 };
 
-/// array kind is followd by this struct
+/// array kind is followed by this struct
 pub const Array = extern struct {
     typ: u32,
     index_type: u32,
@@ -149,13 +149,13 @@ pub const FuncLinkage = enum {
     external,
 };
 
-/// var kind is followd by a single Var struct to describe additional
+/// var kind is followed by a single Var struct to describe additional
 /// information related to the variable such as its linkage
 pub const Var = extern struct {
     linkage: u32,
 };
 
-/// datasec kind is followed by multible VarSecInfo to describe all Var kind
+/// datasec kind is followed by multiple VarSecInfo to describe all Var kind
 /// types it contains along with it's in-section offset as well as size.
 pub const VarSecInfo = extern struct {
     typ: u32,
diff --git a/lib/std/os/linux/seccomp.zig b/lib/std/os/linux/seccomp.zig
index 23dbb6ee38..f10cb84aa0 100644
--- a/lib/std/os/linux/seccomp.zig
+++ b/lib/std/os/linux/seccomp.zig
@@ -65,7 +65,7 @@
 //!
 //! Unfortunately, there is no easy solution for issue 5. The most reliable
 //! strategy is to keep testing; test newer Zig versions, different libcs,
-//! different distros, and design your filter to accomidate all of them.
+//! different distros, and design your filter to accommodate all of them.
 //! Alternatively, you could inject a filter at runtime. Since filters are
 //! preserved across execve(2), a filter could be setup before executing your
 //! program, without your program having any knowledge of this happening. This
diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig
index 6d654f1c20..22ffc850e6 100644
--- a/lib/std/os/windows.zig
+++ b/lib/std/os/windows.zig
@@ -2114,7 +2114,7 @@ pub fn loadWinsockExtensionFunction(comptime T: type, sock: ws2_32.SOCKET, guid:
 /// and you get an unexpected error.
 pub fn unexpectedError(err: Win32Error) std.os.UnexpectedError {
     if (std.os.unexpected_error_tracing) {
-        // 614 is the length of the longest windows error desciption
+        // 614 is the length of the longest windows error description
         var buf_wstr: [614]WCHAR = undefined;
         var buf_utf8: [614]u8 = undefined;
         const len = kernel32.FormatMessageW(
diff --git a/lib/std/packed_int_array.zig b/lib/std/packed_int_array.zig
index 8004d223f7..10d8af0575 100644
--- a/lib/std/packed_int_array.zig
+++ b/lib/std/packed_int_array.zig
@@ -33,7 +33,7 @@ pub fn PackedIntIo(comptime Int: type, comptime endian: Endian) type {
     // In the worst case, this is the number of bytes we need to touch
     // to read or write a value, as bits. To calculate for int_bits > 1,
     // set aside 2 bits to touch the first and last bytes, then divide
-    // by 8 to see how many bytes can be filled up inbetween.
+    // by 8 to see how many bytes can be filled up in between.
     const max_io_bits = switch (int_bits) {
         0 => 0,
         1 => 8,
@@ -298,7 +298,7 @@ pub fn PackedIntSliceEndian(comptime Int: type, comptime endian: Endian) type {
         }
 
         /// Initialize a packed slice using the memory at `bytes`, with `int_count`
-        /// elements. `bytes` must be large enough to accomodate the requested
+        /// elements. `bytes` must be large enough to accommodate the requested
         /// count.
         pub fn init(bytes: []u8, int_count: usize) Self {
             debug.assert(bytes.len >= bytesRequired(int_count));
diff --git a/lib/std/pdb.zig b/lib/std/pdb.zig
index fdd162a34f..5bc836b08e 100644
--- a/lib/std/pdb.zig
+++ b/lib/std/pdb.zig
@@ -912,7 +912,7 @@ const Msf = struct {
         const stream_sizes = try allocator.alloc(u32, stream_count);
         defer allocator.free(stream_sizes);
 
-        // Microsoft's implementation uses @as(u32, -1) for inexistant streams.
+        // Microsoft's implementation uses @as(u32, -1) for inexistent streams.
         // These streams are not used, but still participate in the file
         // and must be taken into account when resolving stream indices.
         const Nil = 0xFFFFFFFF;
diff --git a/lib/std/simd.zig b/lib/std/simd.zig
index c2e0d9f972..905925fb78 100644
--- a/lib/std/simd.zig
+++ b/lib/std/simd.zig
@@ -159,7 +159,7 @@ pub fn interlace(vecs: anytype) @Vector(vectorLength(@TypeOf(vecs[0])) * vecs.le
 }
 
 /// The contents of `interlaced` is evenly split between vec_count vectors that are returned as an array. They "take turns",
-/// recieving one element from `interlaced` at a time.
+/// receiving one element from `interlaced` at a time.
 pub fn deinterlace(
     comptime vec_count: usize,
     interlaced: anytype,
diff --git a/lib/std/target/csky.zig b/lib/std/target/csky.zig
index 10386b31b8..0a2812c6d9 100644
--- a/lib/std/target/csky.zig
+++ b/lib/std/target/csky.zig
@@ -214,7 +214,7 @@ pub const all_features = blk: {
     };
     result[@enumToInt(Feature.dsp_silan)] = .{
         .llvm_name = "dsp_silan",
-        .description = "Enable DSP Silan instrutions",
+        .description = "Enable DSP Silan instructions",
         .dependencies = featureSet(&[_]Feature{}),
     };
     result[@enumToInt(Feature.dspe60)] = .{
@@ -224,7 +224,7 @@ pub const all_features = blk: {
     };
     result[@enumToInt(Feature.dspv2)] = .{
         .llvm_name = "dspv2",
-        .description = "Enable DSP V2.0 instrutions",
+        .description = "Enable DSP V2.0 instructions",
         .dependencies = featureSet(&[_]Feature{}),
     };
     result[@enumToInt(Feature.e1)] = .{
@@ -243,7 +243,7 @@ pub const all_features = blk: {
     };
     result[@enumToInt(Feature.edsp)] = .{
         .llvm_name = "edsp",
-        .description = "Enable DSP instrutions",
+        .description = "Enable DSP instructions",
         .dependencies = featureSet(&[_]Feature{}),
     };
     result[@enumToInt(Feature.elrw)] = .{
@@ -333,12 +333,12 @@ pub const all_features = blk: {
     };
     result[@enumToInt(Feature.hwdiv)] = .{
         .llvm_name = "hwdiv",
-        .description = "Enable divide instrutions",
+        .description = "Enable divide instructions",
         .dependencies = featureSet(&[_]Feature{}),
     };
     result[@enumToInt(Feature.istack)] = .{
         .llvm_name = "istack",
-        .description = "Enable interrput attribute",
+        .description = "Enable interrupt attribute",
         .dependencies = featureSet(&[_]Feature{}),
     };
     result[@enumToInt(Feature.java)] = .{
@@ -362,7 +362,7 @@ pub const all_features = blk: {
     };
     result[@enumToInt(Feature.multiple_stld)] = .{
         .llvm_name = "multiple_stld",
-        .description = "Enable multiple load/store instrutions",
+        .description = "Enable multiple load/store instructions",
         .dependencies = featureSet(&[_]Feature{}),
     };
     result[@enumToInt(Feature.nvic)] = .{
@@ -372,7 +372,7 @@ pub const all_features = blk: {
     };
     result[@enumToInt(Feature.pushpop)] = .{
         .llvm_name = "pushpop",
-        .description = "Enable push/pop instrutions",
+        .description = "Enable push/pop instructions",
         .dependencies = featureSet(&[_]Feature{}),
     };
     result[@enumToInt(Feature.smart)] = .{
diff --git a/lib/std/time.zig b/lib/std/time.zig
index 5f86c25b8d..3eb342fa85 100644
--- a/lib/std/time.zig
+++ b/lib/std/time.zig
@@ -258,7 +258,7 @@ pub const Instant = struct {
 
 /// A monotonic, high performance timer.
 ///
-/// Timer.start() is used to initalize the timer
+/// Timer.start() is used to initialize the timer
 /// and gives the caller an opportunity to check for the existence of a supported clock.
 /// Once a supported clock is discovered,
 /// it is assumed that it will be available for the duration of the Timer's use.
diff --git a/lib/std/valgrind/memcheck.zig b/lib/std/valgrind/memcheck.zig
index 489abac2f6..25081510cd 100644
--- a/lib/std/valgrind/memcheck.zig
+++ b/lib/std/valgrind/memcheck.zig
@@ -77,7 +77,7 @@ pub fn discard(blkindex: usize) bool {
 }
 
 /// Check that memory at qzz.ptr is addressable for qzz.len bytes.
-/// If suitable addressibility is not established, Valgrind prints an
+/// If suitable addressability is not established, Valgrind prints an
 /// error message and returns the address of the first offending byte.
 /// Otherwise it returns zero.
 pub fn checkMemIsAddressable(qzz: []u8) usize {
@@ -85,7 +85,7 @@ pub fn checkMemIsAddressable(qzz: []u8) usize {
 }
 
 /// Check that memory at qzz.ptr is addressable and defined for
-/// qzz.len bytes.  If suitable addressibility and definedness are not
+/// qzz.len bytes.  If suitable addressability and definedness are not
 /// established, Valgrind prints an error message and returns the
 /// address of the first offending byte.  Otherwise it returns zero.
 pub fn checkMemIsDefined(qzz: []u8) usize {
diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig
index 5ad8f8a07e..9176e14480 100644
--- a/lib/std/zig/parser_test.zig
+++ b/lib/std/zig/parser_test.zig
@@ -239,7 +239,7 @@ test "zig fmt: file ends in comment after var decl" {
     );
 }
 
-test "zig fmt: if statment" {
+test "zig fmt: if statement" {
     try testCanonical(
         \\test "" {
         \\    if (optional()) |some|
@@ -529,7 +529,7 @@ test "zig fmt: remove empty lines at start/end of block" {
     );
 }
 
-test "zig fmt: allow empty line before commment at start of block" {
+test "zig fmt: allow empty line before comment at start of block" {
     try testCanonical(
         \\test {
         \\
@@ -4371,7 +4371,7 @@ test "zig fmt: same line doc comment returns error" {
         \\const Foo = struct{
         \\    bar: u32, /// comment
         \\    foo: u32, /// comment
-        \\    /// commment
+        \\    /// comment
         \\};
         \\
         \\const a = 42; /// comment
diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig
index d8a8c8e9ed..367d06f7c6 100644
--- a/lib/std/zig/render.zig
+++ b/lib/std/zig/render.zig
@@ -681,7 +681,7 @@ fn renderExpression(gpa: Allocator, ais: *Ais, tree: Ast, node: Ast.Node.Index,
 
             try renderToken(ais, tree, switch_token, .space); // switch keyword
             try renderToken(ais, tree, switch_token + 1, .none); // lparen
-            try renderExpression(gpa, ais, tree, condition, .none); // condtion expression
+            try renderExpression(gpa, ais, tree, condition, .none); // condition expression
             try renderToken(ais, tree, rparen, .space); // rparen
 
             ais.pushIndentNextLine();

From 6ae19fa48d7853430ba45f2de90550cd48cf5b4c Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Sat, 29 Apr 2023 17:44:56 +0100
Subject: [PATCH 023/725] std.c: add essential freebsd's capsicum api subset.

---
 lib/std/c/freebsd.zig | 39 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/lib/std/c/freebsd.zig b/lib/std/c/freebsd.zig
index e5cdc0e715..28e95d0414 100644
--- a/lib/std/c/freebsd.zig
+++ b/lib/std/c/freebsd.zig
@@ -476,10 +476,43 @@ pub const sockaddr = extern struct {
 
 pub const CAP_RIGHTS_VERSION = 0;
 
-pub const cap_rights = extern struct {
-    rights: [CAP_RIGHTS_VERSION + 2]u64,
+pub const cap_rights_t = extern struct {
+    cr_rights: [CAP_RIGHTS_VERSION + 2]u64,
 };
 
+pub const CAP = struct {
+    pub fn RIGHT(idx: u6, bit: u64) u64 {
+        return (@intCast(u64, 1) << (57 + idx)) | bit;
+    }
+    pub const READ = CAP.RIGHT(0, 0x0000000000000001);
+    pub const WRITE = CAP.RIGHT(0, 0x0000000000000002);
+    pub const SEEK_TELL = CAP.RIGHT(0, 0x0000000000000004);
+    pub const SEEK = CAP.SEEK_TELL | 0x0000000000000008;
+    pub const PREAD = CAP.SEEK | CAP.READ;
+    pub const PWRITE = CAP.SEEK | CAP.WRITE;
+    pub const MMAP = CAP.RIGHT(0, 0x0000000000000010);
+    pub const MMAP_R = CAP.MMAP | CAP.SEEK | CAP.READ;
+    pub const MMAP_W = CAP.MMAP | CAP.SEEK | CAP.WRITE;
+    pub const MMAP_X = CAP.MMAP | CAP.SEEK | 0x0000000000000020;
+    pub const MMAP_RW = CAP.MMAP_R | CAP.MMAP_W;
+    pub const MMAP_RX = CAP.MMAP_R | CAP.MMAP_X;
+    pub const MMAP_WX = CAP.MMAP_W | CAP.MMAP_X;
+    pub const MMAP_RWX = CAP.MMAP_R | CAP.MMAP_W | CAP.MMAP_X;
+    pub const CREATE = CAP.RIGHT(0, 0x0000000000000040);
+    pub const FEXECVE = CAP.RIGHT(0, 0x0000000000000080);
+    pub const FSYNC = CAP.RIGHT(0, 0x0000000000000100);
+    pub const FTRUNCATE = CAP.RIGHT(0, 0x0000000000000200);
+};
+
+pub extern "c" fn __cap_rights_init(version: c_int, rights: ?*cap_rights_t, ...) ?*cap_rights_t;
+pub extern "c" fn __cap_rights_set(rights: ?*cap_rights_t, ...) ?*cap_rights_t;
+pub extern "c" fn __cap_rights_clear(rights: ?*cap_rights_t, ...) ?*cap_rights_t;
+pub extern "c" fn __cap_rights_merge(dst: ?*cap_rights_t, src: ?*const cap_rights_t) ?*cap_rights_t;
+pub extern "c" fn __cap_rights_remove(dst: ?*cap_rights_t, src: ?*const cap_rights_t) ?*cap_rights_t;
+pub extern "c" fn __cap_rights_contains(dst: ?*const cap_rights_t, src: ?*const cap_rights_t) bool;
+pub extern "c" fn __cap_rights_is_set(rights: ?*const cap_rights_t, ...) bool;
+pub extern "c" fn __cap_rights_is_valid(rights: ?*const cap_rights_t) bool;
+
 pub const kinfo_file = extern struct {
     /// Size of this record.
     /// A zero value is for the sentinel record at the end of an array.
@@ -581,7 +614,7 @@ pub const kinfo_file = extern struct {
     // Reserved for future use.
     _spare: c_int,
     /// Capability rights.
-    cap_rights: cap_rights,
+    cap_rights: cap_rights_t,
     /// Reserved for future cap_rights
     _cap_spare: u64,
     /// Path to file, if any.

From e963793e37f93d84f1e5295d309ebe0c738b663d Mon Sep 17 00:00:00 2001
From: Mason Remaley 
Date: Mon, 1 May 2023 01:03:46 -0400
Subject: [PATCH 024/725] Updates std.meta.intToEnum to support non-exhaustive
 enums (#15491)

This was preventing `std.json` from deserializing non-exhaustive enums.
---
 lib/std/meta.zig | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/lib/std/meta.zig b/lib/std/meta.zig
index 97c2ff4fb0..7be3b71347 100644
--- a/lib/std/meta.zig
+++ b/lib/std/meta.zig
@@ -965,18 +965,33 @@ test "intToEnum with error return" {
         A,
         B,
     };
+    const E3 = enum(i8) { A, _ };
 
     var zero: u8 = 0;
     var one: u16 = 1;
     try testing.expect(intToEnum(E1, zero) catch unreachable == E1.A);
     try testing.expect(intToEnum(E2, one) catch unreachable == E2.B);
+    try testing.expect(intToEnum(E3, zero) catch unreachable == E3.A);
+    try testing.expect(intToEnum(E3, 127) catch unreachable == @intToEnum(E3, 127));
+    try testing.expect(intToEnum(E3, -128) catch unreachable == @intToEnum(E3, -128));
     try testing.expectError(error.InvalidEnumTag, intToEnum(E1, one));
+    try testing.expectError(error.InvalidEnumTag, intToEnum(E3, 128));
+    try testing.expectError(error.InvalidEnumTag, intToEnum(E3, -129));
 }
 
 pub const IntToEnumError = error{InvalidEnumTag};
 
 pub fn intToEnum(comptime EnumTag: type, tag_int: anytype) IntToEnumError!EnumTag {
-    inline for (@typeInfo(EnumTag).Enum.fields) |f| {
+    const enum_info = @typeInfo(EnumTag).Enum;
+
+    if (!enum_info.is_exhaustive) {
+        if (std.math.cast(enum_info.tag_type, tag_int)) |tag| {
+            return @intToEnum(EnumTag, tag);
+        }
+        return error.InvalidEnumTag;
+    }
+
+    inline for (enum_info.fields) |f| {
         const this_tag_value = @field(EnumTag, f.name);
         if (tag_int == @enumToInt(this_tag_value)) {
             return this_tag_value;

From 0f0f005e927be5e9d813e40c8c6c6580fd4f5697 Mon Sep 17 00:00:00 2001
From: Martin Wickham 
Date: Sat, 4 Jun 2022 17:34:21 -0500
Subject: [PATCH 025/725] std.windows: use posix semantics to delete files, if
 available

Justification: When a file is deleted on Windows, it may not be
immediately removed from the directory. This can cause problems
with future scans of that directory, which will see the partially
deleted file. Under some workloads and system configurations,
Windows files may appear to be deleted immediately.

This is the PR with requested fixup. Thanks to @SpexGuy for the
original PR.
---
 lib/std/os/windows.zig | 75 +++++++++++++++++++++++++++++++-----------
 1 file changed, 56 insertions(+), 19 deletions(-)

diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig
index 22ffc850e6..37ec7f09a3 100644
--- a/lib/std/os/windows.zig
+++ b/lib/std/os/windows.zig
@@ -937,25 +937,50 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil
         .DELETE_PENDING => return,
         else => return unexpectedStatus(rc),
     }
-    var file_dispo = FILE_DISPOSITION_INFORMATION{
-        .DeleteFile = TRUE,
-    };
-    rc = ntdll.NtSetInformationFile(
-        tmp_handle,
-        &io,
-        &file_dispo,
-        @sizeOf(FILE_DISPOSITION_INFORMATION),
-        .FileDispositionInformation,
-    );
-    CloseHandle(tmp_handle);
-    switch (rc) {
-        .SUCCESS => return,
-        .DIRECTORY_NOT_EMPTY => return error.DirNotEmpty,
-        .INVALID_PARAMETER => unreachable,
-        .CANNOT_DELETE => return error.AccessDenied,
-        .MEDIA_WRITE_PROTECTED => return error.AccessDenied,
-        .ACCESS_DENIED => return error.AccessDenied,
-        else => return unexpectedStatus(rc),
+    defer CloseHandle(tmp_handle);
+    if (comptime builtin.target.os.version_range.windows.min.isAtLeast(.win10_rs1)) {
+        // Deletion with posix semantics.
+        var info = FILE_DISPOSITION_INFORMATION_EX{
+            .Flags = FILE_DISPOSITION_DELETE |
+                FILE_DISPOSITION_POSIX_SEMANTICS |
+                FILE_DISPOSITION_ON_CLOSE |
+                FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE,
+        };
+
+        rc = ntdll.NtSetInformationFile(
+            tmp_handle,
+            &io,
+            &info,
+            @sizeOf(FILE_DISPOSITION_INFORMATION_EX),
+            .FileDispositionInformationEx,
+        );
+        switch (rc) {
+            .SUCCESS => {},
+            .CANNOT_DELETE => return error.FileBusy, // file is currently mapped
+            else => return unexpectedStatus(rc),
+        }
+    } else {
+        // Deletion with file pending semantics, which requires waiting or moving
+        // files to get them removed (from here).
+        var file_dispo = FILE_DISPOSITION_INFORMATION{
+            .DeleteFile = TRUE,
+        };
+        rc = ntdll.NtSetInformationFile(
+            tmp_handle,
+            &io,
+            &file_dispo,
+            @sizeOf(FILE_DISPOSITION_INFORMATION),
+            .FileDispositionInformation,
+        );
+        switch (rc) {
+            .SUCCESS => {},
+            .DIRECTORY_NOT_EMPTY => return error.DirNotEmpty,
+            .INVALID_PARAMETER => unreachable,
+            .CANNOT_DELETE => return error.AccessDenied,
+            .MEDIA_WRITE_PROTECTED => return error.AccessDenied,
+            .ACCESS_DENIED => return error.AccessDenied,
+            else => return unexpectedStatus(rc),
+        }
     }
 }
 
@@ -2397,6 +2422,18 @@ pub const FILE_NAME_INFORMATION = extern struct {
     FileName: [1]WCHAR,
 };
 
+pub const FILE_DISPOSITION_INFORMATION_EX = extern struct {
+    /// combination of FILE_DISPOSITION_* flags
+    Flags: ULONG,
+};
+
+const FILE_DISPOSITION_DO_NOT_DELETE: ULONG = 0x00000000;
+const FILE_DISPOSITION_DELETE: ULONG = 0x00000001;
+const FILE_DISPOSITION_POSIX_SEMANTICS: ULONG = 0x00000002;
+const FILE_DISPOSITION_FORCE_IMAGE_SECTION_CHECK: ULONG = 0x00000004;
+const FILE_DISPOSITION_ON_CLOSE: ULONG = 0x00000008;
+const FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE: ULONG = 0x00000010;
+
 pub const FILE_RENAME_INFORMATION = extern struct {
     ReplaceIfExists: BOOLEAN,
     RootDirectory: ?HANDLE,

From be50dbf1ce03f5dc5deef86882ab3505363b683a Mon Sep 17 00:00:00 2001
From: Jan Philipp Hafer 
Date: Mon, 1 May 2023 12:45:41 +0200
Subject: [PATCH 026/725] apply suggestion by user @xEgoist

FILE_DISPOSITION_ON_CLOSE is used to set/clear the FILE_DELETE_ON_CLOSE,
but we do not use that anymore and FILE_DISPOSITION_POSIX_SEMANTICS
already implies unmapping of the handle and removal fo it on close.
---
 lib/std/os/windows.zig | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig
index 37ec7f09a3..e6ce95713c 100644
--- a/lib/std/os/windows.zig
+++ b/lib/std/os/windows.zig
@@ -943,7 +943,6 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil
         var info = FILE_DISPOSITION_INFORMATION_EX{
             .Flags = FILE_DISPOSITION_DELETE |
                 FILE_DISPOSITION_POSIX_SEMANTICS |
-                FILE_DISPOSITION_ON_CLOSE |
                 FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE,
         };
 

From 7594d2c0977497c81db0c394f775832689bad492 Mon Sep 17 00:00:00 2001
From: Jan Philipp Hafer 
Date: Mon, 1 May 2023 18:22:28 +0200
Subject: [PATCH 027/725] address review by user @squeek502

---
 lib/std/fs/test.zig    | 35 +++++++++++++++++++++++++++--------
 lib/std/os/windows.zig | 25 +++++++++++--------------
 2 files changed, 38 insertions(+), 22 deletions(-)

diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig
index 15c8307f58..660ec0c381 100644
--- a/lib/std/fs/test.zig
+++ b/lib/std/fs/test.zig
@@ -1416,23 +1416,42 @@ test "File.PermissionsUnix" {
     try testing.expect(!permissions_unix.unixHas(.other, .execute));
 }
 
-test "delete a read-only file on windows" {
-    if (builtin.os.tag != .windows) return error.SkipZigTest;
+test "delete a read-only file on windows with file pending semantics" {
+    if (builtin.os.tag != .windows or builtin.target.os.version_range.windows.min.isAtLeast(.win10_rs1))
+        return error.SkipZigTest;
+
+    var tmp = tmpDir(.{});
+    defer tmp.cleanup();
+    {
+        const file = try tmp.dir.createFile("test_file", .{ .read = true });
+        defer file.close();
+        // Create a file and make it read-only
+        const metadata = try file.metadata();
+        var permissions = metadata.permissions();
+        permissions.setReadOnly(true);
+        try file.setPermissions(permissions);
+        try testing.expectError(error.AccessDenied, tmp.dir.deleteFile("test_file"));
+        // Now make the file not read-only
+        permissions.setReadOnly(false);
+        try file.setPermissions(permissions);
+    }
+    try tmp.dir.deleteFile("test_file");
+}
+
+test "delete a read-only file on windows with posix semantis" {
+    if (builtin.os.tag != .windows or !builtin.target.os.version_range.windows.min.isAtLeast(.win10_rs1))
+        return error.SkipZigTest;
 
     var tmp = tmpDir(.{});
     defer tmp.cleanup();
     const file = try tmp.dir.createFile("test_file", .{ .read = true });
+    defer file.close();
     // Create a file and make it read-only
     const metadata = try file.metadata();
     var permissions = metadata.permissions();
     permissions.setReadOnly(true);
     try file.setPermissions(permissions);
-    try testing.expectError(error.AccessDenied, tmp.dir.deleteFile("test_file"));
-    // Now make the file not read-only
-    permissions.setReadOnly(false);
-    try file.setPermissions(permissions);
-    file.close();
-    try tmp.dir.deleteFile("test_file");
+    try tmp.dir.deleteFile("test_file"); // file is unmapped and deleted once last handle closed
 }
 
 test "delete a setAsCwd directory on Windows" {
diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig
index e6ce95713c..ef2643792e 100644
--- a/lib/std/os/windows.zig
+++ b/lib/std/os/windows.zig
@@ -938,6 +938,7 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil
         else => return unexpectedStatus(rc),
     }
     defer CloseHandle(tmp_handle);
+
     if (comptime builtin.target.os.version_range.windows.min.isAtLeast(.win10_rs1)) {
         // Deletion with posix semantics.
         var info = FILE_DISPOSITION_INFORMATION_EX{
@@ -953,17 +954,13 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil
             @sizeOf(FILE_DISPOSITION_INFORMATION_EX),
             .FileDispositionInformationEx,
         );
-        switch (rc) {
-            .SUCCESS => {},
-            .CANNOT_DELETE => return error.FileBusy, // file is currently mapped
-            else => return unexpectedStatus(rc),
-        }
     } else {
         // Deletion with file pending semantics, which requires waiting or moving
         // files to get them removed (from here).
         var file_dispo = FILE_DISPOSITION_INFORMATION{
             .DeleteFile = TRUE,
         };
+
         rc = ntdll.NtSetInformationFile(
             tmp_handle,
             &io,
@@ -971,15 +968,15 @@ pub fn DeleteFile(sub_path_w: []const u16, options: DeleteFileOptions) DeleteFil
             @sizeOf(FILE_DISPOSITION_INFORMATION),
             .FileDispositionInformation,
         );
-        switch (rc) {
-            .SUCCESS => {},
-            .DIRECTORY_NOT_EMPTY => return error.DirNotEmpty,
-            .INVALID_PARAMETER => unreachable,
-            .CANNOT_DELETE => return error.AccessDenied,
-            .MEDIA_WRITE_PROTECTED => return error.AccessDenied,
-            .ACCESS_DENIED => return error.AccessDenied,
-            else => return unexpectedStatus(rc),
-        }
+    }
+    switch (rc) {
+        .SUCCESS => {},
+        .DIRECTORY_NOT_EMPTY => return error.DirNotEmpty,
+        .INVALID_PARAMETER => unreachable,
+        .CANNOT_DELETE => return error.AccessDenied,
+        .MEDIA_WRITE_PROTECTED => return error.AccessDenied,
+        .ACCESS_DENIED => return error.AccessDenied,
+        else => return unexpectedStatus(rc),
     }
 }
 

From 28923474401051a9aa0bddd60904b9be64943dba Mon Sep 17 00:00:00 2001
From: jcalabro 
Date: Mon, 1 May 2023 12:57:05 -0400
Subject: [PATCH 028/725] Fix PBKDF2 docstring comment

---
 lib/std/crypto/pbkdf2.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/std/crypto/pbkdf2.zig b/lib/std/crypto/pbkdf2.zig
index 7c6df5444d..115fd38b3d 100644
--- a/lib/std/crypto/pbkdf2.zig
+++ b/lib/std/crypto/pbkdf2.zig
@@ -49,7 +49,7 @@ const WeakParametersError = std.crypto.errors.WeakParametersError;
 ///         Larger iteration counts improve security by increasing the time required to compute
 ///         the dk. It is common to tune this parameter to achieve approximately 100ms.
 ///
-/// Prf: Pseudo-random function to use. A common choice is `std.crypto.auth.hmac.HmacSha256`.
+/// Prf: Pseudo-random function to use. A common choice is `std.crypto.auth.hmac.sha2.HmacSha256`.
 pub fn pbkdf2(dk: []u8, password: []const u8, salt: []const u8, rounds: u32, comptime Prf: type) (WeakParametersError || OutputTooLongError)!void {
     if (rounds < 1) return error.WeakParameters;
 

From 7c9891d7b7d09060630693231702f27669dd0dc9 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Wed, 26 Apr 2023 22:51:35 -0400
Subject: [PATCH 029/725] x86_64: use std.log for debug logging

---
 src/arch/x86_64/CodeGen.zig | 137 +++++++++++++++++++++++++-----------
 src/print_air.zig           |  12 +++-
 2 files changed, 104 insertions(+), 45 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index be972d7aea..b862252561 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -7,6 +7,8 @@ const leb128 = std.leb;
 const link = @import("../../link.zig");
 const log = std.log.scoped(.codegen);
 const tracking_log = std.log.scoped(.tracking);
+const verbose_tracking_log = std.log.scoped(.verbose_tracking);
+const wip_mir_log = std.log.scoped(.wip_mir);
 const math = std.math;
 const mem = std.mem;
 const trace = @import("../../tracy.zig").trace;
@@ -48,9 +50,6 @@ const sse = abi.RegisterClass.sse;
 
 const InnerError = CodeGenError || error{OutOfRegisters};
 
-const debug_wip_mir = false;
-const debug_tracking = false;
-
 gpa: Allocator,
 air: Air,
 liveness: Liveness,
@@ -575,12 +574,6 @@ pub fn generate(
     assert(fn_owner_decl.has_tv);
     const fn_type = fn_owner_decl.ty;
 
-    if (debug_wip_mir) {
-        const stderr = std.io.getStdErr().writer();
-        fn_owner_decl.renderFullyQualifiedName(mod, stderr) catch {};
-        stderr.writeAll(":\n") catch {};
-    }
-
     const gpa = bin_file.allocator;
     var function = Self{
         .gpa = gpa,
@@ -614,6 +607,8 @@ pub fn generate(
         if (builtin.mode == .Debug) function.mir_to_air_map.deinit(gpa);
     }
 
+    wip_mir_log.debug("{}:", .{function.fmtDecl(module_fn.owner_decl)});
+
     try function.frame_allocs.resize(gpa, FrameIndex.named_count);
     function.frame_allocs.set(
         @enumToInt(FrameIndex.stack_frame),
@@ -715,48 +710,104 @@ pub fn generate(
     }
 }
 
-fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {
-    if (!debug_wip_mir) return;
-    const stderr = std.io.getStdErr().writer();
+const FormatDeclData = struct {
+    mod: *Module,
+    decl_index: Module.Decl.Index,
+};
+fn formatDecl(
+    data: FormatDeclData,
+    comptime _: []const u8,
+    _: std.fmt.FormatOptions,
+    writer: anytype,
+) @TypeOf(writer).Error!void {
+    try data.mod.declPtr(data.decl_index).renderFullyQualifiedName(data.mod, writer);
+}
+fn fmtDecl(self: *Self, decl_index: Module.Decl.Index) std.fmt.Formatter(formatDecl) {
+    return .{ .data = .{
+        .mod = self.bin_file.options.module.?,
+        .decl_index = decl_index,
+    } };
+}
 
+const FormatAirData = struct {
+    self: *Self,
+    inst: Air.Inst.Index,
+};
+fn formatAir(
+    data: FormatAirData,
+    comptime _: []const u8,
+    _: std.fmt.FormatOptions,
+    writer: anytype,
+) @TypeOf(writer).Error!void {
+    @import("../../print_air.zig").dumpInst(
+        data.inst,
+        data.self.bin_file.options.module.?,
+        data.self.air,
+        data.self.liveness,
+    );
+}
+fn fmtAir(self: *Self, inst: Air.Inst.Index) std.fmt.Formatter(formatAir) {
+    return .{ .data = .{ .self = self, .inst = inst } };
+}
+
+const FormatWipMirData = struct {
+    self: *Self,
+    inst: Mir.Inst.Index,
+};
+fn formatWipMir(
+    data: FormatWipMirData,
+    comptime _: []const u8,
+    _: std.fmt.FormatOptions,
+    writer: anytype,
+) @TypeOf(writer).Error!void {
     var lower = Lower{
-        .allocator = self.gpa,
+        .allocator = data.self.gpa,
         .mir = .{
-            .instructions = self.mir_instructions.slice(),
-            .extra = self.mir_extra.items,
+            .instructions = data.self.mir_instructions.slice(),
+            .extra = data.self.mir_extra.items,
             .frame_locs = (std.MultiArrayList(Mir.FrameLoc){}).slice(),
         },
-        .target = self.target,
-        .src_loc = self.src_loc,
+        .target = data.self.target,
+        .src_loc = data.self.src_loc,
     };
-    for (lower.lowerMir(inst) catch |err| switch (err) {
+    for (lower.lowerMir(data.self.mir_instructions.get(data.inst)) catch |err| switch (err) {
         error.LowerFail => {
             defer {
-                lower.err_msg.?.deinit(self.gpa);
+                lower.err_msg.?.deinit(data.self.gpa);
                 lower.err_msg = null;
             }
-            try stderr.print("{s}\n", .{lower.err_msg.?.msg});
+            try writer.writeAll(lower.err_msg.?.msg);
             return;
         },
-        error.InvalidInstruction, error.CannotEncode => |e| {
-            try stderr.writeAll(switch (e) {
-                error.InvalidInstruction => "CodeGen failed to find a viable instruction.\n",
-                error.CannotEncode => "CodeGen failed to encode the instruction.\n",
+        error.OutOfMemory, error.InvalidInstruction, error.CannotEncode => |e| {
+            try writer.writeAll(switch (e) {
+                error.OutOfMemory => "Out of memory",
+                error.InvalidInstruction => "CodeGen failed to find a viable instruction.",
+                error.CannotEncode => "CodeGen failed to encode the instruction.",
             });
             return;
         },
         else => |e| return e,
-    }) |lower_inst| {
-        try stderr.print("  | {}\n", .{lower_inst});
-    }
+    }) |lower_inst| try writer.print("  | {}", .{lower_inst});
+}
+fn fmtWipMir(self: *Self, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {
+    return .{ .data = .{ .self = self, .inst = inst } };
 }
 
-fn dumpTracking(self: *Self) !void {
-    if (!debug_tracking) return;
-    const stderr = std.io.getStdErr().writer();
-
-    var it = self.inst_tracking.iterator();
-    while (it.next()) |entry| try stderr.print("%{d} = {}\n", .{ entry.key_ptr.*, entry.value_ptr.* });
+const FormatTrackingData = struct {
+    self: *Self,
+};
+fn formatTracking(
+    data: FormatTrackingData,
+    comptime _: []const u8,
+    _: std.fmt.FormatOptions,
+    writer: anytype,
+) @TypeOf(writer).Error!void {
+    var it = data.self.inst_tracking.iterator();
+    while (it.next()) |entry| try writer.print("\n%{d} = {}", .{ entry.key_ptr.*, entry.value_ptr.* });
+}
+fn fmtTracking(self: *Self) std.fmt.Formatter(formatTracking) {
+    return .{ .data = .{ .self = self } };
 }
 
 fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
@@ -764,7 +815,14 @@ fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
     try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
     const result_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);
     self.mir_instructions.appendAssumeCapacity(inst);
-    self.dumpWipMir(inst) catch {};
+    switch (inst.tag) {
+        else => wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)}),
+        .dbg_line,
+        .dbg_prologue_end,
+        .dbg_epilogue_begin,
+        .dead,
+        => {},
+    }
     return result_index;
 }
 
@@ -1186,13 +1244,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
         }
 
         if (self.liveness.isUnused(inst) and !self.air.mustLower(inst)) continue;
-        if (debug_wip_mir) @import("../../print_air.zig").dumpInst(
-            inst,
-            self.bin_file.options.module.?,
-            self.air,
-            self.liveness,
-        );
-        self.dumpTracking() catch {};
+        wip_mir_log.debug("{}", .{self.fmtAir(inst)});
+        verbose_tracking_log.debug("{}", .{self.fmtTracking()});
 
         const old_air_bookkeeping = self.air_bookkeeping;
         try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
@@ -1453,7 +1506,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             }
         }
     }
-    self.dumpTracking() catch {};
+    verbose_tracking_log.debug("{}", .{self.fmtTracking()});
 }
 
 fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
diff --git a/src/print_air.zig b/src/print_air.zig
index 2d7995842f..d90d31ec67 100644
--- a/src/print_air.zig
+++ b/src/print_air.zig
@@ -94,14 +94,20 @@ const Writer = struct {
         for (w.air.instructions.items(.tag), 0..) |tag, i| {
             const inst = @intCast(Air.Inst.Index, i);
             switch (tag) {
-                .constant, .const_ty => try w.writeInst(s, inst),
+                .constant, .const_ty => {
+                    try w.writeInst(s, inst);
+                    try s.writeByte('\n');
+                },
                 else => continue,
             }
         }
     }
 
     fn writeBody(w: *Writer, s: anytype, body: []const Air.Inst.Index) @TypeOf(s).Error!void {
-        for (body) |inst| try w.writeInst(s, inst);
+        for (body) |inst| {
+            try w.writeInst(s, inst);
+            try s.writeByte('\n');
+        }
     }
 
     fn writeInst(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
@@ -336,7 +342,7 @@ const Writer = struct {
             .work_group_id,
             => try w.writeWorkDimension(s, inst),
         }
-        try s.writeAll(")\n");
+        try s.writeByte(')');
     }
 
     fn writeBinOp(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {

From 4ec49da5f6a6a8e77cdb66b8f814718bf11fffef Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Fri, 28 Apr 2023 20:39:38 -0400
Subject: [PATCH 030/725] x86_64: implement a bunch of floating point stuff

---
 src/arch/x86_64/CodeGen.zig   | 438 +++++++++++++++++++++++++++-------
 src/arch/x86_64/Encoding.zig  |  27 ++-
 src/arch/x86_64/Lower.zig     |   6 +
 src/arch/x86_64/Mir.zig       |  12 +
 src/arch/x86_64/encoder.zig   |   2 +-
 src/arch/x86_64/encodings.zig |  14 ++
 src/codegen.zig               | 103 ++++++--
 7 files changed, 486 insertions(+), 116 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index b862252561..d9c7298c95 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -1297,9 +1297,10 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .ceil,
             .round,
             .trunc_float,
-            .neg,
             => try self.airUnaryMath(inst),
 
+            .neg => try self.airNeg(inst),
+
             .add_with_overflow => try self.airAddSubWithOverflow(inst),
             .sub_with_overflow => try self.airAddSubWithOverflow(inst),
             .mul_with_overflow => try self.airMulWithOverflow(inst),
@@ -1881,7 +1882,7 @@ pub fn spillRegisters(self: *Self, registers: []const Register) !void {
 /// allocated. A second call to `copyToTmpRegister` may return the same register.
 /// This can have a side effect of spilling instructions to the stack to free up a register.
 fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
-    const reg: Register = try self.register_manager.allocReg(null, try self.regClassForType(ty));
+    const reg = try self.register_manager.allocReg(null, try self.regClassForType(ty));
     try self.genSetReg(reg, ty, mcv);
     return reg;
 }
@@ -1924,16 +1925,48 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
 
 fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    _ = ty_op;
-    return self.fail("TODO implement airFptrunc for {}", .{self.target.cpu.arch});
-    // return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
+    const dst_ty = self.air.typeOfIndex(inst);
+    const src_ty = self.air.typeOf(ty_op.operand);
+    if (dst_ty.floatBits(self.target.*) != 32 or src_ty.floatBits(self.target.*) != 64 or
+        !Target.x86.featureSetHas(self.target.cpu.features, .sse2))
+        return self.fail("TODO implement airFptrunc from {} to {}", .{
+            src_ty.fmt(self.bin_file.options.module.?),
+            dst_ty.fmt(self.bin_file.options.module.?),
+        });
+
+    const src_mcv = try self.resolveInst(ty_op.operand);
+    const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+        src_mcv
+    else
+        try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
+    const dst_lock = self.register_manager.lockReg(dst_mcv.register);
+    defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+
+    try self.genBinOpMir(.cvtsd2ss, src_ty, dst_mcv, src_mcv);
+    return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
 }
 
 fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    _ = ty_op;
-    return self.fail("TODO implement airFpext for {}", .{self.target.cpu.arch});
-    // return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
+    const dst_ty = self.air.typeOfIndex(inst);
+    const src_ty = self.air.typeOf(ty_op.operand);
+    if (dst_ty.floatBits(self.target.*) != 64 or src_ty.floatBits(self.target.*) != 32 or
+        !Target.x86.featureSetHas(self.target.cpu.features, .sse2))
+        return self.fail("TODO implement airFpext from {} to {}", .{
+            src_ty.fmt(self.bin_file.options.module.?),
+            dst_ty.fmt(self.bin_file.options.module.?),
+        });
+
+    const src_mcv = try self.resolveInst(ty_op.operand);
+    const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
+        src_mcv
+    else
+        try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
+    const dst_lock = self.register_manager.lockReg(dst_mcv.register);
+    defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+
+    try self.genBinOpMir(.cvtss2sd, src_ty, dst_mcv, src_mcv);
+    return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
 }
 
 fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
@@ -3953,10 +3986,65 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
 }
 
+fn airNeg(self: *Self, inst: Air.Inst.Index) !void {
+    const un_op = self.air.instructions.items(.data)[inst].un_op;
+    const ty = self.air.typeOf(un_op);
+    const ty_bits = ty.floatBits(self.target.*);
+
+    var arena = std.heap.ArenaAllocator.init(self.gpa);
+    defer arena.deinit();
+
+    const ExpectedContents = union {
+        f16: Value.Payload.Float_16,
+        f32: Value.Payload.Float_32,
+        f64: Value.Payload.Float_64,
+        f80: Value.Payload.Float_80,
+        f128: Value.Payload.Float_128,
+    };
+    var stack align(@alignOf(ExpectedContents)) =
+        std.heap.stackFallback(@sizeOf(ExpectedContents), arena.allocator());
+
+    var vec_pl = Type.Payload.Array{
+        .base = .{ .tag = .vector },
+        .data = .{
+            .len = @divExact(128, ty_bits),
+            .elem_type = ty,
+        },
+    };
+    const vec_ty = Type.initPayload(&vec_pl.base);
+
+    var sign_pl = Value.Payload.SubValue{
+        .base = .{ .tag = .repeated },
+        .data = try Value.floatToValue(-0.0, stack.get(), ty, self.target.*),
+    };
+    const sign_val = Value.initPayload(&sign_pl.base);
+
+    const sign_mcv = try self.genTypedValue(.{ .ty = vec_ty, .val = sign_val });
+
+    const src_mcv = try self.resolveInst(un_op);
+    const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
+        src_mcv
+    else
+        try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
+    const dst_lock = self.register_manager.lockReg(dst_mcv.register);
+    defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+
+    try self.genBinOpMir(switch (ty_bits) {
+        32 => .xorps,
+        64 => .xorpd,
+        else => return self.fail("TODO implement airNeg for {}", .{
+            ty.fmt(self.bin_file.options.module.?),
+        }),
+    }, vec_ty, dst_mcv, sign_mcv);
+    return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
+}
+
 fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     _ = un_op;
-    return self.fail("TODO implement airUnaryMath for {}", .{self.target.cpu.arch});
+    return self.fail("TODO implement airUnaryMath for {}", .{
+        self.air.instructions.items(.tag)[inst],
+    });
     //return self.finishAir(inst, result, .{ un_op, .none, .none });
 }
 
@@ -4109,7 +4197,6 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro
 fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
     const elem_ty = self.air.typeOfIndex(inst);
-    const elem_size = elem_ty.abiSize(self.target.*);
     const result: MCValue = result: {
         if (!elem_ty.hasRuntimeBitsIgnoreComptime()) break :result .none;
 
@@ -4117,14 +4204,20 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
         const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
         defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
 
+        const ptr_ty = self.air.typeOf(ty_op.operand);
+        const elem_size = elem_ty.abiSize(self.target.*);
+
+        const elem_rc = try self.regClassForType(elem_ty);
+        const ptr_rc = try self.regClassForType(ptr_ty);
+
         const ptr_mcv = try self.resolveInst(ty_op.operand);
-        const dst_mcv = if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr_mcv))
+        const dst_mcv = if (elem_size <= 8 and elem_rc.supersetOf(ptr_rc) and
+            self.reuseOperand(inst, ty_op.operand, 0, ptr_mcv))
             // The MCValue that holds the pointer can be re-used as the value.
             ptr_mcv
         else
             try self.allocRegOrMem(inst, true);
 
-        const ptr_ty = self.air.typeOf(ty_op.operand);
         if (ptr_ty.ptrInfo().data.host_size > 0) {
             try self.packedLoad(dst_mcv, ptr_ty, ptr_mcv);
         } else {
@@ -4346,17 +4439,9 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
 
         switch (src_mcv) {
             .load_frame => |frame_addr| {
-                const field_abi_size = @intCast(u32, field_ty.abiSize(self.target.*));
-                const limb_abi_size = @min(field_abi_size, 8);
-                const limb_abi_bits = limb_abi_size * 8;
-                const field_byte_off = @intCast(i32, field_off / limb_abi_bits * limb_abi_size);
-                const field_bit_off = field_off % limb_abi_bits;
-
-                if (field_bit_off == 0) {
-                    const off_mcv = MCValue{ .load_frame = .{
-                        .index = frame_addr.index,
-                        .off = frame_addr.off + field_byte_off,
-                    } };
+                if (field_off % 8 == 0) {
+                    const off_mcv =
+                        src_mcv.address().offset(@intCast(i32, @divExact(field_off, 8))).deref();
                     if (self.reuseOperand(inst, operand, 0, src_mcv)) break :result off_mcv;
 
                     const dst_mcv = try self.allocRegOrMem(inst, true);
@@ -4364,6 +4449,12 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
                     break :result dst_mcv;
                 }
 
+                const field_abi_size = @intCast(u32, field_ty.abiSize(self.target.*));
+                const limb_abi_size = @min(field_abi_size, 8);
+                const limb_abi_bits = limb_abi_size * 8;
+                const field_byte_off = @intCast(i32, field_off / limb_abi_bits * limb_abi_size);
+                const field_bit_off = field_off % limb_abi_bits;
+
                 if (field_abi_size > 8) {
                     return self.fail("TODO implement struct_field_val with large packed field", .{});
                 }
@@ -5181,24 +5272,69 @@ fn genBinOp(
     switch (tag) {
         .add,
         .addwrap,
-        => try self.genBinOpMir(switch (lhs_ty.tag()) {
+        => try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
             else => .add,
-            .f32 => .addss,
-            .f64 => .addsd,
+            .Float => switch (lhs_ty.floatBits(self.target.*)) {
+                32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
+                    .addss
+                else
+                    return self.fail("TODO implement genBinOp for {s} {} without sse", .{
+                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                    }),
+                64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
+                    .addsd
+                else
+                    return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
+                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                    }),
+                else => return self.fail("TODO implement genBinOp for {s} {}", .{
+                    @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                }),
+            },
         }, lhs_ty, dst_mcv, src_mcv),
 
         .sub,
         .subwrap,
-        => try self.genBinOpMir(switch (lhs_ty.tag()) {
+        => try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
             else => .sub,
-            .f32 => .subss,
-            .f64 => .subsd,
+            .Float => switch (lhs_ty.floatBits(self.target.*)) {
+                32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
+                    .subss
+                else
+                    return self.fail("TODO implement genBinOp for {s} {} without sse", .{
+                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                    }),
+                64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
+                    .subsd
+                else
+                    return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
+                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                    }),
+                else => return self.fail("TODO implement genBinOp for {s} {}", .{
+                    @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                }),
+            },
         }, lhs_ty, dst_mcv, src_mcv),
 
-        .mul => try self.genBinOpMir(switch (lhs_ty.tag()) {
-            .f32 => .mulss,
-            .f64 => .mulsd,
+        .mul => try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
             else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
+            .Float => switch (lhs_ty.floatBits(self.target.*)) {
+                32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
+                    .mulss
+                else
+                    return self.fail("TODO implement genBinOp for {s} {} without sse", .{
+                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                    }),
+                64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
+                    .mulsd
+                else
+                    return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
+                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                    }),
+                else => return self.fail("TODO implement genBinOp for {s} {}", .{
+                    @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                }),
+            },
         }, lhs_ty, dst_mcv, src_mcv),
 
         .div_float,
@@ -5206,12 +5342,27 @@ fn genBinOp(
         .div_trunc,
         .div_floor,
         => {
-            try self.genBinOpMir(switch (lhs_ty.tag()) {
-                .f32 => .divss,
-                .f64 => .divsd,
+            try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
                 else => return self.fail("TODO implement genBinOp for {s} {}", .{
                     @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
                 }),
+                .Float => switch (lhs_ty.floatBits(self.target.*)) {
+                    32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
+                        .divss
+                    else
+                        return self.fail("TODO implement genBinOp for {s} {} without sse", .{
+                            @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                        }),
+                    64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
+                        .divsd
+                    else
+                        return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
+                            @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                        }),
+                    else => return self.fail("TODO implement genBinOp for {s} {}", .{
+                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                    }),
+                },
             }, lhs_ty, dst_mcv, src_mcv);
             switch (tag) {
                 .div_float,
@@ -5222,16 +5373,18 @@ fn genBinOp(
                 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse4_1)) {
                     const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
                     const dst_alias = registerAlias(dst_mcv.register, abi_size);
-                    try self.asmRegisterRegisterImmediate(switch (lhs_ty.tag()) {
-                        .f32 => .roundss,
-                        .f64 => .roundsd,
+                    try self.asmRegisterRegisterImmediate(switch (lhs_ty.floatBits(self.target.*)) {
+                        32 => .roundss,
+                        64 => .roundsd,
                         else => unreachable,
                     }, dst_alias, dst_alias, Immediate.u(switch (tag) {
                         .div_trunc => 0b1_0_11,
                         .div_floor => 0b1_0_01,
                         else => unreachable,
                     }));
-                } else return self.fail("TODO implement round without sse4_1", .{}),
+                } else return self.fail("TODO implement genBinOp for {s} {} without sse4_1", .{
+                    @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                }),
                 else => unreachable,
             }
         },
@@ -5453,39 +5606,68 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
                         )),
                     else => unreachable,
                 },
-                .register_offset,
                 .eflags,
+                .register_offset,
                 .memory,
+                .indirect,
                 .load_direct,
                 .lea_direct,
                 .load_got,
                 .lea_got,
                 .load_tlv,
                 .lea_tlv,
+                .load_frame,
                 .lea_frame,
                 => {
-                    assert(abi_size <= 8);
+                    blk: {
+                        return self.asmRegisterMemory(
+                            mir_tag,
+                            registerAlias(dst_reg, abi_size),
+                            Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
+                                .memory => |addr| .{
+                                    .base = .{ .reg = .ds },
+                                    .disp = math.cast(i32, addr) orelse break :blk,
+                                },
+                                .indirect => |reg_off| .{
+                                    .base = .{ .reg = reg_off.reg },
+                                    .disp = reg_off.off,
+                                },
+                                .load_frame => |frame_addr| .{
+                                    .base = .{ .frame = frame_addr.index },
+                                    .disp = frame_addr.off,
+                                },
+                                else => break :blk,
+                            }),
+                        );
+                    }
+
                     const dst_reg_lock = self.register_manager.lockReg(dst_reg);
                     defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock);
 
-                    const reg = try self.copyToTmpRegister(ty, src_mcv);
-                    return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = reg });
-                },
-                .indirect, .load_frame => try self.asmRegisterMemory(
-                    mir_tag,
-                    registerAlias(dst_reg, abi_size),
-                    Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
-                        .indirect => |reg_off| .{
-                            .base = .{ .reg = reg_off.reg },
-                            .disp = reg_off.off,
+                    switch (src_mcv) {
+                        .eflags,
+                        .register_offset,
+                        .lea_direct,
+                        .lea_got,
+                        .lea_tlv,
+                        .lea_frame,
+                        => {
+                            const reg = try self.copyToTmpRegister(ty, src_mcv);
+                            return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = reg });
                         },
-                        .load_frame => |frame_addr| .{
-                            .base = .{ .frame = frame_addr.index },
-                            .disp = frame_addr.off,
+                        .memory,
+                        .load_direct,
+                        .load_got,
+                        .load_tlv,
+                        => {
+                            const addr_reg = try self.copyToTmpRegister(ty, src_mcv.address());
+                            return self.genBinOpMir(mir_tag, ty, dst_mcv, .{
+                                .indirect = .{ .reg = addr_reg },
+                            });
                         },
                         else => unreachable,
-                    }),
-                ),
+                    }
+                },
             }
         },
         .memory, .indirect, .load_got, .load_direct, .load_tlv, .load_frame => {
@@ -6175,10 +6357,25 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
     defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
     const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
-    try self.genBinOpMir(switch (ty.tag()) {
+    try self.genBinOpMir(switch (ty.zigTypeTag()) {
         else => .cmp,
-        .f32 => .ucomiss,
-        .f64 => .ucomisd,
+        .Float => switch (ty.floatBits(self.target.*)) {
+            32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
+                .ucomiss
+            else
+                return self.fail("TODO implement airCmp for {} without sse", .{
+                    ty.fmt(self.bin_file.options.module.?),
+                }),
+            64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
+                .ucomisd
+            else
+                return self.fail("TODO implement airCmp for {} without sse2", .{
+                    ty.fmt(self.bin_file.options.module.?),
+                }),
+            else => return self.fail("TODO implement airCmp for {}", .{
+                ty.fmt(self.bin_file.options.module.?),
+            }),
+        },
     }, ty, dst_mcv, src_mcv);
 
     const signedness = if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned;
@@ -7608,7 +7805,8 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
         const dst_rc = try self.regClassForType(dst_ty);
         const src_rc = try self.regClassForType(src_ty);
         const operand = try self.resolveInst(ty_op.operand);
-        if (dst_rc.eql(src_rc) and self.reuseOperand(inst, ty_op.operand, 0, operand)) break :result operand;
+        if (dst_rc.supersetOf(src_rc) and self.reuseOperand(inst, ty_op.operand, 0, operand))
+            break :result operand;
 
         const operand_lock = switch (operand) {
             .register => |reg| self.register_manager.lockReg(reg),
@@ -7648,9 +7846,59 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
 
 fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    _ = ty_op;
-    return self.fail("TODO implement airIntToFloat for {}", .{self.target.cpu.arch});
-    //return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
+
+    const src_ty = self.air.typeOf(ty_op.operand);
+    const src_bits = @intCast(u32, src_ty.bitSize(self.target.*));
+    const src_signedness =
+        if (src_ty.isAbiInt()) src_ty.intInfo(self.target.*).signedness else .unsigned;
+    const dst_ty = self.air.typeOfIndex(inst);
+
+    const src_size = std.math.divCeil(u32, @max(switch (src_signedness) {
+        .signed => src_bits,
+        .unsigned => src_bits + 1,
+    }, 32), 8) catch unreachable;
+    if (src_size > 8) return self.fail("TODO implement airIntToFloat from {} to {}", .{
+        src_ty.fmt(self.bin_file.options.module.?),
+        dst_ty.fmt(self.bin_file.options.module.?),
+    });
+
+    const src_mcv = try self.resolveInst(ty_op.operand);
+    const src_reg = switch (src_mcv) {
+        .register => |reg| reg,
+        else => try self.copyToTmpRegister(src_ty, src_mcv),
+    };
+    const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
+    defer self.register_manager.unlockReg(src_lock);
+
+    if (src_bits < src_size * 8) try self.truncateRegister(src_ty, src_reg);
+
+    const dst_reg = try self.register_manager.allocReg(inst, try self.regClassForType(dst_ty));
+    const dst_mcv = MCValue{ .register = dst_reg };
+    const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
+    defer self.register_manager.unlockReg(dst_lock);
+
+    try self.asmRegisterRegister(switch (dst_ty.floatBits(self.target.*)) {
+        32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
+            .cvtsi2ss
+        else
+            return self.fail("TODO implement airIntToFloat from {} to {} without sse", .{
+                src_ty.fmt(self.bin_file.options.module.?),
+                dst_ty.fmt(self.bin_file.options.module.?),
+            }),
+        64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
+            .cvtsi2sd
+        else
+            return self.fail("TODO implement airIntToFloat from {} to {} without sse2", .{
+                src_ty.fmt(self.bin_file.options.module.?),
+                dst_ty.fmt(self.bin_file.options.module.?),
+            }),
+        else => return self.fail("TODO implement airIntToFloat from {} to {}", .{
+            src_ty.fmt(self.bin_file.options.module.?),
+            dst_ty.fmt(self.bin_file.options.module.?),
+        }),
+    }, dst_reg.to128(), registerAlias(src_reg, src_size));
+
+    return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
 }
 
 fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
@@ -8717,6 +8965,7 @@ fn resolveCallingConventionValues(
         },
         .C => {
             var param_reg_i: usize = 0;
+            var param_sse_reg_i: usize = 0;
             result.stack_align = 16;
 
             switch (self.target.os.tag) {
@@ -8734,26 +8983,39 @@ fn resolveCallingConventionValues(
                 // TODO: is this even possible for C calling convention?
                 result.return_value = InstTracking.init(.none);
             } else {
-                const ret_reg = abi.getCAbiIntReturnRegs(self.target.*)[0];
-                const ret_ty_size = @intCast(u31, ret_ty.abiSize(self.target.*));
-                if (ret_ty_size <= 8) {
-                    const aliased_reg = registerAlias(ret_reg, ret_ty_size);
-                    result.return_value = .{ .short = .{ .register = aliased_reg }, .long = .none };
-                } else {
-                    const ret_indirect_reg = abi.getCAbiIntParamRegs(self.target.*)[param_reg_i];
-                    param_reg_i += 1;
-                    result.return_value = .{
-                        .short = .{ .indirect = .{ .reg = ret_reg } },
-                        .long = .{ .indirect = .{ .reg = ret_indirect_reg } },
-                    };
+                const classes = switch (self.target.os.tag) {
+                    .windows => &[1]abi.Class{abi.classifyWindows(ret_ty, self.target.*)},
+                    else => mem.sliceTo(&abi.classifySystemV(ret_ty, self.target.*, .ret), .none),
+                };
+                if (classes.len > 1) {
+                    return self.fail("TODO handle multiple classes per type", .{});
                 }
+                const ret_reg = abi.getCAbiIntReturnRegs(self.target.*)[0];
+                result.return_value = switch (classes[0]) {
+                    .integer => InstTracking.init(.{ .register = registerAlias(
+                        ret_reg,
+                        @intCast(u32, ret_ty.abiSize(self.target.*)),
+                    ) }),
+                    .float, .sse => InstTracking.init(.{ .register = .xmm0 }),
+                    .memory => ret: {
+                        const ret_indirect_reg = abi.getCAbiIntParamRegs(self.target.*)[param_reg_i];
+                        param_reg_i += 1;
+                        break :ret .{
+                            .short = .{ .indirect = .{ .reg = ret_reg } },
+                            .long = .{ .indirect = .{ .reg = ret_indirect_reg } },
+                        };
+                    },
+                    else => |class| return self.fail("TODO handle calling convention class {s}", .{
+                        @tagName(class),
+                    }),
+                };
             }
 
             // Input params
             for (param_types, result.args) |ty, *arg| {
                 assert(ty.hasRuntimeBitsIgnoreComptime());
 
-                const classes: []const abi.Class = switch (self.target.os.tag) {
+                const classes = switch (self.target.os.tag) {
                     .windows => &[1]abi.Class{abi.classifyWindows(ty, self.target.*)},
                     else => mem.sliceTo(&abi.classifySystemV(ty, self.target.*, .arg), .none),
                 };
@@ -8761,13 +9023,29 @@ fn resolveCallingConventionValues(
                     return self.fail("TODO handle multiple classes per type", .{});
                 }
                 switch (classes[0]) {
-                    .integer => blk: {
-                        if (param_reg_i >= abi.getCAbiIntParamRegs(self.target.*).len) break :blk;
-                        const param_reg = abi.getCAbiIntParamRegs(self.target.*)[param_reg_i];
+                    .integer => if (param_reg_i < abi.getCAbiIntParamRegs(self.target.*).len) {
+                        arg.* = .{ .register = abi.getCAbiIntParamRegs(self.target.*)[param_reg_i] };
                         param_reg_i += 1;
-                        arg.* = .{ .register = param_reg };
                         continue;
                     },
+                    .float, .sse => switch (self.target.os.tag) {
+                        .windows => if (param_reg_i < 4) {
+                            arg.* = .{ .register = @intToEnum(
+                                Register,
+                                @enumToInt(Register.xmm0) + param_reg_i,
+                            ) };
+                            param_reg_i += 1;
+                            continue;
+                        },
+                        else => if (param_sse_reg_i < 8) {
+                            arg.* = .{ .register = @intToEnum(
+                                Register,
+                                @enumToInt(Register.xmm0) + param_sse_reg_i,
+                            ) };
+                            param_sse_reg_i += 1;
+                            continue;
+                        },
+                    },
                     .memory => {}, // fallthrough
                     else => |class| return self.fail("TODO handle calling convention class {s}", .{
                         @tagName(class),
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index a977af7842..5cb7f7a2d9 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -58,7 +58,7 @@ pub fn findByMnemonic(
     next: for (mnemonic_to_encodings_map[@enumToInt(mnemonic)]) |data| {
         switch (data.mode) {
             .rex => if (!rex_required) continue,
-            .long, .sse2_long => {},
+            .long, .sse_long, .sse2_long => {},
             else => if (rex_required) continue,
         }
         for (input_ops, data.ops) |input_op, data_op|
@@ -90,7 +90,7 @@ pub fn findByOpcode(opc: []const u8, prefixes: struct {
         if (prefixes.rex.w) {
             switch (data.mode) {
                 .short, .fpu, .sse, .sse2, .sse4_1, .none => continue,
-                .long, .sse2_long, .rex => {},
+                .long, .sse_long, .sse2_long, .rex => {},
             }
         } else if (prefixes.rex.present and !prefixes.rex.isSet()) {
             switch (data.mode) {
@@ -138,7 +138,7 @@ pub fn modRmExt(encoding: Encoding) u3 {
 pub fn operandBitSize(encoding: Encoding) u64 {
     switch (encoding.data.mode) {
         .short => return 16,
-        .long, .sse2_long => return 64,
+        .long, .sse_long, .sse2_long => return 64,
         else => {},
     }
     const bit_size: u64 = switch (encoding.data.op_en) {
@@ -163,7 +163,7 @@ pub fn format(
     _ = options;
     _ = fmt;
     switch (encoding.data.mode) {
-        .long, .sse2_long => try writer.writeAll("REX.W + "),
+        .long, .sse_long, .sse2_long => try writer.writeAll("REX.W + "),
         else => {},
     }
 
@@ -269,21 +269,25 @@ pub const Mnemonic = enum {
     // SSE
     addss,
     cmpss,
+    cvtsi2ss,
     divss,
     maxss, minss,
     movss,
     mulss,
     subss,
     ucomiss,
+    xorps,
     // SSE2
     addsd,
     //cmpsd,
+    cvtsd2ss, cvtsi2sd, cvtss2sd,
     divsd,
     maxsd, minsd,
     movq, //movd, movsd,
     mulsd,
     subsd,
     ucomisd,
+    xorpd,
     // SSE4.1
     roundss,
     roundsd,
@@ -318,7 +322,7 @@ pub const Op = enum {
     m,
     moffs,
     sreg,
-    xmm, xmm_m32, xmm_m64,
+    xmm, xmm_m32, xmm_m64, xmm_m128,
     // zig fmt: on
 
     pub fn fromOperand(operand: Instruction.Operand) Op {
@@ -400,7 +404,7 @@ pub const Op = enum {
             .imm32, .imm32s, .eax, .r32, .m32, .rm32, .rel32, .xmm_m32 => 32,
             .imm64, .rax, .r64, .m64, .rm64, .xmm_m64 => 64,
             .m80 => 80,
-            .m128, .xmm => 128,
+            .m128, .xmm, .xmm_m128 => 128,
         };
     }
 
@@ -423,8 +427,8 @@ pub const Op = enum {
             .al, .ax, .eax, .rax,
             .r8, .r16, .r32, .r64,
             .rm8, .rm16, .rm32, .rm64,
-            .xmm, .xmm_m32, .xmm_m64,
-            =>  true,
+            .xmm, .xmm_m32, .xmm_m64, .xmm_m128,
+            => true,
             else => false,
         };
         // zig fmt: on
@@ -449,7 +453,7 @@ pub const Op = enum {
             .rm8, .rm16, .rm32, .rm64,
             .m8, .m16, .m32, .m64, .m80, .m128,
             .m,
-            .xmm_m32, .xmm_m64,
+            .xmm_m32, .xmm_m64, .xmm_m128,
             =>  true,
             else => false,
         };
@@ -470,13 +474,13 @@ pub const Op = enum {
             .r8, .r16, .r32, .r64 => .general_purpose,
             .rm8, .rm16, .rm32, .rm64 => .general_purpose,
             .sreg => .segment,
-            .xmm, .xmm_m32, .xmm_m64 => .floating_point,
+            .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .floating_point,
         };
     }
 
     pub fn isFloatingPointRegister(op: Op) bool {
         return switch (op) {
-            .xmm, .xmm_m32, .xmm_m64 => true,
+            .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => true,
             else => false,
         };
     }
@@ -535,6 +539,7 @@ pub const Mode = enum {
     rex,
     long,
     sse,
+    sse_long,
     sse2,
     sse2_long,
     sse4_1,
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index af0146c6e1..a961100687 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -95,6 +95,7 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
 
         .addss,
         .cmpss,
+        .cvtsi2ss,
         .divss,
         .maxss,
         .minss,
@@ -103,8 +104,12 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
         .roundss,
         .subss,
         .ucomiss,
+        .xorps,
         .addsd,
         .cmpsd,
+        .cvtsd2ss,
+        .cvtsi2sd,
+        .cvtss2sd,
         .divsd,
         .maxsd,
         .minsd,
@@ -113,6 +118,7 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
         .roundsd,
         .subsd,
         .ucomisd,
+        .xorpd,
         => try lower.mirGeneric(inst),
 
         .cmps,
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index c8703373d2..c14338b13d 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -170,6 +170,8 @@ pub const Inst = struct {
         addss,
         /// Compare scalar single-precision floating-point values
         cmpss,
+        /// Convert doubleword integer to scalar single-precision floating-point value
+        cvtsi2ss,
         /// Divide scalar single-precision floating-point values
         divss,
         /// Return maximum single-precision floating-point value
@@ -186,10 +188,18 @@ pub const Inst = struct {
         subss,
         /// Unordered compare scalar single-precision floating-point values
         ucomiss,
+        /// Bitwise logical xor of packed single precision floating-point values
+        xorps,
         /// Add double precision floating point values
         addsd,
         /// Compare scalar double-precision floating-point values
         cmpsd,
+        /// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
+        cvtsd2ss,
+        /// Convert doubleword integer to scalar double-precision floating-point value
+        cvtsi2sd,
+        /// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
+        cvtss2sd,
         /// Divide scalar double-precision floating-point values
         divsd,
         /// Return maximum double-precision floating-point value
@@ -206,6 +216,8 @@ pub const Inst = struct {
         subsd,
         /// Unordered compare scalar double-precision floating-point values
         ucomisd,
+        /// Bitwise logical xor of packed double precision floating-point values
+        xorpd,
 
         /// Compare string operands
         cmps,
diff --git a/src/arch/x86_64/encoder.zig b/src/arch/x86_64/encoder.zig
index 329dfca924..4c900697f5 100644
--- a/src/arch/x86_64/encoder.zig
+++ b/src/arch/x86_64/encoder.zig
@@ -323,7 +323,7 @@ pub const Instruction = struct {
         var rex = Rex{};
         rex.present = inst.encoding.data.mode == .rex;
         switch (inst.encoding.data.mode) {
-            .long, .sse2_long => rex.w = true,
+            .long, .sse_long, .sse2_long => rex.w = true,
             else => {},
         }
 
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index 333bdceea8..ac427c3633 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -834,6 +834,9 @@ pub const table = [_]Entry{
 
     .{ .cmpss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0xf3, 0x0f, 0xc2 }, 0, .sse },
 
+    .{ .cvtsi2ss, .rm, &.{ .xmm, .rm32 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .sse },
+    .{ .cvtsi2ss, .rm, &.{ .xmm, .rm64 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .sse_long },
+
     .{ .divss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .sse },
 
     .{ .maxss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5f }, 0, .sse },
@@ -849,11 +852,20 @@ pub const table = [_]Entry{
 
     .{ .ucomiss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x0f, 0x2e }, 0, .sse },
 
+    .{ .xorps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x57 }, 0, .sse },
+
     // SSE2
     .{ .addsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x58 }, 0, .sse2 },
 
     .{ .cmpsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0xf2, 0x0f, 0xc2 }, 0, .sse2 },
 
+    .{ .cvtsd2ss, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .sse2 },
+
+    .{ .cvtsi2sd, .rm, &.{ .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .sse2 },
+    .{ .cvtsi2sd, .rm, &.{ .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .sse2_long },
+
+    .{ .cvtss2sd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5a }, 0, .sse2 },
+
     .{ .divsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5e }, 0, .sse2 },
 
     .{ .maxsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5f }, 0, .sse2 },
@@ -878,6 +890,8 @@ pub const table = [_]Entry{
 
     .{ .ucomisd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x2e }, 0, .sse2 },
 
+    .{ .xorpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .sse2 },
+
     // SSE4.1
     .{ .roundss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .sse4_1 },
     .{ .roundsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .sse4_1 },
diff --git a/src/codegen.zig b/src/codegen.zig
index bf80a90cc3..0043b38a5b 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -291,6 +291,20 @@ pub fn generateSymbol(
             },
         },
         .Pointer => switch (typed_value.val.tag()) {
+            .null_value => {
+                switch (target.cpu.arch.ptrBitWidth()) {
+                    32 => {
+                        mem.writeInt(u32, try code.addManyAsArray(4), 0, endian);
+                        if (typed_value.ty.isSlice()) try code.appendNTimes(0xaa, 4);
+                    },
+                    64 => {
+                        mem.writeInt(u64, try code.addManyAsArray(8), 0, endian);
+                        if (typed_value.ty.isSlice()) try code.appendNTimes(0xaa, 8);
+                    },
+                    else => unreachable,
+                }
+                return Result.ok;
+            },
             .zero, .one, .int_u64, .int_big_positive => {
                 switch (target.cpu.arch.ptrBitWidth()) {
                     32 => {
@@ -397,30 +411,15 @@ pub fn generateSymbol(
                     },
                 }
             },
-            .elem_ptr => {
-                const elem_ptr = typed_value.val.castTag(.elem_ptr).?.data;
-                const elem_size = typed_value.ty.childType().abiSize(target);
-                const addend = @intCast(u32, elem_ptr.index * elem_size);
-                const array_ptr = elem_ptr.array_ptr;
-
-                switch (array_ptr.tag()) {
-                    .decl_ref => {
-                        const decl_index = array_ptr.castTag(.decl_ref).?.data;
-                        return lowerDeclRef(bin_file, src_loc, typed_value, decl_index, code, debug_output, .{
-                            .parent_atom_index = reloc_info.parent_atom_index,
-                            .addend = (reloc_info.addend orelse 0) + addend,
-                        });
-                    },
-                    else => return Result{
-                        .fail = try ErrorMsg.create(
-                            bin_file.allocator,
-                            src_loc,
-                            "TODO implement generateSymbol for pointer type value: '{s}'",
-                            .{@tagName(typed_value.val.tag())},
-                        ),
-                    },
-                }
-            },
+            .elem_ptr => return lowerParentPtr(
+                bin_file,
+                src_loc,
+                typed_value,
+                typed_value.val,
+                code,
+                debug_output,
+                reloc_info,
+            ),
             else => return Result{
                 .fail = try ErrorMsg.create(
                     bin_file.allocator,
@@ -838,9 +837,62 @@ pub fn generateSymbol(
     }
 }
 
+fn lowerParentPtr(
+    bin_file: *link.File,
+    src_loc: Module.SrcLoc,
+    typed_value: TypedValue,
+    parent_ptr: Value,
+    code: *std.ArrayList(u8),
+    debug_output: DebugInfoOutput,
+    reloc_info: RelocInfo,
+) CodeGenError!Result {
+    const target = bin_file.options.target;
+
+    switch (parent_ptr.tag()) {
+        .elem_ptr => {
+            const elem_ptr = parent_ptr.castTag(.elem_ptr).?.data;
+            return lowerParentPtr(
+                bin_file,
+                src_loc,
+                typed_value,
+                elem_ptr.array_ptr,
+                code,
+                debug_output,
+                reloc_info.offset(@intCast(u32, elem_ptr.index * elem_ptr.elem_ty.abiSize(target))),
+            );
+        },
+        .decl_ref => {
+            const decl_index = parent_ptr.castTag(.decl_ref).?.data;
+            return lowerDeclRef(
+                bin_file,
+                src_loc,
+                typed_value,
+                decl_index,
+                code,
+                debug_output,
+                reloc_info,
+            );
+        },
+        else => |t| {
+            return Result{
+                .fail = try ErrorMsg.create(
+                    bin_file.allocator,
+                    src_loc,
+                    "TODO implement lowerParentPtr for type '{s}'",
+                    .{@tagName(t)},
+                ),
+            };
+        },
+    }
+}
+
 const RelocInfo = struct {
     parent_atom_index: u32,
     addend: ?u32 = null,
+
+    fn offset(ri: RelocInfo, addend: u32) RelocInfo {
+        return .{ .parent_atom_index = ri.parent_atom_index, .addend = (ri.addend orelse 0) + addend };
+    }
 };
 
 fn lowerDeclRef(
@@ -1095,6 +1147,9 @@ pub fn genTypedValue(
             .Slice => {},
             else => {
                 switch (typed_value.val.tag()) {
+                    .null_value => {
+                        return GenResult.mcv(.{ .immediate = 0 });
+                    },
                     .int_u64 => {
                         return GenResult.mcv(.{ .immediate = typed_value.val.toUnsignedInt(target) });
                     },

From 6de457211fe4f261fab6c30f652f833ed2a144b5 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Thu, 27 Apr 2023 03:24:48 -0400
Subject: [PATCH 031/725] behavior: update affected tests for the x86_64
 backend

---
 test/behavior/basic.zig           | 12 ++++++++----
 test/behavior/bugs/13069.zig      |  2 ++
 test/behavior/cast.zig            |  7 +++++--
 test/behavior/enum.zig            |  2 ++
 test/behavior/eval.zig            |  2 ++
 test/behavior/floatop.zig         | 12 ++++++++----
 test/behavior/fn.zig              |  4 ++++
 test/behavior/generics.zig        |  4 ++++
 test/behavior/math.zig            |  2 ++
 test/behavior/maximum_minimum.zig |  4 ++++
 test/behavior/pointers.zig        |  5 ++++-
 test/behavior/struct.zig          |  3 ++-
 test/behavior/switch.zig          |  2 ++
 test/behavior/union.zig           | 14 ++++++++++++++
 test/behavior/vector.zig          |  4 ++++
 15 files changed, 67 insertions(+), 12 deletions(-)

diff --git a/test/behavior/basic.zig b/test/behavior/basic.zig
index 437b1b1373..86fc61c2c9 100644
--- a/test/behavior/basic.zig
+++ b/test/behavior/basic.zig
@@ -363,7 +363,8 @@ fn hereIsAnOpaqueType(ptr: *OpaqueA) *OpaqueA {
 }
 
 test "take address of parameter" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -391,7 +392,8 @@ test "array 2D const double ptr" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
 
     const rect_2d_vertexes = [_][1]f32{
@@ -405,7 +407,8 @@ test "array 2D const double ptr with offset" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
 
     const rect_2d_vertexes = [_][2]f32{
@@ -419,7 +422,8 @@ test "array 3D const double ptr with offset" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
 
     const rect_3d_vertexes = [_][2][2]f32{
diff --git a/test/behavior/bugs/13069.zig b/test/behavior/bugs/13069.zig
index 41c5906ee6..1c2526ef2a 100644
--- a/test/behavior/bugs/13069.zig
+++ b/test/behavior/bugs/13069.zig
@@ -6,6 +6,8 @@ test {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     var opt_x: ?[3]f32 = [_]f32{0.0} ** 3;
 
diff --git a/test/behavior/cast.zig b/test/behavior/cast.zig
index fc364da0b8..3c1d26f284 100644
--- a/test/behavior/cast.zig
+++ b/test/behavior/cast.zig
@@ -95,7 +95,9 @@ test "comptime_int @intToFloat" {
 
 test "@intToFloat" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
+
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
@@ -634,7 +636,8 @@ test "vector casts" {
 }
 
 test "@floatCast cast down" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/enum.zig b/test/behavior/enum.zig
index 316275d6ac..de5c0efb8e 100644
--- a/test/behavior/enum.zig
+++ b/test/behavior/enum.zig
@@ -940,6 +940,8 @@ test "constant enum initialization with differing sizes" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     try test3_1(test3_foo);
     try test3_2(test3_bar);
diff --git a/test/behavior/eval.zig b/test/behavior/eval.zig
index 3566bdacb0..3a10a288e3 100644
--- a/test/behavior/eval.zig
+++ b/test/behavior/eval.zig
@@ -535,6 +535,8 @@ test "static eval list init" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     try expect(static_vec3.data[2] == 1.0);
     try expect(vec3(0.0, 0.0, 3.0).data[2] == 3.0);
diff --git a/test/behavior/floatop.zig b/test/behavior/floatop.zig
index 3fc2249ccc..700b419fce 100644
--- a/test/behavior/floatop.zig
+++ b/test/behavior/floatop.zig
@@ -904,7 +904,8 @@ test "negation f16" {
 }
 
 test "negation f32" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -924,7 +925,8 @@ test "negation f32" {
 }
 
 test "negation f64" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1060,7 +1062,8 @@ test "nan negation f16" {
 }
 
 test "nan negation f32" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1080,7 +1083,8 @@ test "nan negation f32" {
 
 test "nan negation f64" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/fn.zig b/test/behavior/fn.zig
index 4ff5e20378..71b7b36c21 100644
--- a/test/behavior/fn.zig
+++ b/test/behavior/fn.zig
@@ -338,6 +338,8 @@ test "function call with anon list literal" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
@@ -358,6 +360,8 @@ test "function call with anon list literal - 2D" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
diff --git a/test/behavior/generics.zig b/test/behavior/generics.zig
index 0e002b2016..c5168e420b 100644
--- a/test/behavior/generics.zig
+++ b/test/behavior/generics.zig
@@ -59,6 +59,8 @@ test "fn with comptime args" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     try expect(gimmeTheBigOne(1234, 5678) == 5678);
     try expect(shouldCallSameInstance(34, 12) == 34);
@@ -69,6 +71,8 @@ test "anytype params" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     try expect(max_i32(12, 34) == 34);
     try expect(max_f64(1.2, 3.4) == 3.4);
diff --git a/test/behavior/math.zig b/test/behavior/math.zig
index 846f72e301..b7dd6b9287 100644
--- a/test/behavior/math.zig
+++ b/test/behavior/math.zig
@@ -203,6 +203,8 @@ test "float equality" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const x: f64 = 0.012;
     const y: f64 = x + 1.0;
diff --git a/test/behavior/maximum_minimum.zig b/test/behavior/maximum_minimum.zig
index d538e2db65..9a4ae40eef 100644
--- a/test/behavior/maximum_minimum.zig
+++ b/test/behavior/maximum_minimum.zig
@@ -8,6 +8,8 @@ test "@max" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
@@ -54,6 +56,8 @@ test "@min" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
diff --git a/test/behavior/pointers.zig b/test/behavior/pointers.zig
index 016b59b613..0bd8388660 100644
--- a/test/behavior/pointers.zig
+++ b/test/behavior/pointers.zig
@@ -206,7 +206,8 @@ test "allowzero pointer and slice" {
 }
 
 test "assign null directly to C pointer and test null equality" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -345,6 +346,8 @@ test "pointer sentinel with +inf" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
diff --git a/test/behavior/struct.zig b/test/behavior/struct.zig
index c3609e4a0a..e533e34cc3 100644
--- a/test/behavior/struct.zig
+++ b/test/behavior/struct.zig
@@ -744,7 +744,8 @@ var g_foo: S0 = S0.init();
 
 test "packed struct with fp fields" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/switch.zig b/test/behavior/switch.zig
index b69add6fdf..3924d1d6c1 100644
--- a/test/behavior/switch.zig
+++ b/test/behavior/switch.zig
@@ -231,6 +231,8 @@ test "switch prong with variable" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     try switchProngWithVarFn(SwitchProngWithVarEnum{ .One = 13 });
     try switchProngWithVarFn(SwitchProngWithVarEnum{ .Two = 13.0 });
diff --git a/test/behavior/union.zig b/test/behavior/union.zig
index 025eb4f6bd..76c5b09a89 100644
--- a/test/behavior/union.zig
+++ b/test/behavior/union.zig
@@ -14,6 +14,8 @@ test "basic unions with floats" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     var foo = FooWithFloats{ .int = 1 };
     try expect(foo.int == 1);
@@ -29,6 +31,8 @@ test "init union with runtime value - floats" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     var foo: FooWithFloats = undefined;
 
@@ -216,6 +220,8 @@ test "union with specified enum tag" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     try doTest();
     comptime try doTest();
@@ -225,6 +231,8 @@ test "packed union generates correctly aligned type" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const U = packed union {
         f1: *const fn () error{TestUnexpectedResult}!void,
@@ -903,6 +911,8 @@ test "anonymous union literal syntax" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         const Number = union {
@@ -1055,6 +1065,8 @@ test "containers with single-field enums" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         const A = union(enum) { f1 };
@@ -1515,6 +1527,8 @@ test "reinterpreting enum value inside packed union" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const U = packed union {
         tag: enum { a, b },
diff --git a/test/behavior/vector.zig b/test/behavior/vector.zig
index f6521f04c3..01c76310d7 100644
--- a/test/behavior/vector.zig
+++ b/test/behavior/vector.zig
@@ -321,6 +321,7 @@ test "load vector elements via comptime index" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
@@ -342,6 +343,7 @@ test "store vector elements via comptime index" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
@@ -369,6 +371,7 @@ test "load vector elements via runtime index" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
@@ -390,6 +393,7 @@ test "store vector elements via runtime index" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {

From 1fd48815c6e22b266b318f58c6b3c828b20ace80 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Fri, 28 Apr 2023 01:56:18 -0400
Subject: [PATCH 032/725] x86_64: cleanup unneeded code

---
 src/arch/x86_64/CodeGen.zig | 136 ++++++++++++++++--------------------
 1 file changed, 59 insertions(+), 77 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index d9c7298c95..915b36f267 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -6595,35 +6595,26 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
         if (Air.refToIndex(pl_op.operand)) |op_inst| self.processDeath(op_inst);
     }
 
-    const outer_state = try self.saveState();
-    {
-        self.scope_generation += 1;
-        const inner_state = try self.saveState();
+    self.scope_generation += 1;
+    const state = try self.saveState();
 
-        for (liveness_cond_br.then_deaths) |operand| self.processDeath(operand);
-        try self.genBody(then_body);
-        try self.restoreState(inner_state, &.{}, .{
-            .emit_instructions = false,
-            .update_tracking = true,
-            .resurrect = true,
-            .close_scope = true,
-        });
-
-        try self.performReloc(reloc);
-
-        for (liveness_cond_br.else_deaths) |operand| self.processDeath(operand);
-        try self.genBody(else_body);
-        try self.restoreState(inner_state, &.{}, .{
-            .emit_instructions = false,
-            .update_tracking = true,
-            .resurrect = true,
-            .close_scope = true,
-        });
-    }
-    try self.restoreState(outer_state, &.{}, .{
+    for (liveness_cond_br.then_deaths) |operand| self.processDeath(operand);
+    try self.genBody(then_body);
+    try self.restoreState(state, &.{}, .{
         .emit_instructions = false,
-        .update_tracking = false,
-        .resurrect = false,
+        .update_tracking = true,
+        .resurrect = true,
+        .close_scope = true,
+    });
+
+    try self.performReloc(reloc);
+
+    for (liveness_cond_br.else_deaths) |operand| self.processDeath(operand);
+    try self.genBody(else_body);
+    try self.restoreState(state, &.{}, .{
+        .emit_instructions = false,
+        .update_tracking = true,
+        .resurrect = true,
         .close_scope = true,
     });
 
@@ -6996,64 +6987,55 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
         if (Air.refToIndex(pl_op.operand)) |op_inst| self.processDeath(op_inst);
     }
 
-    const outer_state = try self.saveState();
-    {
-        self.scope_generation += 1;
-        const inner_state = try self.saveState();
+    self.scope_generation += 1;
+    const state = try self.saveState();
 
-        while (case_i < switch_br.data.cases_len) : (case_i += 1) {
-            const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
-            const items = @ptrCast(
-                []const Air.Inst.Ref,
-                self.air.extra[case.end..][0..case.data.items_len],
-            );
-            const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];
-            extra_index = case.end + items.len + case_body.len;
+    while (case_i < switch_br.data.cases_len) : (case_i += 1) {
+        const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
+        const items = @ptrCast(
+            []const Air.Inst.Ref,
+            self.air.extra[case.end..][0..case.data.items_len],
+        );
+        const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];
+        extra_index = case.end + items.len + case_body.len;
 
-            var relocs = try self.gpa.alloc(u32, items.len);
-            defer self.gpa.free(relocs);
+        var relocs = try self.gpa.alloc(u32, items.len);
+        defer self.gpa.free(relocs);
 
-            for (items, relocs) |item, *reloc| {
-                try self.spillEflagsIfOccupied();
-                const item_mcv = try self.resolveInst(item);
-                try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);
-                reloc.* = try self.asmJccReloc(undefined, .ne);
-            }
-
-            for (liveness.deaths[case_i]) |operand| self.processDeath(operand);
-
-            try self.genBody(case_body);
-            try self.restoreState(inner_state, &.{}, .{
-                .emit_instructions = false,
-                .update_tracking = true,
-                .resurrect = true,
-                .close_scope = true,
-            });
-
-            for (relocs) |reloc| try self.performReloc(reloc);
+        for (items, relocs) |item, *reloc| {
+            try self.spillEflagsIfOccupied();
+            const item_mcv = try self.resolveInst(item);
+            try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);
+            reloc.* = try self.asmJccReloc(undefined, .ne);
         }
 
-        if (switch_br.data.else_body_len > 0) {
-            const else_body = self.air.extra[extra_index..][0..switch_br.data.else_body_len];
+        for (liveness.deaths[case_i]) |operand| self.processDeath(operand);
 
-            const else_deaths = liveness.deaths.len - 1;
-            for (liveness.deaths[else_deaths]) |operand| self.processDeath(operand);
+        try self.genBody(case_body);
+        try self.restoreState(state, &.{}, .{
+            .emit_instructions = false,
+            .update_tracking = true,
+            .resurrect = true,
+            .close_scope = true,
+        });
 
-            try self.genBody(else_body);
-            try self.restoreState(inner_state, &.{}, .{
-                .emit_instructions = false,
-                .update_tracking = true,
-                .resurrect = true,
-                .close_scope = true,
-            });
-        }
+        for (relocs) |reloc| try self.performReloc(reloc);
+    }
+
+    if (switch_br.data.else_body_len > 0) {
+        const else_body = self.air.extra[extra_index..][0..switch_br.data.else_body_len];
+
+        const else_deaths = liveness.deaths.len - 1;
+        for (liveness.deaths[else_deaths]) |operand| self.processDeath(operand);
+
+        try self.genBody(else_body);
+        try self.restoreState(state, &.{}, .{
+            .emit_instructions = false,
+            .update_tracking = true,
+            .resurrect = true,
+            .close_scope = true,
+        });
     }
-    try self.restoreState(outer_state, &.{}, .{
-        .emit_instructions = false,
-        .update_tracking = false,
-        .resurrect = false,
-        .close_scope = true,
-    });
 
     // We already took care of pl_op.operand earlier, so we're going to pass .none here
     return self.finishAir(inst, .unreach, .{ .none, .none, .none });

From db76ae82607b941373064b05a348950802a21217 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Fri, 28 Apr 2023 01:56:45 -0400
Subject: [PATCH 033/725] x86_64: fix emitting f80 globals

---
 src/codegen.zig           | 22 ++++++++--------------
 test/behavior/floatop.zig |  1 -
 test/behavior/math.zig    |  1 -
 3 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/src/codegen.zig b/src/codegen.zig
index 0043b38a5b..f967566034 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -91,12 +91,10 @@ pub fn generateFunction(
 
 fn writeFloat(comptime F: type, f: F, target: Target, endian: std.builtin.Endian, code: []u8) void {
     _ = target;
-    const Int = @Type(.{ .Int = .{
-        .signedness = .unsigned,
-        .bits = @typeInfo(F).Float.bits,
-    } });
+    const bits = @typeInfo(F).Float.bits;
+    const Int = @Type(.{ .Int = .{ .signedness = .unsigned, .bits = bits } });
     const int = @bitCast(Int, f);
-    mem.writeInt(Int, code[0..@sizeOf(Int)], int, endian);
+    mem.writeInt(Int, code[0..@divExact(bits, 8)], int, endian);
 }
 
 pub fn generateLazySymbol(
@@ -187,18 +185,14 @@ pub fn generateSymbol(
             };
         },
         .Float => {
-            const float_bits = typed_value.ty.floatBits(target);
-            switch (float_bits) {
+            switch (typed_value.ty.floatBits(target)) {
                 16 => writeFloat(f16, typed_value.val.toFloat(f16), target, endian, try code.addManyAsArray(2)),
                 32 => writeFloat(f32, typed_value.val.toFloat(f32), target, endian, try code.addManyAsArray(4)),
                 64 => writeFloat(f64, typed_value.val.toFloat(f64), target, endian, try code.addManyAsArray(8)),
-                80 => return Result{
-                    .fail = try ErrorMsg.create(
-                        bin_file.allocator,
-                        src_loc,
-                        "TODO handle f80 in generateSymbol",
-                        .{},
-                    ),
+                80 => {
+                    writeFloat(f80, typed_value.val.toFloat(f80), target, endian, try code.addManyAsArray(10));
+                    const abi_size = math.cast(usize, typed_value.ty.abiSize(target)) orelse return error.Overflow;
+                    try code.appendNTimes(0, abi_size - 10);
                 },
                 128 => writeFloat(f128, typed_value.val.toFloat(f128), target, endian, try code.addManyAsArray(16)),
                 else => unreachable,
diff --git a/test/behavior/floatop.zig b/test/behavior/floatop.zig
index 700b419fce..f713cd035c 100644
--- a/test/behavior/floatop.zig
+++ b/test/behavior/floatop.zig
@@ -1023,7 +1023,6 @@ test "comptime fixed-width float zero divided by zero produces NaN" {
 
 test "comptime fixed-width float non-zero divided by zero produces signed Inf" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/math.zig b/test/behavior/math.zig
index b7dd6b9287..03a92a83e9 100644
--- a/test/behavior/math.zig
+++ b/test/behavior/math.zig
@@ -1535,7 +1535,6 @@ test "signed zeros are represented properly" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const S = struct {

From 50f96c2949bd7d396eaaf391ffd5fd55e59ebac1 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Fri, 28 Apr 2023 02:24:29 -0400
Subject: [PATCH 034/725] x86_64: fix stack realignment

---
 src/arch/x86_64/CodeGen.zig | 7 ++++---
 test/behavior/align.zig     | 5 -----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 915b36f267..955a0ba843 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -1673,15 +1673,16 @@ fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
     const frame_allocs_slice = self.frame_allocs.slice();
     const frame_size = frame_allocs_slice.items(.abi_size);
     const frame_align = frame_allocs_slice.items(.abi_align);
+
+    const stack_frame_align = &frame_align[@enumToInt(FrameIndex.stack_frame)];
+    stack_frame_align.* = @max(stack_frame_align.*, alloc.abi_align);
+
     for (self.free_frame_indices.keys(), 0..) |frame_index, free_i| {
         const abi_size = frame_size[@enumToInt(frame_index)];
         if (abi_size != alloc.abi_size) continue;
         const abi_align = &frame_align[@enumToInt(frame_index)];
         abi_align.* = @max(abi_align.*, alloc.abi_align);
 
-        const stack_frame_align = &frame_align[@enumToInt(FrameIndex.stack_frame)];
-        stack_frame_align.* = @max(stack_frame_align.*, alloc.abi_align);
-
         _ = self.free_frame_indices.swapRemoveAt(free_i);
         return frame_index;
     }
diff --git a/test/behavior/align.zig b/test/behavior/align.zig
index 7755cdaa7d..bfc9997dd2 100644
--- a/test/behavior/align.zig
+++ b/test/behavior/align.zig
@@ -282,7 +282,6 @@ fn give() anyerror!u128 {
 
 test "page aligned array on stack" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
@@ -491,10 +490,6 @@ test "read 128-bit field from default aligned struct in global memory" {
 }
 
 test "struct field explicit alignment" {
-    if (builtin.zig_backend == .stage2_x86_64) {
-        // Careful enabling this test, fails randomly.
-        return error.SkipZigTest;
-    }
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From aaef5288f886c86e4257c5c67122a0be8a64c134 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Fri, 28 Apr 2023 17:35:17 -0400
Subject: [PATCH 035/725] x86_64: fix 128-bit cmpxchg

---
 src/arch/x86_64/CodeGen.zig | 134 ++++++++++++++++++------------------
 1 file changed, 67 insertions(+), 67 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 955a0ba843..99b3c913a0 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -219,9 +219,9 @@ pub const MCValue = union(enum) {
             .dead,
             .undef,
             .immediate,
+            .eflags,
             .register,
             .register_offset,
-            .eflags,
             .register_overflow,
             .lea_direct,
             .lea_got,
@@ -297,6 +297,41 @@ pub const MCValue = union(enum) {
         };
     }
 
+    fn mem(mcv: MCValue, ptr_size: Memory.PtrSize) Memory {
+        return switch (mcv) {
+            .none,
+            .unreach,
+            .dead,
+            .undef,
+            .immediate,
+            .eflags,
+            .register,
+            .register_offset,
+            .register_overflow,
+            .load_direct,
+            .lea_direct,
+            .load_got,
+            .lea_got,
+            .load_tlv,
+            .lea_tlv,
+            .lea_frame,
+            .reserved_frame,
+            => unreachable,
+            .memory => |addr| if (math.cast(i32, @bitCast(i64, addr))) |small_addr|
+                Memory.sib(ptr_size, .{ .base = .{ .reg = .ds }, .disp = small_addr })
+            else
+                Memory.moffs(.ds, addr),
+            .indirect => |reg_off| Memory.sib(ptr_size, .{
+                .base = .{ .reg = reg_off.reg },
+                .disp = reg_off.off,
+            }),
+            .load_frame => |frame_addr| Memory.sib(ptr_size, .{
+                .base = .{ .frame = frame_addr.index },
+                .disp = frame_addr.off,
+            }),
+        };
+    }
+
     pub fn format(
         mcv: MCValue,
         comptime _: []const u8,
@@ -7936,70 +7971,50 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
     const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
 
     const ptr_ty = self.air.typeOf(extra.ptr);
-    const ptr_mcv = try self.resolveInst(extra.ptr);
     const val_ty = self.air.typeOf(extra.expected_value);
     const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
 
     try self.spillRegisters(&.{ .rax, .rdx, .rbx, .rcx });
     const regs_lock = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdx, .rbx, .rcx });
-    for (regs_lock) |lock| self.register_manager.unlockReg(lock);
+    defer for (regs_lock) |lock| self.register_manager.unlockReg(lock);
 
     const exp_mcv = try self.resolveInst(extra.expected_value);
-    if (val_abi_size > 8) switch (exp_mcv) {
-        .load_frame => |frame_addr| {
-            try self.genSetReg(.rax, Type.usize, .{ .load_frame = .{
-                .index = frame_addr.index,
-                .off = frame_addr.off + 0,
-            } });
-            try self.genSetReg(.rdx, Type.usize, .{ .load_frame = .{
-                .index = frame_addr.index,
-                .off = frame_addr.off + 8,
-            } });
-        },
-        else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),
+    if (val_abi_size > 8) {
+        try self.genSetReg(.rax, Type.usize, exp_mcv);
+        try self.genSetReg(.rdx, Type.usize, exp_mcv.address().offset(8).deref());
     } else try self.genSetReg(.rax, val_ty, exp_mcv);
-    const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
-    defer self.register_manager.unlockReg(rax_lock);
 
     const new_mcv = try self.resolveInst(extra.new_value);
-    const new_reg: Register = if (val_abi_size > 8) switch (new_mcv) {
-        .load_frame => |frame_addr| new: {
-            try self.genSetReg(.rbx, Type.usize, .{ .load_frame = .{
-                .index = frame_addr.index,
-                .off = frame_addr.off + 0,
-            } });
-            try self.genSetReg(.rcx, Type.usize, .{ .load_frame = .{
-                .index = frame_addr.index,
-                .off = frame_addr.off + 8,
-            } });
-            break :new undefined;
-        },
-        else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),
+    const new_reg = if (val_abi_size > 8) new: {
+        try self.genSetReg(.rbx, Type.usize, new_mcv);
+        try self.genSetReg(.rcx, Type.usize, new_mcv.address().offset(8).deref());
+        break :new null;
     } else try self.copyToTmpRegister(val_ty, new_mcv);
-    const new_lock = self.register_manager.lockRegAssumeUnused(new_reg);
-    defer self.register_manager.unlockReg(new_lock);
+    const new_lock = if (new_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null;
+    defer if (new_lock) |lock| self.register_manager.unlockReg(lock);
 
+    const ptr_mcv = try self.resolveInst(extra.ptr);
     const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
     const ptr_mem = switch (ptr_mcv) {
-        .register => |reg| Memory.sib(ptr_size, .{ .base = .{ .reg = reg } }),
-        .lea_frame => |frame_addr| Memory.sib(ptr_size, .{
-            .base = .{ .frame = frame_addr.index },
-            .disp = frame_addr.off,
+        .immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(ptr_size),
+        else => Memory.sib(ptr_size, .{
+            .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
         }),
-        else => Memory.sib(ptr_size, .{ .base = .{
-            .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
-        } }),
     };
-    const mem_lock = switch (ptr_mem.base()) {
+    switch (ptr_mem) {
+        .sib, .rip => {},
+        .moffs => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
+    }
+    const ptr_lock = switch (ptr_mem.base()) {
         .none, .frame => null,
         .reg => |reg| self.register_manager.lockReg(reg),
     };
-    defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
+    defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
 
     try self.spillEflagsIfOccupied();
     if (val_abi_size <= 8) {
         _ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
-            .r = registerAlias(new_reg, val_abi_size),
+            .r = registerAlias(new_reg.?, val_abi_size),
             .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
         } } });
     } else {
@@ -8017,24 +8032,9 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
         }
 
         const dst_mcv = try self.allocRegOrMem(inst, false);
-        try self.genSetMem(
-            .{ .frame = dst_mcv.load_frame.index },
-            dst_mcv.load_frame.off + 16,
-            Type.bool,
-            .{ .eflags = .ne },
-        );
-        try self.genSetMem(
-            .{ .frame = dst_mcv.load_frame.index },
-            dst_mcv.load_frame.off + 8,
-            Type.usize,
-            .{ .register = .rdx },
-        );
-        try self.genSetMem(
-            .{ .frame = dst_mcv.load_frame.index },
-            dst_mcv.load_frame.off + 0,
-            Type.usize,
-            .{ .register = .rax },
-        );
+        try self.genCopy(Type.usize, dst_mcv, .{ .register = .rax });
+        try self.genCopy(Type.usize, dst_mcv.address().offset(8).deref(), .{ .register = .rdx });
+        try self.genCopy(Type.bool, dst_mcv.address().offset(16).deref(), .{ .eflags = .ne });
         break :result dst_mcv;
     };
     return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
@@ -8065,15 +8065,15 @@ fn atomicOp(
     const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
     const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
     const ptr_mem = switch (ptr_mcv) {
-        .register => |reg| Memory.sib(ptr_size, .{ .base = .{ .reg = reg } }),
-        .lea_frame => |frame_addr| Memory.sib(ptr_size, .{
-            .base = .{ .frame = frame_addr.index },
-            .disp = frame_addr.off,
+        .immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(ptr_size),
+        else => Memory.sib(ptr_size, .{
+            .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
         }),
-        else => Memory.sib(ptr_size, .{ .base = .{
-            .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
-        } }),
     };
+    switch (ptr_mem) {
+        .sib, .rip => {},
+        .moffs => return self.fail("TODO airCmpxchg with {s}", .{@tagName(ptr_mcv)}),
+    }
     const mem_lock = switch (ptr_mem.base()) {
         .none, .frame => null,
         .reg => |reg| self.register_manager.lockReg(reg),

From 00ae3592e6d054276b61404808d016dc2ef7ced5 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Fri, 28 Apr 2023 19:38:18 -0400
Subject: [PATCH 036/725] test_runner: use const to control verbose output

---
 lib/test_runner.zig | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/lib/test_runner.zig b/lib/test_runner.zig
index 20d7b02e26..f5bb0150a7 100644
--- a/lib/test_runner.zig
+++ b/lib/test_runner.zig
@@ -234,25 +234,33 @@ pub fn log(
 /// work-in-progress backends can handle it.
 pub fn mainSimple() anyerror!void {
     const enable_print = false;
+    const print_all = false;
 
     var passed: u64 = 0;
     var skipped: u64 = 0;
     var failed: u64 = 0;
     const stderr = if (enable_print) std.io.getStdErr() else {};
     for (builtin.test_functions) |test_fn| {
+        if (enable_print and print_all) {
+            stderr.writeAll(test_fn.name) catch {};
+            stderr.writeAll("... ") catch {};
+        }
         test_fn.func() catch |err| {
-            if (enable_print) stderr.writeAll(test_fn.name) catch {};
+            if (enable_print and !print_all) {
+                stderr.writeAll(test_fn.name) catch {};
+                stderr.writeAll("... ") catch {};
+            }
             if (err != error.SkipZigTest) {
-                if (enable_print) stderr.writeAll("... FAIL\n") catch {};
+                if (enable_print) stderr.writeAll("FAIL\n") catch {};
                 failed += 1;
                 if (!enable_print) return err;
                 continue;
             }
-            if (enable_print) stderr.writeAll("... SKIP\n") catch {};
+            if (enable_print) stderr.writeAll("SKIP\n") catch {};
             skipped += 1;
             continue;
         };
-        //if (enable_print) stderr.writeAll("... PASS\n") catch {};
+        if (enable_print and print_all) stderr.writeAll("PASS\n") catch {};
         passed += 1;
     }
     if (enable_print) {

From c3889600424e3720dc07b22397129f82f2111d72 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Fri, 28 Apr 2023 20:02:28 -0400
Subject: [PATCH 037/725] x86_64: fix large not and atomicrmw

---
 src/arch/x86_64/CodeGen.zig | 200 ++++++++++++++++--------------------
 test/behavior/atomics.zig   |   1 -
 test/behavior/math.zig      |   1 -
 3 files changed, 86 insertions(+), 116 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 99b3c913a0..670dc6840c 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -2050,13 +2050,10 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
                         registerAlias(src_reg, min_abi_size),
                     );
                 },
-                .load_frame => |frame_addr| try self.asmRegisterMemory(
+                .memory, .indirect, .load_frame => try self.asmRegisterMemory(
                     tag,
                     dst_alias,
-                    Memory.sib(Memory.PtrSize.fromSize(min_abi_size), .{
-                        .base = .{ .frame = frame_addr.index },
-                        .disp = frame_addr.off,
-                    }),
+                    src_mcv.mem(Memory.PtrSize.fromSize(min_abi_size)),
                 ),
                 else => return self.fail("TODO airIntCast from {s} to {s}", .{
                     @tagName(src_mcv),
@@ -2738,19 +2735,9 @@ fn genIntMulDivOpMir(
     };
     switch (mat_rhs) {
         .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),
-        .indirect, .load_frame => try self.asmMemory(
+        .memory, .indirect, .load_frame => try self.asmMemory(
             tag,
-            Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (mat_rhs) {
-                .indirect => |reg_off| .{
-                    .base = .{ .reg = reg_off.reg },
-                    .disp = reg_off.off,
-                },
-                .load_frame => |frame_addr| .{
-                    .base = .{ .frame = frame_addr.index },
-                    .disp = frame_addr.off,
-                },
-                else => unreachable,
-            }),
+            mat_rhs.mem(Memory.PtrSize.fromSize(abi_size)),
         ),
         else => unreachable,
     }
@@ -4628,9 +4615,6 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
     if (src_ty.zigTypeTag() == .Vector) {
         return self.fail("TODO implement genUnOp for {}", .{src_ty.fmt(self.bin_file.options.module.?)});
     }
-    if (src_ty.abiSize(self.target.*) > 8) {
-        return self.fail("TODO implement genUnOp for {}", .{src_ty.fmt(self.bin_file.options.module.?)});
-    }
 
     switch (src_mcv) {
         .eflags => |cc| switch (tag) {
@@ -4646,13 +4630,13 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
     };
     defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
 
-    const dst_mcv: MCValue = if (maybe_inst) |inst|
-        if (self.reuseOperand(inst, src_air, 0, src_mcv))
-            src_mcv
-        else
-            try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv)
-    else
-        .{ .register = try self.copyToTmpRegister(src_ty, src_mcv) };
+    const dst_mcv: MCValue = dst: {
+        if (maybe_inst) |inst| if (self.reuseOperand(inst, src_air, 0, src_mcv)) break :dst src_mcv;
+
+        const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, maybe_inst, true);
+        try self.genCopy(src_ty, dst_mcv, src_mcv);
+        break :dst dst_mcv;
+    };
     const dst_lock = switch (dst_mcv) {
         .register => |reg| self.register_manager.lockReg(reg),
         else => null,
@@ -4661,19 +4645,33 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
 
     switch (tag) {
         .not => {
+            const limb_abi_size = @intCast(u16, @min(src_ty.abiSize(self.target.*), 8));
             const int_info = if (src_ty.tag() == .bool)
                 std.builtin.Type.Int{ .signedness = .unsigned, .bits = 1 }
             else
                 src_ty.intInfo(self.target.*);
-            const extra_bits = self.regExtraBits(src_ty);
-            if (int_info.signedness == .unsigned and extra_bits > 0) {
-                const mask = (@as(u64, 1) << @intCast(u6, src_ty.bitSize(self.target.*))) - 1;
-                try self.genBinOpMir(.xor, src_ty, dst_mcv, .{ .immediate = mask });
-            } else try self.genUnOpMir(.not, src_ty, dst_mcv);
+            var byte_off: i32 = 0;
+            while (byte_off * 8 < int_info.bits) : (byte_off += limb_abi_size) {
+                var limb_pl = Type.Payload.Bits{
+                    .base = .{ .tag = switch (int_info.signedness) {
+                        .signed => .int_signed,
+                        .unsigned => .int_unsigned,
+                    } },
+                    .data = @intCast(u16, @min(int_info.bits - byte_off * 8, limb_abi_size * 8)),
+                };
+                const limb_ty = Type.initPayload(&limb_pl.base);
+                const limb_mcv = switch (byte_off) {
+                    0 => dst_mcv,
+                    else => dst_mcv.address().offset(byte_off).deref(),
+                };
+
+                if (limb_pl.base.tag == .int_unsigned and self.regExtraBits(limb_ty) > 0) {
+                    const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_pl.data);
+                    try self.genBinOpMir(.xor, limb_ty, limb_mcv, .{ .immediate = mask });
+                } else try self.genUnOpMir(.not, limb_ty, limb_mcv);
+            }
         },
-
         .neg => try self.genUnOpMir(.neg, src_ty, dst_mcv),
-
         else => unreachable,
     }
     return dst_mcv;
@@ -4714,17 +4712,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue
         },
         .indirect, .load_frame => try self.asmMemory(
             mir_tag,
-            Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (dst_mcv) {
-                .indirect => |reg_off| .{
-                    .base = .{ .reg = reg_off.reg },
-                    .disp = reg_off.off,
-                },
-                .load_frame => |frame_addr| .{
-                    .base = .{ .frame = frame_addr.index },
-                    .disp = frame_addr.off,
-                },
-                else => unreachable,
-            }),
+            dst_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
         ),
     }
 }
@@ -8179,12 +8167,9 @@ fn atomicOp(
                             registerAlias(val_reg, cmov_abi_size),
                             cc,
                         ),
-                        .load_frame => |frame_addr| try self.asmCmovccRegisterMemory(
+                        .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
                             registerAlias(tmp_reg, cmov_abi_size),
-                            Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{
-                                .base = .{ .frame = frame_addr.index },
-                                .disp = frame_addr.off,
-                            }),
+                            val_mcv.mem(Memory.PtrSize.fromSize(cmov_abi_size)),
                             cc,
                         ),
                         else => {
@@ -8207,72 +8192,62 @@ fn atomicOp(
         } else {
             try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{
                 .base = ptr_mem.sib.base,
-                .scale_index = ptr_mem.sib.scale_index,
+                .scale_index = ptr_mem.scaleIndex(),
                 .disp = ptr_mem.sib.disp + 0,
             }));
             try self.asmRegisterMemory(.mov, .rdx, Memory.sib(.qword, .{
                 .base = ptr_mem.sib.base,
-                .scale_index = ptr_mem.sib.scale_index,
+                .scale_index = ptr_mem.scaleIndex(),
                 .disp = ptr_mem.sib.disp + 8,
             }));
             const loop = @intCast(u32, self.mir_instructions.len);
-            switch (val_mcv) {
-                .load_frame => |frame_addr| {
-                    const val_lo_mem = Memory.sib(.qword, .{
-                        .base = .{ .frame = frame_addr.index },
-                        .disp = frame_addr.off + 0,
-                    });
-                    const val_hi_mem = Memory.sib(.qword, .{
-                        .base = .{ .frame = frame_addr.index },
-                        .disp = frame_addr.off + 8,
-                    });
-
-                    if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
-                        try self.asmRegisterRegister(.mov, .rbx, .rax);
-                        try self.asmRegisterRegister(.mov, .rcx, .rdx);
-                    }
-                    if (rmw_op) |op| switch (op) {
-                        .Xchg => {
-                            try self.asmRegisterMemory(.mov, .rbx, val_lo_mem);
-                            try self.asmRegisterMemory(.mov, .rcx, val_hi_mem);
-                        },
-                        .Add => {
-                            try self.asmRegisterMemory(.add, .rbx, val_lo_mem);
-                            try self.asmRegisterMemory(.adc, .rcx, val_hi_mem);
-                        },
-                        .Sub => {
-                            try self.asmRegisterMemory(.sub, .rbx, val_lo_mem);
-                            try self.asmRegisterMemory(.sbb, .rcx, val_hi_mem);
-                        },
-                        .And => {
-                            try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);
-                            try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);
-                        },
-                        .Nand => {
-                            try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);
-                            try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);
-                            try self.asmRegister(.not, .rbx);
-                            try self.asmRegister(.not, .rcx);
-                        },
-                        .Or => {
-                            try self.asmRegisterMemory(.@"or", .rbx, val_lo_mem);
-                            try self.asmRegisterMemory(.@"or", .rcx, val_hi_mem);
-                        },
-                        .Xor => {
-                            try self.asmRegisterMemory(.xor, .rbx, val_lo_mem);
-                            try self.asmRegisterMemory(.xor, .rcx, val_hi_mem);
-                        },
-                        else => return self.fail(
-                            "TODO implement x86 atomic loop for large abi {s}",
-                            .{@tagName(op)},
-                        ),
-                    };
-                },
-                else => return self.fail(
-                    "TODO implement x86 atomic loop for large abi {s}",
-                    .{@tagName(val_mcv)},
-                ),
+            const val_mem_mcv: MCValue = switch (val_mcv) {
+                .memory, .indirect, .load_frame => val_mcv,
+                else => .{ .indirect = .{
+                    .reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),
+                } },
+            };
+            const val_lo_mem = val_mem_mcv.mem(.qword);
+            const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword);
+            if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
+                try self.asmRegisterRegister(.mov, .rbx, .rax);
+                try self.asmRegisterRegister(.mov, .rcx, .rdx);
             }
+            if (rmw_op) |op| switch (op) {
+                .Xchg => {
+                    try self.asmRegisterMemory(.mov, .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.mov, .rcx, val_hi_mem);
+                },
+                .Add => {
+                    try self.asmRegisterMemory(.add, .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.adc, .rcx, val_hi_mem);
+                },
+                .Sub => {
+                    try self.asmRegisterMemory(.sub, .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.sbb, .rcx, val_hi_mem);
+                },
+                .And => {
+                    try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);
+                },
+                .Nand => {
+                    try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);
+                    try self.asmRegister(.not, .rbx);
+                    try self.asmRegister(.not, .rcx);
+                },
+                .Or => {
+                    try self.asmRegisterMemory(.@"or", .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.@"or", .rcx, val_hi_mem);
+                },
+                .Xor => {
+                    try self.asmRegisterMemory(.xor, .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.xor, .rcx, val_hi_mem);
+                },
+                else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{
+                    val_ty.fmt(self.bin_file.options.module.?), @tagName(op),
+                }),
+            };
             _ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{
                 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
             } });
@@ -9177,15 +9152,16 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
 }
 
 fn regBitSize(self: *Self, ty: Type) u64 {
+    const abi_size = ty.abiSize(self.target.*);
     return switch (ty.zigTypeTag()) {
-        else => switch (ty.abiSize(self.target.*)) {
+        else => switch (abi_size) {
             1 => 8,
             2 => 16,
             3...4 => 32,
             5...8 => 64,
             else => unreachable,
         },
-        .Float => switch (ty.abiSize(self.target.*)) {
+        .Float => switch (abi_size) {
             1...16 => 128,
             17...32 => 256,
             else => unreachable,
@@ -9197,10 +9173,6 @@ fn regExtraBits(self: *Self, ty: Type) u64 {
     return self.regBitSize(ty) - ty.bitSize(self.target.*);
 }
 
-fn hasAvxSupport(target: Target) bool {
-    return Target.x86.featureSetHasAny(target.cpu.features, .{ .avx, .avx2 });
-}
-
 fn getSymbolIndexForDecl(self: *Self, decl_index: Module.Decl.Index) !u32 {
     if (self.bin_file.cast(link.File.MachO)) |macho_file| {
         const atom = try macho_file.getOrCreateAtomForDecl(decl_index);
diff --git a/test/behavior/atomics.zig b/test/behavior/atomics.zig
index 19afa79683..56854d43d8 100644
--- a/test/behavior/atomics.zig
+++ b/test/behavior/atomics.zig
@@ -112,7 +112,6 @@ test "128-bit cmpxchg" {
     if (!supports_128_bit_atomics) return error.SkipZigTest;
 
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
diff --git a/test/behavior/math.zig b/test/behavior/math.zig
index 03a92a83e9..01b927b913 100644
--- a/test/behavior/math.zig
+++ b/test/behavior/math.zig
@@ -379,7 +379,6 @@ fn testBinaryNot(x: u16) !void {
 
 test "binary not 128-bit" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From 3c2636a83dbb964f80f93131b30222ad3889a7e9 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sat, 29 Apr 2023 17:58:09 -0400
Subject: [PATCH 038/725] x86_64: implement more forms of wide mul with
 overflow

---
 src/arch/x86_64/CodeGen.zig | 263 +++++++++++++++++-------------------
 1 file changed, 122 insertions(+), 141 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 670dc6840c..a09759951d 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -2434,12 +2434,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
 
                 const frame_index =
                     try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
-                try self.genSetFrameTruncatedOverflowCompare(
-                    tuple_ty,
-                    frame_index,
-                    partial_mcv.register,
-                    cc,
-                );
+                try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
                 break :result .{ .load_frame = .{ .index = frame_index } };
             },
             else => unreachable,
@@ -2511,12 +2506,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
 
                 const frame_index =
                     try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
-                try self.genSetFrameTruncatedOverflowCompare(
-                    tuple_ty,
-                    frame_index,
-                    partial_mcv.register,
-                    cc,
-                );
+                try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
                 break :result .{ .load_frame = .{ .index = frame_index } };
             },
             else => unreachable,
@@ -2529,173 +2519,164 @@ fn genSetFrameTruncatedOverflowCompare(
     self: *Self,
     tuple_ty: Type,
     frame_index: FrameIndex,
-    reg: Register,
+    src_mcv: MCValue,
     cc: Condition,
 ) !void {
-    const reg_lock = self.register_manager.lockReg(reg);
-    defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
+    const src_lock = switch (src_mcv) {
+        .register => |reg| self.register_manager.lockReg(reg),
+        else => null,
+    };
+    defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
 
     const ty = tuple_ty.structFieldType(0);
     const int_info = ty.intInfo(self.target.*);
-    const extended_ty = switch (int_info.signedness) {
-        .signed => Type.isize,
-        .unsigned => ty,
+
+    var hi_limb_pl = Type.Payload.Bits{
+        .base = .{ .tag = switch (int_info.signedness) {
+            .signed => .int_signed,
+            .unsigned => .int_unsigned,
+        } },
+        .data = (int_info.bits - 1) % 64 + 1,
     };
+    const hi_limb_ty = Type.initPayload(&hi_limb_pl.base);
+
+    var rest_pl = Type.Payload.Bits{
+        .base = .{ .tag = .int_unsigned },
+        .data = int_info.bits - hi_limb_pl.data,
+    };
+    const rest_ty = Type.initPayload(&rest_pl.base);
 
     const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp);
-    const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
-    defer for (temp_regs_locks) |rreg| {
-        self.register_manager.unlockReg(rreg);
-    };
+    const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
+    defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
 
     const overflow_reg = temp_regs[0];
     try self.asmSetccRegister(overflow_reg.to8(), cc);
 
     const scratch_reg = temp_regs[1];
-    try self.genSetReg(scratch_reg, extended_ty, .{ .register = reg });
-    try self.truncateRegister(ty, scratch_reg);
-    try self.genBinOpMir(
-        .cmp,
-        extended_ty,
-        .{ .register = reg },
-        .{ .register = scratch_reg },
-    );
+    const hi_limb_off = if (int_info.bits <= 64) 0 else (int_info.bits - 1) / 64 * 8;
+    const hi_limb_mcv = if (hi_limb_off > 0)
+        src_mcv.address().offset(int_info.bits / 64 * 8).deref()
+    else
+        src_mcv;
+    try self.genSetReg(scratch_reg, hi_limb_ty, hi_limb_mcv);
+    try self.truncateRegister(hi_limb_ty, scratch_reg);
+    try self.genBinOpMir(.cmp, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
 
     const eq_reg = temp_regs[2];
     try self.asmSetccRegister(eq_reg.to8(), .ne);
-    try self.genBinOpMir(
-        .@"or",
-        Type.u8,
-        .{ .register = overflow_reg },
-        .{ .register = eq_reg },
-    );
+    try self.genBinOpMir(.@"or", Type.u8, .{ .register = overflow_reg }, .{ .register = eq_reg });
 
+    const payload_off = @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*));
+    if (hi_limb_off > 0) try self.genSetMem(.{ .frame = frame_index }, payload_off, rest_ty, src_mcv);
+    try self.genSetMem(
+        .{ .frame = frame_index },
+        payload_off + hi_limb_off,
+        hi_limb_ty,
+        .{ .register = scratch_reg },
+    );
     try self.genSetMem(
         .{ .frame = frame_index },
         @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
         tuple_ty.structFieldType(1),
         .{ .register = overflow_reg.to8() },
     );
-    try self.genSetMem(
-        .{ .frame = frame_index },
-        @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
-        ty,
-        .{ .register = scratch_reg },
-    );
 }
 
 fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
     const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
     const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
-    const result: MCValue = result: {
-        const dst_ty = self.air.typeOf(bin_op.lhs);
-        switch (dst_ty.zigTypeTag()) {
-            .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}),
-            .Int => {
-                try self.spillEflagsIfOccupied();
+    const dst_ty = self.air.typeOf(bin_op.lhs);
+    const result: MCValue = switch (dst_ty.zigTypeTag()) {
+        .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}),
+        .Int => result: {
+            try self.spillEflagsIfOccupied();
+            try self.spillRegisters(&.{ .rax, .rdx });
 
-                const dst_info = dst_ty.intInfo(self.target.*);
-                const cc: Condition = switch (dst_info.signedness) {
-                    .unsigned => .c,
-                    .signed => .o,
+            const dst_info = dst_ty.intInfo(self.target.*);
+            const cc: Condition = switch (dst_info.signedness) {
+                .unsigned => .c,
+                .signed => .o,
+            };
+
+            const lhs_active_bits = self.activeIntBits(bin_op.lhs);
+            const rhs_active_bits = self.activeIntBits(bin_op.rhs);
+            var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) {
+                .signed => .int_signed,
+                .unsigned => .int_unsigned,
+            } }, .data = math.max3(lhs_active_bits, rhs_active_bits, dst_info.bits / 2) };
+            const src_ty = Type.initPayload(&src_pl.base);
+
+            const lhs = try self.resolveInst(bin_op.lhs);
+            const rhs = try self.resolveInst(bin_op.rhs);
+
+            const tuple_ty = self.air.typeOfIndex(inst);
+            const extra_bits = if (dst_info.bits <= 64)
+                self.regExtraBits(dst_ty)
+            else
+                dst_info.bits % 64;
+            const partial_mcv = if (dst_info.signedness == .signed and extra_bits > 0) dst: {
+                const rhs_lock: ?RegisterLock = switch (rhs) {
+                    .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
+                    else => null,
                 };
+                defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
 
-                const tuple_ty = self.air.typeOfIndex(inst);
-                if (dst_info.bits >= 8 and math.isPowerOfTwo(dst_info.bits)) {
-                    var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) {
-                        .signed => .int_signed,
-                        .unsigned => .int_unsigned,
-                    } }, .data = math.max3(
-                        self.activeIntBits(bin_op.lhs),
-                        self.activeIntBits(bin_op.rhs),
-                        dst_info.bits / 2,
-                    ) };
-                    const src_ty = Type.initPayload(&src_pl.base);
+                const dst_reg: Register = blk: {
+                    if (lhs.isRegister()) break :blk lhs.register;
+                    break :blk try self.copyToTmpRegister(dst_ty, lhs);
+                };
+                const dst_mcv = MCValue{ .register = dst_reg };
+                const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
+                defer self.register_manager.unlockReg(dst_reg_lock);
 
-                    try self.spillRegisters(&.{ .rax, .rdx });
-                    const lhs = try self.resolveInst(bin_op.lhs);
-                    const rhs = try self.resolveInst(bin_op.rhs);
+                const rhs_mcv: MCValue = blk: {
+                    if (rhs.isRegister() or rhs.isMemory()) break :blk rhs;
+                    break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) };
+                };
+                const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) {
+                    .register => |reg| self.register_manager.lockReg(reg),
+                    else => null,
+                };
+                defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock);
 
-                    const partial_mcv = try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs);
-                    switch (partial_mcv) {
-                        .register => |reg| {
-                            self.eflags_inst = inst;
-                            break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
-                        },
-                        else => {},
-                    }
+                try self.genIntMulComplexOpMir(Type.isize, dst_mcv, rhs_mcv);
+                break :dst dst_mcv;
+            } else try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs);
 
-                    // For now, this is the only supported multiply that doesn't fit in a register.
-                    assert(dst_info.bits == 128 and src_pl.data == 64);
+            switch (partial_mcv) {
+                .register => |reg| if (extra_bits == 0) {
+                    self.eflags_inst = inst;
+                    break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
+                } else {
                     const frame_index =
                         try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
-                    try self.genSetMem(
-                        .{ .frame = frame_index },
-                        @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
-                        tuple_ty.structFieldType(1),
-                        .{ .immediate = 0 }, // overflow is impossible for 64-bit*64-bit -> 128-bit
-                    );
-                    try self.genSetMem(
-                        .{ .frame = frame_index },
-                        @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
-                        tuple_ty.structFieldType(0),
-                        partial_mcv,
-                    );
+                    try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
                     break :result .{ .load_frame = .{ .index = frame_index } };
-                }
+                },
+                // For now, this is the only supported multiply that doesn't fit in a register.
+                else => assert(dst_info.bits <= 128 and src_pl.data == 64),
+            }
 
-                const dst_reg: Register = dst_reg: {
-                    switch (dst_info.signedness) {
-                        .signed => {
-                            const lhs = try self.resolveInst(bin_op.lhs);
-                            const rhs = try self.resolveInst(bin_op.rhs);
-
-                            const rhs_lock: ?RegisterLock = switch (rhs) {
-                                .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
-                                else => null,
-                            };
-                            defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
-
-                            const dst_reg: Register = blk: {
-                                if (lhs.isRegister()) break :blk lhs.register;
-                                break :blk try self.copyToTmpRegister(dst_ty, lhs);
-                            };
-                            const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
-                            defer self.register_manager.unlockReg(dst_reg_lock);
-
-                            const rhs_mcv: MCValue = blk: {
-                                if (rhs.isRegister() or rhs.isMemory()) break :blk rhs;
-                                break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) };
-                            };
-                            const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) {
-                                .register => |reg| self.register_manager.lockReg(reg),
-                                else => null,
-                            };
-                            defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock);
-
-                            try self.genIntMulComplexOpMir(Type.isize, .{ .register = dst_reg }, rhs_mcv);
-
-                            break :dst_reg dst_reg;
-                        },
-                        .unsigned => {
-                            try self.spillRegisters(&.{ .rax, .rdx });
-
-                            const lhs = try self.resolveInst(bin_op.lhs);
-                            const rhs = try self.resolveInst(bin_op.rhs);
-
-                            const dst_mcv = try self.genMulDivBinOp(.mul, null, dst_ty, dst_ty, lhs, rhs);
-                            break :dst_reg dst_mcv.register;
-                        },
-                    }
-                };
-
-                const frame_index =
-                    try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
-                try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, dst_reg, cc);
-                break :result .{ .load_frame = .{ .index = frame_index } };
-            },
-            else => unreachable,
-        }
+            const frame_index =
+                try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
+            if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
+                try self.genSetMem(
+                    .{ .frame = frame_index },
+                    @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
+                    tuple_ty.structFieldType(0),
+                    partial_mcv,
+                );
+                try self.genSetMem(
+                    .{ .frame = frame_index },
+                    @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
+                    tuple_ty.structFieldType(1),
+                    .{ .immediate = 0 }, // overflow is impossible for 64-bit*64-bit -> 128-bit
+                );
+            } else try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
+            break :result .{ .load_frame = .{ .index = frame_index } };
+        },
+        else => unreachable,
     };
     return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
 }

From c81878978a41f117818ac5b4918cd952f123bad7 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sat, 29 Apr 2023 17:56:48 -0400
Subject: [PATCH 039/725] x86_64: optimize wide mul with overflow

---
 src/arch/x86_64/CodeGen.zig | 61 ++++++++++++++++++++++---------------
 1 file changed, 36 insertions(+), 25 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index a09759951d..7f0d07cf9b 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -2520,7 +2520,7 @@ fn genSetFrameTruncatedOverflowCompare(
     tuple_ty: Type,
     frame_index: FrameIndex,
     src_mcv: MCValue,
-    cc: Condition,
+    overflow_cc: ?Condition,
 ) !void {
     const src_lock = switch (src_mcv) {
         .register => |reg| self.register_manager.lockReg(reg),
@@ -2551,7 +2551,7 @@ fn genSetFrameTruncatedOverflowCompare(
     defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
 
     const overflow_reg = temp_regs[0];
-    try self.asmSetccRegister(overflow_reg.to8(), cc);
+    if (overflow_cc) |cc| try self.asmSetccRegister(overflow_reg.to8(), cc);
 
     const scratch_reg = temp_regs[1];
     const hi_limb_off = if (int_info.bits <= 64) 0 else (int_info.bits - 1) / 64 * 8;
@@ -2564,8 +2564,10 @@ fn genSetFrameTruncatedOverflowCompare(
     try self.genBinOpMir(.cmp, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
 
     const eq_reg = temp_regs[2];
-    try self.asmSetccRegister(eq_reg.to8(), .ne);
-    try self.genBinOpMir(.@"or", Type.u8, .{ .register = overflow_reg }, .{ .register = eq_reg });
+    if (overflow_cc) |_| {
+        try self.asmSetccRegister(eq_reg.to8(), .ne);
+        try self.genBinOpMir(.@"or", Type.u8, .{ .register = overflow_reg }, .{ .register = eq_reg });
+    }
 
     const payload_off = @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*));
     if (hi_limb_off > 0) try self.genSetMem(.{ .frame = frame_index }, payload_off, rest_ty, src_mcv);
@@ -2579,7 +2581,7 @@ fn genSetFrameTruncatedOverflowCompare(
         .{ .frame = frame_index },
         @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
         tuple_ty.structFieldType(1),
-        .{ .register = overflow_reg.to8() },
+        if (overflow_cc) |_| .{ .register = overflow_reg.to8() } else .{ .eflags = .ne },
     );
 }
 
@@ -2654,27 +2656,36 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
                     try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
                     break :result .{ .load_frame = .{ .index = frame_index } };
                 },
-                // For now, this is the only supported multiply that doesn't fit in a register.
-                else => assert(dst_info.bits <= 128 and src_pl.data == 64),
-            }
+                else => {
+                    // For now, this is the only supported multiply that doesn't fit in a register,
+                    // so cc being set is impossible.
 
-            const frame_index =
-                try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
-            if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
-                try self.genSetMem(
-                    .{ .frame = frame_index },
-                    @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
-                    tuple_ty.structFieldType(0),
-                    partial_mcv,
-                );
-                try self.genSetMem(
-                    .{ .frame = frame_index },
-                    @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
-                    tuple_ty.structFieldType(1),
-                    .{ .immediate = 0 }, // overflow is impossible for 64-bit*64-bit -> 128-bit
-                );
-            } else try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
-            break :result .{ .load_frame = .{ .index = frame_index } };
+                    assert(dst_info.bits <= 128 and src_pl.data == 64);
+
+                    const frame_index =
+                        try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
+                    if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
+                        try self.genSetMem(
+                            .{ .frame = frame_index },
+                            @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
+                            tuple_ty.structFieldType(0),
+                            partial_mcv,
+                        );
+                        try self.genSetMem(
+                            .{ .frame = frame_index },
+                            @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
+                            tuple_ty.structFieldType(1),
+                            .{ .immediate = 0 },
+                        );
+                    } else try self.genSetFrameTruncatedOverflowCompare(
+                        tuple_ty,
+                        frame_index,
+                        partial_mcv,
+                        null,
+                    );
+                    break :result .{ .load_frame = .{ .index = frame_index } };
+                },
+            }
         },
         else => unreachable,
     };

From 10a4c2269d110d636e7817677fb50c6f418bff34 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sat, 29 Apr 2023 17:57:23 -0400
Subject: [PATCH 040/725] x86_64: enable normal start/test_runner logic on more
 targets

---
 lib/std/start.zig   | 2 +-
 lib/test_runner.zig | 1 -
 test/src/Cases.zig  | 6 ++++++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/std/start.zig b/lib/std/start.zig
index 45e2e7da43..4177c9ba01 100644
--- a/lib/std/start.zig
+++ b/lib/std/start.zig
@@ -18,7 +18,7 @@ const start_sym_name = if (native_arch.isMIPS()) "__start" else "_start";
 // Until then, we have simplified logic here for self-hosted. TODO remove this once
 // self-hosted is capable enough to handle all of the real start.zig logic.
 pub const simplified_logic =
-    (builtin.zig_backend == .stage2_x86_64 and (builtin.link_libc or builtin.os.tag == .plan9)) or
+    (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .plan9) or
     builtin.zig_backend == .stage2_x86 or
     builtin.zig_backend == .stage2_aarch64 or
     builtin.zig_backend == .stage2_arm or
diff --git a/lib/test_runner.zig b/lib/test_runner.zig
index f5bb0150a7..8e29d90433 100644
--- a/lib/test_runner.zig
+++ b/lib/test_runner.zig
@@ -13,7 +13,6 @@ var fba = std.heap.FixedBufferAllocator.init(&cmdline_buffer);
 
 pub fn main() void {
     if (builtin.zig_backend == .stage2_wasm or
-        (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag != .linux) or
         builtin.zig_backend == .stage2_aarch64)
     {
         return mainSimple() catch @panic("test failure");
diff --git a/test/src/Cases.zig b/test/src/Cases.zig
index defe248fe6..4b023f45b0 100644
--- a/test/src/Cases.zig
+++ b/test/src/Cases.zig
@@ -396,6 +396,12 @@ fn addFromDirInner(
                     // Other backends don't support new liveness format
                     continue;
                 }
+                if (backend == .stage2 and target.getOsTag() == .macos and
+                    target.getCpuArch() == .x86_64 and builtin.cpu.arch == .aarch64)
+                {
+                    // Rosetta has issues with ZLD
+                    continue;
+                }
 
                 const next = ctx.cases.items.len;
                 try ctx.cases.append(.{

From f37ca3fa7370c501c630c53b370fecdeb313e3be Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sat, 29 Apr 2023 19:31:34 -0400
Subject: [PATCH 041/725] link: cleanup lazy alignment

This gets the alignment from the code that creates a lazy symbol instead
of guessing it at every use.
---
 src/arch/x86_64/CodeGen.zig |  6 ------
 src/codegen.zig             |  8 ++++----
 src/link/Coff.zig           | 18 +++++-------------
 src/link/Elf.zig            | 12 ++++--------
 src/link/MachO.zig          | 14 +++++---------
 5 files changed, 18 insertions(+), 40 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 7f0d07cf9b..eee89e9ded 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -6416,7 +6416,6 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
     if (self.bin_file.cast(link.File.Elf)) |elf_file| {
         const atom_index = try elf_file.getOrCreateAtomForLazySymbol(
             .{ .kind = .const_data, .ty = Type.anyerror },
-            4, // dword alignment
         );
         const atom = elf_file.getAtom(atom_index);
         _ = try atom.getOrCreateOffsetTableEntry(elf_file);
@@ -6429,14 +6428,12 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
     } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
         const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
             .{ .kind = .const_data, .ty = Type.anyerror },
-            4, // dword alignment
         );
         const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
     } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
         const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
             .{ .kind = .const_data, .ty = Type.anyerror },
-            4, // dword alignment
         );
         const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
@@ -8504,7 +8501,6 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
     if (self.bin_file.cast(link.File.Elf)) |elf_file| {
         const atom_index = try elf_file.getOrCreateAtomForLazySymbol(
             .{ .kind = .const_data, .ty = Type.anyerror },
-            4, // dword alignment
         );
         const atom = elf_file.getAtom(atom_index);
         _ = try atom.getOrCreateOffsetTableEntry(elf_file);
@@ -8517,14 +8513,12 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
     } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
         const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
             .{ .kind = .const_data, .ty = Type.anyerror },
-            4, // dword alignment
         );
         const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
     } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
         const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
             .{ .kind = .const_data, .ty = Type.anyerror },
-            4, // dword alignment
         );
         const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
diff --git a/src/codegen.zig b/src/codegen.zig
index f967566034..690e96d25c 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -104,7 +104,7 @@ pub fn generateLazySymbol(
     code: *std.ArrayList(u8),
     debug_output: DebugInfoOutput,
     reloc_info: RelocInfo,
-) CodeGenError!Result {
+) CodeGenError!struct { res: Result, alignment: u32 } {
     _ = debug_output;
     _ = reloc_info;
 
@@ -133,13 +133,13 @@ pub fn generateLazySymbol(
             code.appendAssumeCapacity(0);
         }
         mem.writeInt(u32, code.items[offset..][0..4], @intCast(u32, code.items.len), endian);
-        return Result.ok;
-    } else return .{ .fail = try ErrorMsg.create(
+        return .{ .res = Result.ok, .alignment = 4 };
+    } else return .{ .res = .{ .fail = try ErrorMsg.create(
         bin_file.allocator,
         src_loc,
         "TODO implement generateLazySymbol for {s} {}",
         .{ @tagName(lazy_sym.kind), lazy_sym.ty.fmt(mod) },
-    ) };
+    ) }, .alignment = undefined };
 }
 
 pub fn generateSymbol(
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 0af681bb5e..d20d17f2b1 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -145,7 +145,6 @@ const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex,
 const LazySymbolMetadata = struct {
     text_atom: ?Atom.Index = null,
     rdata_atom: ?Atom.Index = null,
-    alignment: u32,
 };
 
 const DeclMetadata = struct {
@@ -1195,13 +1194,11 @@ fn updateLazySymbol(self: *Coff, decl: Module.Decl.OptionalIndex, metadata: Lazy
         link.File.LazySymbol.initDecl(.code, decl, mod),
         atom,
         self.text_section_index.?,
-        metadata.alignment,
     );
     if (metadata.rdata_atom) |atom| try self.updateLazySymbolAtom(
         link.File.LazySymbol.initDecl(.const_data, decl, mod),
         atom,
         self.rdata_section_index.?,
-        metadata.alignment,
     );
 }
 
@@ -1210,7 +1207,6 @@ fn updateLazySymbolAtom(
     sym: link.File.LazySymbol,
     atom_index: Atom.Index,
     section_index: u16,
-    required_alignment: u32,
 ) !void {
     const gpa = self.base.allocator;
     const mod = self.base.options.module.?;
@@ -1238,7 +1234,7 @@ fn updateLazySymbolAtom(
     const res = try codegen.generateLazySymbol(&self.base, src, sym, &code_buffer, .none, .{
         .parent_atom_index = local_sym_index,
     });
-    const code = switch (res) {
+    const code = switch (res.res) {
         .ok => code_buffer.items,
         .fail => |em| {
             log.err("{s}", .{em.msg});
@@ -1252,11 +1248,11 @@ fn updateLazySymbolAtom(
     symbol.section_number = @intToEnum(coff.SectionNumber, section_index + 1);
     symbol.type = .{ .complex_type = .NULL, .base_type = .NULL };
 
-    const vaddr = try self.allocateAtom(atom_index, code_len, required_alignment);
+    const vaddr = try self.allocateAtom(atom_index, code_len, res.alignment);
     errdefer self.freeAtom(atom_index);
 
     log.debug("allocated atom for {s} at 0x{x}", .{ name, vaddr });
-    log.debug("  (required alignment 0x{x})", .{required_alignment});
+    log.debug("  (required alignment 0x{x})", .{res.alignment});
 
     atom.size = code_len;
     symbol.value = vaddr;
@@ -1265,14 +1261,10 @@ fn updateLazySymbolAtom(
     try self.writeAtom(atom_index, code);
 }
 
-pub fn getOrCreateAtomForLazySymbol(
-    self: *Coff,
-    sym: link.File.LazySymbol,
-    alignment: u32,
-) !Atom.Index {
+pub fn getOrCreateAtomForLazySymbol(self: *Coff, sym: link.File.LazySymbol) !Atom.Index {
     const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
     errdefer _ = self.lazy_syms.pop();
-    if (!gop.found_existing) gop.value_ptr.* = .{ .alignment = alignment };
+    if (!gop.found_existing) gop.value_ptr.* = .{};
     const atom = switch (sym.kind) {
         .code => &gop.value_ptr.text_atom,
         .const_data => &gop.value_ptr.rdata_atom,
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 48d952b6cc..b9c113f834 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -67,7 +67,6 @@ const Section = struct {
 const LazySymbolMetadata = struct {
     text_atom: ?Atom.Index = null,
     rodata_atom: ?Atom.Index = null,
-    alignment: u32,
 };
 
 const DeclMetadata = struct {
@@ -2377,10 +2376,10 @@ pub fn freeDecl(self: *Elf, decl_index: Module.Decl.Index) void {
     }
 }
 
-pub fn getOrCreateAtomForLazySymbol(self: *Elf, sym: File.LazySymbol, alignment: u32) !Atom.Index {
+pub fn getOrCreateAtomForLazySymbol(self: *Elf, sym: File.LazySymbol) !Atom.Index {
     const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
     errdefer _ = self.lazy_syms.pop();
-    if (!gop.found_existing) gop.value_ptr.* = .{ .alignment = alignment };
+    if (!gop.found_existing) gop.value_ptr.* = .{};
     const atom = switch (sym.kind) {
         .code => &gop.value_ptr.text_atom,
         .const_data => &gop.value_ptr.rodata_atom,
@@ -2663,13 +2662,11 @@ fn updateLazySymbol(self: *Elf, decl: Module.Decl.OptionalIndex, metadata: LazyS
         File.LazySymbol.initDecl(.code, decl, mod),
         atom,
         self.text_section_index.?,
-        metadata.alignment,
     );
     if (metadata.rodata_atom) |atom| try self.updateLazySymbolAtom(
         File.LazySymbol.initDecl(.const_data, decl, mod),
         atom,
         self.rodata_section_index.?,
-        metadata.alignment,
     );
 }
 
@@ -2678,7 +2675,6 @@ fn updateLazySymbolAtom(
     sym: File.LazySymbol,
     atom_index: Atom.Index,
     shdr_index: u16,
-    required_alignment: u32,
 ) !void {
     const gpa = self.base.allocator;
     const mod = self.base.options.module.?;
@@ -2710,7 +2706,7 @@ fn updateLazySymbolAtom(
     const res = try codegen.generateLazySymbol(&self.base, src, sym, &code_buffer, .none, .{
         .parent_atom_index = local_sym_index,
     });
-    const code = switch (res) {
+    const code = switch (res.res) {
         .ok => code_buffer.items,
         .fail => |em| {
             log.err("{s}", .{em.msg});
@@ -2728,7 +2724,7 @@ fn updateLazySymbolAtom(
         .st_value = 0,
         .st_size = 0,
     };
-    const vaddr = try self.allocateAtom(atom_index, code.len, required_alignment);
+    const vaddr = try self.allocateAtom(atom_index, code.len, res.alignment);
     errdefer self.freeAtom(atom_index);
     log.debug("allocated text block for {s} at 0x{x}", .{ name, vaddr });
 
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 21633dea64..a57742507d 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -238,7 +238,6 @@ const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex,
 const LazySymbolMetadata = struct {
     text_atom: ?Atom.Index = null,
     data_const_atom: ?Atom.Index = null,
-    alignment: u32,
 };
 
 const TlvSymbolTable = std.AutoArrayHashMapUnmanaged(SymbolWithLoc, Atom.Index);
@@ -2043,13 +2042,11 @@ fn updateLazySymbol(self: *MachO, decl: Module.Decl.OptionalIndex, metadata: Laz
         File.LazySymbol.initDecl(.code, decl, mod),
         atom,
         self.text_section_index.?,
-        metadata.alignment,
     );
     if (metadata.data_const_atom) |atom| try self.updateLazySymbolAtom(
         File.LazySymbol.initDecl(.const_data, decl, mod),
         atom,
         self.data_const_section_index.?,
-        metadata.alignment,
     );
 }
 
@@ -2058,7 +2055,6 @@ fn updateLazySymbolAtom(
     sym: File.LazySymbol,
     atom_index: Atom.Index,
     section_index: u8,
-    required_alignment: u32,
 ) !void {
     const gpa = self.base.allocator;
     const mod = self.base.options.module.?;
@@ -2090,7 +2086,7 @@ fn updateLazySymbolAtom(
     const res = try codegen.generateLazySymbol(&self.base, src, sym, &code_buffer, .none, .{
         .parent_atom_index = local_sym_index,
     });
-    const code = switch (res) {
+    const code = switch (res.res) {
         .ok => code_buffer.items,
         .fail => |em| {
             log.err("{s}", .{em.msg});
@@ -2104,11 +2100,11 @@ fn updateLazySymbolAtom(
     symbol.n_sect = section_index + 1;
     symbol.n_desc = 0;
 
-    const vaddr = try self.allocateAtom(atom_index, code.len, required_alignment);
+    const vaddr = try self.allocateAtom(atom_index, code.len, res.alignment);
     errdefer self.freeAtom(atom_index);
 
     log.debug("allocated atom for {s} at 0x{x}", .{ name, vaddr });
-    log.debug("  (required alignment 0x{x}", .{required_alignment});
+    log.debug("  (required alignment 0x{x}", .{res.alignment});
 
     atom.size = code.len;
     symbol.n_value = vaddr;
@@ -2117,10 +2113,10 @@ fn updateLazySymbolAtom(
     try self.writeAtom(atom_index, code);
 }
 
-pub fn getOrCreateAtomForLazySymbol(self: *MachO, sym: File.LazySymbol, alignment: u32) !Atom.Index {
+pub fn getOrCreateAtomForLazySymbol(self: *MachO, sym: File.LazySymbol) !Atom.Index {
     const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
     errdefer _ = self.lazy_syms.pop();
-    if (!gop.found_existing) gop.value_ptr.* = .{ .alignment = alignment };
+    if (!gop.found_existing) gop.value_ptr.* = .{};
     const atom = switch (sym.kind) {
         .code => &gop.value_ptr.text_atom,
         .const_data => &gop.value_ptr.data_const_atom,

From 372bc960b8315ea1ff17b369b0b33485d5e34cfb Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sat, 29 Apr 2023 19:57:44 -0400
Subject: [PATCH 042/725] link: update decl-specific lazy symbols

---
 src/link.zig       |  4 ++--
 src/link/Coff.zig  | 23 ++++++++++++++---------
 src/link/Elf.zig   | 23 ++++++++++++++---------
 src/link/MachO.zig | 17 +++++++++--------
 4 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/src/link.zig b/src/link.zig
index 672a53999f..74e3ca85fc 100644
--- a/src/link.zig
+++ b/src/link.zig
@@ -1120,8 +1120,8 @@ pub const File = struct {
         kind: Kind,
         ty: Type,
 
-        pub fn initDecl(kind: Kind, decl: Module.Decl.OptionalIndex, mod: *Module) LazySymbol {
-            return .{ .kind = kind, .ty = if (decl.unwrap()) |decl_index|
+        pub fn initDecl(kind: Kind, decl: ?Module.Decl.Index, mod: *Module) LazySymbol {
+            return .{ .kind = kind, .ty = if (decl) |decl_index|
                 mod.declPtr(decl_index).val.castTag(.ty).?.data
             else
                 Type.anyerror };
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index d20d17f2b1..deac8d1fd6 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -1136,7 +1136,11 @@ pub fn lowerUnnamedConst(self: *Coff, tv: TypedValue, decl_index: Module.Decl.In
     return atom.getSymbolIndex().?;
 }
 
-pub fn updateDecl(self: *Coff, module: *Module, decl_index: Module.Decl.Index) !void {
+pub fn updateDecl(
+    self: *Coff,
+    module: *Module,
+    decl_index: Module.Decl.Index,
+) link.File.UpdateDeclError!void {
     if (build_options.skip_non_native and builtin.object_format != .coff) {
         @panic("Attempted to compile for object format that was disabled by build configuration");
     }
@@ -1146,6 +1150,8 @@ pub fn updateDecl(self: *Coff, module: *Module, decl_index: Module.Decl.Index) !
     const tracy = trace(@src());
     defer tracy.end();
 
+    try self.updateLazySymbol(decl_index);
+
     const decl = module.declPtr(decl_index);
 
     if (decl.val.tag() == .extern_fn) {
@@ -1188,7 +1194,8 @@ pub fn updateDecl(self: *Coff, module: *Module, decl_index: Module.Decl.Index) !
     return self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-fn updateLazySymbol(self: *Coff, decl: Module.Decl.OptionalIndex, metadata: LazySymbolMetadata) !void {
+fn updateLazySymbol(self: *Coff, decl: ?Module.Decl.Index) !void {
+    const metadata = self.lazy_syms.get(Module.Decl.OptionalIndex.init(decl)) orelse return;
     const mod = self.base.options.module.?;
     if (metadata.text_atom) |atom| try self.updateLazySymbolAtom(
         link.File.LazySymbol.initDecl(.code, decl, mod),
@@ -1402,7 +1409,7 @@ pub fn updateDeclExports(
     module: *Module,
     decl_index: Module.Decl.Index,
     exports: []const *Module.Export,
-) !void {
+) link.File.UpdateDeclExportsError!void {
     if (build_options.skip_non_native and builtin.object_format != .coff) {
         @panic("Attempted to compile for object format that was disabled by build configuration");
     }
@@ -1599,12 +1606,10 @@ pub fn flushModule(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod
 
     // Most lazy symbols can be updated when the corresponding decl is,
     // so we only have to worry about the one without an associated decl.
-    if (self.lazy_syms.get(.none)) |metadata| {
-        self.updateLazySymbol(.none, metadata) catch |err| switch (err) {
-            error.CodegenFail => return error.FlushFailure,
-            else => |e| return e,
-        };
-    }
+    self.updateLazySymbol(null) catch |err| switch (err) {
+        error.CodegenFail => return error.FlushFailure,
+        else => |e| return e,
+    };
 
     const gpa = self.base.allocator;
 
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index b9c113f834..29ae97150e 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -1034,12 +1034,10 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
 
     // Most lazy symbols can be updated when the corresponding decl is,
     // so we only have to worry about the one without an associated decl.
-    if (self.lazy_syms.get(.none)) |metadata| {
-        self.updateLazySymbol(.none, metadata) catch |err| switch (err) {
-            error.CodegenFail => return error.FlushFailure,
-            else => |e| return e,
-        };
-    }
+    self.updateLazySymbol(null) catch |err| switch (err) {
+        error.CodegenFail => return error.FlushFailure,
+        else => |e| return e,
+    };
 
     // TODO This linker code currently assumes there is only 1 compilation unit and it
     // corresponds to the Zig source code.
@@ -2579,7 +2577,11 @@ pub fn updateFunc(self: *Elf, module: *Module, func: *Module.Fn, air: Air, liven
     return self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-pub fn updateDecl(self: *Elf, module: *Module, decl_index: Module.Decl.Index) !void {
+pub fn updateDecl(
+    self: *Elf,
+    module: *Module,
+    decl_index: Module.Decl.Index,
+) File.UpdateDeclError!void {
     if (build_options.skip_non_native and builtin.object_format != .elf) {
         @panic("Attempted to compile for object format that was disabled by build configuration");
     }
@@ -2590,6 +2592,8 @@ pub fn updateDecl(self: *Elf, module: *Module, decl_index: Module.Decl.Index) !v
     const tracy = trace(@src());
     defer tracy.end();
 
+    try self.updateLazySymbol(decl_index);
+
     const decl = module.declPtr(decl_index);
 
     if (decl.val.tag() == .extern_fn) {
@@ -2656,7 +2660,8 @@ pub fn updateDecl(self: *Elf, module: *Module, decl_index: Module.Decl.Index) !v
     return self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-fn updateLazySymbol(self: *Elf, decl: Module.Decl.OptionalIndex, metadata: LazySymbolMetadata) !void {
+fn updateLazySymbol(self: *Elf, decl: ?Module.Decl.Index) !void {
+    const metadata = self.lazy_syms.get(Module.Decl.OptionalIndex.init(decl)) orelse return;
     const mod = self.base.options.module.?;
     if (metadata.text_atom) |atom| try self.updateLazySymbolAtom(
         File.LazySymbol.initDecl(.code, decl, mod),
@@ -2810,7 +2815,7 @@ pub fn updateDeclExports(
     module: *Module,
     decl_index: Module.Decl.Index,
     exports: []const *Module.Export,
-) !void {
+) File.UpdateDeclExportsError!void {
     if (build_options.skip_non_native and builtin.object_format != .elf) {
         @panic("Attempted to compile for object format that was disabled by build configuration");
     }
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index a57742507d..5136bd84a2 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -495,12 +495,10 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
 
     // Most lazy symbols can be updated when the corresponding decl is,
     // so we only have to worry about the one without an associated decl.
-    if (self.lazy_syms.get(.none)) |metadata| {
-        self.updateLazySymbol(.none, metadata) catch |err| switch (err) {
-            error.CodegenFail => return error.FlushFailure,
-            else => |e| return e,
-        };
-    }
+    self.updateLazySymbol(null) catch |err| switch (err) {
+        error.CodegenFail => return error.FlushFailure,
+        else => |e| return e,
+    };
 
     const module = self.base.options.module orelse return error.LinkingWithoutZigSourceUnimplemented;
 
@@ -1962,6 +1960,8 @@ pub fn updateDecl(self: *MachO, module: *Module, decl_index: Module.Decl.Index)
     const tracy = trace(@src());
     defer tracy.end();
 
+    try self.updateLazySymbol(decl_index);
+
     const decl = module.declPtr(decl_index);
 
     if (decl.val.tag() == .extern_fn) {
@@ -2036,7 +2036,8 @@ pub fn updateDecl(self: *MachO, module: *Module, decl_index: Module.Decl.Index)
     try self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-fn updateLazySymbol(self: *MachO, decl: Module.Decl.OptionalIndex, metadata: LazySymbolMetadata) !void {
+fn updateLazySymbol(self: *MachO, decl: ?Module.Decl.Index) !void {
+    const metadata = self.lazy_syms.get(Module.Decl.OptionalIndex.init(decl)) orelse return;
     const mod = self.base.options.module.?;
     if (metadata.text_atom) |atom| try self.updateLazySymbolAtom(
         File.LazySymbol.initDecl(.code, decl, mod),
@@ -2353,7 +2354,7 @@ pub fn updateDeclExports(
     module: *Module,
     decl_index: Module.Decl.Index,
     exports: []const *Module.Export,
-) !void {
+) File.UpdateDeclExportsError!void {
     if (build_options.skip_non_native and builtin.object_format != .macho) {
         @panic("Attempted to compile for object format that was disabled by build configuration");
     }

From 19bd7d12b0186f0e45c77c564251d6355966b2ef Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sun, 30 Apr 2023 07:30:32 -0400
Subject: [PATCH 043/725] x86_64: factor out lazy_sym

---
 src/arch/x86_64/CodeGen.zig | 28 ++++++++++------------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index eee89e9ded..99fb516b45 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -6413,10 +6413,10 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
     const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
     defer self.register_manager.unlockReg(addr_lock);
 
+    const mod = self.bin_file.options.module.?;
+    const lazy_sym = link.File.LazySymbol.initDecl(.const_data, null, mod);
     if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-        const atom_index = try elf_file.getOrCreateAtomForLazySymbol(
-            .{ .kind = .const_data, .ty = Type.anyerror },
-        );
+        const atom_index = try elf_file.getOrCreateAtomForLazySymbol(lazy_sym);
         const atom = elf_file.getAtom(atom_index);
         _ = try atom.getOrCreateOffsetTableEntry(elf_file);
         const got_addr = atom.getOffsetTableAddress(elf_file);
@@ -6426,15 +6426,11 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
             Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
         );
     } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-        const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
-            .{ .kind = .const_data, .ty = Type.anyerror },
-        );
+        const atom_index = try coff_file.getOrCreateAtomForLazySymbol(lazy_sym);
         const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
     } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-        const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
-            .{ .kind = .const_data, .ty = Type.anyerror },
-        );
+        const atom_index = try macho_file.getOrCreateAtomForLazySymbol(lazy_sym);
         const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
     } else {
@@ -8498,10 +8494,10 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
     const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
     defer self.register_manager.unlockReg(addr_lock);
 
+    const mod = self.bin_file.options.module.?;
+    const lazy_sym = link.File.LazySymbol.initDecl(.const_data, null, mod);
     if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-        const atom_index = try elf_file.getOrCreateAtomForLazySymbol(
-            .{ .kind = .const_data, .ty = Type.anyerror },
-        );
+        const atom_index = try elf_file.getOrCreateAtomForLazySymbol(lazy_sym);
         const atom = elf_file.getAtom(atom_index);
         _ = try atom.getOrCreateOffsetTableEntry(elf_file);
         const got_addr = atom.getOffsetTableAddress(elf_file);
@@ -8511,15 +8507,11 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
             Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
         );
     } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-        const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
-            .{ .kind = .const_data, .ty = Type.anyerror },
-        );
+        const atom_index = try coff_file.getOrCreateAtomForLazySymbol(lazy_sym);
         const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
     } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-        const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
-            .{ .kind = .const_data, .ty = Type.anyerror },
-        );
+        const atom_index = try macho_file.getOrCreateAtomForLazySymbol(lazy_sym);
         const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
     } else {

From 0489a63a432b31ce0fd9d6ec6c8e2fb72a920573 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sun, 30 Apr 2023 21:13:29 -0400
Subject: [PATCH 044/725] Sema: use trap for backends that don't support
 panic_fn

Debuggers also catch trap, but the code is not allowed to continue.
---
 src/Sema.zig | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/Sema.zig b/src/Sema.zig
index 71a1215dcd..79f2fd7fca 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -23287,8 +23287,7 @@ fn panicWithMsg(
     const arena = sema.arena;
 
     if (!mod.backendSupportsFeature(.panic_fn)) {
-        _ = try block.addNoOp(.breakpoint);
-        _ = try block.addNoOp(.unreach);
+        _ = try block.addNoOp(.trap);
         return;
     }
     const panic_fn = try sema.getBuiltin("panic");
@@ -23336,8 +23335,7 @@ fn panicUnwrapError(
 
     {
         if (!sema.mod.backendSupportsFeature(.panic_unwrap_error)) {
-            _ = try fail_block.addNoOp(.breakpoint);
-            _ = try fail_block.addNoOp(.unreach);
+            _ = try fail_block.addNoOp(.trap);
         } else {
             const panic_fn = try sema.getBuiltin("panicUnwrapError");
             const err = try fail_block.addTyOp(unwrap_err_tag, Type.anyerror, operand);
@@ -23462,8 +23460,7 @@ fn safetyCheckFormatted(
     defer fail_block.instructions.deinit(gpa);
 
     if (!sema.mod.backendSupportsFeature(.safety_check_formatted)) {
-        _ = try fail_block.addNoOp(.breakpoint);
-        _ = try fail_block.addNoOp(.unreach);
+        _ = try fail_block.addNoOp(.trap);
     } else {
         const panic_fn = try sema.getBuiltin(func);
         _ = try sema.analyzeCall(&fail_block, panic_fn, sema.src, sema.src, .auto, false, args, null);

From 47a34d038dd114533c3fcb130e03249ad846fe9c Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sun, 30 Apr 2023 21:47:19 -0400
Subject: [PATCH 045/725] x86_64: implement tagName

---
 src/arch/x86_64/CodeGen.zig | 292 ++++++++++++++++++++++++++++++++----
 src/codegen.zig             |  40 ++++-
 src/link/Coff.zig           |  33 ++--
 src/link/Elf.zig            |  31 ++--
 src/link/MachO.zig          |  33 ++--
 test/behavior/enum.zig      |   5 -
 test/behavior/memset.zig    |   2 -
 test/behavior/type.zig      |   2 -
 test/behavior/union.zig     |   1 -
 9 files changed, 368 insertions(+), 71 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 99fb516b45..71dcbfa461 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -56,7 +56,10 @@ liveness: Liveness,
 bin_file: *link.File,
 debug_output: DebugInfoOutput,
 target: *const std.Target,
-mod_fn: *const Module.Fn,
+owner: union(enum) {
+    mod_fn: *const Module.Fn,
+    decl: Module.Decl.Index,
+},
 err_msg: ?*ErrorMsg,
 args: []MCValue,
 ret_mcv: InstTracking,
@@ -617,7 +620,7 @@ pub fn generate(
         .target = &bin_file.options.target,
         .bin_file = bin_file,
         .debug_output = debug_output,
-        .mod_fn = module_fn,
+        .owner = .{ .mod_fn = module_fn },
         .err_msg = null,
         .args = undefined, // populated after `resolveCallingConventionValues`
         .ret_mcv = undefined, // populated after `resolveCallingConventionValues`
@@ -745,6 +748,92 @@ pub fn generate(
     }
 }
 
+pub fn generateLazy(
+    bin_file: *link.File,
+    src_loc: Module.SrcLoc,
+    lazy_sym: link.File.LazySymbol,
+    code: *std.ArrayList(u8),
+    debug_output: DebugInfoOutput,
+) CodeGenError!Result {
+    const gpa = bin_file.allocator;
+    var function = Self{
+        .gpa = gpa,
+        .air = undefined,
+        .liveness = undefined,
+        .target = &bin_file.options.target,
+        .bin_file = bin_file,
+        .debug_output = debug_output,
+        .owner = .{ .decl = lazy_sym.ty.getOwnerDecl() },
+        .err_msg = null,
+        .args = undefined,
+        .ret_mcv = undefined,
+        .fn_type = undefined,
+        .arg_index = undefined,
+        .src_loc = src_loc,
+        .end_di_line = undefined, // no debug info yet
+        .end_di_column = undefined, // no debug info yet
+    };
+    defer {
+        function.mir_instructions.deinit(gpa);
+        function.mir_extra.deinit(gpa);
+    }
+
+    function.genLazy(lazy_sym) catch |err| switch (err) {
+        error.CodegenFail => return Result{ .fail = function.err_msg.? },
+        error.OutOfRegisters => return Result{
+            .fail = try ErrorMsg.create(bin_file.allocator, src_loc, "CodeGen ran out of registers. This is a bug in the Zig compiler.", .{}),
+        },
+        else => |e| return e,
+    };
+
+    var mir = Mir{
+        .instructions = function.mir_instructions.toOwnedSlice(),
+        .extra = try function.mir_extra.toOwnedSlice(bin_file.allocator),
+        .frame_locs = function.frame_locs.toOwnedSlice(),
+    };
+    defer mir.deinit(bin_file.allocator);
+
+    var emit = Emit{
+        .lower = .{
+            .allocator = bin_file.allocator,
+            .mir = mir,
+            .target = &bin_file.options.target,
+            .src_loc = src_loc,
+        },
+        .bin_file = bin_file,
+        .debug_output = debug_output,
+        .code = code,
+        .prev_di_pc = undefined, // no debug info yet
+        .prev_di_line = undefined, // no debug info yet
+        .prev_di_column = undefined, // no debug info yet
+    };
+    defer emit.deinit();
+    emit.emitMir() catch |err| switch (err) {
+        error.LowerFail, error.EmitFail => return Result{ .fail = emit.lower.err_msg.? },
+        error.InvalidInstruction, error.CannotEncode => |e| {
+            const msg = switch (e) {
+                error.InvalidInstruction => "CodeGen failed to find a viable instruction.",
+                error.CannotEncode => "CodeGen failed to encode the instruction.",
+            };
+            return Result{
+                .fail = try ErrorMsg.create(
+                    bin_file.allocator,
+                    src_loc,
+                    "{s} This is a bug in the Zig compiler.",
+                    .{msg},
+                ),
+            };
+        },
+        else => |e| return e,
+    };
+
+    if (function.err_msg) |em| {
+        return Result{ .fail = em };
+    } else {
+        return Result.ok;
+    }
+}
+
 const FormatDeclData = struct {
     mod: *Module,
     decl_index: Module.Decl.Index,
@@ -1545,6 +1634,103 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
     verbose_tracking_log.debug("{}", .{self.fmtTracking()});
 }
 
+fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
+    switch (lazy_sym.ty.zigTypeTag()) {
+        .Enum => {
+            const enum_ty = lazy_sym.ty;
+            wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(self.bin_file.options.module.?)});
+
+            const param_regs = abi.getCAbiIntParamRegs(self.target.*);
+            const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);
+            defer for (param_locks) |lock| self.register_manager.unlockReg(lock);
+
+            const ret_reg = param_regs[0];
+            const enum_mcv = MCValue{ .register = param_regs[1] };
+
+            var exitlude_jump_relocs = try self.gpa.alloc(u32, enum_ty.enumFieldCount());
+            defer self.gpa.free(exitlude_jump_relocs);
+
+            const data_reg = try self.register_manager.allocReg(null, gp);
+            const data_lock = self.register_manager.lockRegAssumeUnused(data_reg);
+            defer self.register_manager.unlockReg(data_lock);
+
+            const data_lazy_sym = link.File.LazySymbol{ .kind = .const_data, .ty = enum_ty };
+            if (self.bin_file.cast(link.File.Elf)) |elf_file| {
+                const atom_index = elf_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
+                    return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+                const atom = elf_file.getAtom(atom_index);
+                _ = try atom.getOrCreateOffsetTableEntry(elf_file);
+                const got_addr = atom.getOffsetTableAddress(elf_file);
+                try self.asmRegisterMemory(
+                    .mov,
+                    data_reg.to64(),
+                    Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
+                );
+            } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
+                const atom_index = coff_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
+                    return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+                const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
+                try self.genSetReg(data_reg, Type.usize, .{ .lea_got = sym_index });
+            } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
+                const atom_index = macho_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
+                    return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+                const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
+                try self.genSetReg(data_reg, Type.usize, .{ .lea_got = sym_index });
+            } else {
+                return self.fail("TODO implement {s} for {}", .{
+                    @tagName(lazy_sym.kind),
+                    lazy_sym.ty.fmt(self.bin_file.options.module.?),
+                });
+            }
+
+            var data_off: i32 = 0;
+            for (
+                exitlude_jump_relocs,
+                enum_ty.enumFields().keys(),
+                0..,
+            ) |*exitlude_jump_reloc, tag_name, index| {
+                var tag_pl = Value.Payload.U32{
+                    .base = .{ .tag = .enum_field_index },
+                    .data = @intCast(u32, index),
+                };
+                const tag_val = Value.initPayload(&tag_pl.base);
+                const tag_mcv = try self.genTypedValue(.{ .ty = enum_ty, .val = tag_val });
+                try self.genBinOpMir(.cmp, enum_ty, enum_mcv, tag_mcv);
+                const skip_reloc = try self.asmJccReloc(undefined, .ne);
+
+                try self.genSetMem(
+                    .{ .reg = ret_reg },
+                    0,
+                    Type.usize,
+                    .{ .register_offset = .{ .reg = data_reg, .off = data_off } },
+                );
+                try self.genSetMem(.{ .reg = ret_reg }, 8, Type.usize, .{ .immediate = tag_name.len });
+
+                exitlude_jump_reloc.* = try self.asmJmpReloc(undefined);
+                try self.performReloc(skip_reloc);
+
+                data_off += @intCast(i32, tag_name.len + 1);
+            }
+
+            try self.airTrap();
+
+            for (exitlude_jump_relocs) |reloc| try self.performReloc(reloc);
+            try self.asmOpOnly(.ret);
+        },
+        else => return self.fail(
+            "TODO implement {s} for {}",
+            .{ @tagName(lazy_sym.kind), lazy_sym.ty.fmt(self.bin_file.options.module.?) },
+        ),
+    }
+}
+
+fn getOwnerDecl(self: *const Self) Module.Decl.Index {
+    return switch (self.owner) {
+        .mod_fn => |mod_fn| mod_fn.owner_decl,
+        .decl => |index| index,
+    };
+}
+
 fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
     const reg = value.getReg() orelse return;
     if (self.register_manager.isRegFree(reg)) {
@@ -6020,7 +6206,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
 
         const ty = self.air.typeOfIndex(inst);
         const src_index = self.air.instructions.items(.data)[inst].arg.src_index;
-        const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index);
+        const name = self.owner.mod_fn.getParamName(self.bin_file.options.module.?, src_index);
         try self.genArgDbgInfo(ty, name, dst_mcv);
 
         break :result dst_mcv;
@@ -6044,7 +6230,7 @@ fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void {
                 //},
                 else => unreachable, // not a valid function parameter
             };
-            try dw.genArgDbgInfo(name, ty, self.mod_fn.owner_decl, loc);
+            try dw.genArgDbgInfo(name, ty, self.getOwnerDecl(), loc);
         },
         .plan9 => {},
         .none => {},
@@ -6085,7 +6271,7 @@ fn genVarDbgInfo(
                     break :blk .nop;
                 },
             };
-            try dw.genVarDbgInfo(name, ty, self.mod_fn.owner_decl, is_ptr, loc);
+            try dw.genVarDbgInfo(name, ty, self.getOwnerDecl(), is_ptr, loc);
         },
         .plan9 => {},
         .none => {},
@@ -6243,7 +6429,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
             const decl_name = mem.sliceTo(mod.declPtr(extern_fn.owner_decl).name, 0);
             const lib_name = mem.sliceTo(extern_fn.lib_name, 0);
             if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-                const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
+                const atom_index = try self.getSymbolIndexForDecl(self.getOwnerDecl());
                 const sym_index = try coff_file.getGlobalSymbol(decl_name, lib_name);
                 _ = try self.addInst(.{
                     .tag = .mov_linker,
@@ -6257,7 +6443,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
                 try self.asmRegister(.call, .rax);
             } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
                 const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);
-                const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
+                const atom_index = try self.getSymbolIndexForDecl(self.getOwnerDecl());
                 _ = try self.addInst(.{
                     .tag = .call_extern,
                     .ops = undefined,
@@ -6416,7 +6602,8 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
     const mod = self.bin_file.options.module.?;
     const lazy_sym = link.File.LazySymbol.initDecl(.const_data, null, mod);
     if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-        const atom_index = try elf_file.getOrCreateAtomForLazySymbol(lazy_sym);
+        const atom_index = elf_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
         const atom = elf_file.getAtom(atom_index);
         _ = try atom.getOrCreateOffsetTableEntry(elf_file);
         const got_addr = atom.getOffsetTableAddress(elf_file);
@@ -6426,11 +6613,13 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
             Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
         );
     } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-        const atom_index = try coff_file.getOrCreateAtomForLazySymbol(lazy_sym);
+        const atom_index = coff_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
         const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
     } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-        const atom_index = try macho_file.getOrCreateAtomForLazySymbol(lazy_sym);
+        const atom_index = macho_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
         const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
     } else {
@@ -7530,7 +7719,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                         }),
                     ),
                 .load_direct => |sym_index| if (try self.movMirTag(ty) == .mov) {
-                    const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
+                    const atom_index = try self.getSymbolIndexForDecl(self.getOwnerDecl());
                     _ = try self.addInst(.{
                         .tag = .mov_linker,
                         .ops = .direct_reloc,
@@ -7557,7 +7746,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             );
         },
         .lea_direct, .lea_got => |sym_index| {
-            const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
+            const atom_index = try self.getSymbolIndexForDecl(self.getOwnerDecl());
             _ = try self.addInst(.{
                 .tag = switch (src_mcv) {
                     .lea_direct => .lea_linker,
@@ -7577,7 +7766,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             });
         },
         .lea_tlv => |sym_index| {
-            const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
+            const atom_index = try self.getSymbolIndexForDecl(self.getOwnerDecl());
             if (self.bin_file.cast(link.File.MachO)) |_| {
                 _ = try self.addInst(.{
                     .tag = .lea_linker,
@@ -8475,10 +8664,64 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
 
 fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
+    const inst_ty = self.air.typeOfIndex(inst);
+    const enum_ty = self.air.typeOf(un_op);
+
+    // We need a properly aligned and sized call frame to be able to call this function.
+    {
+        const needed_call_frame = FrameAlloc.init(.{
+            .size = inst_ty.abiSize(self.target.*),
+            .alignment = inst_ty.abiAlignment(self.target.*),
+        });
+        const frame_allocs_slice = self.frame_allocs.slice();
+        const stack_frame_size =
+            &frame_allocs_slice.items(.abi_size)[@enumToInt(FrameIndex.call_frame)];
+        stack_frame_size.* = @max(stack_frame_size.*, needed_call_frame.abi_size);
+        const stack_frame_align =
+            &frame_allocs_slice.items(.abi_align)[@enumToInt(FrameIndex.call_frame)];
+        stack_frame_align.* = @max(stack_frame_align.*, needed_call_frame.abi_align);
+    }
+
+    try self.spillEflagsIfOccupied();
+    try self.spillRegisters(abi.getCallerPreservedRegs(self.target.*));
+
+    const param_regs = abi.getCAbiIntParamRegs(self.target.*);
+
+    const dst_mcv = try self.allocRegOrMem(inst, false);
+    try self.genSetReg(param_regs[0], Type.usize, dst_mcv.address());
+
     const operand = try self.resolveInst(un_op);
-    _ = operand;
-    return self.fail("TODO implement airTagName for x86_64", .{});
-    //return self.finishAir(inst, result, .{ un_op, .none, .none });
+    try self.genSetReg(param_regs[1], enum_ty, operand);
+
+    const mod = self.bin_file.options.module.?;
+    const lazy_sym = link.File.LazySymbol.initDecl(.code, enum_ty.getOwnerDecl(), mod);
+    if (self.bin_file.cast(link.File.Elf)) |elf_file| {
+        const atom_index = elf_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+        const atom = elf_file.getAtom(atom_index);
+        _ = try atom.getOrCreateOffsetTableEntry(elf_file);
+        const got_addr = atom.getOffsetTableAddress(elf_file);
+        try self.asmMemory(
+            .call,
+            Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
+        );
+    } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
+        const atom_index = coff_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+        const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
+        try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
+        try self.asmRegister(.call, .rax);
+    } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
+        const atom_index = macho_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+        const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
+        try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
+        try self.asmRegister(.call, .rax);
+    } else {
+        return self.fail("TODO implement airTagName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
+    }
+
+    return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
 }
 
 fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
@@ -8497,7 +8740,8 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
     const mod = self.bin_file.options.module.?;
     const lazy_sym = link.File.LazySymbol.initDecl(.const_data, null, mod);
     if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-        const atom_index = try elf_file.getOrCreateAtomForLazySymbol(lazy_sym);
+        const atom_index = elf_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
         const atom = elf_file.getAtom(atom_index);
         _ = try atom.getOrCreateOffsetTableEntry(elf_file);
         const got_addr = atom.getOffsetTableAddress(elf_file);
@@ -8507,11 +8751,13 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
             Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
         );
     } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-        const atom_index = try coff_file.getOrCreateAtomForLazySymbol(lazy_sym);
+        const atom_index = coff_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
         const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
     } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-        const atom_index = try macho_file.getOrCreateAtomForLazySymbol(lazy_sym);
+        const atom_index = macho_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
         const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
     } else {
@@ -8833,12 +9079,7 @@ fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCV
 }
 
 fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
-    const mcv: MCValue = switch (try codegen.genTypedValue(
-        self.bin_file,
-        self.src_loc,
-        arg_tv,
-        self.mod_fn.owner_decl,
-    )) {
+    return switch (try codegen.genTypedValue(self.bin_file, self.src_loc, arg_tv, self.getOwnerDecl())) {
         .mcv => |mcv| switch (mcv) {
             .none => .none,
             .undef => .undef,
@@ -8853,7 +9094,6 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
             return error.CodegenFail;
         },
     };
-    return mcv;
 }
 
 const CallMCValues = struct {
diff --git a/src/codegen.zig b/src/codegen.zig
index 690e96d25c..078feb409d 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -7,6 +7,7 @@ const link = @import("link.zig");
 const log = std.log.scoped(.codegen);
 const mem = std.mem;
 const math = std.math;
+const target_util = @import("target.zig");
 const trace = @import("tracy.zig").trace;
 
 const Air = @import("Air.zig");
@@ -89,6 +90,19 @@ pub fn generateFunction(
     }
 }
 
+pub fn generateLazyFunction(
+    bin_file: *link.File,
+    src_loc: Module.SrcLoc,
+    lazy_sym: link.File.LazySymbol,
+    code: *std.ArrayList(u8),
+    debug_output: DebugInfoOutput,
+) CodeGenError!Result {
+    switch (bin_file.options.target.cpu.arch) {
+        .x86_64 => return @import("arch/x86_64/CodeGen.zig").generateLazy(bin_file, src_loc, lazy_sym, code, debug_output),
+        else => unreachable,
+    }
+}
+
 fn writeFloat(comptime F: type, f: F, target: Target, endian: std.builtin.Endian, code: []u8) void {
     _ = target;
     const bits = @typeInfo(F).Float.bits;
@@ -101,11 +115,11 @@ pub fn generateLazySymbol(
     bin_file: *link.File,
     src_loc: Module.SrcLoc,
     lazy_sym: link.File.LazySymbol,
+    alignment: *u32,
     code: *std.ArrayList(u8),
     debug_output: DebugInfoOutput,
     reloc_info: RelocInfo,
-) CodeGenError!struct { res: Result, alignment: u32 } {
-    _ = debug_output;
+) CodeGenError!Result {
     _ = reloc_info;
 
     const tracy = trace(@src());
@@ -120,7 +134,13 @@ pub fn generateLazySymbol(
         lazy_sym.ty.fmt(mod),
     });
 
-    if (lazy_sym.kind == .const_data and lazy_sym.ty.isAnyError()) {
+    if (lazy_sym.kind == .code) {
+        alignment.* = target_util.defaultFunctionAlignment(target);
+        return generateLazyFunction(bin_file, src_loc, lazy_sym, code, debug_output);
+    }
+
+    if (lazy_sym.ty.isAnyError()) {
+        alignment.* = 4;
         const err_names = mod.error_name_list.items;
         mem.writeInt(u32, try code.addManyAsArray(4), @intCast(u32, err_names.len), endian);
         var offset = code.items.len;
@@ -133,13 +153,21 @@ pub fn generateLazySymbol(
             code.appendAssumeCapacity(0);
         }
         mem.writeInt(u32, code.items[offset..][0..4], @intCast(u32, code.items.len), endian);
-        return .{ .res = Result.ok, .alignment = 4 };
-    } else return .{ .res = .{ .fail = try ErrorMsg.create(
+        return Result.ok;
+    } else if (lazy_sym.ty.zigTypeTag() == .Enum) {
+        alignment.* = 1;
+        for (lazy_sym.ty.enumFields().keys()) |tag_name| {
+            try code.ensureUnusedCapacity(tag_name.len + 1);
+            code.appendSliceAssumeCapacity(tag_name);
+            code.appendAssumeCapacity(0);
+        }
+        return Result.ok;
+    } else return .{ .fail = try ErrorMsg.create(
         bin_file.allocator,
         src_loc,
         "TODO implement generateLazySymbol for {s} {}",
         .{ @tagName(lazy_sym.kind), lazy_sym.ty.fmt(mod) },
-    ) }, .alignment = undefined };
+    ) };
 }
 
 pub fn generateSymbol(
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index deac8d1fd6..41d4617e53 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -1218,6 +1218,7 @@ fn updateLazySymbolAtom(
     const gpa = self.base.allocator;
     const mod = self.base.options.module.?;
 
+    var required_alignment: u32 = undefined;
     var code_buffer = std.ArrayList(u8).init(gpa);
     defer code_buffer.deinit();
 
@@ -1238,10 +1239,16 @@ fn updateLazySymbolAtom(
             .parent_decl_node = undefined,
             .lazy = .unneeded,
         };
-    const res = try codegen.generateLazySymbol(&self.base, src, sym, &code_buffer, .none, .{
-        .parent_atom_index = local_sym_index,
-    });
-    const code = switch (res.res) {
+    const res = try codegen.generateLazySymbol(
+        &self.base,
+        src,
+        sym,
+        &required_alignment,
+        &code_buffer,
+        .none,
+        .{ .parent_atom_index = local_sym_index },
+    );
+    const code = switch (res) {
         .ok => code_buffer.items,
         .fail => |em| {
             log.err("{s}", .{em.msg});
@@ -1255,11 +1262,11 @@ fn updateLazySymbolAtom(
     symbol.section_number = @intToEnum(coff.SectionNumber, section_index + 1);
     symbol.type = .{ .complex_type = .NULL, .base_type = .NULL };
 
-    const vaddr = try self.allocateAtom(atom_index, code_len, res.alignment);
+    const vaddr = try self.allocateAtom(atom_index, code_len, required_alignment);
     errdefer self.freeAtom(atom_index);
 
     log.debug("allocated atom for {s} at 0x{x}", .{ name, vaddr });
-    log.debug("  (required alignment 0x{x})", .{res.alignment});
+    log.debug("  (required alignment 0x{x})", .{required_alignment});
 
     atom.size = code_len;
     symbol.value = vaddr;
@@ -1270,14 +1277,20 @@ fn updateLazySymbolAtom(
 
 pub fn getOrCreateAtomForLazySymbol(self: *Coff, sym: link.File.LazySymbol) !Atom.Index {
     const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
-    errdefer _ = self.lazy_syms.pop();
+    errdefer _ = if (!gop.found_existing) self.lazy_syms.pop();
     if (!gop.found_existing) gop.value_ptr.* = .{};
-    const atom = switch (sym.kind) {
+    const atom_ptr = switch (sym.kind) {
         .code => &gop.value_ptr.text_atom,
         .const_data => &gop.value_ptr.rdata_atom,
     };
-    if (atom.* == null) atom.* = try self.createAtom();
-    return atom.*.?;
+    if (atom_ptr.*) |atom| return atom;
+    const atom = try self.createAtom();
+    atom_ptr.* = atom;
+    try self.updateLazySymbolAtom(sym, atom, switch (sym.kind) {
+        .code => self.text_section_index.?,
+        .const_data => self.rdata_section_index.?,
+    });
+    return atom;
 }
 
 pub fn getOrCreateAtomForDecl(self: *Coff, decl_index: Module.Decl.Index) !Atom.Index {
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index 29ae97150e..ec113e5c8a 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -2376,14 +2376,20 @@ pub fn freeDecl(self: *Elf, decl_index: Module.Decl.Index) void {
 
 pub fn getOrCreateAtomForLazySymbol(self: *Elf, sym: File.LazySymbol) !Atom.Index {
     const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
-    errdefer _ = self.lazy_syms.pop();
+    errdefer _ = if (!gop.found_existing) self.lazy_syms.pop();
     if (!gop.found_existing) gop.value_ptr.* = .{};
-    const atom = switch (sym.kind) {
+    const atom_ptr = switch (sym.kind) {
         .code => &gop.value_ptr.text_atom,
         .const_data => &gop.value_ptr.rodata_atom,
     };
-    if (atom.* == null) atom.* = try self.createAtom();
-    return atom.*.?;
+    if (atom_ptr.*) |atom| return atom;
+    const atom = try self.createAtom();
+    atom_ptr.* = atom;
+    try self.updateLazySymbolAtom(sym, atom, switch (sym.kind) {
+        .code => self.text_section_index.?,
+        .const_data => self.rodata_section_index.?,
+    });
+    return atom;
 }
 
 pub fn getOrCreateAtomForDecl(self: *Elf, decl_index: Module.Decl.Index) !Atom.Index {
@@ -2684,6 +2690,7 @@ fn updateLazySymbolAtom(
     const gpa = self.base.allocator;
     const mod = self.base.options.module.?;
 
+    var required_alignment: u32 = undefined;
     var code_buffer = std.ArrayList(u8).init(gpa);
     defer code_buffer.deinit();
 
@@ -2708,10 +2715,16 @@ fn updateLazySymbolAtom(
             .parent_decl_node = undefined,
             .lazy = .unneeded,
         };
-    const res = try codegen.generateLazySymbol(&self.base, src, sym, &code_buffer, .none, .{
-        .parent_atom_index = local_sym_index,
-    });
-    const code = switch (res.res) {
+    const res = try codegen.generateLazySymbol(
+        &self.base,
+        src,
+        sym,
+        &required_alignment,
+        &code_buffer,
+        .none,
+        .{ .parent_atom_index = local_sym_index },
+    );
+    const code = switch (res) {
         .ok => code_buffer.items,
         .fail => |em| {
             log.err("{s}", .{em.msg});
@@ -2729,7 +2742,7 @@ fn updateLazySymbolAtom(
         .st_value = 0,
         .st_size = 0,
     };
-    const vaddr = try self.allocateAtom(atom_index, code.len, res.alignment);
+    const vaddr = try self.allocateAtom(atom_index, code.len, required_alignment);
     errdefer self.freeAtom(atom_index);
     log.debug("allocated text block for {s} at 0x{x}", .{ name, vaddr });
 
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 5136bd84a2..c14168c66b 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -2060,6 +2060,7 @@ fn updateLazySymbolAtom(
     const gpa = self.base.allocator;
     const mod = self.base.options.module.?;
 
+    var required_alignment: u32 = undefined;
     var code_buffer = std.ArrayList(u8).init(gpa);
     defer code_buffer.deinit();
 
@@ -2084,10 +2085,16 @@ fn updateLazySymbolAtom(
             .parent_decl_node = undefined,
             .lazy = .unneeded,
         };
-    const res = try codegen.generateLazySymbol(&self.base, src, sym, &code_buffer, .none, .{
-        .parent_atom_index = local_sym_index,
-    });
-    const code = switch (res.res) {
+    const res = try codegen.generateLazySymbol(
+        &self.base,
+        src,
+        sym,
+        &required_alignment,
+        &code_buffer,
+        .none,
+        .{ .parent_atom_index = local_sym_index },
+    );
+    const code = switch (res) {
         .ok => code_buffer.items,
         .fail => |em| {
             log.err("{s}", .{em.msg});
@@ -2101,11 +2108,11 @@ fn updateLazySymbolAtom(
     symbol.n_sect = section_index + 1;
     symbol.n_desc = 0;
 
-    const vaddr = try self.allocateAtom(atom_index, code.len, res.alignment);
+    const vaddr = try self.allocateAtom(atom_index, code.len, required_alignment);
     errdefer self.freeAtom(atom_index);
 
     log.debug("allocated atom for {s} at 0x{x}", .{ name, vaddr });
-    log.debug("  (required alignment 0x{x}", .{res.alignment});
+    log.debug("  (required alignment 0x{x}", .{required_alignment});
 
     atom.size = code.len;
     symbol.n_value = vaddr;
@@ -2116,14 +2123,20 @@ fn updateLazySymbolAtom(
 
 pub fn getOrCreateAtomForLazySymbol(self: *MachO, sym: File.LazySymbol) !Atom.Index {
     const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
-    errdefer _ = self.lazy_syms.pop();
+    errdefer _ = if (!gop.found_existing) self.lazy_syms.pop();
     if (!gop.found_existing) gop.value_ptr.* = .{};
-    const atom = switch (sym.kind) {
+    const atom_ptr = switch (sym.kind) {
         .code => &gop.value_ptr.text_atom,
         .const_data => &gop.value_ptr.data_const_atom,
     };
-    if (atom.* == null) atom.* = try self.createAtom();
-    return atom.*.?;
+    if (atom_ptr.*) |atom| return atom;
+    const atom = try self.createAtom();
+    atom_ptr.* = atom;
+    try self.updateLazySymbolAtom(sym, atom, switch (sym.kind) {
+        .code => self.text_section_index.?,
+        .const_data => self.data_const_section_index.?,
+    });
+    return atom;
 }
 
 fn updateThreadlocalVariable(self: *MachO, module: *Module, decl_index: Module.Decl.Index) !void {
diff --git a/test/behavior/enum.zig b/test/behavior/enum.zig
index de5c0efb8e..26b941bcdc 100644
--- a/test/behavior/enum.zig
+++ b/test/behavior/enum.zig
@@ -981,7 +981,6 @@ fn test3_2(f: Test3Foo) !void {
 }
 
 test "@tagName" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -997,7 +996,6 @@ fn testEnumTagNameBare(n: anytype) []const u8 {
 const BareNumber = enum { One, Two, Three };
 
 test "@tagName non-exhaustive enum" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1008,7 +1006,6 @@ test "@tagName non-exhaustive enum" {
 const NonExhaustive = enum(u8) { A, B, _ };
 
 test "@tagName is null-terminated" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1023,7 +1020,6 @@ test "@tagName is null-terminated" {
 }
 
 test "tag name with assigned enum values" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1113,7 +1109,6 @@ test "enum literal in array literal" {
 
 test "tag name functions are unique" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/memset.zig b/test/behavior/memset.zig
index 2cc390a3c9..89e01a0e56 100644
--- a/test/behavior/memset.zig
+++ b/test/behavior/memset.zig
@@ -114,7 +114,6 @@ test "memset with large array element, runtime known" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
 
     const A = [128]u64;
     var buf: [5]A = undefined;
@@ -132,7 +131,6 @@ test "memset with large array element, comptime known" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
 
     const A = [128]u64;
     var buf: [5]A = undefined;
diff --git a/test/behavior/type.zig b/test/behavior/type.zig
index 0d309b9a6e..695a81b1a3 100644
--- a/test/behavior/type.zig
+++ b/test/behavior/type.zig
@@ -258,7 +258,6 @@ test "Type.ErrorSet" {
 
 test "Type.Struct" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -383,7 +382,6 @@ test "Type.Enum" {
 
 test "Type.Union" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
 
diff --git a/test/behavior/union.zig b/test/behavior/union.zig
index 76c5b09a89..f90a336e76 100644
--- a/test/behavior/union.zig
+++ b/test/behavior/union.zig
@@ -1094,7 +1094,6 @@ test "containers with single-field enums" {
 test "@unionInit on union with tag but no fields" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 

From 0bdfb288ccc58807355221d3e86451c33e390d01 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Mon, 1 May 2023 00:39:31 -0400
Subject: [PATCH 046/725] x86_64: workaround tagName linker issues

Pass extra pointer param with a linker ref when calling the lazy tagName
function to workaround not being able to lower linker refs during
codegen of a lazy func.
---
 src/arch/x86_64/CodeGen.zig | 79 ++++++++++++++++---------------------
 1 file changed, 34 insertions(+), 45 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 71dcbfa461..6f71e0b810 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -1641,48 +1641,16 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
             wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(self.bin_file.options.module.?)});
 
             const param_regs = abi.getCAbiIntParamRegs(self.target.*);
-            const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);
+            const param_locks = self.register_manager.lockRegsAssumeUnused(3, param_regs[0..3].*);
             defer for (param_locks) |lock| self.register_manager.unlockReg(lock);
 
             const ret_reg = param_regs[0];
             const enum_mcv = MCValue{ .register = param_regs[1] };
+            const data_mcv = MCValue{ .register = param_regs[2] };
 
             var exitlude_jump_relocs = try self.gpa.alloc(u32, enum_ty.enumFieldCount());
             defer self.gpa.free(exitlude_jump_relocs);
 
-            const data_reg = try self.register_manager.allocReg(null, gp);
-            const data_lock = self.register_manager.lockRegAssumeUnused(data_reg);
-            defer self.register_manager.unlockReg(data_lock);
-
-            const data_lazy_sym = link.File.LazySymbol{ .kind = .const_data, .ty = enum_ty };
-            if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-                const atom_index = elf_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
-                    return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-                const atom = elf_file.getAtom(atom_index);
-                _ = try atom.getOrCreateOffsetTableEntry(elf_file);
-                const got_addr = atom.getOffsetTableAddress(elf_file);
-                try self.asmRegisterMemory(
-                    .mov,
-                    data_reg.to64(),
-                    Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
-                );
-            } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-                const atom_index = coff_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
-                    return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-                const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
-                try self.genSetReg(data_reg, Type.usize, .{ .lea_got = sym_index });
-            } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-                const atom_index = macho_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
-                    return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-                const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
-                try self.genSetReg(data_reg, Type.usize, .{ .lea_got = sym_index });
-            } else {
-                return self.fail("TODO implement {s} for {}", .{
-                    @tagName(lazy_sym.kind),
-                    lazy_sym.ty.fmt(self.bin_file.options.module.?),
-                });
-            }
-
             var data_off: i32 = 0;
             for (
                 exitlude_jump_relocs,
@@ -1698,12 +1666,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
                 try self.genBinOpMir(.cmp, enum_ty, enum_mcv, tag_mcv);
                 const skip_reloc = try self.asmJccReloc(undefined, .ne);
 
-                try self.genSetMem(
-                    .{ .reg = ret_reg },
-                    0,
-                    Type.usize,
-                    .{ .register_offset = .{ .reg = data_reg, .off = data_off } },
-                );
+                try self.genSetMem(.{ .reg = ret_reg }, 0, Type.usize, data_mcv.offset(data_off));
                 try self.genSetMem(.{ .reg = ret_reg }, 8, Type.usize, .{ .immediate = tag_name.len });
 
                 exitlude_jump_reloc.* = try self.asmJmpReloc(undefined);
@@ -8663,6 +8626,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
 }
 
 fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
+    const mod = self.bin_file.options.module.?;
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     const inst_ty = self.air.typeOfIndex(inst);
     const enum_ty = self.air.typeOf(un_op);
@@ -8693,10 +8657,35 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
     const operand = try self.resolveInst(un_op);
     try self.genSetReg(param_regs[1], enum_ty, operand);
 
-    const mod = self.bin_file.options.module.?;
-    const lazy_sym = link.File.LazySymbol.initDecl(.code, enum_ty.getOwnerDecl(), mod);
+    const data_lazy_sym = link.File.LazySymbol.initDecl(.const_data, enum_ty.getOwnerDecl(), mod);
     if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-        const atom_index = elf_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+        const atom_index = elf_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+        const atom = elf_file.getAtom(atom_index);
+        _ = try atom.getOrCreateOffsetTableEntry(elf_file);
+        const got_addr = atom.getOffsetTableAddress(elf_file);
+        try self.asmRegisterMemory(
+            .mov,
+            param_regs[2],
+            Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
+        );
+    } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
+        const atom_index = coff_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+        const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
+        try self.genSetReg(param_regs[2], Type.usize, .{ .lea_got = sym_index });
+    } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
+        const atom_index = macho_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+        const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
+        try self.genSetReg(param_regs[2], Type.usize, .{ .lea_got = sym_index });
+    } else {
+        return self.fail("TODO implement airTagName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
+    }
+
+    const code_lazy_sym = link.File.LazySymbol.initDecl(.code, enum_ty.getOwnerDecl(), mod);
+    if (self.bin_file.cast(link.File.Elf)) |elf_file| {
+        const atom_index = elf_file.getOrCreateAtomForLazySymbol(code_lazy_sym) catch |err|
             return self.fail("{s} creating lazy symbol", .{@errorName(err)});
         const atom = elf_file.getAtom(atom_index);
         _ = try atom.getOrCreateOffsetTableEntry(elf_file);
@@ -8706,13 +8695,13 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
             Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
         );
     } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-        const atom_index = coff_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+        const atom_index = coff_file.getOrCreateAtomForLazySymbol(code_lazy_sym) catch |err|
             return self.fail("{s} creating lazy symbol", .{@errorName(err)});
         const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
         try self.asmRegister(.call, .rax);
     } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-        const atom_index = macho_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+        const atom_index = macho_file.getOrCreateAtomForLazySymbol(code_lazy_sym) catch |err|
             return self.fail("{s} creating lazy symbol", .{@errorName(err)});
         const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });

From 7064d7dbf0157aed9b497e1158243e472082633f Mon Sep 17 00:00:00 2001
From: Jakub Konka 
Date: Mon, 1 May 2023 09:54:42 +0200
Subject: [PATCH 047/725] Revert "x86_64: workaround tagName linker issues"

This reverts commit aac97b92532e7492b9145e1562e31c2e1fa66c15.
---
 src/arch/x86_64/CodeGen.zig | 79 +++++++++++++++++++++----------------
 1 file changed, 45 insertions(+), 34 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 6f71e0b810..71dcbfa461 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -1641,16 +1641,48 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
             wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(self.bin_file.options.module.?)});
 
             const param_regs = abi.getCAbiIntParamRegs(self.target.*);
-            const param_locks = self.register_manager.lockRegsAssumeUnused(3, param_regs[0..3].*);
+            const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);
             defer for (param_locks) |lock| self.register_manager.unlockReg(lock);
 
             const ret_reg = param_regs[0];
             const enum_mcv = MCValue{ .register = param_regs[1] };
-            const data_mcv = MCValue{ .register = param_regs[2] };
 
             var exitlude_jump_relocs = try self.gpa.alloc(u32, enum_ty.enumFieldCount());
             defer self.gpa.free(exitlude_jump_relocs);
 
+            const data_reg = try self.register_manager.allocReg(null, gp);
+            const data_lock = self.register_manager.lockRegAssumeUnused(data_reg);
+            defer self.register_manager.unlockReg(data_lock);
+
+            const data_lazy_sym = link.File.LazySymbol{ .kind = .const_data, .ty = enum_ty };
+            if (self.bin_file.cast(link.File.Elf)) |elf_file| {
+                const atom_index = elf_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
+                    return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+                const atom = elf_file.getAtom(atom_index);
+                _ = try atom.getOrCreateOffsetTableEntry(elf_file);
+                const got_addr = atom.getOffsetTableAddress(elf_file);
+                try self.asmRegisterMemory(
+                    .mov,
+                    data_reg.to64(),
+                    Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
+                );
+            } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
+                const atom_index = coff_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
+                    return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+                const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
+                try self.genSetReg(data_reg, Type.usize, .{ .lea_got = sym_index });
+            } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
+                const atom_index = macho_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
+                    return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+                const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
+                try self.genSetReg(data_reg, Type.usize, .{ .lea_got = sym_index });
+            } else {
+                return self.fail("TODO implement {s} for {}", .{
+                    @tagName(lazy_sym.kind),
+                    lazy_sym.ty.fmt(self.bin_file.options.module.?),
+                });
+            }
+
             var data_off: i32 = 0;
             for (
                 exitlude_jump_relocs,
@@ -1666,7 +1698,12 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
                 try self.genBinOpMir(.cmp, enum_ty, enum_mcv, tag_mcv);
                 const skip_reloc = try self.asmJccReloc(undefined, .ne);
 
-                try self.genSetMem(.{ .reg = ret_reg }, 0, Type.usize, data_mcv.offset(data_off));
+                try self.genSetMem(
+                    .{ .reg = ret_reg },
+                    0,
+                    Type.usize,
+                    .{ .register_offset = .{ .reg = data_reg, .off = data_off } },
+                );
                 try self.genSetMem(.{ .reg = ret_reg }, 8, Type.usize, .{ .immediate = tag_name.len });
 
                 exitlude_jump_reloc.* = try self.asmJmpReloc(undefined);
@@ -8626,7 +8663,6 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
 }
 
 fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
-    const mod = self.bin_file.options.module.?;
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     const inst_ty = self.air.typeOfIndex(inst);
     const enum_ty = self.air.typeOf(un_op);
@@ -8657,35 +8693,10 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
     const operand = try self.resolveInst(un_op);
     try self.genSetReg(param_regs[1], enum_ty, operand);
 
-    const data_lazy_sym = link.File.LazySymbol.initDecl(.const_data, enum_ty.getOwnerDecl(), mod);
+    const mod = self.bin_file.options.module.?;
+    const lazy_sym = link.File.LazySymbol.initDecl(.code, enum_ty.getOwnerDecl(), mod);
     if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-        const atom_index = elf_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
-            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-        const atom = elf_file.getAtom(atom_index);
-        _ = try atom.getOrCreateOffsetTableEntry(elf_file);
-        const got_addr = atom.getOffsetTableAddress(elf_file);
-        try self.asmRegisterMemory(
-            .mov,
-            param_regs[2],
-            Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
-        );
-    } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-        const atom_index = coff_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
-            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-        const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
-        try self.genSetReg(param_regs[2], Type.usize, .{ .lea_got = sym_index });
-    } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-        const atom_index = macho_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
-            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-        const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
-        try self.genSetReg(param_regs[2], Type.usize, .{ .lea_got = sym_index });
-    } else {
-        return self.fail("TODO implement airTagName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
-    }
-
-    const code_lazy_sym = link.File.LazySymbol.initDecl(.code, enum_ty.getOwnerDecl(), mod);
-    if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-        const atom_index = elf_file.getOrCreateAtomForLazySymbol(code_lazy_sym) catch |err|
+        const atom_index = elf_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
             return self.fail("{s} creating lazy symbol", .{@errorName(err)});
         const atom = elf_file.getAtom(atom_index);
         _ = try atom.getOrCreateOffsetTableEntry(elf_file);
@@ -8695,13 +8706,13 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
             Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
         );
     } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-        const atom_index = coff_file.getOrCreateAtomForLazySymbol(code_lazy_sym) catch |err|
+        const atom_index = coff_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
             return self.fail("{s} creating lazy symbol", .{@errorName(err)});
         const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
         try self.asmRegister(.call, .rax);
     } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-        const atom_index = macho_file.getOrCreateAtomForLazySymbol(code_lazy_sym) catch |err|
+        const atom_index = macho_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
             return self.fail("{s} creating lazy symbol", .{@errorName(err)});
         const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
         try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });

From 565f8979cc38d46b8b905f3a8b7db03238779ffc Mon Sep 17 00:00:00 2001
From: Jakub Konka 
Date: Mon, 1 May 2023 13:41:42 +0200
Subject: [PATCH 048/725] link: fix accessing source atom's symbol index in
 codegen

Since the owner can either be a `Decl` or a `LazySymbol` we need
to preserve this information at the codegen generate function level
so that we can then correctly work out the corresponding `Atom`
in the linker.
---
 src/arch/x86_64/CodeGen.zig | 94 +++++++++++++++++++++++--------------
 src/link/MachO.zig          |  7 +--
 2 files changed, 62 insertions(+), 39 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 71dcbfa461..d5c5938ba0 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -56,10 +56,7 @@ liveness: Liveness,
 bin_file: *link.File,
 debug_output: DebugInfoOutput,
 target: *const std.Target,
-owner: union(enum) {
-    mod_fn: *const Module.Fn,
-    decl: Module.Decl.Index,
-},
+owner: Owner,
 err_msg: ?*ErrorMsg,
 args: []MCValue,
 ret_mcv: InstTracking,
@@ -111,6 +108,44 @@ const mir_to_air_map_init = if (builtin.mode == .Debug) std.AutoHashMapUnmanaged
 const FrameAddr = struct { index: FrameIndex, off: i32 = 0 };
 const RegisterOffset = struct { reg: Register, off: i32 = 0 };
 
+const Owner = union(enum) {
+    mod_fn: *const Module.Fn,
+    lazy_sym: link.File.LazySymbol,
+
+    fn getOwnerDecl(owner: Owner) Module.Decl.Index {
+        return switch (owner) {
+            .mod_fn => |mod_fn| mod_fn.owner_decl,
+            .lazy_sym => |lazy_sym| lazy_sym.ty.getOwnerDecl(),
+        };
+    }
+
+    fn getSymbolIndex(owner: Owner, ctx: *Self) !u32 {
+        switch (owner) {
+            .mod_fn => |mod_fn| {
+                const decl_index = mod_fn.owner_decl;
+                if (ctx.bin_file.cast(link.File.MachO)) |macho_file| {
+                    const atom = try macho_file.getOrCreateAtomForDecl(decl_index);
+                    return macho_file.getAtom(atom).getSymbolIndex().?;
+                } else if (ctx.bin_file.cast(link.File.Coff)) |coff_file| {
+                    const atom = try coff_file.getOrCreateAtomForDecl(decl_index);
+                    return coff_file.getAtom(atom).getSymbolIndex().?;
+                } else unreachable;
+            },
+            .lazy_sym => |lazy_sym| {
+                if (ctx.bin_file.cast(link.File.MachO)) |macho_file| {
+                    const atom = macho_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+                        return ctx.fail("{s} creating lazy symbol", .{@errorName(err)});
+                    return macho_file.getAtom(atom).getSymbolIndex().?;
+                } else if (ctx.bin_file.cast(link.File.Coff)) |coff_file| {
+                    const atom = coff_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+                        return ctx.fail("{s} creating lazy symbol", .{@errorName(err)});
+                    return coff_file.getAtom(atom).getSymbolIndex().?;
+                } else unreachable;
+            },
+        }
+    }
+};
+
 pub const MCValue = union(enum) {
     /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
     /// TODO Look into deleting this tag and using `dead` instead, since every use
@@ -763,7 +798,7 @@ pub fn generateLazy(
         .target = &bin_file.options.target,
         .bin_file = bin_file,
         .debug_output = debug_output,
-        .owner = .{ .decl = lazy_sym.ty.getOwnerDecl() },
+        .owner = .{ .lazy_sym = lazy_sym },
         .err_msg = null,
         .args = undefined,
         .ret_mcv = undefined,
@@ -1724,13 +1759,6 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
     }
 }
 
-fn getOwnerDecl(self: *const Self) Module.Decl.Index {
-    return switch (self.owner) {
-        .mod_fn => |mod_fn| mod_fn.owner_decl,
-        .decl => |index| index,
-    };
-}
-
 fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
     const reg = value.getReg() orelse return;
     if (self.register_manager.isRegFree(reg)) {
@@ -6230,7 +6258,10 @@ fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void {
                 //},
                 else => unreachable, // not a valid function parameter
             };
-            try dw.genArgDbgInfo(name, ty, self.getOwnerDecl(), loc);
+            // TODO: this might need adjusting like the linkers do.
+            // Instead of flattening the owner and passing Decl.Index here we may
+            // want to special case LazySymbol in DWARF linker too.
+            try dw.genArgDbgInfo(name, ty, self.owner.getOwnerDecl(), loc);
         },
         .plan9 => {},
         .none => {},
@@ -6271,7 +6302,10 @@ fn genVarDbgInfo(
                     break :blk .nop;
                 },
             };
-            try dw.genVarDbgInfo(name, ty, self.getOwnerDecl(), is_ptr, loc);
+            // TODO: this might need adjusting like the linkers do.
+            // Instead of flattening the owner and passing Decl.Index here we may
+            // want to special case LazySymbol in DWARF linker too.
+            try dw.genVarDbgInfo(name, ty, self.owner.getOwnerDecl(), is_ptr, loc);
         },
         .plan9 => {},
         .none => {},
@@ -6403,12 +6437,14 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
                     .base = .{ .reg = .ds },
                     .disp = @intCast(i32, got_addr),
                 }));
-            } else if (self.bin_file.cast(link.File.Coff)) |_| {
-                const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);
+            } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
+                const atom = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
+                const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
                 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
                 try self.asmRegister(.call, .rax);
-            } else if (self.bin_file.cast(link.File.MachO)) |_| {
-                const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);
+            } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
+                const atom = try macho_file.getOrCreateAtomForDecl(func.owner_decl);
+                const sym_index = macho_file.getAtom(atom).getSymbolIndex().?;
                 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
                 try self.asmRegister(.call, .rax);
             } else if (self.bin_file.cast(link.File.Plan9)) |p9| {
@@ -6429,7 +6465,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
             const decl_name = mem.sliceTo(mod.declPtr(extern_fn.owner_decl).name, 0);
             const lib_name = mem.sliceTo(extern_fn.lib_name, 0);
             if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-                const atom_index = try self.getSymbolIndexForDecl(self.getOwnerDecl());
+                const atom_index = try self.owner.getSymbolIndex(self);
                 const sym_index = try coff_file.getGlobalSymbol(decl_name, lib_name);
                 _ = try self.addInst(.{
                     .tag = .mov_linker,
@@ -6442,8 +6478,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
                 });
                 try self.asmRegister(.call, .rax);
             } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
+                const atom_index = try self.owner.getSymbolIndex(self);
                 const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);
-                const atom_index = try self.getSymbolIndexForDecl(self.getOwnerDecl());
                 _ = try self.addInst(.{
                     .tag = .call_extern,
                     .ops = undefined,
@@ -7719,7 +7755,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                         }),
                     ),
                 .load_direct => |sym_index| if (try self.movMirTag(ty) == .mov) {
-                    const atom_index = try self.getSymbolIndexForDecl(self.getOwnerDecl());
+                    const atom_index = try self.owner.getSymbolIndex(self);
                     _ = try self.addInst(.{
                         .tag = .mov_linker,
                         .ops = .direct_reloc,
@@ -7746,7 +7782,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             );
         },
         .lea_direct, .lea_got => |sym_index| {
-            const atom_index = try self.getSymbolIndexForDecl(self.getOwnerDecl());
+            const atom_index = try self.owner.getSymbolIndex(self);
             _ = try self.addInst(.{
                 .tag = switch (src_mcv) {
                     .lea_direct => .lea_linker,
@@ -7766,7 +7802,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             });
         },
         .lea_tlv => |sym_index| {
-            const atom_index = try self.getSymbolIndexForDecl(self.getOwnerDecl());
+            const atom_index = try self.owner.getSymbolIndex(self);
             if (self.bin_file.cast(link.File.MachO)) |_| {
                 _ = try self.addInst(.{
                     .tag = .lea_linker,
@@ -9079,7 +9115,7 @@ fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCV
 }
 
 fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
-    return switch (try codegen.genTypedValue(self.bin_file, self.src_loc, arg_tv, self.getOwnerDecl())) {
+    return switch (try codegen.genTypedValue(self.bin_file, self.src_loc, arg_tv, self.owner.getOwnerDecl())) {
         .mcv => |mcv| switch (mcv) {
             .none => .none,
             .undef => .undef,
@@ -9390,13 +9426,3 @@ fn regBitSize(self: *Self, ty: Type) u64 {
 fn regExtraBits(self: *Self, ty: Type) u64 {
     return self.regBitSize(ty) - ty.bitSize(self.target.*);
 }
-
-fn getSymbolIndexForDecl(self: *Self, decl_index: Module.Decl.Index) !u32 {
-    if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-        const atom = try macho_file.getOrCreateAtomForDecl(decl_index);
-        return macho_file.getAtom(atom).getSymbolIndex().?;
-    } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-        const atom = try coff_file.getOrCreateAtomForDecl(decl_index);
-        return coff_file.getAtom(atom).getSymbolIndex().?;
-    } else unreachable;
-}
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index c14168c66b..458b283d4f 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -2112,7 +2112,7 @@ fn updateLazySymbolAtom(
     errdefer self.freeAtom(atom_index);
 
     log.debug("allocated atom for {s} at 0x{x}", .{ name, vaddr });
-    log.debug("  (required alignment 0x{x}", .{required_alignment});
+    log.debug("  (required alignment 0x{x})", .{required_alignment});
 
     atom.size = code.len;
     symbol.n_value = vaddr;
@@ -4157,9 +4157,6 @@ pub fn logSymtab(self: *MachO) void {
 
     log.debug("stubs entries:", .{});
     log.debug("{}", .{self.stub_table});
-
-    // log.debug("threadlocal entries:", .{});
-    // log.debug("{}", .{self.tlv_table});
 }
 
 pub fn logAtoms(self: *MachO) void {
@@ -4199,6 +4196,6 @@ pub fn logAtom(self: *MachO, atom_index: Atom.Index) void {
         sym.n_value,
         atom.size,
         atom.file,
-        sym.n_sect,
+        sym.n_sect + 1,
     });
 }

From 5e7f3d5daaea1e6f5e835648ffd085797b667824 Mon Sep 17 00:00:00 2001
From: Jakub Konka 
Date: Mon, 1 May 2023 19:34:55 +0200
Subject: [PATCH 049/725] x86_64: disable advanced memset tests on Windows

---
 test/behavior/memset.zig | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/behavior/memset.zig b/test/behavior/memset.zig
index 89e01a0e56..076e1f17d9 100644
--- a/test/behavior/memset.zig
+++ b/test/behavior/memset.zig
@@ -114,6 +114,7 @@ test "memset with large array element, runtime known" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
 
     const A = [128]u64;
     var buf: [5]A = undefined;
@@ -131,6 +132,7 @@ test "memset with large array element, comptime known" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
 
     const A = [128]u64;
     var buf: [5]A = undefined;

From db88b414722e698a392ec65a3ef46730341aea25 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Mon, 1 May 2023 16:58:55 -0400
Subject: [PATCH 050/725] x86_64: fix switch multi-prongs and mul/div flags
 clobber

---
 src/arch/x86_64/CodeGen.zig     | 10 ++++++----
 test/behavior/inline_switch.zig |  1 -
 test/behavior/switch.zig        |  4 ----
 test/behavior/union.zig         |  1 -
 4 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index d5c5938ba0..8e61af1c9f 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -2435,6 +2435,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
         } };
         const src_ty = Type.initPayload(&src_pl.base);
 
+        try self.spillEflagsIfOccupied();
         try self.spillRegisters(&.{ .rax, .rdx });
         const lhs = try self.resolveInst(bin_op.lhs);
         const rhs = try self.resolveInst(bin_op.rhs);
@@ -7236,15 +7237,16 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
         var relocs = try self.gpa.alloc(u32, items.len);
         defer self.gpa.free(relocs);
 
-        for (items, relocs) |item, *reloc| {
-            try self.spillEflagsIfOccupied();
+        try self.spillEflagsIfOccupied();
+        for (items, relocs, 0..) |item, *reloc, i| {
             const item_mcv = try self.resolveInst(item);
             try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);
-            reloc.* = try self.asmJccReloc(undefined, .ne);
+            reloc.* = try self.asmJccReloc(undefined, if (i < relocs.len - 1) .e else .ne);
         }
 
         for (liveness.deaths[case_i]) |operand| self.processDeath(operand);
 
+        for (relocs[0 .. relocs.len - 1]) |reloc| try self.performReloc(reloc);
         try self.genBody(case_body);
         try self.restoreState(state, &.{}, .{
             .emit_instructions = false,
@@ -7253,7 +7255,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
             .close_scope = true,
         });
 
-        for (relocs) |reloc| try self.performReloc(reloc);
+        try self.performReloc(relocs[relocs.len - 1]);
     }
 
     if (switch_br.data.else_body_len > 0) {
diff --git a/test/behavior/inline_switch.zig b/test/behavior/inline_switch.zig
index dcd603c94f..6e5013d83b 100644
--- a/test/behavior/inline_switch.zig
+++ b/test/behavior/inline_switch.zig
@@ -94,7 +94,6 @@ test "inline else error" {
 
 test "inline else enum" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const E2 = enum(u8) { a = 2, b = 3, c = 4, d = 5 };
diff --git a/test/behavior/switch.zig b/test/behavior/switch.zig
index 3924d1d6c1..5e2d6d28c1 100644
--- a/test/behavior/switch.zig
+++ b/test/behavior/switch.zig
@@ -88,7 +88,6 @@ fn nonConstSwitch(foo: SwitchStatementFoo) !void {
 const SwitchStatementFoo = enum { A, B, C, D };
 
 test "switch with multiple expressions" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const x = switch (returnsFive()) {
@@ -275,7 +274,6 @@ fn testSwitchEnumPtrCapture() !void {
 }
 
 test "switch handles all cases of number" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     try testSwitchHandleAllCases();
@@ -455,7 +453,6 @@ test "else prong of switch on error set excludes other cases" {
 }
 
 test "switch prongs with error set cases make a new error set type for capture value" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -532,7 +529,6 @@ test "switch with null and T peer types and inferred result location type" {
 test "switch prongs with cases with identical payload types" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const Union = union(enum) {
diff --git a/test/behavior/union.zig b/test/behavior/union.zig
index f90a336e76..3dd8919935 100644
--- a/test/behavior/union.zig
+++ b/test/behavior/union.zig
@@ -1288,7 +1288,6 @@ test "return an extern union from C calling convention" {
 test "noreturn field in union" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const U = union(enum) {

From 3b1ea390a301dbdc992043d97cf618a94e8801de Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Mon, 1 May 2023 19:18:49 -0400
Subject: [PATCH 051/725] x86_64: cleanup lazy symbols

In theory fixes updating lazy symbols during incremental compilation.
---
 src/arch/x86_64/CodeGen.zig | 190 +++++++++++++++---------------------
 src/link/Coff.zig           |  77 ++++++++-------
 src/link/Elf.zig            |  75 +++++++-------
 src/link/MachO.zig          |  78 ++++++++-------
 4 files changed, 207 insertions(+), 213 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 8e61af1c9f..a658103c1a 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -112,7 +112,7 @@ const Owner = union(enum) {
     mod_fn: *const Module.Fn,
     lazy_sym: link.File.LazySymbol,
 
-    fn getOwnerDecl(owner: Owner) Module.Decl.Index {
+    fn getDecl(owner: Owner) Module.Decl.Index {
         return switch (owner) {
             .mod_fn => |mod_fn| mod_fn.owner_decl,
             .lazy_sym => |lazy_sym| lazy_sym.ty.getOwnerDecl(),
@@ -1688,35 +1688,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
             const data_reg = try self.register_manager.allocReg(null, gp);
             const data_lock = self.register_manager.lockRegAssumeUnused(data_reg);
             defer self.register_manager.unlockReg(data_lock);
-
-            const data_lazy_sym = link.File.LazySymbol{ .kind = .const_data, .ty = enum_ty };
-            if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-                const atom_index = elf_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
-                    return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-                const atom = elf_file.getAtom(atom_index);
-                _ = try atom.getOrCreateOffsetTableEntry(elf_file);
-                const got_addr = atom.getOffsetTableAddress(elf_file);
-                try self.asmRegisterMemory(
-                    .mov,
-                    data_reg.to64(),
-                    Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
-                );
-            } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-                const atom_index = coff_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
-                    return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-                const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
-                try self.genSetReg(data_reg, Type.usize, .{ .lea_got = sym_index });
-            } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-                const atom_index = macho_file.getOrCreateAtomForLazySymbol(data_lazy_sym) catch |err|
-                    return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-                const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
-                try self.genSetReg(data_reg, Type.usize, .{ .lea_got = sym_index });
-            } else {
-                return self.fail("TODO implement {s} for {}", .{
-                    @tagName(lazy_sym.kind),
-                    lazy_sym.ty.fmt(self.bin_file.options.module.?),
-                });
-            }
+            try self.genLazySymbolRef(.lea, data_reg, .{ .kind = .const_data, .ty = enum_ty });
 
             var data_off: i32 = 0;
             for (
@@ -6262,7 +6234,7 @@ fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void {
             // TODO: this might need adjusting like the linkers do.
             // Instead of flattening the owner and passing Decl.Index here we may
             // want to special case LazySymbol in DWARF linker too.
-            try dw.genArgDbgInfo(name, ty, self.owner.getOwnerDecl(), loc);
+            try dw.genArgDbgInfo(name, ty, self.owner.getDecl(), loc);
         },
         .plan9 => {},
         .none => {},
@@ -6306,7 +6278,7 @@ fn genVarDbgInfo(
             // TODO: this might need adjusting like the linkers do.
             // Instead of flattening the owner and passing Decl.Index here we may
             // want to special case LazySymbol in DWARF linker too.
-            try dw.genVarDbgInfo(name, ty, self.owner.getOwnerDecl(), is_ptr, loc);
+            try dw.genVarDbgInfo(name, ty, self.owner.getDecl(), is_ptr, loc);
         },
         .plan9 => {},
         .none => {},
@@ -6630,38 +6602,13 @@ fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {
 }
 
 fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
+    const mod = self.bin_file.options.module.?;
     const un_op = self.air.instructions.items(.data)[inst].un_op;
 
     const addr_reg = try self.register_manager.allocReg(null, gp);
     const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
     defer self.register_manager.unlockReg(addr_lock);
-
-    const mod = self.bin_file.options.module.?;
-    const lazy_sym = link.File.LazySymbol.initDecl(.const_data, null, mod);
-    if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-        const atom_index = elf_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
-            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-        const atom = elf_file.getAtom(atom_index);
-        _ = try atom.getOrCreateOffsetTableEntry(elf_file);
-        const got_addr = atom.getOffsetTableAddress(elf_file);
-        try self.asmRegisterMemory(
-            .mov,
-            addr_reg.to64(),
-            Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
-        );
-    } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-        const atom_index = coff_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
-            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-        const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
-        try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
-    } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-        const atom_index = macho_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
-            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-        const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
-        try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
-    } else {
-        return self.fail("TODO implement airCmpLtErrorsLen for x86_64 {s}", .{@tagName(self.bin_file.tag)});
-    }
+    try self.genLazySymbolRef(.lea, addr_reg, link.File.LazySymbol.initDecl(.const_data, null, mod));
 
     try self.spillEflagsIfOccupied();
     self.eflags_inst = inst;
@@ -7999,6 +7946,67 @@ fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue)
     });
 }
 
+fn genLazySymbolRef(
+    self: *Self,
+    comptime tag: Mir.Inst.Tag,
+    reg: Register,
+    lazy_sym: link.File.LazySymbol,
+) InnerError!void {
+    if (self.bin_file.cast(link.File.Elf)) |elf_file| {
+        const atom_index = elf_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+        const atom = elf_file.getAtom(atom_index);
+        _ = try atom.getOrCreateOffsetTableEntry(elf_file);
+        const got_addr = atom.getOffsetTableAddress(elf_file);
+        const got_mem =
+            Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) });
+        switch (tag) {
+            .lea, .mov => try self.asmRegisterMemory(.mov, reg.to64(), got_mem),
+            .call => try self.asmMemory(.call, got_mem),
+            else => unreachable,
+        }
+        switch (tag) {
+            .lea, .call => {},
+            .mov => try self.asmRegisterMemory(
+                tag,
+                reg.to64(),
+                Memory.sib(.qword, .{ .base = .{ .reg = reg.to64() } }),
+            ),
+            else => unreachable,
+        }
+    } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
+        const atom_index = coff_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+        const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
+        switch (tag) {
+            .lea, .call => try self.genSetReg(reg, Type.usize, .{ .lea_got = sym_index }),
+            .mov => try self.genSetReg(reg, Type.usize, .{ .load_got = sym_index }),
+            else => unreachable,
+        }
+        switch (tag) {
+            .lea, .mov => {},
+            .call => try self.asmRegister(.call, reg),
+            else => unreachable,
+        }
+    } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
+        const atom_index = macho_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
+            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
+        const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
+        switch (tag) {
+            .lea, .call => try self.genSetReg(reg, Type.usize, .{ .lea_got = sym_index }),
+            .mov => try self.genSetReg(reg, Type.usize, .{ .load_got = sym_index }),
+            else => unreachable,
+        }
+        switch (tag) {
+            .lea, .mov => {},
+            .call => try self.asmRegister(.call, reg),
+            else => unreachable,
+        }
+    } else {
+        return self.fail("TODO implement genLazySymbol for x86_64 {s}", .{@tagName(self.bin_file.tag)});
+    }
+}
+
 fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     const result = result: {
@@ -8701,6 +8709,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
 }
 
 fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
+    const mod = self.bin_file.options.module.?;
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     const inst_ty = self.air.typeOfIndex(inst);
     const enum_ty = self.air.typeOf(un_op);
@@ -8731,38 +8740,17 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
     const operand = try self.resolveInst(un_op);
     try self.genSetReg(param_regs[1], enum_ty, operand);
 
-    const mod = self.bin_file.options.module.?;
-    const lazy_sym = link.File.LazySymbol.initDecl(.code, enum_ty.getOwnerDecl(), mod);
-    if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-        const atom_index = elf_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
-            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-        const atom = elf_file.getAtom(atom_index);
-        _ = try atom.getOrCreateOffsetTableEntry(elf_file);
-        const got_addr = atom.getOffsetTableAddress(elf_file);
-        try self.asmMemory(
-            .call,
-            Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
-        );
-    } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-        const atom_index = coff_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
-            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-        const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
-        try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
-        try self.asmRegister(.call, .rax);
-    } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-        const atom_index = macho_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
-            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-        const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
-        try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
-        try self.asmRegister(.call, .rax);
-    } else {
-        return self.fail("TODO implement airTagName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
-    }
+    try self.genLazySymbolRef(
+        .call,
+        .rax,
+        link.File.LazySymbol.initDecl(.code, enum_ty.getOwnerDecl(), mod),
+    );
 
     return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
 }
 
 fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
+    const mod = self.bin_file.options.module.?;
     const un_op = self.air.instructions.items(.data)[inst].un_op;
 
     const err_ty = self.air.typeOf(un_op);
@@ -8774,33 +8762,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
     const addr_reg = try self.register_manager.allocReg(null, gp);
     const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
     defer self.register_manager.unlockReg(addr_lock);
-
-    const mod = self.bin_file.options.module.?;
-    const lazy_sym = link.File.LazySymbol.initDecl(.const_data, null, mod);
-    if (self.bin_file.cast(link.File.Elf)) |elf_file| {
-        const atom_index = elf_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
-            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-        const atom = elf_file.getAtom(atom_index);
-        _ = try atom.getOrCreateOffsetTableEntry(elf_file);
-        const got_addr = atom.getOffsetTableAddress(elf_file);
-        try self.asmRegisterMemory(
-            .mov,
-            addr_reg.to64(),
-            Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
-        );
-    } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
-        const atom_index = coff_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
-            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-        const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
-        try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
-    } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
-        const atom_index = macho_file.getOrCreateAtomForLazySymbol(lazy_sym) catch |err|
-            return self.fail("{s} creating lazy symbol", .{@errorName(err)});
-        const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
-        try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
-    } else {
-        return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
-    }
+    try self.genLazySymbolRef(.lea, addr_reg, link.File.LazySymbol.initDecl(.const_data, null, mod));
 
     const start_reg = try self.register_manager.allocReg(null, gp);
     const start_lock = self.register_manager.lockRegAssumeUnused(start_reg);
@@ -9117,7 +9079,7 @@ fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCV
 }
 
 fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
-    return switch (try codegen.genTypedValue(self.bin_file, self.src_loc, arg_tv, self.owner.getOwnerDecl())) {
+    return switch (try codegen.genTypedValue(self.bin_file, self.src_loc, arg_tv, self.owner.getDecl())) {
         .mcv => |mcv| switch (mcv) {
             .none => .none,
             .undef => .undef,
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
index 41d4617e53..81e8c57bdd 100644
--- a/src/link/Coff.zig
+++ b/src/link/Coff.zig
@@ -143,8 +143,11 @@ const Section = struct {
 const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex, LazySymbolMetadata);
 
 const LazySymbolMetadata = struct {
-    text_atom: ?Atom.Index = null,
-    rdata_atom: ?Atom.Index = null,
+    const State = enum { unused, pending_flush, flushed };
+    text_atom: Atom.Index = undefined,
+    rdata_atom: Atom.Index = undefined,
+    text_state: State = .unused,
+    rdata_state: State = .unused,
 };
 
 const DeclMetadata = struct {
@@ -1150,8 +1153,6 @@ pub fn updateDecl(
     const tracy = trace(@src());
     defer tracy.end();
 
-    try self.updateLazySymbol(decl_index);
-
     const decl = module.declPtr(decl_index);
 
     if (decl.val.tag() == .extern_fn) {
@@ -1194,21 +1195,6 @@ pub fn updateDecl(
     return self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-fn updateLazySymbol(self: *Coff, decl: ?Module.Decl.Index) !void {
-    const metadata = self.lazy_syms.get(Module.Decl.OptionalIndex.init(decl)) orelse return;
-    const mod = self.base.options.module.?;
-    if (metadata.text_atom) |atom| try self.updateLazySymbolAtom(
-        link.File.LazySymbol.initDecl(.code, decl, mod),
-        atom,
-        self.text_section_index.?,
-    );
-    if (metadata.rdata_atom) |atom| try self.updateLazySymbolAtom(
-        link.File.LazySymbol.initDecl(.const_data, decl, mod),
-        atom,
-        self.rdata_section_index.?,
-    );
-}
-
 fn updateLazySymbolAtom(
     self: *Coff,
     sym: link.File.LazySymbol,
@@ -1279,14 +1265,19 @@ pub fn getOrCreateAtomForLazySymbol(self: *Coff, sym: link.File.LazySymbol) !Ato
     const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
     errdefer _ = if (!gop.found_existing) self.lazy_syms.pop();
     if (!gop.found_existing) gop.value_ptr.* = .{};
-    const atom_ptr = switch (sym.kind) {
-        .code => &gop.value_ptr.text_atom,
-        .const_data => &gop.value_ptr.rdata_atom,
+    const metadata: struct { atom: *Atom.Index, state: *LazySymbolMetadata.State } = switch (sym.kind) {
+        .code => .{ .atom = &gop.value_ptr.text_atom, .state = &gop.value_ptr.text_state },
+        .const_data => .{ .atom = &gop.value_ptr.rdata_atom, .state = &gop.value_ptr.rdata_state },
     };
-    if (atom_ptr.*) |atom| return atom;
-    const atom = try self.createAtom();
-    atom_ptr.* = atom;
-    try self.updateLazySymbolAtom(sym, atom, switch (sym.kind) {
+    switch (metadata.state.*) {
+        .unused => metadata.atom.* = try self.createAtom(),
+        .pending_flush => return metadata.atom.*,
+        .flushed => {},
+    }
+    metadata.state.* = .pending_flush;
+    const atom = metadata.atom.*;
+    // anyerror needs to be deferred until flushModule
+    if (sym.getDecl() != .none) try self.updateLazySymbolAtom(sym, atom, switch (sym.kind) {
         .code => self.text_section_index.?,
         .const_data => self.rdata_section_index.?,
     });
@@ -1617,15 +1608,35 @@ pub fn flushModule(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Nod
     sub_prog_node.activate();
     defer sub_prog_node.end();
 
-    // Most lazy symbols can be updated when the corresponding decl is,
-    // so we only have to worry about the one without an associated decl.
-    self.updateLazySymbol(null) catch |err| switch (err) {
-        error.CodegenFail => return error.FlushFailure,
-        else => |e| return e,
-    };
-
     const gpa = self.base.allocator;
 
+    const module = self.base.options.module orelse return error.LinkingWithoutZigSourceUnimplemented;
+
+    if (self.lazy_syms.getPtr(.none)) |metadata| {
+        // Most lazy symbols can be updated on first use, but
+        // anyerror needs to wait for everything to be flushed.
+        if (metadata.text_state != .unused) self.updateLazySymbolAtom(
+            link.File.LazySymbol.initDecl(.code, null, module),
+            metadata.text_atom,
+            self.text_section_index.?,
+        ) catch |err| return switch (err) {
+            error.CodegenFail => error.FlushFailure,
+            else => |e| e,
+        };
+        if (metadata.rdata_state != .unused) self.updateLazySymbolAtom(
+            link.File.LazySymbol.initDecl(.const_data, null, module),
+            metadata.rdata_atom,
+            self.rdata_section_index.?,
+        ) catch |err| return switch (err) {
+            error.CodegenFail => error.FlushFailure,
+            else => |e| e,
+        };
+    }
+    for (self.lazy_syms.values()) |*metadata| {
+        if (metadata.text_state != .unused) metadata.text_state = .flushed;
+        if (metadata.rdata_state != .unused) metadata.rdata_state = .flushed;
+    }
+
     while (self.unresolved.popOrNull()) |entry| {
         assert(entry.value); // We only expect imports generated by the incremental linker for now.
         const global = self.globals.items[entry.key];
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
index ec113e5c8a..724ec76500 100644
--- a/src/link/Elf.zig
+++ b/src/link/Elf.zig
@@ -65,8 +65,11 @@ const Section = struct {
 };
 
 const LazySymbolMetadata = struct {
-    text_atom: ?Atom.Index = null,
-    rodata_atom: ?Atom.Index = null,
+    const State = enum { unused, pending_flush, flushed };
+    text_atom: Atom.Index = undefined,
+    rodata_atom: Atom.Index = undefined,
+    text_state: State = .unused,
+    rodata_state: State = .unused,
 };
 
 const DeclMetadata = struct {
@@ -1032,17 +1035,35 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
     sub_prog_node.activate();
     defer sub_prog_node.end();
 
-    // Most lazy symbols can be updated when the corresponding decl is,
-    // so we only have to worry about the one without an associated decl.
-    self.updateLazySymbol(null) catch |err| switch (err) {
-        error.CodegenFail => return error.FlushFailure,
-        else => |e| return e,
-    };
-
     // TODO This linker code currently assumes there is only 1 compilation unit and it
     // corresponds to the Zig source code.
     const module = self.base.options.module orelse return error.LinkingWithoutZigSourceUnimplemented;
 
+    if (self.lazy_syms.getPtr(.none)) |metadata| {
+        // Most lazy symbols can be updated on first use, but
+        // anyerror needs to wait for everything to be flushed.
+        if (metadata.text_state != .unused) self.updateLazySymbolAtom(
+            File.LazySymbol.initDecl(.code, null, module),
+            metadata.text_atom,
+            self.text_section_index.?,
+        ) catch |err| return switch (err) {
+            error.CodegenFail => error.FlushFailure,
+            else => |e| e,
+        };
+        if (metadata.rodata_state != .unused) self.updateLazySymbolAtom(
+            File.LazySymbol.initDecl(.const_data, null, module),
+            metadata.rodata_atom,
+            self.rodata_section_index.?,
+        ) catch |err| return switch (err) {
+            error.CodegenFail => error.FlushFailure,
+            else => |e| e,
+        };
+    }
+    for (self.lazy_syms.values()) |*metadata| {
+        if (metadata.text_state != .unused) metadata.text_state = .flushed;
+        if (metadata.rodata_state != .unused) metadata.rodata_state = .flushed;
+    }
+
     const target_endian = self.base.options.target.cpu.arch.endian();
     const foreign_endian = target_endian != builtin.cpu.arch.endian();
 
@@ -2378,14 +2399,19 @@ pub fn getOrCreateAtomForLazySymbol(self: *Elf, sym: File.LazySymbol) !Atom.Inde
     const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
     errdefer _ = if (!gop.found_existing) self.lazy_syms.pop();
     if (!gop.found_existing) gop.value_ptr.* = .{};
-    const atom_ptr = switch (sym.kind) {
-        .code => &gop.value_ptr.text_atom,
-        .const_data => &gop.value_ptr.rodata_atom,
+    const metadata: struct { atom: *Atom.Index, state: *LazySymbolMetadata.State } = switch (sym.kind) {
+        .code => .{ .atom = &gop.value_ptr.text_atom, .state = &gop.value_ptr.text_state },
+        .const_data => .{ .atom = &gop.value_ptr.rodata_atom, .state = &gop.value_ptr.rodata_state },
     };
-    if (atom_ptr.*) |atom| return atom;
-    const atom = try self.createAtom();
-    atom_ptr.* = atom;
-    try self.updateLazySymbolAtom(sym, atom, switch (sym.kind) {
+    switch (metadata.state.*) {
+        .unused => metadata.atom.* = try self.createAtom(),
+        .pending_flush => return metadata.atom.*,
+        .flushed => {},
+    }
+    metadata.state.* = .pending_flush;
+    const atom = metadata.atom.*;
+    // anyerror needs to be deferred until flushModule
+    if (sym.getDecl() != .none) try self.updateLazySymbolAtom(sym, atom, switch (sym.kind) {
         .code => self.text_section_index.?,
         .const_data => self.rodata_section_index.?,
     });
@@ -2598,8 +2624,6 @@ pub fn updateDecl(
     const tracy = trace(@src());
     defer tracy.end();
 
-    try self.updateLazySymbol(decl_index);
-
     const decl = module.declPtr(decl_index);
 
     if (decl.val.tag() == .extern_fn) {
@@ -2666,21 +2690,6 @@ pub fn updateDecl(
     return self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-fn updateLazySymbol(self: *Elf, decl: ?Module.Decl.Index) !void {
-    const metadata = self.lazy_syms.get(Module.Decl.OptionalIndex.init(decl)) orelse return;
-    const mod = self.base.options.module.?;
-    if (metadata.text_atom) |atom| try self.updateLazySymbolAtom(
-        File.LazySymbol.initDecl(.code, decl, mod),
-        atom,
-        self.text_section_index.?,
-    );
-    if (metadata.rodata_atom) |atom| try self.updateLazySymbolAtom(
-        File.LazySymbol.initDecl(.const_data, decl, mod),
-        atom,
-        self.rodata_section_index.?,
-    );
-}
-
 fn updateLazySymbolAtom(
     self: *Elf,
     sym: File.LazySymbol,
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
index 458b283d4f..a346ec756f 100644
--- a/src/link/MachO.zig
+++ b/src/link/MachO.zig
@@ -236,8 +236,11 @@ const is_hot_update_compatible = switch (builtin.target.os.tag) {
 const LazySymbolTable = std.AutoArrayHashMapUnmanaged(Module.Decl.OptionalIndex, LazySymbolMetadata);
 
 const LazySymbolMetadata = struct {
-    text_atom: ?Atom.Index = null,
-    data_const_atom: ?Atom.Index = null,
+    const State = enum { unused, pending_flush, flushed };
+    text_atom: Atom.Index = undefined,
+    data_const_atom: Atom.Index = undefined,
+    text_state: State = .unused,
+    data_const_state: State = .unused,
 };
 
 const TlvSymbolTable = std.AutoArrayHashMapUnmanaged(SymbolWithLoc, Atom.Index);
@@ -493,15 +496,33 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
     sub_prog_node.activate();
     defer sub_prog_node.end();
 
-    // Most lazy symbols can be updated when the corresponding decl is,
-    // so we only have to worry about the one without an associated decl.
-    self.updateLazySymbol(null) catch |err| switch (err) {
-        error.CodegenFail => return error.FlushFailure,
-        else => |e| return e,
-    };
-
     const module = self.base.options.module orelse return error.LinkingWithoutZigSourceUnimplemented;
 
+    if (self.lazy_syms.getPtr(.none)) |metadata| {
+        // Most lazy symbols can be updated on first use, but
+        // anyerror needs to wait for everything to be flushed.
+        if (metadata.text_state != .unused) self.updateLazySymbolAtom(
+            File.LazySymbol.initDecl(.code, null, module),
+            metadata.text_atom,
+            self.text_section_index.?,
+        ) catch |err| return switch (err) {
+            error.CodegenFail => error.FlushFailure,
+            else => |e| e,
+        };
+        if (metadata.data_const_state != .unused) self.updateLazySymbolAtom(
+            File.LazySymbol.initDecl(.const_data, null, module),
+            metadata.data_const_atom,
+            self.data_const_section_index.?,
+        ) catch |err| return switch (err) {
+            error.CodegenFail => error.FlushFailure,
+            else => |e| e,
+        };
+    }
+    for (self.lazy_syms.values()) |*metadata| {
+        if (metadata.text_state != .unused) metadata.text_state = .flushed;
+        if (metadata.data_const_state != .unused) metadata.data_const_state = .flushed;
+    }
+
     if (self.d_sym) |*d_sym| {
         try d_sym.dwarf.flushModule(module);
     }
@@ -1960,8 +1981,6 @@ pub fn updateDecl(self: *MachO, module: *Module, decl_index: Module.Decl.Index)
     const tracy = trace(@src());
     defer tracy.end();
 
-    try self.updateLazySymbol(decl_index);
-
     const decl = module.declPtr(decl_index);
 
     if (decl.val.tag() == .extern_fn) {
@@ -2036,21 +2055,6 @@ pub fn updateDecl(self: *MachO, module: *Module, decl_index: Module.Decl.Index)
     try self.updateDeclExports(module, decl_index, module.getDeclExports(decl_index));
 }
 
-fn updateLazySymbol(self: *MachO, decl: ?Module.Decl.Index) !void {
-    const metadata = self.lazy_syms.get(Module.Decl.OptionalIndex.init(decl)) orelse return;
-    const mod = self.base.options.module.?;
-    if (metadata.text_atom) |atom| try self.updateLazySymbolAtom(
-        File.LazySymbol.initDecl(.code, decl, mod),
-        atom,
-        self.text_section_index.?,
-    );
-    if (metadata.data_const_atom) |atom| try self.updateLazySymbolAtom(
-        File.LazySymbol.initDecl(.const_data, decl, mod),
-        atom,
-        self.data_const_section_index.?,
-    );
-}
-
 fn updateLazySymbolAtom(
     self: *MachO,
     sym: File.LazySymbol,
@@ -2125,14 +2129,22 @@ pub fn getOrCreateAtomForLazySymbol(self: *MachO, sym: File.LazySymbol) !Atom.In
     const gop = try self.lazy_syms.getOrPut(self.base.allocator, sym.getDecl());
     errdefer _ = if (!gop.found_existing) self.lazy_syms.pop();
     if (!gop.found_existing) gop.value_ptr.* = .{};
-    const atom_ptr = switch (sym.kind) {
-        .code => &gop.value_ptr.text_atom,
-        .const_data => &gop.value_ptr.data_const_atom,
+    const metadata: struct { atom: *Atom.Index, state: *LazySymbolMetadata.State } = switch (sym.kind) {
+        .code => .{ .atom = &gop.value_ptr.text_atom, .state = &gop.value_ptr.text_state },
+        .const_data => .{
+            .atom = &gop.value_ptr.data_const_atom,
+            .state = &gop.value_ptr.data_const_state,
+        },
     };
-    if (atom_ptr.*) |atom| return atom;
-    const atom = try self.createAtom();
-    atom_ptr.* = atom;
-    try self.updateLazySymbolAtom(sym, atom, switch (sym.kind) {
+    switch (metadata.state.*) {
+        .unused => metadata.atom.* = try self.createAtom(),
+        .pending_flush => return metadata.atom.*,
+        .flushed => {},
+    }
+    metadata.state.* = .pending_flush;
+    const atom = metadata.atom.*;
+    // anyerror needs to be deferred until flushModule
+    if (sym.getDecl() != .none) try self.updateLazySymbolAtom(sym, atom, switch (sym.kind) {
         .code => self.text_section_index.?,
         .const_data => self.data_const_section_index.?,
     });

From 6f1336a50c6c2d9498bb9b3fb291cb305688c1b9 Mon Sep 17 00:00:00 2001
From: r00ster91 
Date: Tue, 2 May 2023 04:41:25 +0200
Subject: [PATCH 052/725] autodoc: make the help modal toggleable

Now you can simply press "?" again to toggle the help modal instead of
requiring Esc. Both Esc and "?" work.
---
 lib/docs/index.html |  2 +-
 lib/docs/main.js    | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/lib/docs/index.html b/lib/docs/index.html
index 6b9f95a7d0..36cfcc55cc 100644
--- a/lib/docs/index.html
+++ b/lib/docs/index.html
@@ -875,7 +875,7 @@
       

         

           
Keyboard Shortcuts

-          
?
Show this help modal

+          
?
Toggle this help modal

           
s
Focus the search field

           

             
↑
Move up in search results

diff --git a/lib/docs/main.js b/lib/docs/main.js
index 14bcf0f544..504c708b45 100644
--- a/lib/docs/main.js
+++ b/lib/docs/main.js
@@ -4044,9 +4044,14 @@ function addDeclToSearchResults(decl, declIndex, modNames, item, list, stack) {
         }
         break;
       case "?":
-        ev.preventDefault();
-        ev.stopPropagation();
-        showHelpModal();
+        // toggle the help modal
+        if (!domHelpModal.classList.contains("hidden")) {
+            onEscape(ev);
+        } else {
+            ev.preventDefault();
+            ev.stopPropagation();
+            showHelpModal();
+        }
         break;
     }
   }
@@ -4850,4 +4855,4 @@ function RadixTree() {
 
 // BUT!
 
-// We want to be able to search "Hash", for example!
\ No newline at end of file
+// We want to be able to search "Hash", for example!

From eab4cd7a66803809ca94a03adc8bbdefc87bd60b Mon Sep 17 00:00:00 2001
From: r00ster91 
Date: Tue, 2 May 2023 04:43:06 +0200
Subject: [PATCH 053/725] autodoc: type "?" instead of opening help modal if
 search selected

The question mark character can appear in identifiers as part of the
`@"syntax"` so we should allow typing it. Now, when the search is
selected, "?" is entered instead. It also shouldn't be that common in
general for the user to want to open the help modal.
---
 lib/docs/main.js | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/docs/main.js b/lib/docs/main.js
index 504c708b45..bca4d6c84e 100644
--- a/lib/docs/main.js
+++ b/lib/docs/main.js
@@ -127,16 +127,20 @@ const NAV_MODES = {
   window.guideSearch = guidesSearchIndex;
   parseGuides();
 
-  
+  // identifiers can contain '?' so we want to allow typing
+  // the question mark when the search is focused instead of toggling the help modal
+  let canToggleHelpModal = true;
 
   domSearch.disabled = false;
   domSearch.addEventListener("keydown", onSearchKeyDown, false);
   domSearch.addEventListener("focus", ev => {
     domSearchPlaceholder.classList.add("hidden");
+    canToggleHelpModal = false;
   });
   domSearch.addEventListener("blur", ev => {
     if (domSearch.value.length == 0)
       domSearchPlaceholder.classList.remove("hidden");
+    canToggleHelpModal = true;
   });
   domSectSearchAllResultsLink.addEventListener('click', onClickSearchShowAllResults, false);
   function onClickSearchShowAllResults(ev) {
@@ -4044,6 +4048,8 @@ function addDeclToSearchResults(decl, declIndex, modNames, item, list, stack) {
         }
         break;
       case "?":
+        if (!canToggleHelpModal) break;
+
         // toggle the help modal
         if (!domHelpModal.classList.contains("hidden")) {
             onEscape(ev);

From e9cbdb2cfd6ccc51b50e97e7df419451aa8a1c95 Mon Sep 17 00:00:00 2001
From: DraagrenKirneh 
Date: Tue, 2 May 2023 13:43:41 +0200
Subject: [PATCH 054/725] Add a random generated prefix to the unix socket_path
 to guard against multiple tests/threads running the same test at the same
 time

---
 lib/std/net/test.zig | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/lib/std/net/test.zig b/lib/std/net/test.zig
index f59d73d621..9923e4932b 100644
--- a/lib/std/net/test.zig
+++ b/lib/std/net/test.zig
@@ -1,4 +1,4 @@
-const std = @import("../std.zig");
+const std = @import("std");
 const builtin = @import("builtin");
 const net = std.net;
 const mem = std.mem;
@@ -300,22 +300,23 @@ test "listen on a unix socket, send bytes, receive bytes" {
     var server = net.StreamServer.init(.{});
     defer server.deinit();
 
-    const socket_path = "socket.unix";
+    var socket_path = try generateFileName("socket.unix");
+    defer testing.allocator.free(socket_path);
 
     var socket_addr = try net.Address.initUnix(socket_path);
     defer std.fs.cwd().deleteFile(socket_path) catch {};
     try server.listen(socket_addr);
 
     const S = struct {
-        fn clientFn() !void {
-            const socket = try net.connectUnixSocket(socket_path);
+        fn clientFn(path: []const u8) !void {
+            const socket = try net.connectUnixSocket(path);
             defer socket.close();
 
             _ = try socket.writer().writeAll("Hello world!");
         }
     };
 
-    const t = try std.Thread.spawn(.{}, S.clientFn, .{});
+    const t = try std.Thread.spawn(.{}, S.clientFn, .{socket_path});
     defer t.join();
 
     var client = try server.accept();
@@ -326,3 +327,13 @@ test "listen on a unix socket, send bytes, receive bytes" {
     try testing.expectEqual(@as(usize, 12), n);
     try testing.expectEqualSlices(u8, "Hello world!", buf[0..n]);
 }
+
+fn generateFileName(base_name: []const u8) ![]const u8 {
+    const random_bytes_count = 12;
+    const sub_path_len = comptime std.fs.base64_encoder.calcSize(random_bytes_count);
+    var random_bytes: [12]u8 = undefined;
+    std.crypto.random.bytes(&random_bytes);
+    var sub_path: [sub_path_len]u8 = undefined;
+    _ = std.fs.base64_encoder.encode(&sub_path, &random_bytes);
+    return std.fmt.allocPrint(testing.allocator, "{s}-{s}", .{ sub_path[0..], base_name });
+}

From 152c7b1885a0ed8e5a2435ef7b51568b357eaea4 Mon Sep 17 00:00:00 2001
From: mlugg 
Date: Sun, 30 Apr 2023 10:30:40 +0100
Subject: [PATCH 055/725] Implement multi-argument @min/@max and notice bounds

Resolves: #14039
---
 src/AstGen.zig                    |  63 +++++---
 src/BuiltinFn.zig                 |   4 +-
 src/Sema.zig                      | 253 ++++++++++++++++++++++++------
 src/Zir.zig                       |  16 +-
 src/arch/x86_64/CodeGen.zig       |   8 +-
 src/print_zir.zig                 |   2 +
 src/type.zig                      |  67 +++++++-
 test/behavior/maximum_minimum.zig |  56 +++++++
 8 files changed, 394 insertions(+), 75 deletions(-)

diff --git a/src/AstGen.zig b/src/AstGen.zig
index aece3eafec..2568b89980 100644
--- a/src/AstGen.zig
+++ b/src/AstGen.zig
@@ -7907,6 +7907,48 @@ fn typeOf(
     return rvalue(gz, ri, typeof_inst, node);
 }
 
+fn minMax(
+    gz: *GenZir,
+    scope: *Scope,
+    ri: ResultInfo,
+    node: Ast.Node.Index,
+    args: []const Ast.Node.Index,
+    comptime op: enum { min, max },
+) InnerError!Zir.Inst.Ref {
+    const astgen = gz.astgen;
+    if (args.len < 2) {
+        return astgen.failNode(node, "expected at least 2 arguments, found 0", .{});
+    }
+    if (args.len == 2) {
+        const tag: Zir.Inst.Tag = switch (op) {
+            .min => .min,
+            .max => .max,
+        };
+        const a = try expr(gz, scope, .{ .rl = .none }, args[0]);
+        const b = try expr(gz, scope, .{ .rl = .none }, args[1]);
+        const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{
+            .lhs = a,
+            .rhs = b,
+        });
+        return rvalue(gz, ri, result, node);
+    }
+    const payload_index = try addExtra(astgen, Zir.Inst.NodeMultiOp{
+        .src_node = gz.nodeIndexToRelative(node),
+    });
+    var extra_index = try reserveExtra(gz.astgen, args.len);
+    for (args) |arg| {
+        const arg_ref = try expr(gz, scope, .{ .rl = .none }, arg);
+        astgen.extra.items[extra_index] = @enumToInt(arg_ref);
+        extra_index += 1;
+    }
+    const tag: Zir.Inst.Extended = switch (op) {
+        .min => .min_multi,
+        .max => .max_multi,
+    };
+    const result = try gz.addExtendedMultiOpPayloadIndex(tag, payload_index, args.len);
+    return rvalue(gz, ri, result, node);
+}
+
 fn builtinCall(
     gz: *GenZir,
     scope: *Scope,
@@ -7997,6 +8039,8 @@ fn builtinCall(
         .TypeOf     => return typeOf(   gz, scope, ri, node, params),
         .union_init => return unionInit(gz, scope, ri, node, params),
         .c_import   => return cImport(  gz, scope,     node, params[0]),
+        .min        => return minMax(   gz, scope, ri, node, params, .min),
+        .max        => return minMax(   gz, scope, ri, node, params, .max),
         // zig fmt: on
 
         .@"export" => {
@@ -8358,25 +8402,6 @@ fn builtinCall(
             return rvalue(gz, ri, result, node);
         },
 
-        .max => {
-            const a = try expr(gz, scope, .{ .rl = .none }, params[0]);
-            const b = try expr(gz, scope, .{ .rl = .none }, params[1]);
-            const result = try gz.addPlNode(.max, node, Zir.Inst.Bin{
-                .lhs = a,
-                .rhs = b,
-            });
-            return rvalue(gz, ri, result, node);
-        },
-        .min => {
-            const a = try expr(gz, scope, .{ .rl = .none }, params[0]);
-            const b = try expr(gz, scope, .{ .rl = .none }, params[1]);
-            const result = try gz.addPlNode(.min, node, Zir.Inst.Bin{
-                .lhs = a,
-                .rhs = b,
-            });
-            return rvalue(gz, ri, result, node);
-        },
-
         .add_with_overflow => return overflowArithmetic(gz, scope, ri, node, params, .add_with_overflow),
         .sub_with_overflow => return overflowArithmetic(gz, scope, ri, node, params, .sub_with_overflow),
         .mul_with_overflow => return overflowArithmetic(gz, scope, ri, node, params, .mul_with_overflow),
diff --git a/src/BuiltinFn.zig b/src/BuiltinFn.zig
index d31a1e7c25..426939afdf 100644
--- a/src/BuiltinFn.zig
+++ b/src/BuiltinFn.zig
@@ -608,7 +608,7 @@ pub const list = list: {
             "@max",
             .{
                 .tag = .max,
-                .param_count = 2,
+                .param_count = null,
             },
         },
         .{
@@ -629,7 +629,7 @@ pub const list = list: {
             "@min",
             .{
                 .tag = .min,
-                .param_count = 2,
+                .param_count = null,
             },
         },
         .{
diff --git a/src/Sema.zig b/src/Sema.zig
index 79f2fd7fca..13f4d684a1 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -1137,6 +1137,8 @@ fn analyzeBodyInner(
                     .asm_expr              => try sema.zirAsm(               block, extended, true),
                     .typeof_peer           => try sema.zirTypeofPeer(        block, extended),
                     .compile_log           => try sema.zirCompileLog(               extended),
+                    .min_multi             => try sema.zirMinMaxMulti(       block, extended, .min),
+                    .max_multi             => try sema.zirMinMaxMulti(       block, extended, .max),
                     .add_with_overflow     => try sema.zirOverflowArithmetic(block, extended, extended.opcode),
                     .sub_with_overflow     => try sema.zirOverflowArithmetic(block, extended, extended.opcode),
                     .mul_with_overflow     => try sema.zirOverflowArithmetic(block, extended, extended.opcode),
@@ -12143,7 +12145,7 @@ fn zirShl(
                 lhs_ty,
                 try lhs_ty.maxInt(sema.arena, target),
             );
-            const rhs_limited = try sema.analyzeMinMax(block, rhs_src, rhs, max_int, .min, rhs_src, rhs_src);
+            const rhs_limited = try sema.analyzeMinMax(block, rhs_src, .min, &.{ rhs, max_int }, &.{ rhs_src, rhs_src });
             break :rhs try sema.intCast(block, src, lhs_ty, rhs_src, rhs_limited, rhs_src, false);
         } else {
             break :rhs rhs;
@@ -21752,64 +21754,223 @@ fn zirMinMax(
     const rhs = try sema.resolveInst(extra.rhs);
     try sema.checkNumericType(block, lhs_src, sema.typeOf(lhs));
     try sema.checkNumericType(block, rhs_src, sema.typeOf(rhs));
-    return sema.analyzeMinMax(block, src, lhs, rhs, air_tag, lhs_src, rhs_src);
+    return sema.analyzeMinMax(block, src, air_tag, &.{ lhs, rhs }, &.{ lhs_src, rhs_src });
+}
+
+fn zirMinMaxMulti(
+    sema: *Sema,
+    block: *Block,
+    extended: Zir.Inst.Extended.InstData,
+    comptime air_tag: Air.Inst.Tag,
+) CompileError!Air.Inst.Ref {
+    const extra = sema.code.extraData(Zir.Inst.NodeMultiOp, extended.operand);
+    const src_node = extra.data.src_node;
+    const src = LazySrcLoc.nodeOffset(src_node);
+    const operands = sema.code.refSlice(extra.end, extended.small);
+
+    const air_refs = try sema.arena.alloc(Air.Inst.Ref, operands.len);
+    const operand_srcs = try sema.arena.alloc(LazySrcLoc, operands.len);
+
+    for (operands, air_refs, operand_srcs, 0..) |zir_ref, *air_ref, *op_src, i| {
+        op_src.* = switch (i) {
+            0 => .{ .node_offset_builtin_call_arg0 = src_node },
+            1 => .{ .node_offset_builtin_call_arg1 = src_node },
+            2 => .{ .node_offset_builtin_call_arg2 = src_node },
+            3 => .{ .node_offset_builtin_call_arg3 = src_node },
+            4 => .{ .node_offset_builtin_call_arg4 = src_node },
+            5 => .{ .node_offset_builtin_call_arg5 = src_node },
+            else => src, // TODO: better source location
+        };
+        air_ref.* = try sema.resolveInst(zir_ref);
+        try sema.checkNumericType(block, op_src.*, sema.typeOf(air_ref.*));
+    }
+
+    return sema.analyzeMinMax(block, src, air_tag, air_refs, operand_srcs);
 }
 
 fn analyzeMinMax(
     sema: *Sema,
     block: *Block,
     src: LazySrcLoc,
-    lhs: Air.Inst.Ref,
-    rhs: Air.Inst.Ref,
     comptime air_tag: Air.Inst.Tag,
-    lhs_src: LazySrcLoc,
-    rhs_src: LazySrcLoc,
+    operands: []const Air.Inst.Ref,
+    operand_srcs: []const LazySrcLoc,
 ) CompileError!Air.Inst.Ref {
-    const simd_op = try sema.checkSimdBinOp(block, src, lhs, rhs, lhs_src, rhs_src);
+    assert(operands.len == operand_srcs.len);
+    assert(operands.len > 0);
 
-    // TODO @max(max_int, undefined) should return max_int
+    if (operands.len == 1) return operands[0];
 
-    const runtime_src = if (simd_op.lhs_val) |lhs_val| rs: {
-        if (lhs_val.isUndef()) return sema.addConstUndef(simd_op.result_ty);
-
-        const rhs_val = simd_op.rhs_val orelse break :rs rhs_src;
-
-        if (rhs_val.isUndef()) return sema.addConstUndef(simd_op.result_ty);
-
-        try sema.resolveLazyValue(lhs_val);
-        try sema.resolveLazyValue(rhs_val);
-
-        const opFunc = switch (air_tag) {
-            .min => Value.numberMin,
-            .max => Value.numberMax,
-            else => unreachable,
-        };
-        const target = sema.mod.getTarget();
-        const vec_len = simd_op.len orelse {
-            const result_val = opFunc(lhs_val, rhs_val, target);
-            return sema.addConstant(simd_op.result_ty, result_val);
-        };
-        var lhs_buf: Value.ElemValueBuffer = undefined;
-        var rhs_buf: Value.ElemValueBuffer = undefined;
-        const elems = try sema.arena.alloc(Value, vec_len);
-        for (elems, 0..) |*elem, i| {
-            const lhs_elem_val = lhs_val.elemValueBuffer(sema.mod, i, &lhs_buf);
-            const rhs_elem_val = rhs_val.elemValueBuffer(sema.mod, i, &rhs_buf);
-            elem.* = opFunc(lhs_elem_val, rhs_elem_val, target);
-        }
-        return sema.addConstant(
-            simd_op.result_ty,
-            try Value.Tag.aggregate.create(sema.arena, elems),
-        );
-    } else rs: {
-        if (simd_op.rhs_val) |rhs_val| {
-            if (rhs_val.isUndef()) return sema.addConstUndef(simd_op.result_ty);
-        }
-        break :rs lhs_src;
+    const mod = sema.mod;
+    const target = mod.getTarget();
+    const opFunc = switch (air_tag) {
+        .min => Value.numberMin,
+        .max => Value.numberMax,
+        else => unreachable,
     };
 
+    // First, find all comptime-known arguments, and get their min/max
+    var runtime_known = try std.DynamicBitSet.initFull(sema.arena, operands.len);
+    var cur_minmax: ?Air.Inst.Ref = null;
+    var cur_minmax_src: LazySrcLoc = undefined; // defined if cur_minmax not null
+    for (operands, operand_srcs, 0..) |operand, operand_src, operand_idx| {
+        // Resolve the value now to avoid redundant calls to `checkSimdBinOp` - we'll have to call
+        // it in the runtime path anyway since the result type may have been refined
+        const uncasted_operand_val = (try sema.resolveMaybeUndefVal(operand)) orelse continue;
+        if (cur_minmax) |cur| {
+            const simd_op = try sema.checkSimdBinOp(block, src, cur, operand, cur_minmax_src, operand_src);
+            const cur_val = simd_op.lhs_val.?; // cur_minmax is comptime-known
+            const operand_val = simd_op.rhs_val.?; // we checked the operand was resolvable above
+
+            runtime_known.unset(operand_idx);
+
+            if (cur_val.isUndef()) continue; // result is also undef
+            if (operand_val.isUndef()) {
+                cur_minmax = try sema.addConstUndef(simd_op.result_ty);
+                continue;
+            }
+
+            try sema.resolveLazyValue(cur_val);
+            try sema.resolveLazyValue(operand_val);
+
+            const vec_len = simd_op.len orelse {
+                const result_val = opFunc(cur_val, operand_val, target);
+                cur_minmax = try sema.addConstant(simd_op.result_ty, result_val);
+                continue;
+            };
+            var lhs_buf: Value.ElemValueBuffer = undefined;
+            var rhs_buf: Value.ElemValueBuffer = undefined;
+            const elems = try sema.arena.alloc(Value, vec_len);
+            for (elems, 0..) |*elem, i| {
+                const lhs_elem_val = cur_val.elemValueBuffer(mod, i, &lhs_buf);
+                const rhs_elem_val = operand_val.elemValueBuffer(mod, i, &rhs_buf);
+                elem.* = opFunc(lhs_elem_val, rhs_elem_val, target);
+            }
+            cur_minmax = try sema.addConstant(
+                simd_op.result_ty,
+                try Value.Tag.aggregate.create(sema.arena, elems),
+            );
+        } else {
+            runtime_known.unset(operand_idx);
+            cur_minmax = try sema.addConstant(sema.typeOf(operand), uncasted_operand_val);
+            cur_minmax_src = operand_src;
+        }
+    }
+
+    const comptime_refined_ty: ?Type = if (cur_minmax) |ct_minmax_ref| refined: {
+        // Refine the comptime-known result type based on the operation
+        const val = (try sema.resolveMaybeUndefVal(ct_minmax_ref)).?;
+        const orig_ty = sema.typeOf(ct_minmax_ref);
+        const refined_ty = if (orig_ty.zigTypeTag() == .Vector) blk: {
+            const elem_ty = orig_ty.childType();
+            const len = orig_ty.vectorLen();
+
+            if (len == 0) break :blk orig_ty;
+            if (elem_ty.isAnyFloat()) break :blk orig_ty; // can't refine floats
+
+            var cur_min: Value = try val.elemValue(mod, sema.arena, 0);
+            var cur_max: Value = cur_min;
+            for (1..len) |idx| {
+                const elem_val = try val.elemValue(mod, sema.arena, idx);
+                if (elem_val.isUndef()) break :blk orig_ty; // can't refine undef
+                if (Value.order(elem_val, cur_min, target).compare(.lt)) cur_min = elem_val;
+                if (Value.order(elem_val, cur_max, target).compare(.gt)) cur_max = elem_val;
+            }
+
+            const refined_elem_ty = try Type.intFittingRange(target, sema.arena, cur_min, cur_max);
+            break :blk try Type.vector(sema.arena, len, refined_elem_ty);
+        } else blk: {
+            if (orig_ty.isAnyFloat()) break :blk orig_ty; // can't refine floats
+            if (val.isUndef()) break :blk orig_ty; // can't refine undef
+            break :blk try Type.intFittingRange(target, sema.arena, val, val);
+        };
+
+        // Apply the refined type to the current value - this isn't strictly necessary in the
+        // runtime case since we'll refine again afterwards, but keeping things as small as possible
+        // will allow us to emit more optimal AIR (if all the runtime operands have smaller types
+        // than the non-refined comptime type).
+        if (!refined_ty.eql(orig_ty, mod)) {
+            if (std.debug.runtime_safety) {
+                assert(try sema.intFitsInType(val, refined_ty, null));
+            }
+            cur_minmax = try sema.addConstant(refined_ty, val);
+        }
+
+        break :refined refined_ty;
+    } else null;
+
+    const runtime_idx = runtime_known.findFirstSet() orelse return cur_minmax.?;
+    const runtime_src = operand_srcs[runtime_idx];
     try sema.requireRuntimeBlock(block, src, runtime_src);
-    return block.addBinOp(air_tag, simd_op.lhs, simd_op.rhs);
+
+    // Now, iterate over runtime operands, emitting a min/max instruction for each. We'll refine the
+    // type again at the end, based on the comptime-known bound.
+
+    // If the comptime-known part is undef we can avoid emitting actual instructions later
+    const known_undef = if (cur_minmax) |operand| blk: {
+        const val = (try sema.resolveMaybeUndefVal(operand)).?;
+        break :blk val.isUndef();
+    } else false;
+
+    if (cur_minmax == null) {
+        // No comptime operands - use the first operand as the starting value
+        assert(runtime_idx == 0);
+        cur_minmax = operands[0];
+        cur_minmax_src = runtime_src;
+        runtime_known.unset(0); // don't look at this operand in the loop below
+    }
+
+    var it = runtime_known.iterator(.{});
+    while (it.next()) |idx| {
+        const lhs = cur_minmax.?;
+        const lhs_src = cur_minmax_src;
+        const rhs = operands[idx];
+        const rhs_src = operand_srcs[idx];
+        const simd_op = try sema.checkSimdBinOp(block, src, lhs, rhs, lhs_src, rhs_src);
+        if (known_undef) {
+            cur_minmax = try sema.addConstant(simd_op.result_ty, Value.undef);
+        } else {
+            cur_minmax = try block.addBinOp(air_tag, simd_op.lhs, simd_op.rhs);
+        }
+    }
+
+    if (comptime_refined_ty) |comptime_ty| refine: {
+        // Finally, refine the type based on the comptime-known bound.
+        if (known_undef) break :refine; // can't refine undef
+        const unrefined_ty = sema.typeOf(cur_minmax.?);
+        const is_vector = unrefined_ty.zigTypeTag() == .Vector;
+        const comptime_elem_ty = if (is_vector) comptime_ty.childType() else comptime_ty;
+        const unrefined_elem_ty = if (is_vector) unrefined_ty.childType() else unrefined_ty;
+
+        if (unrefined_elem_ty.isAnyFloat()) break :refine; // we can't refine floats
+
+        // Compute the final bounds based on the runtime type and the comptime-known bound type
+        const min_val = switch (air_tag) {
+            .min => try unrefined_elem_ty.minInt(sema.arena, target),
+            .max => try comptime_elem_ty.minInt(sema.arena, target), // @max(ct, rt) >= ct
+            else => unreachable,
+        };
+        const max_val = switch (air_tag) {
+            .min => try comptime_elem_ty.maxInt(sema.arena, target), // @min(ct, rt) <= ct
+            .max => try unrefined_elem_ty.maxInt(sema.arena, target),
+            else => unreachable,
+        };
+
+        // Find the smallest type which can contain these bounds
+        const final_elem_ty = try Type.intFittingRange(target, sema.arena, min_val, max_val);
+
+        const final_ty = if (is_vector)
+            try Type.vector(sema.arena, unrefined_ty.vectorLen(), final_elem_ty)
+        else
+            final_elem_ty;
+
+        if (!final_ty.eql(unrefined_ty, mod)) {
+            // We've reduced the type - cast the result down
+            return block.addTyOp(.intcast, final_ty, cur_minmax.?);
+        }
+    }
+
+    return cur_minmax.?;
 }
 
 fn upgradeToArrayPtr(sema: *Sema, block: *Block, ptr: Air.Inst.Ref, len: u64) !Air.Inst.Ref {
diff --git a/src/Zir.zig b/src/Zir.zig
index ab33b625f7..51c90c61cb 100644
--- a/src/Zir.zig
+++ b/src/Zir.zig
@@ -927,10 +927,10 @@ pub const Inst = struct {
         /// Implements the `@memset` builtin.
         /// Uses the `pl_node` union field with payload `Bin`.
         memset,
-        /// Implements the `@min` builtin.
+        /// Implements the `@min` builtin for 2 args.
         /// Uses the `pl_node` union field with payload `Bin`
         min,
-        /// Implements the `@max` builtin.
+        /// Implements the `@max` builtin for 2 args.
         /// Uses the `pl_node` union field with payload `Bin`
         max,
         /// Implements the `@cImport` builtin.
@@ -1905,10 +1905,20 @@ pub const Inst = struct {
         compile_log,
         /// The builtin `@TypeOf` which returns the type after Peer Type Resolution
         /// of one or more params.
-        /// `operand` is payload index to `NodeMultiOp`.
+        /// `operand` is payload index to `TypeOfPeer`.
         /// `small` is `operands_len`.
         /// The AST node is the builtin call.
         typeof_peer,
+        /// Implements the `@min` builtin for more than 2 args.
+        /// `operand` is payload index to `NodeMultiOp`.
+        /// `small` is `operands_len`.
+        /// The AST node is the builtin call.
+        min_multi,
+        /// Implements the `@max` builtin for more than 2 args.
+        /// `operand` is payload index to `NodeMultiOp`.
+        /// `small` is `operands_len`.
+        /// The AST node is the builtin call.
+        max_multi,
         /// Implements the `@addWithOverflow` builtin.
         /// `operand` is payload index to `BinNode`.
         /// `small` is unused.
diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index a658103c1a..dd093508b1 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -4298,7 +4298,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
 
     const val_ty = ptr_info.pointee_type;
     const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
-    const limb_abi_size = @min(val_abi_size, 8);
+    const limb_abi_size: u32 = @min(val_abi_size, 8);
     const limb_abi_bits = limb_abi_size * 8;
     const val_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size);
     const val_bit_off = ptr_info.bit_offset % limb_abi_bits;
@@ -4434,7 +4434,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
     const ptr_info = ptr_ty.ptrInfo().data;
     const src_ty = ptr_ty.childType();
 
-    const limb_abi_size = @min(ptr_info.host_size, 8);
+    const limb_abi_size: u16 = @min(ptr_info.host_size, 8);
     const limb_abi_bits = limb_abi_size * 8;
 
     const src_bit_size = src_ty.bitSize(self.target.*);
@@ -4652,7 +4652,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
                 }
 
                 const field_abi_size = @intCast(u32, field_ty.abiSize(self.target.*));
-                const limb_abi_size = @min(field_abi_size, 8);
+                const limb_abi_size: u32 = @min(field_abi_size, 8);
                 const limb_abi_bits = limb_abi_size * 8;
                 const field_byte_off = @intCast(i32, field_off / limb_abi_bits * limb_abi_size);
                 const field_bit_off = field_off % limb_abi_bits;
@@ -5875,7 +5875,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
         },
         .memory, .indirect, .load_got, .load_direct, .load_tlv, .load_frame => {
             const OpInfo = ?struct { addr_reg: Register, addr_lock: RegisterLock };
-            const limb_abi_size = @min(abi_size, 8);
+            const limb_abi_size: u32 = @min(abi_size, 8);
 
             const dst_info: OpInfo = switch (dst_mcv) {
                 else => unreachable,
diff --git a/src/print_zir.zig b/src/print_zir.zig
index 922366dc85..4e4f13641c 100644
--- a/src/print_zir.zig
+++ b/src/print_zir.zig
@@ -482,6 +482,8 @@ const Writer = struct {
 
             .compile_log => try self.writeNodeMultiOp(stream, extended),
             .typeof_peer => try self.writeTypeofPeer(stream, extended),
+            .min_multi => try self.writeNodeMultiOp(stream, extended),
+            .max_multi => try self.writeNodeMultiOp(stream, extended),
 
             .select => try self.writeSelect(stream, extended),
 
diff --git a/src/type.zig b/src/type.zig
index b25f13342d..c9a6f49d3e 100644
--- a/src/type.zig
+++ b/src/type.zig
@@ -6723,7 +6723,17 @@ pub const Type = extern union {
 
     pub fn smallestUnsignedInt(arena: Allocator, max: u64) !Type {
         const bits = smallestUnsignedBits(max);
-        return switch (bits) {
+        return intWithBits(arena, false, bits);
+    }
+
+    pub fn intWithBits(arena: Allocator, sign: bool, bits: u16) !Type {
+        return if (sign) switch (bits) {
+            8 => initTag(.i8),
+            16 => initTag(.i16),
+            32 => initTag(.i32),
+            64 => initTag(.i64),
+            else => return Tag.int_signed.create(arena, bits),
+        } else switch (bits) {
             1 => initTag(.u1),
             8 => initTag(.u8),
             16 => initTag(.u16),
@@ -6733,6 +6743,61 @@ pub const Type = extern union {
         };
     }
 
+    /// Given a value representing an integer, returns the number of bits necessary to represent
+    /// this value in an integer. If `sign` is true, returns the number of bits necessary in a
+    /// twos-complement integer; otherwise in an unsigned integer.
+    /// Asserts that `val` is not undef. If `val` is negative, asserts that `sign` is true.
+    pub fn intBitsForValue(target: Target, val: Value, sign: bool) u16 {
+        assert(!val.isUndef());
+        switch (val.tag()) {
+            .int_big_positive => {
+                const limbs = val.castTag(.int_big_positive).?.data;
+                const big: std.math.big.int.Const = .{ .limbs = limbs, .positive = true };
+                return @intCast(u16, big.bitCountAbs() + @boolToInt(sign));
+            },
+            .int_big_negative => {
+                const limbs = val.castTag(.int_big_negative).?.data;
+                // Zero is still a possibility, in which case unsigned is fine
+                for (limbs) |limb| {
+                    if (limb != 0) break;
+                } else return 0; // val == 0
+                assert(sign);
+                const big: std.math.big.int.Const = .{ .limbs = limbs, .positive = false };
+                return @intCast(u16, big.bitCountTwosComp());
+            },
+            .int_i64 => {
+                const x = val.castTag(.int_i64).?.data;
+                if (x >= 0) return smallestUnsignedBits(@intCast(u64, x));
+                assert(sign);
+                return smallestUnsignedBits(@intCast(u64, -x - 1)) + 1;
+            },
+            else => {
+                const x = val.toUnsignedInt(target);
+                return smallestUnsignedBits(x) + @boolToInt(sign);
+            },
+        }
+    }
+
+    /// Returns the smallest possible integer type containing both `min` and `max`. Asserts that neither
+    /// value is undef.
+    /// TODO: if #3806 is implemented, this becomes trivial
+    pub fn intFittingRange(target: Target, arena: Allocator, min: Value, max: Value) !Type {
+        assert(!min.isUndef());
+        assert(!max.isUndef());
+
+        if (std.debug.runtime_safety) {
+            assert(Value.order(min, max, target).compare(.lte));
+        }
+
+        const sign = min.orderAgainstZero() == .lt;
+
+        const min_val_bits = intBitsForValue(target, min, sign);
+        const max_val_bits = intBitsForValue(target, max, sign);
+        const bits = @max(min_val_bits, max_val_bits);
+
+        return intWithBits(arena, sign, bits);
+    }
+
     /// This is only used for comptime asserts. Bump this number when you make a change
     /// to packed struct layout to find out all the places in the codebase you need to edit!
     pub const packed_struct_layout_version = 2;
diff --git a/test/behavior/maximum_minimum.zig b/test/behavior/maximum_minimum.zig
index 9a4ae40eef..d7b93c56c0 100644
--- a/test/behavior/maximum_minimum.zig
+++ b/test/behavior/maximum_minimum.zig
@@ -106,3 +106,59 @@ test "@min/@max on lazy values" {
     const size = @max(@sizeOf(A), @sizeOf(B));
     try expect(size == @sizeOf(B));
 }
+
+test "@min/@max more than two arguments" {
+    const x: u32 = 30;
+    const y: u32 = 10;
+    const z: u32 = 20;
+    try expectEqual(@as(u32, 10), @min(x, y, z));
+    try expectEqual(@as(u32, 30), @max(x, y, z));
+}
+
+test "@min/@max more than two vector arguments" {
+    if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+
+    const x: @Vector(2, u32) = .{ 3, 2 };
+    const y: @Vector(2, u32) = .{ 4, 1 };
+    const z: @Vector(2, u32) = .{ 5, 0 };
+    try expectEqual(@Vector(2, u32){ 3, 0 }, @min(x, y, z));
+    try expectEqual(@Vector(2, u32){ 5, 2 }, @max(x, y, z));
+}
+
+test "@min/@max notices bounds" {
+    if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+
+    var x: u16 = 20;
+    const y = 30;
+    var z: u32 = 100;
+    const min = @min(x, y, z);
+    const max = @max(x, y, z);
+    try expectEqual(x, min);
+    try expectEqual(u5, @TypeOf(min));
+    try expectEqual(z, max);
+    try expectEqual(u32, @TypeOf(max));
+}
+
+test "@min/@max notices vector bounds" {
+    if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+
+    var x: @Vector(2, u16) = .{ 140, 40 };
+    const y: @Vector(2, u64) = .{ 5, 100 };
+    var z: @Vector(2, u32) = .{ 10, 300 };
+    const min = @min(x, y, z);
+    const max = @max(x, y, z);
+    try expectEqual(@Vector(2, u32){ 5, 40 }, min);
+    try expectEqual(@Vector(2, u7), @TypeOf(min));
+    try expectEqual(@Vector(2, u32){ 140, 300 }, max);
+    try expectEqual(@Vector(2, u32), @TypeOf(max));
+}

From b9841750f91604020268549fc5e2c6b10c1f8477 Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Tue, 2 May 2023 20:16:19 +0100
Subject: [PATCH 056/725] std.c: adding freebsd's CPU_COUNT macro portage.

---
 lib/std/c/freebsd.zig | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/lib/std/c/freebsd.zig b/lib/std/c/freebsd.zig
index 28e95d0414..352c12c695 100644
--- a/lib/std/c/freebsd.zig
+++ b/lib/std/c/freebsd.zig
@@ -9,6 +9,20 @@ pub const CPU_SETSIZE = 256;
 pub const cpuset_t = extern struct {
     __bits: [(CPU_SETSIZE + (@bitSizeOf(c_long) - 1)) / @bitSizeOf(c_long)]c_long,
 };
+
+// TODO: can eventually serve for the domainset_t's type too.
+fn __BIT_COUNT(bits: []const c_long) c_long {
+    var count: c_long = 0;
+    for (bits) |b| {
+        count += @popCount(b);
+    }
+    return count;
+}
+
+pub fn CPU_COUNT(set: cpuset_t) c_int {
+    return @intCast(c_int, __BIT_COUNT(set.__bits[0..]));
+}
+
 pub const cpulevel_t = c_int;
 pub const cpuwhich_t = c_int;
 pub const id_t = i64;

From b643c5dc917e0f1c52e893bc0046aaddb7ca6fee Mon Sep 17 00:00:00 2001
From: DraagrenKirneh 
Date: Sun, 23 Apr 2023 22:29:23 +0200
Subject: [PATCH 057/725] Change compression detection to use content-type
 instead of the url ending

---
 src/Package.zig | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/Package.zig b/src/Package.zig
index b44cb7fcc6..ab4aaeb41f 100644
--- a/src/Package.zig
+++ b/src/Package.zig
@@ -5,6 +5,7 @@ const std = @import("std");
 const fs = std.fs;
 const mem = std.mem;
 const Allocator = mem.Allocator;
+const ascii = std.ascii;
 const assert = std.debug.assert;
 const log = std.log.scoped(.package);
 const main = @import("main.zig");
@@ -488,11 +489,16 @@ fn fetchAndUnpack(
         try req.start();
         try req.wait();
 
-        if (mem.endsWith(u8, uri.path, ".tar.gz")) {
+        const content_type = req.response.headers.getFirstValue("Content-Type") orelse
+            return report.fail(dep.url_tok, "missing Content-Type for '{s}'", .{uri.path});
+
+        if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
+            ascii.eqlIgnoreCase(content_type, "application/x-gzip"))
+        {
             // I observed the gzip stream to read 1 byte at a time, so I am using a
             // buffered reader on the front of it.
             try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.gzip);
-        } else if (mem.endsWith(u8, uri.path, ".tar.xz")) {
+        } else if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) {
             // I have not checked what buffer sizes the xz decompression implementation uses
             // by default, so the same logic applies for buffering the reader as for gzip.
             try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.xz);

From 14c68e847aeb00233c3a2660ec81deaae70d51da Mon Sep 17 00:00:00 2001
From: Arnau <61841960+Arnau478@users.noreply.github.com>
Date: Tue, 2 May 2023 13:15:29 +0200
Subject: [PATCH 058/725] langref: Documented `extern "..."` use

The use of `extern "..."` for specifying the library that has the definition was undocumented.
Fixes #13906
---
 doc/langref.html.in | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/langref.html.in b/doc/langref.html.in
index a213950a15..69ad0624c4 100644
--- a/doc/langref.html.in
+++ b/doc/langref.html.in
@@ -5184,7 +5184,8 @@ export fn sub(a: i8, b: i8) i8 { return a - b; }
 
 // The extern specifier is used to declare a function that will be resolved
 // at link time, when linking statically, or at runtime, when linking
-// dynamically.
+// dynamically. The quoted identifier after the extern keyword specifies
+// the library that has the function. (e.g. "c" -> libc.so)
 // The callconv specifier changes the calling convention of the function.
 const WINAPI: std.builtin.CallingConvention = if (native_arch == .x86) .Stdcall else .C;
 extern "kernel32" fn ExitProcess(exit_code: u32) callconv(WINAPI) noreturn;

From 855493bb8b395970921494d3a11ccfeaac30c2dc Mon Sep 17 00:00:00 2001
From: Tw 
Date: Fri, 28 Apr 2023 16:06:56 +0800
Subject: [PATCH 059/725] bpf: correct return type of ringbuf_output helper

Signed-off-by: Tw 
---
 lib/std/os/linux/bpf/helpers.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/std/os/linux/bpf/helpers.zig b/lib/std/os/linux/bpf/helpers.zig
index dafd4de69f..6084b01e6c 100644
--- a/lib/std/os/linux/bpf/helpers.zig
+++ b/lib/std/os/linux/bpf/helpers.zig
@@ -143,7 +143,7 @@ pub const seq_printf = @intToPtr(*const fn (m: *kern.SeqFile, fmt: ?*const u8, f
 pub const seq_write = @intToPtr(*const fn (m: *kern.SeqFile, data: ?*const u8, len: u32) c_long, 127);
 pub const sk_cgroup_id = @intToPtr(*const fn (sk: *kern.BpfSock) u64, 128);
 pub const sk_ancestor_cgroup_id = @intToPtr(*const fn (sk: *kern.BpfSock, ancestor_level: c_long) u64, 129);
-pub const ringbuf_output = @intToPtr(*const fn (ringbuf: ?*anyopaque, data: ?*anyopaque, size: u64, flags: u64) ?*anyopaque, 130);
+pub const ringbuf_output = @intToPtr(*const fn (ringbuf: ?*anyopaque, data: ?*anyopaque, size: u64, flags: u64) c_long, 130);
 pub const ringbuf_reserve = @intToPtr(*const fn (ringbuf: ?*anyopaque, size: u64, flags: u64) ?*anyopaque, 131);
 pub const ringbuf_submit = @intToPtr(*const fn (data: ?*anyopaque, flags: u64) void, 132);
 pub const ringbuf_discard = @intToPtr(*const fn (data: ?*anyopaque, flags: u64) void, 133);

From 13eb7251d37759bd47403db304c6120c706fe353 Mon Sep 17 00:00:00 2001
From: Nicolas Sterchele 
Date: Mon, 20 Mar 2023 09:23:10 +0100
Subject: [PATCH 060/725] build: rename std.Build.*Step to std.Build.Step.*

Follow-up actions from #14647

Fixes #14947
---
 lib/std/Build.zig                             | 28 +++++++++----------
 .../{CheckFileStep.zig => Step/CheckFile.zig} | 11 ++++----
 .../CheckObject.zig}                          |  2 +-
 .../{CompileStep.zig => Step/Compile.zig}     |  2 +-
 .../ConfigHeader.zig}                         |  8 +++---
 lib/std/Build/{FmtStep.zig => Step/Fmt.zig}   |  7 ++---
 .../InstallArtifact.zig}                      |  2 +-
 .../InstallDir.zig}                           |  2 +-
 .../InstallFile.zig}                          |  2 +-
 .../{ObjCopyStep.zig => Step/ObjCopy.zig}     |  0
 .../{OptionsStep.zig => Step/Options.zig}     |  2 +-
 .../{RemoveDirStep.zig => Step/RemoveDir.zig} |  2 +-
 lib/std/Build/{RunStep.zig => Step/Run.zig}   |  2 +-
 .../TranslateC.zig}                           |  2 +-
 .../{WriteFileStep.zig => Step/WriteFile.zig} | 12 ++++----
 15 files changed, 40 insertions(+), 44 deletions(-)
 rename lib/std/Build/{CheckFileStep.zig => Step/CheckFile.zig} (98%)
 rename lib/std/Build/{CheckObjectStep.zig => Step/CheckObject.zig} (99%)
 rename lib/std/Build/{CompileStep.zig => Step/Compile.zig} (99%)
 rename lib/std/Build/{ConfigHeaderStep.zig => Step/ConfigHeader.zig} (99%)
 rename lib/std/Build/{FmtStep.zig => Step/Fmt.zig} (98%)
 rename lib/std/Build/{InstallArtifactStep.zig => Step/InstallArtifact.zig} (99%)
 rename lib/std/Build/{InstallDirStep.zig => Step/InstallDir.zig} (99%)
 rename lib/std/Build/{InstallFileStep.zig => Step/InstallFile.zig} (98%)
 rename lib/std/Build/{ObjCopyStep.zig => Step/ObjCopy.zig} (100%)
 rename lib/std/Build/{OptionsStep.zig => Step/Options.zig} (99%)
 rename lib/std/Build/{RemoveDirStep.zig => Step/RemoveDir.zig} (97%)
 rename lib/std/Build/{RunStep.zig => Step/Run.zig} (99%)
 rename lib/std/Build/{TranslateCStep.zig => Step/TranslateC.zig} (99%)
 rename lib/std/Build/{WriteFileStep.zig => Step/WriteFile.zig} (99%)

diff --git a/lib/std/Build.zig b/lib/std/Build.zig
index a2c8a22e32..bda50112b6 100644
--- a/lib/std/Build.zig
+++ b/lib/std/Build.zig
@@ -29,20 +29,20 @@ pub const Builder = Build;
 pub const InstallDirectoryOptions = InstallDirStep.Options;
 
 pub const Step = @import("Build/Step.zig");
-pub const CheckFileStep = @import("Build/CheckFileStep.zig");
-pub const CheckObjectStep = @import("Build/CheckObjectStep.zig");
-pub const ConfigHeaderStep = @import("Build/ConfigHeaderStep.zig");
-pub const FmtStep = @import("Build/FmtStep.zig");
-pub const InstallArtifactStep = @import("Build/InstallArtifactStep.zig");
-pub const InstallDirStep = @import("Build/InstallDirStep.zig");
-pub const InstallFileStep = @import("Build/InstallFileStep.zig");
-pub const ObjCopyStep = @import("Build/ObjCopyStep.zig");
-pub const CompileStep = @import("Build/CompileStep.zig");
-pub const OptionsStep = @import("Build/OptionsStep.zig");
-pub const RemoveDirStep = @import("Build/RemoveDirStep.zig");
-pub const RunStep = @import("Build/RunStep.zig");
-pub const TranslateCStep = @import("Build/TranslateCStep.zig");
-pub const WriteFileStep = @import("Build/WriteFileStep.zig");
+pub const CheckFileStep = @import("Build/Step/CheckFile.zig");
+pub const CheckObjectStep = @import("Build/Step/CheckObject.zig");
+pub const ConfigHeaderStep = @import("Build/Step/ConfigHeader.zig");
+pub const FmtStep = @import("Build/Step/Fmt.zig");
+pub const InstallArtifactStep = @import("Build/Step/InstallArtifact.zig");
+pub const InstallDirStep = @import("Build/Step/InstallDir.zig");
+pub const InstallFileStep = @import("Build/Step/InstallFile.zig");
+pub const ObjCopyStep = @import("Build/Step/ObjCopy.zig");
+pub const CompileStep = @import("Build/Step/Compile.zig");
+pub const OptionsStep = @import("Build/Step/Options.zig");
+pub const RemoveDirStep = @import("Build/Step/RemoveDir.zig");
+pub const RunStep = @import("Build/Step/Run.zig");
+pub const TranslateCStep = @import("Build/Step/TranslateC.zig");
+pub const WriteFileStep = @import("Build/Step/WriteFile.zig");
 
 install_tls: TopLevelStep,
 uninstall_tls: TopLevelStep,
diff --git a/lib/std/Build/CheckFileStep.zig b/lib/std/Build/Step/CheckFile.zig
similarity index 98%
rename from lib/std/Build/CheckFileStep.zig
rename to lib/std/Build/Step/CheckFile.zig
index 1c2b6b7786..ad8b1a25f0 100644
--- a/lib/std/Build/CheckFileStep.zig
+++ b/lib/std/Build/Step/CheckFile.zig
@@ -2,6 +2,11 @@
 //! TODO: make this more flexible, supporting more kinds of checks.
 //! TODO: generalize the code in std.testing.expectEqualStrings and make this
 //! CheckFileStep produce those helpful diagnostics when there is not a match.
+const CheckFileStep = @This();
+const std = @import("std");
+const Step = std.Build.Step;
+const fs = std.fs;
+const mem = std.mem;
 
 step: Step,
 expected_matches: []const []const u8,
@@ -80,9 +85,3 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
         }
     }
 }
-
-const CheckFileStep = @This();
-const std = @import("../std.zig");
-const Step = std.Build.Step;
-const fs = std.fs;
-const mem = std.mem;
diff --git a/lib/std/Build/CheckObjectStep.zig b/lib/std/Build/Step/CheckObject.zig
similarity index 99%
rename from lib/std/Build/CheckObjectStep.zig
rename to lib/std/Build/Step/CheckObject.zig
index e79ce9d3df..431f74eccc 100644
--- a/lib/std/Build/CheckObjectStep.zig
+++ b/lib/std/Build/Step/CheckObject.zig
@@ -1,4 +1,4 @@
-const std = @import("../std.zig");
+const std = @import("std");
 const assert = std.debug.assert;
 const fs = std.fs;
 const macho = std.macho;
diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/Step/Compile.zig
similarity index 99%
rename from lib/std/Build/CompileStep.zig
rename to lib/std/Build/Step/Compile.zig
index d5a135e24b..7627c4e6d0 100644
--- a/lib/std/Build/CompileStep.zig
+++ b/lib/std/Build/Step/Compile.zig
@@ -1,5 +1,5 @@
 const builtin = @import("builtin");
-const std = @import("../std.zig");
+const std = @import("std");
 const mem = std.mem;
 const fs = std.fs;
 const assert = std.debug.assert;
diff --git a/lib/std/Build/ConfigHeaderStep.zig b/lib/std/Build/Step/ConfigHeader.zig
similarity index 99%
rename from lib/std/Build/ConfigHeaderStep.zig
rename to lib/std/Build/Step/ConfigHeader.zig
index c1849b410e..6bfe28ae62 100644
--- a/lib/std/Build/ConfigHeaderStep.zig
+++ b/lib/std/Build/Step/ConfigHeader.zig
@@ -1,3 +1,7 @@
+const std = @import("std");
+const ConfigHeaderStep = @This();
+const Step = std.Build.Step;
+
 pub const Style = union(enum) {
     /// The configure format supported by autotools. It uses `#undef foo` to
     /// mark lines that can be substituted with different values.
@@ -431,7 +435,3 @@ fn renderValueNasm(output: *std.ArrayList(u8), name: []const u8, value: Value) !
         },
     }
 }
-
-const std = @import("../std.zig");
-const ConfigHeaderStep = @This();
-const Step = std.Build.Step;
diff --git a/lib/std/Build/FmtStep.zig b/lib/std/Build/Step/Fmt.zig
similarity index 98%
rename from lib/std/Build/FmtStep.zig
rename to lib/std/Build/Step/Fmt.zig
index 2a82342336..23d5d9e3ff 100644
--- a/lib/std/Build/FmtStep.zig
+++ b/lib/std/Build/Step/Fmt.zig
@@ -1,6 +1,9 @@
 //! This step has two modes:
 //! * Modify mode: directly modify source files, formatting them in place.
 //! * Check mode: fail the step if a non-conforming file is found.
+const std = @import("std");
+const Step = std.Build.Step;
+const FmtStep = @This();
 
 step: Step,
 paths: []const []const u8,
@@ -67,7 +70,3 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
 
     return step.evalChildProcess(argv.items);
 }
-
-const std = @import("../std.zig");
-const Step = std.Build.Step;
-const FmtStep = @This();
diff --git a/lib/std/Build/InstallArtifactStep.zig b/lib/std/Build/Step/InstallArtifact.zig
similarity index 99%
rename from lib/std/Build/InstallArtifactStep.zig
rename to lib/std/Build/Step/InstallArtifact.zig
index 50cf6ff323..fa357a9ae9 100644
--- a/lib/std/Build/InstallArtifactStep.zig
+++ b/lib/std/Build/Step/InstallArtifact.zig
@@ -1,4 +1,4 @@
-const std = @import("../std.zig");
+const std = @import("std");
 const Step = std.Build.Step;
 const CompileStep = std.Build.CompileStep;
 const InstallDir = std.Build.InstallDir;
diff --git a/lib/std/Build/InstallDirStep.zig b/lib/std/Build/Step/InstallDir.zig
similarity index 99%
rename from lib/std/Build/InstallDirStep.zig
rename to lib/std/Build/Step/InstallDir.zig
index d9ea248913..28280dcb7f 100644
--- a/lib/std/Build/InstallDirStep.zig
+++ b/lib/std/Build/Step/InstallDir.zig
@@ -1,4 +1,4 @@
-const std = @import("../std.zig");
+const std = @import("std");
 const mem = std.mem;
 const fs = std.fs;
 const Step = std.Build.Step;
diff --git a/lib/std/Build/InstallFileStep.zig b/lib/std/Build/Step/InstallFile.zig
similarity index 98%
rename from lib/std/Build/InstallFileStep.zig
rename to lib/std/Build/Step/InstallFile.zig
index 011ad48208..b6b66fd1e0 100644
--- a/lib/std/Build/InstallFileStep.zig
+++ b/lib/std/Build/Step/InstallFile.zig
@@ -1,4 +1,4 @@
-const std = @import("../std.zig");
+const std = @import("std");
 const Step = std.Build.Step;
 const FileSource = std.Build.FileSource;
 const InstallDir = std.Build.InstallDir;
diff --git a/lib/std/Build/ObjCopyStep.zig b/lib/std/Build/Step/ObjCopy.zig
similarity index 100%
rename from lib/std/Build/ObjCopyStep.zig
rename to lib/std/Build/Step/ObjCopy.zig
diff --git a/lib/std/Build/OptionsStep.zig b/lib/std/Build/Step/Options.zig
similarity index 99%
rename from lib/std/Build/OptionsStep.zig
rename to lib/std/Build/Step/Options.zig
index a0e72e3695..101c284cf0 100644
--- a/lib/std/Build/OptionsStep.zig
+++ b/lib/std/Build/Step/Options.zig
@@ -1,4 +1,4 @@
-const std = @import("../std.zig");
+const std = @import("std");
 const builtin = @import("builtin");
 const fs = std.fs;
 const Step = std.Build.Step;
diff --git a/lib/std/Build/RemoveDirStep.zig b/lib/std/Build/Step/RemoveDir.zig
similarity index 97%
rename from lib/std/Build/RemoveDirStep.zig
rename to lib/std/Build/Step/RemoveDir.zig
index a5bf3c3256..59025a7e91 100644
--- a/lib/std/Build/RemoveDirStep.zig
+++ b/lib/std/Build/Step/RemoveDir.zig
@@ -1,4 +1,4 @@
-const std = @import("../std.zig");
+const std = @import("std");
 const fs = std.fs;
 const Step = std.Build.Step;
 const RemoveDirStep = @This();
diff --git a/lib/std/Build/RunStep.zig b/lib/std/Build/Step/Run.zig
similarity index 99%
rename from lib/std/Build/RunStep.zig
rename to lib/std/Build/Step/Run.zig
index 5d530c7a25..4e973cfd98 100644
--- a/lib/std/Build/RunStep.zig
+++ b/lib/std/Build/Step/Run.zig
@@ -1,4 +1,4 @@
-const std = @import("../std.zig");
+const std = @import("std");
 const builtin = @import("builtin");
 const Step = std.Build.Step;
 const CompileStep = std.Build.CompileStep;
diff --git a/lib/std/Build/TranslateCStep.zig b/lib/std/Build/Step/TranslateC.zig
similarity index 99%
rename from lib/std/Build/TranslateCStep.zig
rename to lib/std/Build/Step/TranslateC.zig
index f2dc23d950..86727ea2f0 100644
--- a/lib/std/Build/TranslateCStep.zig
+++ b/lib/std/Build/Step/TranslateC.zig
@@ -1,4 +1,4 @@
-const std = @import("../std.zig");
+const std = @import("std");
 const Step = std.Build.Step;
 const CompileStep = std.Build.CompileStep;
 const CheckFileStep = std.Build.CheckFileStep;
diff --git a/lib/std/Build/WriteFileStep.zig b/lib/std/Build/Step/WriteFile.zig
similarity index 99%
rename from lib/std/Build/WriteFileStep.zig
rename to lib/std/Build/Step/WriteFile.zig
index dee79af5be..68f7c37c6c 100644
--- a/lib/std/Build/WriteFileStep.zig
+++ b/lib/std/Build/Step/WriteFile.zig
@@ -8,6 +8,11 @@
 //! during the normal build process, but as a utility run by a developer with
 //! intention to update source files, which will then be committed to version
 //! control.
+const std = @import("std");
+const Step = std.Build.Step;
+const fs = std.fs;
+const ArrayList = std.ArrayList;
+const WriteFileStep = @This();
 
 step: Step,
 /// The elements here are pointers because we need stable pointers for the
@@ -284,10 +289,3 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
 
     try step.writeManifest(&man);
 }
-
-const std = @import("../std.zig");
-const Step = std.Build.Step;
-const fs = std.fs;
-const ArrayList = std.ArrayList;
-
-const WriteFileStep = @This();

From 8d6336420b937075e3363f9548adb0092af7f819 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carl=20=C3=85stholm?= 
Date: Tue, 2 May 2023 19:30:01 +0200
Subject: [PATCH 061/725] std.log.defaultLog: remove freestanding compile error

---
 lib/std/log.zig | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/lib/std/log.zig b/lib/std/log.zig
index 48a287a4ee..dc45df9ea3 100644
--- a/lib/std/log.zig
+++ b/lib/std/log.zig
@@ -146,12 +146,6 @@ pub fn defaultLog(
     comptime format: []const u8,
     args: anytype,
 ) void {
-    if (builtin.os.tag == .freestanding)
-        @compileError(
-            \\freestanding targets do not have I/O configured;
-            \\please provide at least an empty `log` function declaration
-        );
-
     const level_txt = comptime message_level.asText();
     const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): ";
     const stderr = std.io.getStdErr().writer();

From f56f5af4039aadd5f9377eb2663e4733c37fabc5 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Tue, 2 May 2023 00:29:39 -0400
Subject: [PATCH 062/725] x86_64: fix global slices

---
 src/codegen.zig             | 12 +++++++-----
 test/behavior/cast.zig      |  4 ----
 test/behavior/generics.zig  |  1 -
 test/behavior/src.zig       |  1 -
 test/behavior/type_info.zig |  4 ----
 5 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/src/codegen.zig b/src/codegen.zig
index 078feb409d..912bf03609 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -1156,11 +1156,13 @@ pub fn genTypedValue(
     const target = bin_file.options.target;
     const ptr_bits = target.cpu.arch.ptrBitWidth();
 
-    if (typed_value.val.castTag(.decl_ref)) |payload| {
-        return genDeclRef(bin_file, src_loc, typed_value, payload.data);
-    }
-    if (typed_value.val.castTag(.decl_ref_mut)) |payload| {
-        return genDeclRef(bin_file, src_loc, typed_value, payload.data.decl_index);
+    if (!typed_value.ty.isSlice()) {
+        if (typed_value.val.castTag(.decl_ref)) |payload| {
+            return genDeclRef(bin_file, src_loc, typed_value, payload.data);
+        }
+        if (typed_value.val.castTag(.decl_ref_mut)) |payload| {
+            return genDeclRef(bin_file, src_loc, typed_value, payload.data.decl_index);
+        }
     }
 
     switch (typed_value.ty.zigTypeTag()) {
diff --git a/test/behavior/cast.zig b/test/behavior/cast.zig
index 3c1d26f284..cbf1712f67 100644
--- a/test/behavior/cast.zig
+++ b/test/behavior/cast.zig
@@ -688,7 +688,6 @@ test "peer cast: error set any anyerror" {
 }
 
 test "peer type resolution: error set supersets" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -718,7 +717,6 @@ test "peer type resolution: error set supersets" {
 }
 
 test "peer type resolution: disjoint error sets" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -748,7 +746,6 @@ test "peer type resolution: disjoint error sets" {
 }
 
 test "peer type resolution: error union and error set" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -782,7 +779,6 @@ test "peer type resolution: error union and error set" {
 }
 
 test "peer type resolution: error union after non-error" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/generics.zig b/test/behavior/generics.zig
index c5168e420b..367f2113d9 100644
--- a/test/behavior/generics.zig
+++ b/test/behavior/generics.zig
@@ -250,7 +250,6 @@ test "function parameter is generic" {
 }
 
 test "generic function instantiation turns into comptime call" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/src.zig b/test/behavior/src.zig
index e6b84e5d56..89a8e424aa 100644
--- a/test/behavior/src.zig
+++ b/test/behavior/src.zig
@@ -14,7 +14,6 @@ const builtin = @import("builtin");
 const expect = std.testing.expect;
 
 test "@src" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
diff --git a/test/behavior/type_info.zig b/test/behavior/type_info.zig
index 5a1ab7c2aa..ad6d5ac42f 100644
--- a/test/behavior/type_info.zig
+++ b/test/behavior/type_info.zig
@@ -159,7 +159,6 @@ fn testArray() !void {
 test "type info: error set, error union info, anyerror" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     try testErrorSet();
@@ -191,7 +190,6 @@ fn testErrorSet() !void {
 test "type info: error set single value" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const TestSet = error.One;
@@ -205,7 +203,6 @@ test "type info: error set single value" {
 test "type info: error set merged" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     const TestSet = error{ One, Two } || error{Three};
@@ -219,7 +216,6 @@ test "type info: error set merged" {
 }
 
 test "type info: enum info" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From 3f5592c114ffbc28e7ffb9e9e411870b31d81dd8 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Tue, 2 May 2023 00:50:38 -0400
Subject: [PATCH 063/725] x86_64: implement slice elem ptr for more MCValue
 tags

---
 src/arch/x86_64/CodeGen.zig | 12 +-----------
 test/behavior/for.zig       |  4 ----
 2 files changed, 1 insertion(+), 15 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index dd093508b1..51518485e4 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -3502,17 +3502,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
     defer self.register_manager.unlockReg(offset_reg_lock);
 
     const addr_reg = try self.register_manager.allocReg(null, gp);
-    switch (slice_mcv) {
-        .load_frame => |frame_addr| try self.asmRegisterMemory(
-            .mov,
-            addr_reg.to64(),
-            Memory.sib(.qword, .{
-                .base = .{ .frame = frame_addr.index },
-                .disp = frame_addr.off,
-            }),
-        ),
-        else => return self.fail("TODO implement slice_elem_ptr when slice is {}", .{slice_mcv}),
-    }
+    try self.genSetReg(addr_reg, Type.usize, slice_mcv);
     // TODO we could allocate register here, but need to expect addr register and potentially
     // offset register.
     try self.genBinOpMir(.add, slice_ptr_field_type, .{ .register = addr_reg }, .{
diff --git a/test/behavior/for.zig b/test/behavior/for.zig
index 0cec2d62aa..b3d82fd255 100644
--- a/test/behavior/for.zig
+++ b/test/behavior/for.zig
@@ -66,7 +66,6 @@ test "ignore lval with underscore (for loop)" {
 }
 
 test "basic for loop" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -306,7 +305,6 @@ test "1-based counter and ptr to array" {
 test "slice and two counters, one is offset and one is runtime" {
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
 
     const slice: []const u8 = "blah";
@@ -335,7 +333,6 @@ test "slice and two counters, one is offset and one is runtime" {
 test "two slices, one captured by-ref" {
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
 
     var buf: [10]u8 = undefined;
@@ -355,7 +352,6 @@ test "two slices, one captured by-ref" {
 test "raw pointer and slice" {
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
 
     var buf: [10]u8 = undefined;

From 3a30b827414926dceccf606d6557e00f1cc6458d Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Tue, 2 May 2023 01:27:21 -0400
Subject: [PATCH 064/725] x86_64: implement fieldParentPtr

---
 src/arch/x86_64/CodeGen.zig | 26 ++++++++++++++++++++++----
 test/behavior/struct.zig    |  1 -
 test/behavior/tuple.zig     |  2 --
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 51518485e4..521307ed6b 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -233,6 +233,13 @@ pub const MCValue = union(enum) {
         };
     }
 
+    fn isRegisterOffset(mcv: MCValue) bool {
+        return switch (mcv) {
+            .register, .register_offset => true,
+            else => false,
+        };
+    }
+
     fn getReg(mcv: MCValue) ?Register {
         return switch (mcv) {
             .register => |reg| reg,
@@ -4772,10 +4779,21 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
 }
 
 fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
-    const ty_op = self.air.instructions.items(.data)[inst].ty_op;
-    _ = ty_op;
-    return self.fail("TODO implement airFieldParentPtr for {}", .{self.target.cpu.arch});
-    //return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
+    const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
+    const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
+
+    const inst_ty = self.air.typeOfIndex(inst);
+    const parent_ty = inst_ty.childType();
+    const field_offset = @intCast(i32, parent_ty.structFieldOffset(extra.field_index, self.target.*));
+
+    const src_mcv = try self.resolveInst(extra.field_ptr);
+    const dst_mcv = if (src_mcv.isRegisterOffset() and
+        self.reuseOperand(inst, extra.field_ptr, 0, src_mcv))
+        src_mcv
+    else
+        try self.copyToRegisterWithInstTracking(inst, inst_ty, src_mcv);
+    const result = dst_mcv.offset(-field_offset);
+    return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none });
 }
 
 fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
diff --git a/test/behavior/struct.zig b/test/behavior/struct.zig
index e533e34cc3..d5972b9161 100644
--- a/test/behavior/struct.zig
+++ b/test/behavior/struct.zig
@@ -1333,7 +1333,6 @@ test "under-aligned struct field" {
 }
 
 test "fieldParentPtr of a zero-bit field" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
 
diff --git a/test/behavior/tuple.zig b/test/behavior/tuple.zig
index 11cc8b2dce..c1e5f40a46 100644
--- a/test/behavior/tuple.zig
+++ b/test/behavior/tuple.zig
@@ -209,7 +209,6 @@ test "initializing anon struct with explicit type" {
 }
 
 test "fieldParentPtr of tuple" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -220,7 +219,6 @@ test "fieldParentPtr of tuple" {
 }
 
 test "fieldParentPtr of anon struct" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From f894ec264b3e2af7e0f66a1ea76b4d06ac6c5e3a Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Tue, 2 May 2023 02:11:23 -0400
Subject: [PATCH 065/725] codegen: fix global nested field_ptr

---
 src/codegen.zig         | 109 ++++++++++++++--------------------------
 test/behavior/union.zig |   1 -
 2 files changed, 39 insertions(+), 71 deletions(-)

diff --git a/src/codegen.zig b/src/codegen.zig
index 912bf03609..b320005494 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -374,66 +374,7 @@ pub fn generateSymbol(
 
                 return Result.ok;
             },
-            .field_ptr => {
-                const field_ptr = typed_value.val.castTag(.field_ptr).?.data;
-                const container_ptr = field_ptr.container_ptr;
-
-                switch (container_ptr.tag()) {
-                    .decl_ref => {
-                        const decl_index = container_ptr.castTag(.decl_ref).?.data;
-                        const decl = mod.declPtr(decl_index);
-                        const addend = blk: {
-                            switch (decl.ty.zigTypeTag()) {
-                                .Struct => {
-                                    const addend = decl.ty.structFieldOffset(field_ptr.field_index, target);
-                                    break :blk @intCast(u32, addend);
-                                },
-                                .Pointer => {
-                                    assert(decl.ty.isSlice());
-                                    var buf: Type.SlicePtrFieldTypeBuffer = undefined;
-                                    const addend = switch (field_ptr.field_index) {
-                                        0 => 0,
-                                        1 => decl.ty.slicePtrFieldType(&buf).abiSize(target),
-                                        else => unreachable,
-                                    };
-                                    break :blk @intCast(u32, addend);
-                                },
-                                else => return Result{
-                                    .fail = try ErrorMsg.create(
-                                        bin_file.allocator,
-                                        src_loc,
-                                        "TODO implement generateSymbol for pointer type value: '{s}'",
-                                        .{@tagName(typed_value.val.tag())},
-                                    ),
-                                },
-                            }
-                        };
-                        return lowerDeclRef(bin_file, src_loc, typed_value, decl_index, code, debug_output, .{
-                            .parent_atom_index = reloc_info.parent_atom_index,
-                            .addend = (reloc_info.addend orelse 0) + addend,
-                        });
-                    },
-                    .field_ptr => {
-                        switch (try generateSymbol(bin_file, src_loc, .{
-                            .ty = typed_value.ty,
-                            .val = container_ptr,
-                        }, code, debug_output, reloc_info)) {
-                            .ok => {},
-                            .fail => |em| return Result{ .fail = em },
-                        }
-                        return Result.ok;
-                    },
-                    else => return Result{
-                        .fail = try ErrorMsg.create(
-                            bin_file.allocator,
-                            src_loc,
-                            "TODO implement generateSymbol for pointer type value: '{s}'",
-                            .{@tagName(typed_value.val.tag())},
-                        ),
-                    },
-                }
-            },
-            .elem_ptr => return lowerParentPtr(
+            .field_ptr, .elem_ptr => return lowerParentPtr(
                 bin_file,
                 src_loc,
                 typed_value,
@@ -846,16 +787,12 @@ pub fn generateSymbol(
             },
             else => unreachable,
         },
-        else => |t| {
-            return Result{
-                .fail = try ErrorMsg.create(
-                    bin_file.allocator,
-                    src_loc,
-                    "TODO implement generateSymbol for type '{s}'",
-                    .{@tagName(t)},
-                ),
-            };
-        },
+        else => |t| return Result{ .fail = try ErrorMsg.create(
+            bin_file.allocator,
+            src_loc,
+            "TODO implement generateSymbol for type '{s}'",
+            .{@tagName(t)},
+        ) },
     }
 }
 
@@ -871,6 +808,38 @@ fn lowerParentPtr(
     const target = bin_file.options.target;
 
     switch (parent_ptr.tag()) {
+        .field_ptr => {
+            const field_ptr = parent_ptr.castTag(.field_ptr).?.data;
+            return lowerParentPtr(
+                bin_file,
+                src_loc,
+                typed_value,
+                field_ptr.container_ptr,
+                code,
+                debug_output,
+                reloc_info.offset(@intCast(u32, switch (field_ptr.container_ty.zigTypeTag()) {
+                    .Pointer => offset: {
+                        assert(field_ptr.container_ty.isSlice());
+                        var buf: Type.SlicePtrFieldTypeBuffer = undefined;
+                        break :offset switch (field_ptr.field_index) {
+                            0 => 0,
+                            1 => field_ptr.container_ty.slicePtrFieldType(&buf).abiSize(target),
+                            else => unreachable,
+                        };
+                    },
+                    .Struct, .Union => field_ptr.container_ty.structFieldOffset(
+                        field_ptr.field_index,
+                        target,
+                    ),
+                    else => return Result{ .fail = try ErrorMsg.create(
+                        bin_file.allocator,
+                        src_loc,
+                        "TODO implement lowerParentPtr for field_ptr with a container of type {}",
+                        .{field_ptr.container_ty.fmt(bin_file.options.module.?)},
+                    ) },
+                })),
+            );
+        },
         .elem_ptr => {
             const elem_ptr = parent_ptr.castTag(.elem_ptr).?.data;
             return lowerParentPtr(
diff --git a/test/behavior/union.zig b/test/behavior/union.zig
index 3dd8919935..d8989c21f0 100644
--- a/test/behavior/union.zig
+++ b/test/behavior/union.zig
@@ -448,7 +448,6 @@ const Foo1 = union(enum) {
 var glbl: Foo1 = undefined;
 
 test "global union with single field is correctly initialized" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO

From 40ef796278bd5ba405099609e03bdeacab7d3154 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Tue, 2 May 2023 02:14:29 -0400
Subject: [PATCH 066/725] x86_64: fix todo message typo

---
 src/arch/x86_64/CodeGen.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 521307ed6b..e2a1076ce9 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -9008,7 +9008,7 @@ fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
     const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
     const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
     _ = extra;
-    return self.fail("TODO implement airAggregateInit for x86_64", .{});
+    return self.fail("TODO implement airUnionInit for x86_64", .{});
     //return self.finishAir(inst, result, .{ extra.init, .none, .none });
 }
 

From 31429a4e8649961624878d11e1bb330107013086 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Tue, 2 May 2023 02:23:50 -0400
Subject: [PATCH 067/725] codegen: handle variable and decl_ref_mut
 consistently

---
 src/codegen.zig           | 71 +++++++++++++++++++++------------------
 test/behavior/eval.zig    |  1 -
 test/behavior/ptrcast.zig |  2 --
 3 files changed, 39 insertions(+), 35 deletions(-)

diff --git a/src/codegen.zig b/src/codegen.zig
index b320005494..9d479b90cd 100644
--- a/src/codegen.zig
+++ b/src/codegen.zig
@@ -341,14 +341,20 @@ pub fn generateSymbol(
                 }
                 return Result.ok;
             },
-            .variable => {
-                const decl = typed_value.val.castTag(.variable).?.data.owner_decl;
-                return lowerDeclRef(bin_file, src_loc, typed_value, decl, code, debug_output, reloc_info);
-            },
-            .decl_ref => {
-                const decl = typed_value.val.castTag(.decl_ref).?.data;
-                return lowerDeclRef(bin_file, src_loc, typed_value, decl, code, debug_output, reloc_info);
-            },
+            .variable, .decl_ref, .decl_ref_mut => |tag| return lowerDeclRef(
+                bin_file,
+                src_loc,
+                typed_value,
+                switch (tag) {
+                    .variable => typed_value.val.castTag(.variable).?.data.owner_decl,
+                    .decl_ref => typed_value.val.castTag(.decl_ref).?.data,
+                    .decl_ref_mut => typed_value.val.castTag(.decl_ref_mut).?.data.decl_index,
+                    else => unreachable,
+                },
+                code,
+                debug_output,
+                reloc_info,
+            ),
             .slice => {
                 const slice = typed_value.val.castTag(.slice).?.data;
 
@@ -787,11 +793,11 @@ pub fn generateSymbol(
             },
             else => unreachable,
         },
-        else => |t| return Result{ .fail = try ErrorMsg.create(
+        else => |tag| return Result{ .fail = try ErrorMsg.create(
             bin_file.allocator,
             src_loc,
             "TODO implement generateSymbol for type '{s}'",
-            .{@tagName(t)},
+            .{@tagName(tag)},
         ) },
     }
 }
@@ -852,28 +858,26 @@ fn lowerParentPtr(
                 reloc_info.offset(@intCast(u32, elem_ptr.index * elem_ptr.elem_ty.abiSize(target))),
             );
         },
-        .decl_ref => {
-            const decl_index = parent_ptr.castTag(.decl_ref).?.data;
-            return lowerDeclRef(
-                bin_file,
-                src_loc,
-                typed_value,
-                decl_index,
-                code,
-                debug_output,
-                reloc_info,
-            );
-        },
-        else => |t| {
-            return Result{
-                .fail = try ErrorMsg.create(
-                    bin_file.allocator,
-                    src_loc,
-                    "TODO implement lowerParentPtr for type '{s}'",
-                    .{@tagName(t)},
-                ),
-            };
-        },
+        .variable, .decl_ref, .decl_ref_mut => |tag| return lowerDeclRef(
+            bin_file,
+            src_loc,
+            typed_value,
+            switch (tag) {
+                .variable => parent_ptr.castTag(.variable).?.data.owner_decl,
+                .decl_ref => parent_ptr.castTag(.decl_ref).?.data,
+                .decl_ref_mut => parent_ptr.castTag(.decl_ref_mut).?.data.decl_index,
+                else => unreachable,
+            },
+            code,
+            debug_output,
+            reloc_info,
+        ),
+        else => |tag| return Result{ .fail = try ErrorMsg.create(
+            bin_file.allocator,
+            src_loc,
+            "TODO implement lowerParentPtr for type '{s}'",
+            .{@tagName(tag)},
+        ) },
     }
 }
 
@@ -1126,6 +1130,9 @@ pub fn genTypedValue(
     const ptr_bits = target.cpu.arch.ptrBitWidth();
 
     if (!typed_value.ty.isSlice()) {
+        if (typed_value.val.castTag(.variable)) |payload| {
+            return genDeclRef(bin_file, src_loc, typed_value, payload.data.owner_decl);
+        }
         if (typed_value.val.castTag(.decl_ref)) |payload| {
             return genDeclRef(bin_file, src_loc, typed_value, payload.data);
         }
diff --git a/test/behavior/eval.zig b/test/behavior/eval.zig
index 3a10a288e3..c7285ef8a8 100644
--- a/test/behavior/eval.zig
+++ b/test/behavior/eval.zig
@@ -1185,7 +1185,6 @@ test "equality of pointers to comptime const" {
 }
 
 test "storing an array of type in a field" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/ptrcast.zig b/test/behavior/ptrcast.zig
index 6a5ae726ae..d52f7ea727 100644
--- a/test/behavior/ptrcast.zig
+++ b/test/behavior/ptrcast.zig
@@ -128,7 +128,6 @@ fn testReinterpretOverAlignedExternStructAsExternStruct() !void {
 test "lower reinterpreted comptime field ptr (with under-aligned fields)" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
@@ -152,7 +151,6 @@ test "lower reinterpreted comptime field ptr (with under-aligned fields)" {
 test "lower reinterpreted comptime field ptr" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 

From 9ccdbca635a3b5a26b65ab8e52533d3acc8f2f5e Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Tue, 2 May 2023 03:24:04 -0400
Subject: [PATCH 068/725] x86_64: implement fabs

---
 src/arch/x86_64/CodeGen.zig   | 20 ++++++++++++++------
 src/arch/x86_64/Encoding.zig  |  6 ++++++
 src/arch/x86_64/Lower.zig     |  6 ++++++
 src/arch/x86_64/Mir.zig       | 12 ++++++++++++
 src/arch/x86_64/encodings.zig | 12 ++++++++++++
 test/behavior/floatop.zig     |  3 ++-
 6 files changed, 52 insertions(+), 7 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index e2a1076ce9..5685357108 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -1458,14 +1458,13 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .log,
             .log2,
             .log10,
-            .fabs,
             .floor,
             .ceil,
             .round,
             .trunc_float,
             => try self.airUnaryMath(inst),
 
-            .neg => try self.airNeg(inst),
+            .neg, .fabs => try self.airFloatSign(inst),
 
             .add_with_overflow => try self.airAddSubWithOverflow(inst),
             .sub_with_overflow => try self.airAddSubWithOverflow(inst),
@@ -4185,7 +4184,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
 }
 
-fn airNeg(self: *Self, inst: Air.Inst.Index) !void {
+fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     const ty = self.air.typeOf(un_op);
     const ty_bits = ty.floatBits(self.target.*);
@@ -4228,10 +4227,19 @@ fn airNeg(self: *Self, inst: Air.Inst.Index) !void {
     const dst_lock = self.register_manager.lockReg(dst_mcv.register);
     defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
+    const tag = self.air.instructions.items(.tag)[inst];
     try self.genBinOpMir(switch (ty_bits) {
-        32 => .xorps,
-        64 => .xorpd,
-        else => return self.fail("TODO implement airNeg for {}", .{
+        32 => switch (tag) {
+            .neg => .xorps,
+            .fabs => .andnps,
+            else => unreachable,
+        },
+        64 => switch (tag) {
+            .neg => .xorpd,
+            .fabs => .andnpd,
+            else => unreachable,
+        },
+        else => return self.fail("TODO implement airFloatSign for {}", .{
             ty.fmt(self.bin_file.options.module.?),
         }),
     }, vec_ty, dst_mcv, sign_mcv);
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index 5cb7f7a2d9..bb1757c91c 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -268,23 +268,29 @@ pub const Mnemonic = enum {
     movd,
     // SSE
     addss,
+    andps,
+    andnps,
     cmpss,
     cvtsi2ss,
     divss,
     maxss, minss,
     movss,
     mulss,
+    orps,
     subss,
     ucomiss,
     xorps,
     // SSE2
     addsd,
+    andpd,
+    andnpd,
     //cmpsd,
     cvtsd2ss, cvtsi2sd, cvtss2sd,
     divsd,
     maxsd, minsd,
     movq, //movd, movsd,
     mulsd,
+    orpd,
     subsd,
     ucomisd,
     xorpd,
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index a961100687..03e395b171 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -94,6 +94,8 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
         .xor,
 
         .addss,
+        .andnps,
+        .andps,
         .cmpss,
         .cvtsi2ss,
         .divss,
@@ -101,11 +103,14 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
         .minss,
         .movss,
         .mulss,
+        .orps,
         .roundss,
         .subss,
         .ucomiss,
         .xorps,
         .addsd,
+        .andnpd,
+        .andpd,
         .cmpsd,
         .cvtsd2ss,
         .cvtsi2sd,
@@ -115,6 +120,7 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
         .minsd,
         .movsd,
         .mulsd,
+        .orpd,
         .roundsd,
         .subsd,
         .ucomisd,
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index c14338b13d..f3d7a5a66f 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -168,6 +168,10 @@ pub const Inst = struct {
 
         /// Add single precision floating point values
         addss,
+        /// Bitwise logical and of packed single precision floating-point values
+        andps,
+        /// Bitwise logical and not of packed single precision floating-point values
+        andnps,
         /// Compare scalar single-precision floating-point values
         cmpss,
         /// Convert doubleword integer to scalar single-precision floating-point value
@@ -182,6 +186,8 @@ pub const Inst = struct {
         movss,
         /// Multiply scalar single-precision floating-point values
         mulss,
+        /// Bitwise logical or of packed single precision floating-point values
+        orps,
         /// Round scalar single-precision floating-point values
         roundss,
         /// Subtract scalar single-precision floating-point values
@@ -192,6 +198,10 @@ pub const Inst = struct {
         xorps,
         /// Add double precision floating point values
         addsd,
+        /// Bitwise logical and not of packed double precision floating-point values
+        andnpd,
+        /// Bitwise logical and of packed double precision floating-point values
+        andpd,
         /// Compare scalar double-precision floating-point values
         cmpsd,
         /// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
@@ -210,6 +220,8 @@ pub const Inst = struct {
         movsd,
         /// Multiply scalar double-precision floating-point values
         mulsd,
+        /// Bitwise logical or of packed double precision floating-point values
+        orpd,
         /// Round scalar double-precision floating-point values
         roundsd,
         /// Subtract scalar double-precision floating-point values
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index ac427c3633..35b2f13fe7 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -832,6 +832,10 @@ pub const table = [_]Entry{
     // SSE
     .{ .addss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .sse },
 
+    .{ .andnps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x55 }, 0, .sse },
+
+    .{ .andps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x54 }, 0, .sse },
+
     .{ .cmpss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0xf3, 0x0f, 0xc2 }, 0, .sse },
 
     .{ .cvtsi2ss, .rm, &.{ .xmm, .rm32 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .sse },
@@ -848,6 +852,8 @@ pub const table = [_]Entry{
 
     .{ .mulss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .sse },
 
+    .{ .orps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .sse },
+
     .{ .subss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .sse },
 
     .{ .ucomiss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x0f, 0x2e }, 0, .sse },
@@ -857,6 +863,10 @@ pub const table = [_]Entry{
     // SSE2
     .{ .addsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x58 }, 0, .sse2 },
 
+    .{ .andnpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x55 }, 0, .sse2 },
+
+    .{ .andpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x54 }, 0, .sse2 },
+
     .{ .cmpsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0xf2, 0x0f, 0xc2 }, 0, .sse2 },
 
     .{ .cvtsd2ss, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .sse2 },
@@ -883,6 +893,8 @@ pub const table = [_]Entry{
 
     .{ .mulsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x59 }, 0, .sse2 },
 
+    .{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .sse2 },
+
     .{ .subsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .sse2 },
 
     .{ .movsd, .rm, &.{ .xmm,     .xmm_m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .sse2 },
diff --git a/test/behavior/floatop.zig b/test/behavior/floatop.zig
index f713cd035c..ecf1473d14 100644
--- a/test/behavior/floatop.zig
+++ b/test/behavior/floatop.zig
@@ -96,7 +96,8 @@ test "negative f128 floatToInt at compile-time" {
 }
 
 test "@sqrt" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From 36a39267b89583ad1dded42a54de90a7f9b5eacf Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Tue, 2 May 2023 21:01:59 -0400
Subject: [PATCH 069/725] x86_64: fix feature confusion

---
 src/arch/x86_64/CodeGen.zig       |  7 ++++++-
 test/behavior/basic.zig           |  8 --------
 test/behavior/bugs/13069.zig      |  2 --
 test/behavior/cast.zig            |  5 -----
 test/behavior/enum.zig            |  2 --
 test/behavior/eval.zig            |  2 --
 test/behavior/floatop.zig         | 10 ----------
 test/behavior/fn.zig              |  4 ----
 test/behavior/generics.zig        |  4 ----
 test/behavior/math.zig            |  2 --
 test/behavior/maximum_minimum.zig |  4 ----
 test/behavior/pointers.zig        |  4 ----
 test/behavior/struct.zig          |  2 --
 test/behavior/switch.zig          |  2 --
 test/behavior/union.zig           | 14 --------------
 15 files changed, 6 insertions(+), 66 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 5685357108..9487cc33c2 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -5879,7 +5879,12 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
                         .load_got,
                         .load_tlv,
                         => {
-                            const addr_reg = try self.copyToTmpRegister(ty, src_mcv.address());
+                            var ptr_pl = Type.Payload.ElemType{
+                                .base = .{ .tag = .single_const_pointer },
+                                .data = ty,
+                            };
+                            const ptr_ty = Type.initPayload(&ptr_pl.base);
+                            const addr_reg = try self.copyToTmpRegister(ptr_ty, src_mcv.address());
                             return self.genBinOpMir(mir_tag, ty, dst_mcv, .{
                                 .indirect = .{ .reg = addr_reg },
                             });
diff --git a/test/behavior/basic.zig b/test/behavior/basic.zig
index 86fc61c2c9..6fdd309371 100644
--- a/test/behavior/basic.zig
+++ b/test/behavior/basic.zig
@@ -363,8 +363,6 @@ fn hereIsAnOpaqueType(ptr: *OpaqueA) *OpaqueA {
 }
 
 test "take address of parameter" {
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -392,8 +390,6 @@ test "array 2D const double ptr" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
 
     const rect_2d_vertexes = [_][1]f32{
@@ -407,8 +403,6 @@ test "array 2D const double ptr with offset" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
 
     const rect_2d_vertexes = [_][2]f32{
@@ -422,8 +416,6 @@ test "array 3D const double ptr with offset" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
 
     const rect_3d_vertexes = [_][2][2]f32{
diff --git a/test/behavior/bugs/13069.zig b/test/behavior/bugs/13069.zig
index 1c2526ef2a..41c5906ee6 100644
--- a/test/behavior/bugs/13069.zig
+++ b/test/behavior/bugs/13069.zig
@@ -6,8 +6,6 @@ test {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     var opt_x: ?[3]f32 = [_]f32{0.0} ** 3;
 
diff --git a/test/behavior/cast.zig b/test/behavior/cast.zig
index cbf1712f67..039e0a3d17 100644
--- a/test/behavior/cast.zig
+++ b/test/behavior/cast.zig
@@ -95,9 +95,6 @@ test "comptime_int @intToFloat" {
 
 test "@intToFloat" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
-
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
@@ -636,8 +633,6 @@ test "vector casts" {
 }
 
 test "@floatCast cast down" {
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/enum.zig b/test/behavior/enum.zig
index 26b941bcdc..5f0037f6dc 100644
--- a/test/behavior/enum.zig
+++ b/test/behavior/enum.zig
@@ -940,8 +940,6 @@ test "constant enum initialization with differing sizes" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     try test3_1(test3_foo);
     try test3_2(test3_bar);
diff --git a/test/behavior/eval.zig b/test/behavior/eval.zig
index c7285ef8a8..b12fcc7afa 100644
--- a/test/behavior/eval.zig
+++ b/test/behavior/eval.zig
@@ -535,8 +535,6 @@ test "static eval list init" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     try expect(static_vec3.data[2] == 1.0);
     try expect(vec3(0.0, 0.0, 3.0).data[2] == 3.0);
diff --git a/test/behavior/floatop.zig b/test/behavior/floatop.zig
index ecf1473d14..b98d782da1 100644
--- a/test/behavior/floatop.zig
+++ b/test/behavior/floatop.zig
@@ -96,8 +96,6 @@ test "negative f128 floatToInt at compile-time" {
 }
 
 test "@sqrt" {
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -905,8 +903,6 @@ test "negation f16" {
 }
 
 test "negation f32" {
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -926,8 +922,6 @@ test "negation f32" {
 }
 
 test "negation f64" {
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1062,8 +1056,6 @@ test "nan negation f16" {
 }
 
 test "nan negation f32" {
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1083,8 +1075,6 @@ test "nan negation f32" {
 
 test "nan negation f64" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/fn.zig b/test/behavior/fn.zig
index 71b7b36c21..4ff5e20378 100644
--- a/test/behavior/fn.zig
+++ b/test/behavior/fn.zig
@@ -338,8 +338,6 @@ test "function call with anon list literal" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
@@ -360,8 +358,6 @@ test "function call with anon list literal - 2D" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
diff --git a/test/behavior/generics.zig b/test/behavior/generics.zig
index 367f2113d9..e7c053e36c 100644
--- a/test/behavior/generics.zig
+++ b/test/behavior/generics.zig
@@ -59,8 +59,6 @@ test "fn with comptime args" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     try expect(gimmeTheBigOne(1234, 5678) == 5678);
     try expect(shouldCallSameInstance(34, 12) == 34);
@@ -71,8 +69,6 @@ test "anytype params" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     try expect(max_i32(12, 34) == 34);
     try expect(max_f64(1.2, 3.4) == 3.4);
diff --git a/test/behavior/math.zig b/test/behavior/math.zig
index 01b927b913..f9c9f43927 100644
--- a/test/behavior/math.zig
+++ b/test/behavior/math.zig
@@ -203,8 +203,6 @@ test "float equality" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const x: f64 = 0.012;
     const y: f64 = x + 1.0;
diff --git a/test/behavior/maximum_minimum.zig b/test/behavior/maximum_minimum.zig
index d7b93c56c0..e6a8553e8c 100644
--- a/test/behavior/maximum_minimum.zig
+++ b/test/behavior/maximum_minimum.zig
@@ -8,8 +8,6 @@ test "@max" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
@@ -56,8 +54,6 @@ test "@min" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
diff --git a/test/behavior/pointers.zig b/test/behavior/pointers.zig
index 0bd8388660..cfce97b550 100644
--- a/test/behavior/pointers.zig
+++ b/test/behavior/pointers.zig
@@ -206,8 +206,6 @@ test "allowzero pointer and slice" {
 }
 
 test "assign null directly to C pointer and test null equality" {
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -346,8 +344,6 @@ test "pointer sentinel with +inf" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
diff --git a/test/behavior/struct.zig b/test/behavior/struct.zig
index d5972b9161..0ca7f70de1 100644
--- a/test/behavior/struct.zig
+++ b/test/behavior/struct.zig
@@ -744,8 +744,6 @@ var g_foo: S0 = S0.init();
 
 test "packed struct with fp fields" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/switch.zig b/test/behavior/switch.zig
index 5e2d6d28c1..a32a762e04 100644
--- a/test/behavior/switch.zig
+++ b/test/behavior/switch.zig
@@ -230,8 +230,6 @@ test "switch prong with variable" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     try switchProngWithVarFn(SwitchProngWithVarEnum{ .One = 13 });
     try switchProngWithVarFn(SwitchProngWithVarEnum{ .Two = 13.0 });
diff --git a/test/behavior/union.zig b/test/behavior/union.zig
index d8989c21f0..41842f5bc5 100644
--- a/test/behavior/union.zig
+++ b/test/behavior/union.zig
@@ -14,8 +14,6 @@ test "basic unions with floats" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     var foo = FooWithFloats{ .int = 1 };
     try expect(foo.int == 1);
@@ -31,8 +29,6 @@ test "init union with runtime value - floats" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     var foo: FooWithFloats = undefined;
 
@@ -220,8 +216,6 @@ test "union with specified enum tag" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     try doTest();
     comptime try doTest();
@@ -231,8 +225,6 @@ test "packed union generates correctly aligned type" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const U = packed union {
         f1: *const fn () error{TestUnexpectedResult}!void,
@@ -910,8 +902,6 @@ test "anonymous union literal syntax" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         const Number = union {
@@ -1064,8 +1054,6 @@ test "containers with single-field enums" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const S = struct {
         const A = union(enum) { f1 };
@@ -1524,8 +1512,6 @@ test "reinterpreting enum value inside packed union" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64 and
-        comptime !std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sse, .sse2, .sse4_1 })) return error.SkipZigTest; // TODO
 
     const U = packed union {
         tag: enum { a, b },

From a5e50891cbd66c740afba722e94f841a2b43c1fc Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Tue, 2 May 2023 21:22:30 -0400
Subject: [PATCH 070/725] x86_64: optimize code size for double neg/abs

---
 src/arch/x86_64/CodeGen.zig | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 9487cc33c2..9e50e6b5b4 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -4229,16 +4229,12 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
 
     const tag = self.air.instructions.items(.tag)[inst];
     try self.genBinOpMir(switch (ty_bits) {
-        32 => switch (tag) {
+        // No point using an extra prefix byte for *pd which performs the same operation.
+        32, 64 => switch (tag) {
             .neg => .xorps,
             .fabs => .andnps,
             else => unreachable,
         },
-        64 => switch (tag) {
-            .neg => .xorpd,
-            .fabs => .andnpd,
-            else => unreachable,
-        },
         else => return self.fail("TODO implement airFloatSign for {}", .{
             ty.fmt(self.bin_file.options.module.?),
         }),

From 6893f90887836584f9377793cca7235d8947a326 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Wed, 3 May 2023 02:37:48 -0400
Subject: [PATCH 071/725] x86_64: implement sqrt

---
 src/arch/x86_64/CodeGen.zig   | 28 +++++++++++++++++++++++++++-
 src/arch/x86_64/Encoding.zig  |  4 ++++
 src/arch/x86_64/Lower.zig     |  4 ++++
 src/arch/x86_64/Mir.zig       |  8 ++++++++
 src/arch/x86_64/encodings.zig |  6 ++++++
 5 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 9e50e6b5b4..ee621ffd87 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -229,6 +229,7 @@ pub const MCValue = union(enum) {
     fn isRegister(mcv: MCValue) bool {
         return switch (mcv) {
             .register => true,
+            .register_offset => |reg_off| return reg_off.off == 0,
             else => false,
         };
     }
@@ -1449,7 +1450,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .shl_sat         => try self.airShlSat(inst),
             .slice           => try self.airSlice(inst),
 
-            .sqrt,
             .sin,
             .cos,
             .tan,
@@ -1464,6 +1464,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .trunc_float,
             => try self.airUnaryMath(inst),
 
+            .sqrt => try self.airSqrt(inst),
             .neg, .fabs => try self.airFloatSign(inst),
 
             .add_with_overflow => try self.airAddSubWithOverflow(inst),
@@ -4242,6 +4243,31 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
 }
 
+fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
+    const un_op = self.air.instructions.items(.data)[inst].un_op;
+    const ty = self.air.typeOf(un_op);
+
+    const src_mcv = try self.resolveInst(un_op);
+    const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
+        src_mcv
+    else
+        try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
+
+    try self.genBinOpMir(switch (ty.zigTypeTag()) {
+        .Float => switch (ty.floatBits(self.target.*)) {
+            32 => .sqrtss,
+            64 => .sqrtsd,
+            else => return self.fail("TODO implement airSqrt for {}", .{
+                ty.fmt(self.bin_file.options.module.?),
+            }),
+        },
+        else => return self.fail("TODO implement airSqrt for {}", .{
+            ty.fmt(self.bin_file.options.module.?),
+        }),
+    }, ty, dst_mcv, src_mcv);
+    return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
+}
+
 fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     _ = un_op;
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index bb1757c91c..13d7b1776d 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -277,6 +277,8 @@ pub const Mnemonic = enum {
     movss,
     mulss,
     orps,
+    sqrtps,
+    sqrtss,
     subss,
     ucomiss,
     xorps,
@@ -291,6 +293,8 @@ pub const Mnemonic = enum {
     movq, //movd, movsd,
     mulsd,
     orpd,
+    sqrtpd,
+    sqrtsd,
     subsd,
     ucomisd,
     xorpd,
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index 03e395b171..b369ba2a6b 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -105,6 +105,8 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
         .mulss,
         .orps,
         .roundss,
+        .sqrtps,
+        .sqrtss,
         .subss,
         .ucomiss,
         .xorps,
@@ -122,6 +124,8 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
         .mulsd,
         .orpd,
         .roundsd,
+        .sqrtpd,
+        .sqrtsd,
         .subsd,
         .ucomisd,
         .xorpd,
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index f3d7a5a66f..95efc0a96c 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -190,7 +190,11 @@ pub const Inst = struct {
         orps,
         /// Round scalar single-precision floating-point values
         roundss,
+        /// Square root of scalar single precision floating-point value
+        sqrtps,
         /// Subtract scalar single-precision floating-point values
+        sqrtss,
+        /// Square root of single precision floating-point values
         subss,
         /// Unordered compare scalar single-precision floating-point values
         ucomiss,
@@ -224,6 +228,10 @@ pub const Inst = struct {
         orpd,
         /// Round scalar double-precision floating-point values
         roundsd,
+        /// Square root of double precision floating-point values
+        sqrtpd,
+        /// Square root of scalar double precision floating-point value
+        sqrtsd,
         /// Subtract scalar double-precision floating-point values
         subsd,
         /// Unordered compare scalar double-precision floating-point values
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index 35b2f13fe7..5c443157c5 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -856,6 +856,9 @@ pub const table = [_]Entry{
 
     .{ .subss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .sse },
 
+    .{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{       0x0f, 0x51 }, 0, .sse },
+    .{ .sqrtss, .rm, &.{ .xmm, .xmm_m32  }, &.{ 0xf3, 0x0f, 0x51 }, 0, .sse },
+
     .{ .ucomiss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x0f, 0x2e }, 0, .sse },
 
     .{ .xorps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x57 }, 0, .sse },
@@ -895,6 +898,9 @@ pub const table = [_]Entry{
 
     .{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .sse2 },
 
+    .{ .sqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .sse2 },
+    .{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0xf2, 0x0f, 0x51 }, 0, .sse2 },
+
     .{ .subsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .sse2 },
 
     .{ .movsd, .rm, &.{ .xmm,     .xmm_m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .sse2 },

From a19faa2481e84e065a8762cb7c7cbf35426929fd Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Wed, 3 May 2023 04:21:40 -0400
Subject: [PATCH 072/725] x86_64: implement movement of more types

 * f16
 * f128
 * vector
---
 src/arch/x86_64/CodeGen.zig   | 221 +++++++++++++++++++++++-----------
 src/arch/x86_64/Encoding.zig  |   6 +-
 src/arch/x86_64/Lower.zig     |  13 ++
 src/arch/x86_64/Mir.zig       |  14 +++
 src/arch/x86_64/encodings.zig |  18 +++
 test/behavior/vector.zig      |   6 -
 6 files changed, 198 insertions(+), 80 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index ee621ffd87..97e672b71f 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -1210,6 +1210,28 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !
     });
 }
 
+fn asmRegisterMemoryImmediate(
+    self: *Self,
+    tag: Mir.Inst.Tag,
+    reg: Register,
+    m: Memory,
+    imm: Immediate,
+) !void {
+    _ = try self.addInst(.{
+        .tag = tag,
+        .ops = switch (m) {
+            .sib => .rmi_sib,
+            .rip => .rmi_rip,
+            else => unreachable,
+        },
+        .data = .{ .rix = .{ .r = reg, .i = @intCast(u8, imm.unsigned), .payload = switch (m) {
+            .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+            .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+            else => unreachable,
+        } } },
+    });
+}
+
 fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !void {
     _ = try self.addInst(.{
         .tag = tag,
@@ -1951,7 +1973,7 @@ fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_
         const ptr_bits = self.target.cpu.arch.ptrBitWidth();
         const ptr_bytes: u64 = @divExact(ptr_bits, 8);
         if (abi_size <= ptr_bytes) {
-            if (self.register_manager.tryAllocReg(inst, try self.regClassForType(elem_ty))) |reg| {
+            if (self.register_manager.tryAllocReg(inst, regClassForType(elem_ty))) |reg| {
                 return MCValue{ .register = registerAlias(reg, abi_size) };
             }
         }
@@ -1961,14 +1983,9 @@ fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_
     return .{ .load_frame = .{ .index = frame_index } };
 }
 
-fn regClassForType(self: *Self, ty: Type) !RegisterManager.RegisterBitSet {
+fn regClassForType(ty: Type) RegisterManager.RegisterBitSet {
     return switch (ty.zigTypeTag()) {
-        .Vector => self.fail("TODO regClassForType for {}", .{ty.fmt(self.bin_file.options.module.?)}),
-        .Float => switch (ty.floatBits(self.target.*)) {
-            32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse)) sse else gp,
-            64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2)) sse else gp,
-            else => gp,
-        },
+        .Float, .Vector => sse,
         else => gp,
     };
 }
@@ -2111,7 +2128,7 @@ pub fn spillRegisters(self: *Self, registers: []const Register) !void {
 /// allocated. A second call to `copyToTmpRegister` may return the same register.
 /// This can have a side effect of spilling instructions to the stack to free up a register.
 fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
-    const reg = try self.register_manager.allocReg(null, try self.regClassForType(ty));
+    const reg = try self.register_manager.allocReg(null, regClassForType(ty));
     try self.genSetReg(reg, ty, mcv);
     return reg;
 }
@@ -2126,7 +2143,7 @@ fn copyToRegisterWithInstTracking(
     ty: Type,
     mcv: MCValue,
 ) !MCValue {
-    const reg: Register = try self.register_manager.allocReg(reg_owner, try self.regClassForType(ty));
+    const reg: Register = try self.register_manager.allocReg(reg_owner, regClassForType(ty));
     try self.genSetReg(reg, ty, mcv);
     return MCValue{ .register = reg };
 }
@@ -2159,8 +2176,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
     if (dst_ty.floatBits(self.target.*) != 32 or src_ty.floatBits(self.target.*) != 64 or
         !Target.x86.featureSetHas(self.target.cpu.features, .sse2))
         return self.fail("TODO implement airFptrunc from {} to {}", .{
-            src_ty.fmt(self.bin_file.options.module.?),
-            dst_ty.fmt(self.bin_file.options.module.?),
+            src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
         });
 
     const src_mcv = try self.resolveInst(ty_op.operand);
@@ -2182,8 +2198,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
     if (dst_ty.floatBits(self.target.*) != 64 or src_ty.floatBits(self.target.*) != 32 or
         !Target.x86.featureSetHas(self.target.cpu.features, .sse2))
         return self.fail("TODO implement airFpext from {} to {}", .{
-            src_ty.fmt(self.bin_file.options.module.?),
-            dst_ty.fmt(self.bin_file.options.module.?),
+            src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
         });
 
     const src_mcv = try self.resolveInst(ty_op.operand);
@@ -4436,8 +4451,8 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
         const ptr_ty = self.air.typeOf(ty_op.operand);
         const elem_size = elem_ty.abiSize(self.target.*);
 
-        const elem_rc = try self.regClassForType(elem_ty);
-        const ptr_rc = try self.regClassForType(ptr_ty);
+        const elem_rc = regClassForType(elem_ty);
+        const ptr_rc = regClassForType(ptr_ty);
 
         const ptr_mcv = try self.resolveInst(ty_op.operand);
         const dst_mcv = if (elem_size <= 8 and elem_rc.supersetOf(ptr_rc) and
@@ -5257,8 +5272,7 @@ fn genMulDivBinOp(
         .mul, .mulwrap => dst_abi_size != src_abi_size and dst_abi_size != src_abi_size * 2,
         .div_trunc, .div_floor, .div_exact, .rem, .mod => dst_abi_size != src_abi_size,
     } or src_abi_size > 8) return self.fail("TODO implement genMulDivBinOp from {} to {}", .{
-        src_ty.fmt(self.bin_file.options.module.?),
-        dst_ty.fmt(self.bin_file.options.module.?),
+        src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
     });
     const ty = if (dst_abi_size <= 8) dst_ty else src_ty;
     const abi_size = if (dst_abi_size <= 8) dst_abi_size else src_abi_size;
@@ -5558,7 +5572,9 @@ fn genBinOp(
         }, lhs_ty, dst_mcv, src_mcv),
 
         .mul => try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
-            else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
+            else => return self.fail("TODO implement genBinOp for {s} {}", .{
+                @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+            }),
             .Float => switch (lhs_ty.floatBits(self.target.*)) {
                 32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
                     .mulss
@@ -5761,9 +5777,13 @@ fn genBinOp(
                     .max => .maxsd,
                     else => unreachable,
                 },
-                else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
+                else => return self.fail("TODO implement genBinOp for {s} {}", .{
+                    @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+                }),
             }, lhs_ty, dst_mcv, src_mcv),
-            else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
+            else => return self.fail("TODO implement genBinOp for {s} {}", .{
+                @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
+            }),
         },
 
         else => unreachable,
@@ -5802,8 +5822,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
                     .Float => {
                         if (!Target.x86.featureSetHas(self.target.cpu.features, .sse))
                             return self.fail("TODO genBinOpMir for {s} {} without sse", .{
-                                @tagName(mir_tag),
-                                ty.fmt(self.bin_file.options.module.?),
+                                @tagName(mir_tag), ty.fmt(self.bin_file.options.module.?),
                             });
                         return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
                     },
@@ -7588,10 +7607,11 @@ fn movMirTag(self: *Self, ty: Type) !Mir.Inst.Tag {
     return switch (ty.zigTypeTag()) {
         else => .mov,
         .Float => switch (ty.floatBits(self.target.*)) {
-            16 => .mov,
-            32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse)) .movss else .mov,
-            64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2)) .movsd else .mov,
-            else => return self.fail("TODO movMirTag for {}", .{
+            16 => unreachable, // needs special handling
+            32 => .movss,
+            64 => .movsd,
+            128 => .movaps,
+            else => return self.fail("TODO movMirTag from {}", .{
                 ty.fmt(self.bin_file.options.module.?),
             }),
         },
@@ -7700,8 +7720,17 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
         },
         .register => |src_reg| if (dst_reg.id() != src_reg.id()) try self.asmRegisterRegister(
             if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))
-                try self.movMirTag(ty)
+                switch (ty.zigTypeTag()) {
+                    else => .mov,
+                    .Float, .Vector => .movaps,
+                }
             else switch (abi_size) {
+                2 => return try self.asmRegisterRegisterImmediate(
+                    if (dst_reg.class() == .floating_point) .pinsrw else .pextrw,
+                    registerAlias(dst_reg, abi_size),
+                    registerAlias(src_reg, abi_size),
+                    Immediate.u(0),
+                ),
                 4 => .movd,
                 8 => .movq,
                 else => return self.fail(
@@ -7712,18 +7741,12 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             registerAlias(dst_reg, abi_size),
             registerAlias(src_reg, abi_size),
         ),
-        .register_offset, .indirect, .load_frame, .lea_frame => try self.asmRegisterMemory(
-            switch (src_mcv) {
-                .register_offset => |reg_off| switch (reg_off.off) {
-                    0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
-                    else => .lea,
-                },
-                .indirect, .load_frame => try self.movMirTag(ty),
-                .lea_frame => .lea,
-                else => unreachable,
-            },
-            registerAlias(dst_reg, abi_size),
-            Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
+        .register_offset,
+        .indirect,
+        .load_frame,
+        .lea_frame,
+        => {
+            const src_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
                 .register_offset, .indirect => |reg_off| .{
                     .base = .{ .reg = reg_off.reg },
                     .disp = reg_off.off,
@@ -7733,20 +7756,51 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                     .disp = frame_addr.off,
                 },
                 else => unreachable,
-            }),
-        ),
+            });
+            if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
+                try self.asmRegisterMemoryImmediate(
+                    .pinsrw,
+                    registerAlias(dst_reg, abi_size),
+                    src_mem,
+                    Immediate.u(0),
+                )
+            else
+                try self.asmRegisterMemory(
+                    switch (src_mcv) {
+                        .register_offset => |reg_off| switch (reg_off.off) {
+                            0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
+                            else => .lea,
+                        },
+                        .indirect, .load_frame => try self.movMirTag(ty),
+                        .lea_frame => .lea,
+                        else => unreachable,
+                    },
+                    registerAlias(dst_reg, abi_size),
+                    src_mem,
+                );
+        },
         .memory, .load_direct, .load_got, .load_tlv => {
             switch (src_mcv) {
-                .memory => |addr| if (math.cast(i32, @bitCast(i64, addr))) |small_addr|
-                    return self.asmRegisterMemory(
-                        try self.movMirTag(ty),
-                        registerAlias(dst_reg, abi_size),
-                        Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
-                            .base = .{ .reg = .ds },
-                            .disp = small_addr,
-                        }),
-                    ),
-                .load_direct => |sym_index| if (try self.movMirTag(ty) == .mov) {
+                .memory => |addr| if (math.cast(i32, @bitCast(i64, addr))) |small_addr| {
+                    const src_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
+                        .base = .{ .reg = .ds },
+                        .disp = small_addr,
+                    });
+                    return if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
+                        self.asmRegisterMemoryImmediate(
+                            .pinsrw,
+                            registerAlias(dst_reg, abi_size),
+                            src_mem,
+                            Immediate.u(0),
+                        )
+                    else
+                        self.asmRegisterMemory(
+                            try self.movMirTag(ty),
+                            registerAlias(dst_reg, abi_size),
+                            src_mem,
+                        );
+                },
+                .load_direct => |sym_index| if (!ty.isRuntimeFloat()) {
                     const atom_index = try self.owner.getSymbolIndex(self);
                     _ = try self.addInst(.{
                         .tag = .mov_linker,
@@ -7767,11 +7821,22 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
             defer self.register_manager.unlockReg(addr_lock);
 
-            try self.asmRegisterMemory(
-                try self.movMirTag(ty),
-                registerAlias(dst_reg, abi_size),
-                Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
-            );
+            const src_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
+                .base = .{ .reg = addr_reg },
+            });
+            if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
+                try self.asmRegisterMemoryImmediate(
+                    .pinsrw,
+                    registerAlias(dst_reg, abi_size),
+                    src_mem,
+                    Immediate.u(0),
+                )
+            else
+                try self.asmRegisterMemory(
+                    try self.movMirTag(ty),
+                    registerAlias(dst_reg, abi_size),
+                    src_mem,
+                );
         },
         .lea_direct, .lea_got => |sym_index| {
             const atom_index = try self.owner.getSymbolIndex(self);
@@ -7864,11 +7929,25 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
             },
         },
         .eflags => |cc| try self.asmSetccMemory(Memory.sib(.byte, .{ .base = base, .disp = disp }), cc),
-        .register => |reg| try self.asmMemoryRegister(
-            try self.movMirTag(ty),
-            Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
-            registerAlias(reg, abi_size),
-        ),
+        .register => |src_reg| {
+            const dst_mem = Memory.sib(
+                Memory.PtrSize.fromSize(abi_size),
+                .{ .base = base, .disp = disp },
+            );
+            if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
+                try self.asmMemoryRegisterImmediate(
+                    .pextrw,
+                    dst_mem,
+                    registerAlias(src_reg, abi_size),
+                    Immediate.u(0),
+                )
+            else
+                try self.asmMemoryRegister(
+                    try self.movMirTag(ty),
+                    dst_mem,
+                    registerAlias(src_reg, abi_size),
+                );
+        },
         .register_overflow => |ro| {
             try self.genSetMem(
                 base,
@@ -8071,8 +8150,8 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
     const src_ty = self.air.typeOf(ty_op.operand);
 
     const result = result: {
-        const dst_rc = try self.regClassForType(dst_ty);
-        const src_rc = try self.regClassForType(src_ty);
+        const dst_rc = regClassForType(dst_ty);
+        const src_rc = regClassForType(src_ty);
         const operand = try self.resolveInst(ty_op.operand);
         if (dst_rc.supersetOf(src_rc) and self.reuseOperand(inst, ty_op.operand, 0, operand))
             break :result operand;
@@ -8127,8 +8206,7 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
         .unsigned => src_bits + 1,
     }, 32), 8) catch unreachable;
     if (src_size > 8) return self.fail("TODO implement airIntToFloat from {} to {}", .{
-        src_ty.fmt(self.bin_file.options.module.?),
-        dst_ty.fmt(self.bin_file.options.module.?),
+        src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
     });
 
     const src_mcv = try self.resolveInst(ty_op.operand);
@@ -8141,7 +8219,7 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
 
     if (src_bits < src_size * 8) try self.truncateRegister(src_ty, src_reg);
 
-    const dst_reg = try self.register_manager.allocReg(inst, try self.regClassForType(dst_ty));
+    const dst_reg = try self.register_manager.allocReg(inst, regClassForType(dst_ty));
     const dst_mcv = MCValue{ .register = dst_reg };
     const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
     defer self.register_manager.unlockReg(dst_lock);
@@ -8151,19 +8229,16 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
             .cvtsi2ss
         else
             return self.fail("TODO implement airIntToFloat from {} to {} without sse", .{
-                src_ty.fmt(self.bin_file.options.module.?),
-                dst_ty.fmt(self.bin_file.options.module.?),
+                src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
             }),
         64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
             .cvtsi2sd
         else
             return self.fail("TODO implement airIntToFloat from {} to {} without sse2", .{
-                src_ty.fmt(self.bin_file.options.module.?),
-                dst_ty.fmt(self.bin_file.options.module.?),
+                src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
             }),
         else => return self.fail("TODO implement airIntToFloat from {} to {}", .{
-            src_ty.fmt(self.bin_file.options.module.?),
-            dst_ty.fmt(self.bin_file.options.module.?),
+            src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
         }),
     }, dst_reg.to128(), registerAlias(src_reg, src_size));
 
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index 13d7b1776d..944fe85458 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -274,9 +274,11 @@ pub const Mnemonic = enum {
     cvtsi2ss,
     divss,
     maxss, minss,
-    movss,
+    movaps, movss, movups,
     mulss,
     orps,
+    pextrw,
+    pinsrw,
     sqrtps,
     sqrtss,
     subss,
@@ -290,7 +292,9 @@ pub const Mnemonic = enum {
     cvtsd2ss, cvtsi2sd, cvtss2sd,
     divsd,
     maxsd, minsd,
+    movapd,
     movq, //movd, movsd,
+    movupd,
     mulsd,
     orpd,
     sqrtpd,
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index b369ba2a6b..4289cfaf2a 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -101,9 +101,13 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
         .divss,
         .maxss,
         .minss,
+        .movaps,
         .movss,
+        .movups,
         .mulss,
         .orps,
+        .pextrw,
+        .pinsrw,
         .roundss,
         .sqrtps,
         .sqrtss,
@@ -198,6 +202,8 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
         .mi_rip_u,
         .lock_mi_sib_u,
         .lock_mi_rip_u,
+        .rmi_sib,
+        .rmi_rip,
         .mri_sib,
         .mri_rip,
         => Immediate.u(i),
@@ -212,6 +218,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
     return lower.mir.resolveFrameLoc(switch (ops) {
         .rm_sib,
         .rm_sib_cc,
+        .rmi_sib,
         .m_sib,
         .m_sib_cc,
         .mi_sib_u,
@@ -227,6 +234,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
 
         .rm_rip,
         .rm_rip_cc,
+        .rmi_rip,
         .m_rip,
         .m_rip_cc,
         .mi_rip_u,
@@ -321,6 +329,11 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .reg = inst.data.rx.r },
             .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
         },
+        .rmi_sib, .rmi_rip => &.{
+            .{ .reg = inst.data.rix.r },
+            .{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
+            .{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
+        },
         .mr_sib, .lock_mr_sib, .mr_rip, .lock_mr_rip => &.{
             .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
             .{ .reg = inst.data.rx.r },
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index 95efc0a96c..6b2db1b696 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -182,12 +182,20 @@ pub const Inst = struct {
         maxss,
         /// Return minimum single-precision floating-point value
         minss,
+        /// Move aligned packed single-precision floating-point values
+        movaps,
         /// Move scalar single-precision floating-point value
         movss,
+        /// Move unaligned packed single-precision floating-point values
+        movups,
         /// Multiply scalar single-precision floating-point values
         mulss,
         /// Bitwise logical or of packed single precision floating-point values
         orps,
+        /// Extract word
+        pextrw,
+        /// Insert word
+        pinsrw,
         /// Round scalar single-precision floating-point values
         roundss,
         /// Square root of scalar single precision floating-point value
@@ -346,6 +354,12 @@ pub const Inst = struct {
         /// Register, memory (RIP) operands with condition code (CC).
         /// Uses `rx_cc` payload.
         rm_rip_cc,
+        /// Register, memory (SIB), immediate (byte) operands.
+        /// Uses `rix` payload with extra data of type `MemorySib`.
+        rmi_sib,
+        /// Register, memory (RIP), immediate (byte) operands.
+        /// Uses `rix` payload with extra data of type `MemoryRip`.
+        rmi_rip,
         /// Single memory (SIB) operand.
         /// Uses `payload` with extra data of type `MemorySib`.
         m_sib,
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index 5c443157c5..f87a110e99 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -847,9 +847,15 @@ pub const table = [_]Entry{
 
     .{ .minss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5d }, 0, .sse },
 
+    .{ .movaps, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .sse },
+    .{ .movaps, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x29 }, 0, .sse },
+
     .{ .movss, .rm, &.{ .xmm,     .xmm_m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .sse },
     .{ .movss, .mr, &.{ .xmm_m32, .xmm     }, &.{ 0xf3, 0x0f, 0x11 }, 0, .sse },
 
+    .{ .movups, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .sse },
+    .{ .movups, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x11 }, 0, .sse },
+
     .{ .mulss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .sse },
 
     .{ .orps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .sse },
@@ -885,6 +891,9 @@ pub const table = [_]Entry{
 
     .{ .minsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .sse2 },
 
+    .{ .movapd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .sse2 },
+    .{ .movapd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x29 }, 0, .sse2 },
+
     .{ .movd, .rm, &.{ .xmm,  .rm32 }, &.{ 0x66, 0x0f, 0x6e }, 0, .sse2 },
     .{ .movd, .mr, &.{ .rm32, .xmm  }, &.{ 0x66, 0x0f, 0x7e }, 0, .sse2 },
 
@@ -894,10 +903,17 @@ pub const table = [_]Entry{
     .{ .movq, .rm, &.{ .xmm,     .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .sse2 },
     .{ .movq, .mr, &.{ .xmm_m64, .xmm     }, &.{ 0x66, 0x0f, 0xd6 }, 0, .sse2 },
 
+    .{ .movupd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .sse2 },
+    .{ .movupd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x11 }, 0, .sse2 },
+
     .{ .mulsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x59 }, 0, .sse2 },
 
     .{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .sse2 },
 
+    .{ .pextrw, .mri, &.{ .r16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .sse2 },
+
+    .{ .pinsrw, .rmi, &.{ .xmm, .rm16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .sse2 },
+
     .{ .sqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .sse2 },
     .{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0xf2, 0x0f, 0x51 }, 0, .sse2 },
 
@@ -911,6 +927,8 @@ pub const table = [_]Entry{
     .{ .xorpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .sse2 },
 
     // SSE4.1
+    .{ .pextrw, .mri, &.{ .rm16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .sse4_1 },
+
     .{ .roundss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .sse4_1 },
     .{ .roundsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .sse4_1 },
 };
diff --git a/test/behavior/vector.zig b/test/behavior/vector.zig
index 01c76310d7..2c55af5f85 100644
--- a/test/behavior/vector.zig
+++ b/test/behavior/vector.zig
@@ -133,7 +133,6 @@ test "vector bit operators" {
 }
 
 test "implicit cast vector to array" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -151,7 +150,6 @@ test "implicit cast vector to array" {
 }
 
 test "array to vector" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -321,7 +319,6 @@ test "load vector elements via comptime index" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
@@ -343,7 +340,6 @@ test "store vector elements via comptime index" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
@@ -371,7 +367,6 @@ test "load vector elements via runtime index" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {
@@ -393,7 +388,6 @@ test "store vector elements via runtime index" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const S = struct {
         fn doTheTest() !void {

From 5def162391da5050761beb3e6efb24b99716fc99 Mon Sep 17 00:00:00 2001
From: Jonathan Marler 
Date: Mon, 6 Mar 2023 18:29:27 -0700
Subject: [PATCH 073/725] fix bug and simplify std.crypto.tls.Client.limitVecs

---
 lib/std/crypto/tls/Client.zig | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/lib/std/crypto/tls/Client.zig b/lib/std/crypto/tls/Client.zig
index c886bc7276..e67fd20925 100644
--- a/lib/std/crypto/tls/Client.zig
+++ b/lib/std/crypto/tls/Client.zig
@@ -1330,18 +1330,15 @@ const VecPut = struct {
 
 /// Limit iovecs to a specific byte size.
 fn limitVecs(iovecs: []std.os.iovec, len: usize) []std.os.iovec {
-    var vec_i: usize = 0;
     var bytes_left: usize = len;
-    while (true) {
-        if (bytes_left >= iovecs[vec_i].iov_len) {
-            bytes_left -= iovecs[vec_i].iov_len;
-            vec_i += 1;
-            if (vec_i == iovecs.len or bytes_left == 0) return iovecs[0..vec_i];
-            continue;
+    for (iovecs, 0..) |*iovec, vec_i| {
+        if (bytes_left <= iovec.iov_len) {
+            iovec.iov_len = bytes_left;
+            return iovecs[0 .. vec_i + 1];
         }
-        iovecs[vec_i].iov_len = bytes_left;
-        return iovecs[0..vec_i];
+        bytes_left -= iovec.iov_len;
     }
+    return iovecs;
 }
 
 /// The priority order here is chosen based on what crypto algorithms Zig has

From 3f3b1a6808113fd5f9b2cec1033009cbb17dc969 Mon Sep 17 00:00:00 2001
From: Veikka Tuominen 
Date: Wed, 3 May 2023 11:49:55 +0300
Subject: [PATCH 074/725] std.Build: use Step.* instead of *Step

Follow up to 13eb7251d37759bd47403db304c6120c706fe353
---
 build.zig                                   |  12 +-
 lib/init-exe/build.zig                      |   2 +-
 lib/std/Build.zig                           | 150 ++++++++---------
 lib/std/Build/Step.zig                      |  67 +++++---
 lib/std/Build/Step/CheckFile.zig            |  12 +-
 lib/std/Build/Step/CheckObject.zig          |  28 ++--
 lib/std/Build/Step/Compile.zig              | 168 ++++++++++----------
 lib/std/Build/Step/ConfigHeader.zig         |  26 +--
 lib/std/Build/Step/Fmt.zig                  |   8 +-
 lib/std/Build/Step/InstallArtifact.zig      |  17 +-
 lib/std/Build/Step/InstallFile.zig          |   8 +-
 lib/std/Build/Step/ObjCopy.zig              |  17 +-
 lib/std/Build/Step/Options.zig              |  25 ++-
 lib/std/Build/Step/RemoveDir.zig            |   8 +-
 lib/std/Build/Step/Run.zig                  | 108 ++++++-------
 lib/std/Build/Step/TranslateC.zig           |  26 ++-
 lib/std/Build/Step/WriteFile.zig            |  38 ++---
 test/link/macho/dead_strip/build.zig        |   2 +-
 test/link/macho/dead_strip_dylibs/build.zig |   2 +-
 test/link/macho/headerpad/build.zig         |   2 +-
 test/link/macho/search_strategy/build.zig   |   2 +-
 test/link/macho/unwind_info/build.zig       |   2 +-
 test/link/macho/uuid/build.zig              |   3 +-
 test/src/Cases.zig                          |   2 +-
 test/src/StackTrace.zig                     |   2 +-
 test/standalone/install_raw_hex/build.zig   |   1 -
 test/tests.zig                              |   2 +-
 27 files changed, 379 insertions(+), 361 deletions(-)

diff --git a/build.zig b/build.zig
index 78345ac9e4..208d06fe1d 100644
--- a/build.zig
+++ b/build.zig
@@ -533,7 +533,7 @@ fn addCompilerStep(
     b: *std.Build,
     optimize: std.builtin.OptimizeMode,
     target: std.zig.CrossTarget,
-) *std.Build.CompileStep {
+) *std.Build.Step.Compile {
     const exe = b.addExecutable(.{
         .name = "zig",
         .root_source_file = .{ .path = "src/main.zig" },
@@ -561,7 +561,7 @@ const exe_cflags = [_][]const u8{
 fn addCmakeCfgOptionsToExe(
     b: *std.Build,
     cfg: CMakeConfig,
-    exe: *std.Build.CompileStep,
+    exe: *std.Build.Step.Compile,
     use_zig_libcxx: bool,
 ) !void {
     if (exe.target.isDarwin()) {
@@ -640,7 +640,7 @@ fn addCmakeCfgOptionsToExe(
     }
 }
 
-fn addStaticLlvmOptionsToExe(exe: *std.Build.CompileStep) !void {
+fn addStaticLlvmOptionsToExe(exe: *std.Build.Step.Compile) !void {
     // Adds the Zig C++ sources which both stage1 and stage2 need.
     //
     // We need this because otherwise zig_clang_cc1_main.cpp ends up pulling
@@ -679,7 +679,7 @@ fn addStaticLlvmOptionsToExe(exe: *std.Build.CompileStep) !void {
 fn addCxxKnownPath(
     b: *std.Build,
     ctx: CMakeConfig,
-    exe: *std.Build.CompileStep,
+    exe: *std.Build.Step.Compile,
     objname: []const u8,
     errtxt: ?[]const u8,
     need_cpp_includes: bool,
@@ -709,7 +709,7 @@ fn addCxxKnownPath(
     }
 }
 
-fn addCMakeLibraryList(exe: *std.Build.CompileStep, list: []const u8) void {
+fn addCMakeLibraryList(exe: *std.Build.Step.Compile, list: []const u8) void {
     var it = mem.tokenize(u8, list, ";");
     while (it.next()) |lib| {
         if (mem.startsWith(u8, lib, "-l")) {
@@ -723,7 +723,7 @@ fn addCMakeLibraryList(exe: *std.Build.CompileStep, list: []const u8) void {
 }
 
 const CMakeConfig = struct {
-    llvm_linkage: std.Build.CompileStep.Linkage,
+    llvm_linkage: std.Build.Step.Compile.Linkage,
     cmake_binary_dir: []const u8,
     cmake_prefix_path: []const u8,
     cmake_static_library_prefix: []const u8,
diff --git a/lib/init-exe/build.zig b/lib/init-exe/build.zig
index abf8654f0f..1221984190 100644
--- a/lib/init-exe/build.zig
+++ b/lib/init-exe/build.zig
@@ -29,7 +29,7 @@ pub fn build(b: *std.Build) void {
     // step when running `zig build`).
     b.installArtifact(exe);
 
-    // This *creates* a RunStep in the build graph, to be executed when another
+    // This *creates* a Run step in the build graph, to be executed when another
     // step is evaluated that depends on it. The next line below will establish
     // such a dependency.
     const run_cmd = b.addRunArtifact(exe);
diff --git a/lib/std/Build.zig b/lib/std/Build.zig
index bda50112b6..ca55d23937 100644
--- a/lib/std/Build.zig
+++ b/lib/std/Build.zig
@@ -21,27 +21,41 @@ const Build = @This();
 
 pub const Cache = @import("Build/Cache.zig");
 
-/// deprecated: use `CompileStep`.
-pub const LibExeObjStep = CompileStep;
+/// deprecated: use `Step.Compile`.
+pub const LibExeObjStep = Step.Compile;
 /// deprecated: use `Build`.
 pub const Builder = Build;
-/// deprecated: use `InstallDirStep.Options`
-pub const InstallDirectoryOptions = InstallDirStep.Options;
+/// deprecated: use `Step.InstallDir.Options`
+pub const InstallDirectoryOptions = Step.InstallDir.Options;
 
 pub const Step = @import("Build/Step.zig");
+/// deprecated: use `Step.CheckFile`.
 pub const CheckFileStep = @import("Build/Step/CheckFile.zig");
+/// deprecated: use `Step.CheckObject`.
 pub const CheckObjectStep = @import("Build/Step/CheckObject.zig");
+/// deprecated: use `Step.ConfigHeader`.
 pub const ConfigHeaderStep = @import("Build/Step/ConfigHeader.zig");
+/// deprecated: use `Step.Fmt`.
 pub const FmtStep = @import("Build/Step/Fmt.zig");
+/// deprecated: use `Step.InstallArtifact`.
 pub const InstallArtifactStep = @import("Build/Step/InstallArtifact.zig");
+/// deprecated: use `Step.InstallDir`.
 pub const InstallDirStep = @import("Build/Step/InstallDir.zig");
+/// deprecated: use `Step.InstallFile`.
 pub const InstallFileStep = @import("Build/Step/InstallFile.zig");
+/// deprecated: use `Step.ObjCopy`.
 pub const ObjCopyStep = @import("Build/Step/ObjCopy.zig");
+/// deprecated: use `Step.Compile`.
 pub const CompileStep = @import("Build/Step/Compile.zig");
+/// deprecated: use `Step.Options`.
 pub const OptionsStep = @import("Build/Step/Options.zig");
+/// deprecated: use `Step.RemoveDir`.
 pub const RemoveDirStep = @import("Build/Step/RemoveDir.zig");
+/// deprecated: use `Step.Run`.
 pub const RunStep = @import("Build/Step/Run.zig");
+/// deprecated: use `Step.TranslateC`.
 pub const TranslateCStep = @import("Build/Step/TranslateC.zig");
+/// deprecated: use `Step.WriteFile`.
 pub const WriteFileStep = @import("Build/Step/WriteFile.zig");
 
 install_tls: TopLevelStep,
@@ -442,8 +456,8 @@ pub fn resolveInstallPrefix(self: *Build, install_prefix: ?[]const u8, dir_list:
     self.h_dir = self.pathJoin(&h_list);
 }
 
-pub fn addOptions(self: *Build) *OptionsStep {
-    return OptionsStep.create(self);
+pub fn addOptions(self: *Build) *Step.Options {
+    return Step.Options.create(self);
 }
 
 pub const ExecutableOptions = struct {
@@ -452,7 +466,7 @@ pub const ExecutableOptions = struct {
     version: ?std.builtin.Version = null,
     target: CrossTarget = .{},
     optimize: std.builtin.Mode = .Debug,
-    linkage: ?CompileStep.Linkage = null,
+    linkage: ?Step.Compile.Linkage = null,
     max_rss: usize = 0,
     link_libc: ?bool = null,
     single_threaded: ?bool = null,
@@ -460,8 +474,8 @@ pub const ExecutableOptions = struct {
     use_lld: ?bool = null,
 };
 
-pub fn addExecutable(b: *Build, options: ExecutableOptions) *CompileStep {
-    return CompileStep.create(b, .{
+pub fn addExecutable(b: *Build, options: ExecutableOptions) *Step.Compile {
+    return Step.Compile.create(b, .{
         .name = options.name,
         .root_source_file = options.root_source_file,
         .version = options.version,
@@ -489,8 +503,8 @@ pub const ObjectOptions = struct {
     use_lld: ?bool = null,
 };
 
-pub fn addObject(b: *Build, options: ObjectOptions) *CompileStep {
-    return CompileStep.create(b, .{
+pub fn addObject(b: *Build, options: ObjectOptions) *Step.Compile {
+    return Step.Compile.create(b, .{
         .name = options.name,
         .root_source_file = options.root_source_file,
         .target = options.target,
@@ -517,8 +531,8 @@ pub const SharedLibraryOptions = struct {
     use_lld: ?bool = null,
 };
 
-pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *CompileStep {
-    return CompileStep.create(b, .{
+pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *Step.Compile {
+    return Step.Compile.create(b, .{
         .name = options.name,
         .root_source_file = options.root_source_file,
         .kind = .lib,
@@ -547,8 +561,8 @@ pub const StaticLibraryOptions = struct {
     use_lld: ?bool = null,
 };
 
-pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *CompileStep {
-    return CompileStep.create(b, .{
+pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *Step.Compile {
+    return Step.Compile.create(b, .{
         .name = options.name,
         .root_source_file = options.root_source_file,
         .kind = .lib,
@@ -579,8 +593,8 @@ pub const TestOptions = struct {
     use_lld: ?bool = null,
 };
 
-pub fn addTest(b: *Build, options: TestOptions) *CompileStep {
-    return CompileStep.create(b, .{
+pub fn addTest(b: *Build, options: TestOptions) *Step.Compile {
+    return Step.Compile.create(b, .{
         .name = options.name,
         .kind = .@"test",
         .root_source_file = options.root_source_file,
@@ -604,8 +618,8 @@ pub const AssemblyOptions = struct {
     max_rss: usize = 0,
 };
 
-pub fn addAssembly(b: *Build, options: AssemblyOptions) *CompileStep {
-    const obj_step = CompileStep.create(b, .{
+pub fn addAssembly(b: *Build, options: AssemblyOptions) *Step.Compile {
+    const obj_step = Step.Compile.create(b, .{
         .name = options.name,
         .kind = .obj,
         .root_source_file = null,
@@ -657,25 +671,25 @@ fn moduleDependenciesToArrayHashMap(arena: Allocator, deps: []const ModuleDepend
     return result;
 }
 
-/// Initializes a RunStep with argv, which must at least have the path to the
+/// Initializes a `Step.Run` with argv, which must at least have the path to the
 /// executable. More command line arguments can be added with `addArg`,
 /// `addArgs`, and `addArtifactArg`.
 /// Be careful using this function, as it introduces a system dependency.
-/// To run an executable built with zig build, see `CompileStep.run`.
-pub fn addSystemCommand(self: *Build, argv: []const []const u8) *RunStep {
+/// To run an executable built with zig build, see `Step.Compile.run`.
+pub fn addSystemCommand(self: *Build, argv: []const []const u8) *Step.Run {
     assert(argv.len >= 1);
-    const run_step = RunStep.create(self, self.fmt("run {s}", .{argv[0]}));
+    const run_step = Step.Run.create(self, self.fmt("run {s}", .{argv[0]}));
     run_step.addArgs(argv);
     return run_step;
 }
 
-/// Creates a `RunStep` with an executable built with `addExecutable`.
-/// Add command line arguments with methods of `RunStep`.
-pub fn addRunArtifact(b: *Build, exe: *CompileStep) *RunStep {
+/// Creates a `Step.Run` with an executable built with `addExecutable`.
+/// Add command line arguments with methods of `Step.Run`.
+pub fn addRunArtifact(b: *Build, exe: *Step.Compile) *Step.Run {
     // It doesn't have to be native. We catch that if you actually try to run it.
     // Consider that this is declarative; the run step may not be run unless a user
     // option is supplied.
-    const run_step = RunStep.create(b, b.fmt("run {s}", .{exe.name}));
+    const run_step = Step.Run.create(b, b.fmt("run {s}", .{exe.name}));
     run_step.addArtifactArg(exe);
 
     if (exe.kind == .@"test") {
@@ -696,14 +710,14 @@ pub fn addRunArtifact(b: *Build, exe: *CompileStep) *RunStep {
 /// when an option found in the input file is missing from `values`.
 pub fn addConfigHeader(
     b: *Build,
-    options: ConfigHeaderStep.Options,
+    options: Step.ConfigHeader.Options,
     values: anytype,
-) *ConfigHeaderStep {
+) *Step.ConfigHeader {
     var options_copy = options;
     if (options_copy.first_ret_addr == null)
         options_copy.first_ret_addr = @returnAddress();
 
-    const config_header_step = ConfigHeaderStep.create(b, options_copy);
+    const config_header_step = Step.ConfigHeader.create(b, options_copy);
     config_header_step.addValues(values);
     return config_header_step;
 }
@@ -734,28 +748,28 @@ pub fn dupePath(self: *Build, bytes: []const u8) []u8 {
     return the_copy;
 }
 
-pub fn addWriteFile(self: *Build, file_path: []const u8, data: []const u8) *WriteFileStep {
+pub fn addWriteFile(self: *Build, file_path: []const u8, data: []const u8) *Step.WriteFile {
     const write_file_step = self.addWriteFiles();
     write_file_step.add(file_path, data);
     return write_file_step;
 }
 
-pub fn addWriteFiles(b: *Build) *WriteFileStep {
-    return WriteFileStep.create(b);
+pub fn addWriteFiles(b: *Build) *Step.WriteFile {
+    return Step.WriteFile.create(b);
 }
 
-pub fn addRemoveDirTree(self: *Build, dir_path: []const u8) *RemoveDirStep {
-    const remove_dir_step = self.allocator.create(RemoveDirStep) catch @panic("OOM");
-    remove_dir_step.* = RemoveDirStep.init(self, dir_path);
+pub fn addRemoveDirTree(self: *Build, dir_path: []const u8) *Step.RemoveDir {
+    const remove_dir_step = self.allocator.create(Step.RemoveDir) catch @panic("OOM");
+    remove_dir_step.* = Step.RemoveDir.init(self, dir_path);
     return remove_dir_step;
 }
 
-pub fn addFmt(b: *Build, options: FmtStep.Options) *FmtStep {
-    return FmtStep.create(b, options);
+pub fn addFmt(b: *Build, options: Step.Fmt.Options) *Step.Fmt {
+    return Step.Fmt.create(b, options);
 }
 
-pub fn addTranslateC(self: *Build, options: TranslateCStep.Options) *TranslateCStep {
-    return TranslateCStep.create(self, options);
+pub fn addTranslateC(self: *Build, options: Step.TranslateC.Options) *Step.TranslateC {
+    return Step.TranslateC.create(self, options);
 }
 
 pub fn getInstallStep(self: *Build) *Step {
@@ -1213,12 +1227,12 @@ fn printCmd(ally: Allocator, cwd: ?[]const u8, argv: []const []const u8) void {
     std.debug.print("{s}\n", .{text});
 }
 
-pub fn installArtifact(self: *Build, artifact: *CompileStep) void {
+pub fn installArtifact(self: *Build, artifact: *Step.Compile) void {
     self.getInstallStep().dependOn(&self.addInstallArtifact(artifact).step);
 }
 
-pub fn addInstallArtifact(self: *Build, artifact: *CompileStep) *InstallArtifactStep {
-    return InstallArtifactStep.create(self, artifact);
+pub fn addInstallArtifact(self: *Build, artifact: *Step.Compile) *Step.InstallArtifact {
+    return Step.InstallArtifact.create(self, artifact);
 }
 
 ///`dest_rel_path` is relative to prefix path
@@ -1240,26 +1254,26 @@ pub fn installLibFile(self: *Build, src_path: []const u8, dest_rel_path: []const
     self.getInstallStep().dependOn(&self.addInstallFileWithDir(.{ .path = src_path }, .lib, dest_rel_path).step);
 }
 
-pub fn addObjCopy(b: *Build, source: FileSource, options: ObjCopyStep.Options) *ObjCopyStep {
-    return ObjCopyStep.create(b, source, options);
+pub fn addObjCopy(b: *Build, source: FileSource, options: Step.ObjCopy.Options) *Step.ObjCopy {
+    return Step.ObjCopy.create(b, source, options);
 }
 
 ///`dest_rel_path` is relative to install prefix path
-pub fn addInstallFile(self: *Build, source: FileSource, dest_rel_path: []const u8) *InstallFileStep {
+pub fn addInstallFile(self: *Build, source: FileSource, dest_rel_path: []const u8) *Step.InstallFile {
     return self.addInstallFileWithDir(source.dupe(self), .prefix, dest_rel_path);
 }
 
 ///`dest_rel_path` is relative to bin path
-pub fn addInstallBinFile(self: *Build, source: FileSource, dest_rel_path: []const u8) *InstallFileStep {
+pub fn addInstallBinFile(self: *Build, source: FileSource, dest_rel_path: []const u8) *Step.InstallFile {
     return self.addInstallFileWithDir(source.dupe(self), .bin, dest_rel_path);
 }
 
 ///`dest_rel_path` is relative to lib path
-pub fn addInstallLibFile(self: *Build, source: FileSource, dest_rel_path: []const u8) *InstallFileStep {
+pub fn addInstallLibFile(self: *Build, source: FileSource, dest_rel_path: []const u8) *Step.InstallFile {
     return self.addInstallFileWithDir(source.dupe(self), .lib, dest_rel_path);
 }
 
-pub fn addInstallHeaderFile(b: *Build, src_path: []const u8, dest_rel_path: []const u8) *InstallFileStep {
+pub fn addInstallHeaderFile(b: *Build, src_path: []const u8, dest_rel_path: []const u8) *Step.InstallFile {
     return b.addInstallFileWithDir(.{ .path = src_path }, .header, dest_rel_path);
 }
 
@@ -1268,22 +1282,22 @@ pub fn addInstallFileWithDir(
     source: FileSource,
     install_dir: InstallDir,
     dest_rel_path: []const u8,
-) *InstallFileStep {
-    return InstallFileStep.create(self, source.dupe(self), install_dir, dest_rel_path);
+) *Step.InstallFile {
+    return Step.InstallFile.create(self, source.dupe(self), install_dir, dest_rel_path);
 }
 
-pub fn addInstallDirectory(self: *Build, options: InstallDirectoryOptions) *InstallDirStep {
-    const install_step = self.allocator.create(InstallDirStep) catch @panic("OOM");
-    install_step.* = InstallDirStep.init(self, options);
+pub fn addInstallDirectory(self: *Build, options: InstallDirectoryOptions) *Step.InstallDir {
+    const install_step = self.allocator.create(Step.InstallDir) catch @panic("OOM");
+    install_step.* = Step.InstallDir.init(self, options);
     return install_step;
 }
 
 pub fn addCheckFile(
     b: *Build,
     file_source: FileSource,
-    options: CheckFileStep.Options,
-) *CheckFileStep {
-    return CheckFileStep.create(b, file_source, options);
+    options: Step.CheckFile.Options,
+) *Step.CheckFile {
+    return Step.CheckFile.create(b, file_source, options);
 }
 
 pub fn pushInstalledFile(self: *Build, dir: InstallDir, dest_rel_path: []const u8) void {
@@ -1453,10 +1467,10 @@ pub fn getInstallPath(self: *Build, dir: InstallDir, dest_rel_path: []const u8)
 pub const Dependency = struct {
     builder: *Build,
 
-    pub fn artifact(d: *Dependency, name: []const u8) *CompileStep {
-        var found: ?*CompileStep = null;
+    pub fn artifact(d: *Dependency, name: []const u8) *Step.Compile {
+        var found: ?*Step.Compile = null;
         for (d.builder.install_tls.step.dependencies.items) |dep_step| {
-            const inst = dep_step.cast(InstallArtifactStep) orelse continue;
+            const inst = dep_step.cast(Step.InstallArtifact) orelse continue;
             if (mem.eql(u8, inst.artifact.name, name)) {
                 if (found != null) panic("artifact name '{s}' is ambiguous", .{name});
                 found = inst.artifact;
@@ -1464,7 +1478,7 @@ pub const Dependency = struct {
         }
         return found orelse {
             for (d.builder.install_tls.step.dependencies.items) |dep_step| {
-                const inst = dep_step.cast(InstallArtifactStep) orelse continue;
+                const inst = dep_step.cast(Step.InstallArtifact) orelse continue;
                 log.info("available artifact: '{s}'", .{inst.artifact.name});
             }
             panic("unable to find artifact '{s}'", .{name});
@@ -1808,17 +1822,5 @@ pub fn hex64(x: u64) [16]u8 {
 }
 
 test {
-    _ = CheckFileStep;
-    _ = CheckObjectStep;
-    _ = FmtStep;
-    _ = InstallArtifactStep;
-    _ = InstallDirStep;
-    _ = InstallFileStep;
-    _ = ObjCopyStep;
-    _ = CompileStep;
-    _ = OptionsStep;
-    _ = RemoveDirStep;
-    _ = RunStep;
-    _ = TranslateCStep;
-    _ = WriteFileStep;
+    _ = Step;
 }
diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig
index bdb500d99c..40c88df2b9 100644
--- a/lib/std/Build/Step.zig
+++ b/lib/std/Build/Step.zig
@@ -94,26 +94,41 @@ pub const Id = enum {
     pub fn Type(comptime id: Id) type {
         return switch (id) {
             .top_level => Build.TopLevelStep,
-            .compile => Build.CompileStep,
-            .install_artifact => Build.InstallArtifactStep,
-            .install_file => Build.InstallFileStep,
-            .install_dir => Build.InstallDirStep,
-            .remove_dir => Build.RemoveDirStep,
-            .fmt => Build.FmtStep,
-            .translate_c => Build.TranslateCStep,
-            .write_file => Build.WriteFileStep,
-            .run => Build.RunStep,
-            .check_file => Build.CheckFileStep,
-            .check_object => Build.CheckObjectStep,
-            .config_header => Build.ConfigHeaderStep,
-            .objcopy => Build.ObjCopyStep,
-            .options => Build.OptionsStep,
+            .compile => Compile,
+            .install_artifact => InstallArtifact,
+            .install_file => InstallFile,
+            .install_dir => InstallDir,
+            .remove_dir => RemoveDir,
+            .fmt => Fmt,
+            .translate_c => TranslateC,
+            .write_file => WriteFile,
+            .run => Run,
+            .check_file => CheckFile,
+            .check_object => CheckObject,
+            .config_header => ConfigHeader,
+            .objcopy => ObjCopy,
+            .options => Options,
             .custom => @compileError("no type available for custom step"),
         };
     }
 };
 
-pub const Options = struct {
+pub const CheckFile = @import("Step/CheckFile.zig");
+pub const CheckObject = @import("Step/CheckObject.zig");
+pub const ConfigHeader = @import("Step/ConfigHeader.zig");
+pub const Fmt = @import("Step/Fmt.zig");
+pub const InstallArtifact = @import("Step/InstallArtifact.zig");
+pub const InstallDir = @import("Step/InstallDir.zig");
+pub const InstallFile = @import("Step/InstallFile.zig");
+pub const ObjCopy = @import("Step/ObjCopy.zig");
+pub const Compile = @import("Step/Compile.zig");
+pub const Options = @import("Step/Options.zig");
+pub const RemoveDir = @import("Step/RemoveDir.zig");
+pub const Run = @import("Step/Run.zig");
+pub const TranslateC = @import("Step/TranslateC.zig");
+pub const WriteFile = @import("Step/WriteFile.zig");
+
+pub const StepOptions = struct {
     id: Id,
     name: []const u8,
     owner: *Build,
@@ -122,7 +137,7 @@ pub const Options = struct {
     max_rss: usize = 0,
 };
 
-pub fn init(options: Options) Step {
+pub fn init(options: StepOptions) Step {
     const arena = options.owner.allocator;
 
     var addresses = [1]usize{0} ** n_debug_stack_frames;
@@ -387,8 +402,8 @@ pub fn evalZigProcess(
     s.result_duration_ns = timer.read();
     s.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0;
 
-    // Special handling for CompileStep that is expecting compile errors.
-    if (s.cast(Build.CompileStep)) |compile| switch (term) {
+    // Special handling for Compile step that is expecting compile errors.
+    if (s.cast(Compile)) |compile| switch (term) {
         .Exited => {
             // Note that the exit code may be 0 in this case due to the
             // compiler server protocol.
@@ -535,3 +550,19 @@ pub fn writeManifest(s: *Step, man: *std.Build.Cache.Manifest) !void {
         };
     }
 }
+
+test {
+    _ = CheckFile;
+    _ = CheckObject;
+    _ = Fmt;
+    _ = InstallArtifact;
+    _ = InstallDir;
+    _ = InstallFile;
+    _ = ObjCopy;
+    _ = Compile;
+    _ = Options;
+    _ = RemoveDir;
+    _ = Run;
+    _ = TranslateC;
+    _ = WriteFile;
+}
diff --git a/lib/std/Build/Step/CheckFile.zig b/lib/std/Build/Step/CheckFile.zig
index ad8b1a25f0..dc359b5654 100644
--- a/lib/std/Build/Step/CheckFile.zig
+++ b/lib/std/Build/Step/CheckFile.zig
@@ -1,8 +1,8 @@
 //! Fail the build step if a file does not match certain checks.
 //! TODO: make this more flexible, supporting more kinds of checks.
 //! TODO: generalize the code in std.testing.expectEqualStrings and make this
-//! CheckFileStep produce those helpful diagnostics when there is not a match.
-const CheckFileStep = @This();
+//! CheckFile step produce those helpful diagnostics when there is not a match.
+const CheckFile = @This();
 const std = @import("std");
 const Step = std.Build.Step;
 const fs = std.fs;
@@ -25,8 +25,8 @@ pub fn create(
     owner: *std.Build,
     source: std.Build.FileSource,
     options: Options,
-) *CheckFileStep {
-    const self = owner.allocator.create(CheckFileStep) catch @panic("OOM");
+) *CheckFile {
+    const self = owner.allocator.create(CheckFile) catch @panic("OOM");
     self.* = .{
         .step = Step.init(.{
             .id = .check_file,
@@ -42,14 +42,14 @@ pub fn create(
     return self;
 }
 
-pub fn setName(self: *CheckFileStep, name: []const u8) void {
+pub fn setName(self: *CheckFile, name: []const u8) void {
     self.step.name = name;
 }
 
 fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     _ = prog_node;
     const b = step.owner;
-    const self = @fieldParentPtr(CheckFileStep, "step", step);
+    const self = @fieldParentPtr(CheckFile, "step", step);
 
     const src_path = self.source.getPath(b);
     const contents = fs.cwd().readFileAlloc(b.allocator, src_path, self.max_bytes) catch |err| {
diff --git a/lib/std/Build/Step/CheckObject.zig b/lib/std/Build/Step/CheckObject.zig
index 431f74eccc..c77dc3de36 100644
--- a/lib/std/Build/Step/CheckObject.zig
+++ b/lib/std/Build/Step/CheckObject.zig
@@ -6,7 +6,7 @@ const math = std.math;
 const mem = std.mem;
 const testing = std.testing;
 
-const CheckObjectStep = @This();
+const CheckObject = @This();
 
 const Allocator = mem.Allocator;
 const Step = std.Build.Step;
@@ -24,9 +24,9 @@ pub fn create(
     owner: *std.Build,
     source: std.Build.FileSource,
     obj_format: std.Target.ObjectFormat,
-) *CheckObjectStep {
+) *CheckObject {
     const gpa = owner.allocator;
-    const self = gpa.create(CheckObjectStep) catch @panic("OOM");
+    const self = gpa.create(CheckObject) catch @panic("OOM");
     self.* = .{
         .step = Step.init(.{
             .id = .check_file,
@@ -47,11 +47,11 @@ pub fn create(
 /// TODO this doesn't actually compare, and there's no apparent reason for it
 /// to depend on the check object step. I don't see why this function should exist,
 /// the caller could just add the run step directly.
-pub fn runAndCompare(self: *CheckObjectStep) *std.Build.RunStep {
+pub fn runAndCompare(self: *CheckObject) *std.Build.Step.Run {
     const dependencies_len = self.step.dependencies.items.len;
     assert(dependencies_len > 0);
     const exe_step = self.step.dependencies.items[dependencies_len - 1];
-    const exe = exe_step.cast(std.Build.CompileStep).?;
+    const exe = exe_step.cast(std.Build.Step.Compile).?;
     const run = self.step.owner.addRunArtifact(exe);
     run.skip_foreign_checks = true;
     run.step.dependOn(&self.step);
@@ -274,15 +274,15 @@ const Check = struct {
 };
 
 /// Creates a new sequence of actions with `phrase` as the first anchor searched phrase.
-pub fn checkStart(self: *CheckObjectStep, phrase: []const u8) void {
+pub fn checkStart(self: *CheckObject, phrase: []const u8) void {
     var new_check = Check.create(self.step.owner.allocator);
     new_check.match(.{ .string = self.step.owner.dupe(phrase) });
     self.checks.append(new_check) catch @panic("OOM");
 }
 
-/// Adds another searched phrase to the latest created Check with `CheckObjectStep.checkStart(...)`.
+/// Adds another searched phrase to the latest created Check with `CheckObject.checkStart(...)`.
 /// Asserts at least one check already exists.
-pub fn checkNext(self: *CheckObjectStep, phrase: []const u8) void {
+pub fn checkNext(self: *CheckObject, phrase: []const u8) void {
     assert(self.checks.items.len > 0);
     const last = &self.checks.items[self.checks.items.len - 1];
     last.match(.{ .string = self.step.owner.dupe(phrase) });
@@ -291,7 +291,7 @@ pub fn checkNext(self: *CheckObjectStep, phrase: []const u8) void {
 /// Like `checkNext()` but takes an additional argument `FileSource` which will be
 /// resolved to a full search query in `make()`.
 pub fn checkNextFileSource(
-    self: *CheckObjectStep,
+    self: *CheckObject,
     phrase: []const u8,
     file_source: std.Build.FileSource,
 ) void {
@@ -300,10 +300,10 @@ pub fn checkNextFileSource(
     last.match(.{ .string = self.step.owner.dupe(phrase), .file_source = file_source });
 }
 
-/// Adds another searched phrase to the latest created Check with `CheckObjectStep.checkStart(...)`
+/// Adds another searched phrase to the latest created Check with `CheckObject.checkStart(...)`
 /// however ensures there is no matching phrase in the output.
 /// Asserts at least one check already exists.
-pub fn checkNotPresent(self: *CheckObjectStep, phrase: []const u8) void {
+pub fn checkNotPresent(self: *CheckObject, phrase: []const u8) void {
     assert(self.checks.items.len > 0);
     const last = &self.checks.items[self.checks.items.len - 1];
     last.notPresent(.{ .string = self.step.owner.dupe(phrase) });
@@ -312,7 +312,7 @@ pub fn checkNotPresent(self: *CheckObjectStep, phrase: []const u8) void {
 /// Creates a new check checking specifically symbol table parsed and dumped from the object
 /// file.
 /// Issuing this check will force parsing and dumping of the symbol table.
-pub fn checkInSymtab(self: *CheckObjectStep) void {
+pub fn checkInSymtab(self: *CheckObject) void {
     self.dump_symtab = true;
     const symtab_label = switch (self.obj_format) {
         .macho => MachODumper.symtab_label,
@@ -325,7 +325,7 @@ pub fn checkInSymtab(self: *CheckObjectStep) void {
 /// on the extracted variables. It will then compare the reduced program with the value of
 /// the expected variable.
 pub fn checkComputeCompare(
-    self: *CheckObjectStep,
+    self: *CheckObject,
     program: []const u8,
     expected: ComputeCompareExpected,
 ) void {
@@ -338,7 +338,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     _ = prog_node;
     const b = step.owner;
     const gpa = b.allocator;
-    const self = @fieldParentPtr(CheckObjectStep, "step", step);
+    const self = @fieldParentPtr(CheckObject, "step", step);
 
     const src_path = self.source.getPath(b);
     const contents = fs.cwd().readFileAllocOptions(
diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig
index 7627c4e6d0..2371f49daf 100644
--- a/lib/std/Build/Step/Compile.zig
+++ b/lib/std/Build/Step/Compile.zig
@@ -18,14 +18,8 @@ const ExecError = std.Build.ExecError;
 const Module = std.Build.Module;
 const VcpkgRoot = std.Build.VcpkgRoot;
 const InstallDir = std.Build.InstallDir;
-const InstallArtifactStep = std.Build.InstallArtifactStep;
 const GeneratedFile = std.Build.GeneratedFile;
-const ObjCopyStep = std.Build.ObjCopyStep;
-const CheckObjectStep = std.Build.CheckObjectStep;
-const RunStep = std.Build.RunStep;
-const OptionsStep = std.Build.OptionsStep;
-const ConfigHeaderStep = std.Build.ConfigHeaderStep;
-const CompileStep = @This();
+const Compile = @This();
 
 pub const base_id: Step.Id = .compile;
 
@@ -211,8 +205,8 @@ want_lto: ?bool = null,
 use_llvm: ?bool,
 use_lld: ?bool,
 
-/// This is an advanced setting that can change the intent of this CompileStep.
-/// If this slice has nonzero length, it means that this CompileStep exists to
+/// This is an advanced setting that can change the intent of this Compile step.
+/// If this slice has nonzero length, it means that this Compile step exists to
 /// check for compile errors and return *success* if they match, and failure
 /// otherwise.
 expect_errors: []const []const u8 = &.{},
@@ -242,7 +236,7 @@ pub const CSourceFile = struct {
 
 pub const LinkObject = union(enum) {
     static_path: FileSource,
-    other_step: *CompileStep,
+    other_step: *Compile,
     system_lib: SystemLib,
     assembly_file: FileSource,
     c_source_file: *CSourceFile,
@@ -273,8 +267,8 @@ const FrameworkLinkInfo = struct {
 pub const IncludeDir = union(enum) {
     raw_path: []const u8,
     raw_path_system: []const u8,
-    other_step: *CompileStep,
-    config_header_step: *ConfigHeaderStep,
+    other_step: *Compile,
+    config_header_step: *Step.ConfigHeader,
 };
 
 pub const Options = struct {
@@ -319,7 +313,7 @@ pub const EmitOption = union(enum) {
     }
 };
 
-pub fn create(owner: *std.Build, options: Options) *CompileStep {
+pub fn create(owner: *std.Build, options: Options) *Compile {
     const name = owner.dupe(options.name);
     const root_src: ?FileSource = if (options.root_source_file) |rsrc| rsrc.dupe(owner) else null;
     if (mem.indexOf(u8, name, "/") != null or mem.indexOf(u8, name, "\\") != null) {
@@ -361,8 +355,8 @@ pub fn create(owner: *std.Build, options: Options) *CompileStep {
         .version = options.version,
     }) catch @panic("OOM");
 
-    const self = owner.allocator.create(CompileStep) catch @panic("OOM");
-    self.* = CompileStep{
+    const self = owner.allocator.create(Compile) catch @panic("OOM");
+    self.* = Compile{
         .strip = null,
         .unwind_tables = null,
         .verbose_link = false,
@@ -459,7 +453,7 @@ pub fn create(owner: *std.Build, options: Options) *CompileStep {
     return self;
 }
 
-pub fn installHeader(cs: *CompileStep, src_path: []const u8, dest_rel_path: []const u8) void {
+pub fn installHeader(cs: *Compile, src_path: []const u8, dest_rel_path: []const u8) void {
     const b = cs.step.owner;
     const install_file = b.addInstallHeaderFile(src_path, dest_rel_path);
     b.getInstallStep().dependOn(&install_file.step);
@@ -472,8 +466,8 @@ pub const InstallConfigHeaderOptions = struct {
 };
 
 pub fn installConfigHeader(
-    cs: *CompileStep,
-    config_header: *ConfigHeaderStep,
+    cs: *Compile,
+    config_header: *Step.ConfigHeader,
     options: InstallConfigHeaderOptions,
 ) void {
     const dest_rel_path = options.dest_rel_path orelse config_header.include_path;
@@ -489,7 +483,7 @@ pub fn installConfigHeader(
 }
 
 pub fn installHeadersDirectory(
-    a: *CompileStep,
+    a: *Compile,
     src_dir_path: []const u8,
     dest_rel_path: []const u8,
 ) void {
@@ -501,8 +495,8 @@ pub fn installHeadersDirectory(
 }
 
 pub fn installHeadersDirectoryOptions(
-    cs: *CompileStep,
-    options: std.Build.InstallDirStep.Options,
+    cs: *Compile,
+    options: std.Build.Step.InstallDir.Options,
 ) void {
     const b = cs.step.owner;
     const install_dir = b.addInstallDirectory(options);
@@ -510,7 +504,7 @@ pub fn installHeadersDirectoryOptions(
     cs.installed_headers.append(&install_dir.step) catch @panic("OOM");
 }
 
-pub fn installLibraryHeaders(cs: *CompileStep, l: *CompileStep) void {
+pub fn installLibraryHeaders(cs: *Compile, l: *Compile) void {
     assert(l.kind == .lib);
     const b = cs.step.owner;
     const install_step = b.getInstallStep();
@@ -533,7 +527,7 @@ pub fn installLibraryHeaders(cs: *CompileStep, l: *CompileStep) void {
     cs.installed_headers.appendSlice(l.installed_headers.items) catch @panic("OOM");
 }
 
-pub fn addObjCopy(cs: *CompileStep, options: ObjCopyStep.Options) *ObjCopyStep {
+pub fn addObjCopy(cs: *Compile, options: Step.ObjCopy.Options) *Step.ObjCopy {
     const b = cs.step.owner;
     var copy = options;
     if (copy.basename == null) {
@@ -554,34 +548,34 @@ pub const run = @compileError("deprecated; use std.Build.addRunArtifact");
 /// which is undesirable when installing an artifact provided by a dependency package.
 pub const install = @compileError("deprecated; use std.Build.installArtifact");
 
-pub fn checkObject(self: *CompileStep) *CheckObjectStep {
-    return CheckObjectStep.create(self.step.owner, self.getOutputSource(), self.target_info.target.ofmt);
+pub fn checkObject(self: *Compile) *Step.CheckObject {
+    return Step.CheckObject.create(self.step.owner, self.getOutputSource(), self.target_info.target.ofmt);
 }
 
-pub fn setLinkerScriptPath(self: *CompileStep, source: FileSource) void {
+pub fn setLinkerScriptPath(self: *Compile, source: FileSource) void {
     const b = self.step.owner;
     self.linker_script = source.dupe(b);
     source.addStepDependencies(&self.step);
 }
 
-pub fn forceUndefinedSymbol(self: *CompileStep, symbol_name: []const u8) void {
+pub fn forceUndefinedSymbol(self: *Compile, symbol_name: []const u8) void {
     const b = self.step.owner;
     self.force_undefined_symbols.put(b.dupe(symbol_name), {}) catch @panic("OOM");
 }
 
-pub fn linkFramework(self: *CompileStep, framework_name: []const u8) void {
+pub fn linkFramework(self: *Compile, framework_name: []const u8) void {
     const b = self.step.owner;
     self.frameworks.put(b.dupe(framework_name), .{}) catch @panic("OOM");
 }
 
-pub fn linkFrameworkNeeded(self: *CompileStep, framework_name: []const u8) void {
+pub fn linkFrameworkNeeded(self: *Compile, framework_name: []const u8) void {
     const b = self.step.owner;
     self.frameworks.put(b.dupe(framework_name), .{
         .needed = true,
     }) catch @panic("OOM");
 }
 
-pub fn linkFrameworkWeak(self: *CompileStep, framework_name: []const u8) void {
+pub fn linkFrameworkWeak(self: *Compile, framework_name: []const u8) void {
     const b = self.step.owner;
     self.frameworks.put(b.dupe(framework_name), .{
         .weak = true,
@@ -589,7 +583,7 @@ pub fn linkFrameworkWeak(self: *CompileStep, framework_name: []const u8) void {
 }
 
 /// Returns whether the library, executable, or object depends on a particular system library.
-pub fn dependsOnSystemLibrary(self: CompileStep, name: []const u8) bool {
+pub fn dependsOnSystemLibrary(self: Compile, name: []const u8) bool {
     if (isLibCLibrary(name)) {
         return self.is_linking_libc;
     }
@@ -605,51 +599,51 @@ pub fn dependsOnSystemLibrary(self: CompileStep, name: []const u8) bool {
     return false;
 }
 
-pub fn linkLibrary(self: *CompileStep, lib: *CompileStep) void {
+pub fn linkLibrary(self: *Compile, lib: *Compile) void {
     assert(lib.kind == .lib);
     self.linkLibraryOrObject(lib);
 }
 
-pub fn isDynamicLibrary(self: *CompileStep) bool {
+pub fn isDynamicLibrary(self: *Compile) bool {
     return self.kind == .lib and self.linkage == Linkage.dynamic;
 }
 
-pub fn isStaticLibrary(self: *CompileStep) bool {
+pub fn isStaticLibrary(self: *Compile) bool {
     return self.kind == .lib and self.linkage != Linkage.dynamic;
 }
 
-pub fn producesPdbFile(self: *CompileStep) bool {
+pub fn producesPdbFile(self: *Compile) bool {
     if (!self.target.isWindows() and !self.target.isUefi()) return false;
     if (self.target.getObjectFormat() == .c) return false;
     if (self.strip == true) return false;
     return self.isDynamicLibrary() or self.kind == .exe or self.kind == .@"test";
 }
 
-pub fn linkLibC(self: *CompileStep) void {
+pub fn linkLibC(self: *Compile) void {
     self.is_linking_libc = true;
 }
 
-pub fn linkLibCpp(self: *CompileStep) void {
+pub fn linkLibCpp(self: *Compile) void {
     self.is_linking_libcpp = true;
 }
 
 /// If the value is omitted, it is set to 1.
 /// `name` and `value` need not live longer than the function call.
-pub fn defineCMacro(self: *CompileStep, name: []const u8, value: ?[]const u8) void {
+pub fn defineCMacro(self: *Compile, name: []const u8, value: ?[]const u8) void {
     const b = self.step.owner;
     const macro = std.Build.constructCMacro(b.allocator, name, value);
     self.c_macros.append(macro) catch @panic("OOM");
 }
 
 /// name_and_value looks like [name]=[value]. If the value is omitted, it is set to 1.
-pub fn defineCMacroRaw(self: *CompileStep, name_and_value: []const u8) void {
+pub fn defineCMacroRaw(self: *Compile, name_and_value: []const u8) void {
     const b = self.step.owner;
     self.c_macros.append(b.dupe(name_and_value)) catch @panic("OOM");
 }
 
 /// This one has no integration with anything, it just puts -lname on the command line.
 /// Prefer to use `linkSystemLibrary` instead.
-pub fn linkSystemLibraryName(self: *CompileStep, name: []const u8) void {
+pub fn linkSystemLibraryName(self: *Compile, name: []const u8) void {
     const b = self.step.owner;
     self.link_objects.append(.{
         .system_lib = .{
@@ -663,7 +657,7 @@ pub fn linkSystemLibraryName(self: *CompileStep, name: []const u8) void {
 
 /// This one has no integration with anything, it just puts -needed-lname on the command line.
 /// Prefer to use `linkSystemLibraryNeeded` instead.
-pub fn linkSystemLibraryNeededName(self: *CompileStep, name: []const u8) void {
+pub fn linkSystemLibraryNeededName(self: *Compile, name: []const u8) void {
     const b = self.step.owner;
     self.link_objects.append(.{
         .system_lib = .{
@@ -677,7 +671,7 @@ pub fn linkSystemLibraryNeededName(self: *CompileStep, name: []const u8) void {
 
 /// Darwin-only. This one has no integration with anything, it just puts -weak-lname on the
 /// command line. Prefer to use `linkSystemLibraryWeak` instead.
-pub fn linkSystemLibraryWeakName(self: *CompileStep, name: []const u8) void {
+pub fn linkSystemLibraryWeakName(self: *Compile, name: []const u8) void {
     const b = self.step.owner;
     self.link_objects.append(.{
         .system_lib = .{
@@ -691,7 +685,7 @@ pub fn linkSystemLibraryWeakName(self: *CompileStep, name: []const u8) void {
 
 /// This links against a system library, exclusively using pkg-config to find the library.
 /// Prefer to use `linkSystemLibrary` instead.
-pub fn linkSystemLibraryPkgConfigOnly(self: *CompileStep, lib_name: []const u8) void {
+pub fn linkSystemLibraryPkgConfigOnly(self: *Compile, lib_name: []const u8) void {
     const b = self.step.owner;
     self.link_objects.append(.{
         .system_lib = .{
@@ -705,7 +699,7 @@ pub fn linkSystemLibraryPkgConfigOnly(self: *CompileStep, lib_name: []const u8)
 
 /// This links against a system library, exclusively using pkg-config to find the library.
 /// Prefer to use `linkSystemLibraryNeeded` instead.
-pub fn linkSystemLibraryNeededPkgConfigOnly(self: *CompileStep, lib_name: []const u8) void {
+pub fn linkSystemLibraryNeededPkgConfigOnly(self: *Compile, lib_name: []const u8) void {
     const b = self.step.owner;
     self.link_objects.append(.{
         .system_lib = .{
@@ -719,7 +713,7 @@ pub fn linkSystemLibraryNeededPkgConfigOnly(self: *CompileStep, lib_name: []cons
 
 /// Run pkg-config for the given library name and parse the output, returning the arguments
 /// that should be passed to zig to link the given library.
-fn runPkgConfig(self: *CompileStep, lib_name: []const u8) ![]const []const u8 {
+fn runPkgConfig(self: *Compile, lib_name: []const u8) ![]const []const u8 {
     const b = self.step.owner;
     const pkg_name = match: {
         // First we have to map the library name to pkg config name. Unfortunately,
@@ -813,19 +807,19 @@ fn runPkgConfig(self: *CompileStep, lib_name: []const u8) ![]const []const u8 {
     return zig_args.toOwnedSlice();
 }
 
-pub fn linkSystemLibrary(self: *CompileStep, name: []const u8) void {
+pub fn linkSystemLibrary(self: *Compile, name: []const u8) void {
     self.linkSystemLibraryInner(name, .{});
 }
 
-pub fn linkSystemLibraryNeeded(self: *CompileStep, name: []const u8) void {
+pub fn linkSystemLibraryNeeded(self: *Compile, name: []const u8) void {
     self.linkSystemLibraryInner(name, .{ .needed = true });
 }
 
-pub fn linkSystemLibraryWeak(self: *CompileStep, name: []const u8) void {
+pub fn linkSystemLibraryWeak(self: *Compile, name: []const u8) void {
     self.linkSystemLibraryInner(name, .{ .weak = true });
 }
 
-fn linkSystemLibraryInner(self: *CompileStep, name: []const u8, opts: struct {
+fn linkSystemLibraryInner(self: *Compile, name: []const u8, opts: struct {
     needed: bool = false,
     weak: bool = false,
 }) void {
@@ -850,7 +844,7 @@ fn linkSystemLibraryInner(self: *CompileStep, name: []const u8, opts: struct {
 }
 
 /// Handy when you have many C/C++ source files and want them all to have the same flags.
-pub fn addCSourceFiles(self: *CompileStep, files: []const []const u8, flags: []const []const u8) void {
+pub fn addCSourceFiles(self: *Compile, files: []const []const u8, flags: []const []const u8) void {
     const b = self.step.owner;
     const c_source_files = b.allocator.create(CSourceFiles) catch @panic("OOM");
 
@@ -864,14 +858,14 @@ pub fn addCSourceFiles(self: *CompileStep, files: []const []const u8, flags: []c
     self.link_objects.append(.{ .c_source_files = c_source_files }) catch @panic("OOM");
 }
 
-pub fn addCSourceFile(self: *CompileStep, file: []const u8, flags: []const []const u8) void {
+pub fn addCSourceFile(self: *Compile, file: []const u8, flags: []const []const u8) void {
     self.addCSourceFileSource(.{
         .args = flags,
         .source = .{ .path = file },
     });
 }
 
-pub fn addCSourceFileSource(self: *CompileStep, source: CSourceFile) void {
+pub fn addCSourceFileSource(self: *Compile, source: CSourceFile) void {
     const b = self.step.owner;
     const c_source_file = b.allocator.create(CSourceFile) catch @panic("OOM");
     c_source_file.* = source.dupe(b);
@@ -879,85 +873,85 @@ pub fn addCSourceFileSource(self: *CompileStep, source: CSourceFile) void {
     source.source.addStepDependencies(&self.step);
 }
 
-pub fn setVerboseLink(self: *CompileStep, value: bool) void {
+pub fn setVerboseLink(self: *Compile, value: bool) void {
     self.verbose_link = value;
 }
 
-pub fn setVerboseCC(self: *CompileStep, value: bool) void {
+pub fn setVerboseCC(self: *Compile, value: bool) void {
     self.verbose_cc = value;
 }
 
-pub fn overrideZigLibDir(self: *CompileStep, dir_path: []const u8) void {
+pub fn overrideZigLibDir(self: *Compile, dir_path: []const u8) void {
     const b = self.step.owner;
     self.zig_lib_dir = b.dupePath(dir_path);
 }
 
-pub fn setMainPkgPath(self: *CompileStep, dir_path: []const u8) void {
+pub fn setMainPkgPath(self: *Compile, dir_path: []const u8) void {
     const b = self.step.owner;
     self.main_pkg_path = b.dupePath(dir_path);
 }
 
-pub fn setLibCFile(self: *CompileStep, libc_file: ?FileSource) void {
+pub fn setLibCFile(self: *Compile, libc_file: ?FileSource) void {
     const b = self.step.owner;
     self.libc_file = if (libc_file) |f| f.dupe(b) else null;
 }
 
 /// Returns the generated executable, library or object file.
 /// To run an executable built with zig build, use `run`, or create an install step and invoke it.
-pub fn getOutputSource(self: *CompileStep) FileSource {
+pub fn getOutputSource(self: *Compile) FileSource {
     return .{ .generated = &self.output_path_source };
 }
 
-pub fn getOutputDirectorySource(self: *CompileStep) FileSource {
+pub fn getOutputDirectorySource(self: *Compile) FileSource {
     return .{ .generated = &self.output_dirname_source };
 }
 
 /// Returns the generated import library. This function can only be called for libraries.
-pub fn getOutputLibSource(self: *CompileStep) FileSource {
+pub fn getOutputLibSource(self: *Compile) FileSource {
     assert(self.kind == .lib);
     return .{ .generated = &self.output_lib_path_source };
 }
 
 /// Returns the generated header file.
 /// This function can only be called for libraries or object files which have `emit_h` set.
-pub fn getOutputHSource(self: *CompileStep) FileSource {
+pub fn getOutputHSource(self: *Compile) FileSource {
     assert(self.kind != .exe and self.kind != .@"test");
     assert(self.emit_h);
     return .{ .generated = &self.output_h_path_source };
 }
 
 /// Returns the generated PDB file. This function can only be called for Windows and UEFI.
-pub fn getOutputPdbSource(self: *CompileStep) FileSource {
+pub fn getOutputPdbSource(self: *Compile) FileSource {
     // TODO: Is this right? Isn't PDB for *any* PE/COFF file?
     assert(self.target.isWindows() or self.target.isUefi());
     return .{ .generated = &self.output_pdb_path_source };
 }
 
-pub fn addAssemblyFile(self: *CompileStep, path: []const u8) void {
+pub fn addAssemblyFile(self: *Compile, path: []const u8) void {
     const b = self.step.owner;
     self.link_objects.append(.{
         .assembly_file = .{ .path = b.dupe(path) },
     }) catch @panic("OOM");
 }
 
-pub fn addAssemblyFileSource(self: *CompileStep, source: FileSource) void {
+pub fn addAssemblyFileSource(self: *Compile, source: FileSource) void {
     const b = self.step.owner;
     const source_duped = source.dupe(b);
     self.link_objects.append(.{ .assembly_file = source_duped }) catch @panic("OOM");
     source_duped.addStepDependencies(&self.step);
 }
 
-pub fn addObjectFile(self: *CompileStep, source_file: []const u8) void {
+pub fn addObjectFile(self: *Compile, source_file: []const u8) void {
     self.addObjectFileSource(.{ .path = source_file });
 }
 
-pub fn addObjectFileSource(self: *CompileStep, source: FileSource) void {
+pub fn addObjectFileSource(self: *Compile, source: FileSource) void {
     const b = self.step.owner;
     self.link_objects.append(.{ .static_path = source.dupe(b) }) catch @panic("OOM");
     source.addStepDependencies(&self.step);
 }
 
-pub fn addObject(self: *CompileStep, obj: *CompileStep) void {
+pub fn addObject(self: *Compile, obj: *Compile) void {
     assert(obj.kind == .obj);
     self.linkLibraryOrObject(obj);
 }
@@ -967,54 +961,54 @@ pub const addIncludeDir = @compileError("deprecated; use addIncludePath");
 pub const addLibPath = @compileError("deprecated, use addLibraryPath");
 pub const addFrameworkDir = @compileError("deprecated, use addFrameworkPath");
 
-pub fn addSystemIncludePath(self: *CompileStep, path: []const u8) void {
+pub fn addSystemIncludePath(self: *Compile, path: []const u8) void {
     const b = self.step.owner;
     self.include_dirs.append(IncludeDir{ .raw_path_system = b.dupe(path) }) catch @panic("OOM");
 }
 
-pub fn addIncludePath(self: *CompileStep, path: []const u8) void {
+pub fn addIncludePath(self: *Compile, path: []const u8) void {
     const b = self.step.owner;
     self.include_dirs.append(IncludeDir{ .raw_path = b.dupe(path) }) catch @panic("OOM");
 }
 
-pub fn addConfigHeader(self: *CompileStep, config_header: *ConfigHeaderStep) void {
+pub fn addConfigHeader(self: *Compile, config_header: *Step.ConfigHeader) void {
     self.step.dependOn(&config_header.step);
     self.include_dirs.append(.{ .config_header_step = config_header }) catch @panic("OOM");
 }
 
-pub fn addLibraryPath(self: *CompileStep, path: []const u8) void {
+pub fn addLibraryPath(self: *Compile, path: []const u8) void {
     const b = self.step.owner;
     self.lib_paths.append(.{ .path = b.dupe(path) }) catch @panic("OOM");
 }
 
-pub fn addLibraryPathDirectorySource(self: *CompileStep, directory_source: FileSource) void {
+pub fn addLibraryPathDirectorySource(self: *Compile, directory_source: FileSource) void {
     self.lib_paths.append(directory_source) catch @panic("OOM");
     directory_source.addStepDependencies(&self.step);
 }
 
-pub fn addRPath(self: *CompileStep, path: []const u8) void {
+pub fn addRPath(self: *Compile, path: []const u8) void {
     const b = self.step.owner;
     self.rpaths.append(.{ .path = b.dupe(path) }) catch @panic("OOM");
 }
 
-pub fn addRPathDirectorySource(self: *CompileStep, directory_source: FileSource) void {
+pub fn addRPathDirectorySource(self: *Compile, directory_source: FileSource) void {
     self.rpaths.append(directory_source) catch @panic("OOM");
     directory_source.addStepDependencies(&self.step);
 }
 
-pub fn addFrameworkPath(self: *CompileStep, dir_path: []const u8) void {
+pub fn addFrameworkPath(self: *Compile, dir_path: []const u8) void {
     const b = self.step.owner;
     self.framework_dirs.append(.{ .path = b.dupe(dir_path) }) catch @panic("OOM");
 }
 
-pub fn addFrameworkPathDirectorySource(self: *CompileStep, directory_source: FileSource) void {
+pub fn addFrameworkPathDirectorySource(self: *Compile, directory_source: FileSource) void {
     self.framework_dirs.append(directory_source) catch @panic("OOM");
     directory_source.addStepDependencies(&self.step);
 }
 
 /// Adds a module to be used with `@import` and exposing it in the current
 /// package's module table using `name`.
-pub fn addModule(cs: *CompileStep, name: []const u8, module: *Module) void {
+pub fn addModule(cs: *Compile, name: []const u8, module: *Module) void {
     const b = cs.step.owner;
     cs.modules.put(b.dupe(name), module) catch @panic("OOM");
 
@@ -1025,17 +1019,17 @@ pub fn addModule(cs: *CompileStep, name: []const u8, module: *Module) void {
 
 /// Adds a module to be used with `@import` without exposing it in the current
 /// package's module table.
-pub fn addAnonymousModule(cs: *CompileStep, name: []const u8, options: std.Build.CreateModuleOptions) void {
+pub fn addAnonymousModule(cs: *Compile, name: []const u8, options: std.Build.CreateModuleOptions) void {
     const b = cs.step.owner;
     const module = b.createModule(options);
     return addModule(cs, name, module);
 }
 
-pub fn addOptions(cs: *CompileStep, module_name: []const u8, options: *OptionsStep) void {
+pub fn addOptions(cs: *Compile, module_name: []const u8, options: *Step.Options) void {
     addModule(cs, module_name, options.createModule());
 }
 
-fn addRecursiveBuildDeps(cs: *CompileStep, module: *Module, done: *std.AutoHashMap(*Module, void)) !void {
+fn addRecursiveBuildDeps(cs: *Compile, module: *Module, done: *std.AutoHashMap(*Module, void)) !void {
     if (done.contains(module)) return;
     try done.put(module, {});
     module.source_file.addStepDependencies(&cs.step);
@@ -1046,7 +1040,7 @@ fn addRecursiveBuildDeps(cs: *CompileStep, module: *Module, done: *std.AutoHashM
 
 /// If Vcpkg was found on the system, it will be added to include and lib
 /// paths for the specified target.
-pub fn addVcpkgPaths(self: *CompileStep, linkage: CompileStep.Linkage) !void {
+pub fn addVcpkgPaths(self: *Compile, linkage: Compile.Linkage) !void {
     const b = self.step.owner;
     // Ideally in the Unattempted case we would call the function recursively
     // after findVcpkgRoot and have only one switch statement, but the compiler
@@ -1082,7 +1076,7 @@ pub fn addVcpkgPaths(self: *CompileStep, linkage: CompileStep.Linkage) !void {
     }
 }
 
-pub fn setExecCmd(self: *CompileStep, args: []const ?[]const u8) void {
+pub fn setExecCmd(self: *Compile, args: []const ?[]const u8) void {
     const b = self.step.owner;
     assert(self.kind == .@"test");
     const duped_args = b.allocator.alloc(?[]u8, args.len) catch @panic("OOM");
@@ -1092,7 +1086,7 @@ pub fn setExecCmd(self: *CompileStep, args: []const ?[]const u8) void {
     self.exec_cmd_args = duped_args;
 }
 
-fn linkLibraryOrObject(self: *CompileStep, other: *CompileStep) void {
+fn linkLibraryOrObject(self: *Compile, other: *Compile) void {
     self.step.dependOn(&other.step);
     self.link_objects.append(.{ .other_step = other }) catch @panic("OOM");
     self.include_dirs.append(.{ .other_step = other }) catch @panic("OOM");
@@ -1103,7 +1097,7 @@ fn linkLibraryOrObject(self: *CompileStep, other: *CompileStep) void {
 }
 
 fn appendModuleArgs(
-    cs: *CompileStep,
+    cs: *Compile,
     zig_args: *ArrayList([]const u8),
 ) error{OutOfMemory}!void {
     const b = cs.step.owner;
@@ -1214,7 +1208,7 @@ fn constructDepString(
 
 fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     const b = step.owner;
-    const self = @fieldParentPtr(CompileStep, "step", step);
+    const self = @fieldParentPtr(Compile, "step", step);
 
     if (self.root_src == null and self.link_objects.items.len == 0) {
         return step.fail("the linker needs one or more objects to link", .{});
@@ -2088,7 +2082,7 @@ const TransitiveDeps = struct {
         }
     }
 
-    fn addInner(td: *TransitiveDeps, other: *CompileStep, dyn: bool) !void {
+    fn addInner(td: *TransitiveDeps, other: *Compile, dyn: bool) !void {
         // Inherit dependency on libc and libc++
         td.is_linking_libcpp = td.is_linking_libcpp or other.is_linking_libcpp;
         td.is_linking_libc = td.is_linking_libc or other.is_linking_libc;
@@ -2128,7 +2122,7 @@ const TransitiveDeps = struct {
     }
 };
 
-fn checkCompileErrors(self: *CompileStep) !void {
+fn checkCompileErrors(self: *Compile) !void {
     // Clear this field so that it does not get printed by the build runner.
     const actual_eb = self.step.result_error_bundle;
     self.step.result_error_bundle = std.zig.ErrorBundle.empty;
diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig
index 6bfe28ae62..a17784c96a 100644
--- a/lib/std/Build/Step/ConfigHeader.zig
+++ b/lib/std/Build/Step/ConfigHeader.zig
@@ -1,5 +1,5 @@
 const std = @import("std");
-const ConfigHeaderStep = @This();
+const ConfigHeader = @This();
 const Step = std.Build.Step;
 
 pub const Style = union(enum) {
@@ -48,8 +48,8 @@ pub const Options = struct {
     first_ret_addr: ?usize = null,
 };
 
-pub fn create(owner: *std.Build, options: Options) *ConfigHeaderStep {
-    const self = owner.allocator.create(ConfigHeaderStep) catch @panic("OOM");
+pub fn create(owner: *std.Build, options: Options) *ConfigHeader {
+    const self = owner.allocator.create(ConfigHeader) catch @panic("OOM");
 
     var include_path: []const u8 = "config.h";
 
@@ -93,21 +93,21 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeaderStep {
     return self;
 }
 
-pub fn addValues(self: *ConfigHeaderStep, values: anytype) void {
+pub fn addValues(self: *ConfigHeader, values: anytype) void {
     return addValuesInner(self, values) catch @panic("OOM");
 }
 
-pub fn getFileSource(self: *ConfigHeaderStep) std.Build.FileSource {
+pub fn getFileSource(self: *ConfigHeader) std.Build.FileSource {
     return .{ .generated = &self.output_file };
 }
 
-fn addValuesInner(self: *ConfigHeaderStep, values: anytype) !void {
+fn addValuesInner(self: *ConfigHeader, values: anytype) !void {
     inline for (@typeInfo(@TypeOf(values)).Struct.fields) |field| {
         try putValue(self, field.name, field.type, @field(values, field.name));
     }
 }
 
-fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v: T) !void {
+fn putValue(self: *ConfigHeader, field_name: []const u8, comptime T: type, v: T) !void {
     switch (@typeInfo(T)) {
         .Null => {
             try self.values.put(field_name, .undef);
@@ -151,31 +151,31 @@ fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v
                 else => {},
             }
 
-            @compileError("unsupported ConfigHeaderStep value type: " ++ @typeName(T));
+            @compileError("unsupported ConfigHeader value type: " ++ @typeName(T));
         },
-        else => @compileError("unsupported ConfigHeaderStep value type: " ++ @typeName(T)),
+        else => @compileError("unsupported ConfigHeader value type: " ++ @typeName(T)),
     }
 }
 
 fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     _ = prog_node;
     const b = step.owner;
-    const self = @fieldParentPtr(ConfigHeaderStep, "step", step);
+    const self = @fieldParentPtr(ConfigHeader, "step", step);
     const gpa = b.allocator;
     const arena = b.allocator;
 
     var man = b.cache.obtain();
     defer man.deinit();
 
-    // Random bytes to make ConfigHeaderStep unique. Refresh this with new
-    // random bytes when ConfigHeaderStep implementation is modified in a
+    // Random bytes to make ConfigHeader unique. Refresh this with new
+    // random bytes when ConfigHeader implementation is modified in a
     // non-backwards-compatible way.
     man.hash.add(@as(u32, 0xdef08d23));
 
     var output = std.ArrayList(u8).init(gpa);
     defer output.deinit();
 
-    const header_text = "This file was generated by ConfigHeaderStep using the Zig Build System.";
+    const header_text = "This file was generated by ConfigHeader using the Zig Build System.";
     const c_generated_line = "/* " ++ header_text ++ " */\n";
     const asm_generated_line = "; " ++ header_text ++ "\n";
 
diff --git a/lib/std/Build/Step/Fmt.zig b/lib/std/Build/Step/Fmt.zig
index 23d5d9e3ff..8e8cc51c0d 100644
--- a/lib/std/Build/Step/Fmt.zig
+++ b/lib/std/Build/Step/Fmt.zig
@@ -3,7 +3,7 @@
 //! * Check mode: fail the step if a non-conforming file is found.
 const std = @import("std");
 const Step = std.Build.Step;
-const FmtStep = @This();
+const Fmt = @This();
 
 step: Step,
 paths: []const []const u8,
@@ -19,8 +19,8 @@ pub const Options = struct {
     check: bool = false,
 };
 
-pub fn create(owner: *std.Build, options: Options) *FmtStep {
-    const self = owner.allocator.create(FmtStep) catch @panic("OOM");
+pub fn create(owner: *std.Build, options: Options) *Fmt {
+    const self = owner.allocator.create(Fmt) catch @panic("OOM");
     const name = if (options.check) "zig fmt --check" else "zig fmt";
     self.* = .{
         .step = Step.init(.{
@@ -47,7 +47,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
 
     const b = step.owner;
     const arena = b.allocator;
-    const self = @fieldParentPtr(FmtStep, "step", step);
+    const self = @fieldParentPtr(Fmt, "step", step);
 
     var argv: std.ArrayListUnmanaged([]const u8) = .{};
     try argv.ensureUnusedCapacity(arena, 2 + 1 + self.paths.len + 2 * self.exclude_paths.len);
diff --git a/lib/std/Build/Step/InstallArtifact.zig b/lib/std/Build/Step/InstallArtifact.zig
index fa357a9ae9..9552a44440 100644
--- a/lib/std/Build/Step/InstallArtifact.zig
+++ b/lib/std/Build/Step/InstallArtifact.zig
@@ -1,24 +1,23 @@
 const std = @import("std");
 const Step = std.Build.Step;
-const CompileStep = std.Build.CompileStep;
 const InstallDir = std.Build.InstallDir;
-const InstallArtifactStep = @This();
+const InstallArtifact = @This();
 const fs = std.fs;
 
 pub const base_id = .install_artifact;
 
 step: Step,
-artifact: *CompileStep,
+artifact: *Step.Compile,
 dest_dir: InstallDir,
 pdb_dir: ?InstallDir,
 h_dir: ?InstallDir,
 /// If non-null, adds additional path components relative to dest_dir, and
-/// overrides the basename of the CompileStep.
+/// overrides the basename of the Compile step.
 dest_sub_path: ?[]const u8,
 
-pub fn create(owner: *std.Build, artifact: *CompileStep) *InstallArtifactStep {
-    const self = owner.allocator.create(InstallArtifactStep) catch @panic("OOM");
-    self.* = InstallArtifactStep{
+pub fn create(owner: *std.Build, artifact: *Step.Compile) *InstallArtifact {
+    const self = owner.allocator.create(InstallArtifact) catch @panic("OOM");
+    self.* = InstallArtifact{
         .step = Step.init(.{
             .id = base_id,
             .name = owner.fmt("install {s}", .{artifact.name}),
@@ -66,7 +65,7 @@ pub fn create(owner: *std.Build, artifact: *CompileStep) *InstallArtifactStep {
 
 fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     _ = prog_node;
-    const self = @fieldParentPtr(InstallArtifactStep, "step", step);
+    const self = @fieldParentPtr(InstallArtifact, "step", step);
     const src_builder = self.artifact.step.owner;
     const dest_builder = step.owner;
 
@@ -90,7 +89,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
         self.artifact.version != null and
         self.artifact.target.wantSharedLibSymLinks())
     {
-        try CompileStep.doAtomicSymLinks(step, full_dest_path, self.artifact.major_only_filename.?, self.artifact.name_only_filename.?);
+        try Step.Compile.doAtomicSymLinks(step, full_dest_path, self.artifact.major_only_filename.?, self.artifact.name_only_filename.?);
     }
     if (self.artifact.isDynamicLibrary() and
         self.artifact.target.isWindows() and
diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig
index b6b66fd1e0..784685dc3a 100644
--- a/lib/std/Build/Step/InstallFile.zig
+++ b/lib/std/Build/Step/InstallFile.zig
@@ -2,7 +2,7 @@ const std = @import("std");
 const Step = std.Build.Step;
 const FileSource = std.Build.FileSource;
 const InstallDir = std.Build.InstallDir;
-const InstallFileStep = @This();
+const InstallFile = @This();
 const assert = std.debug.assert;
 
 pub const base_id = .install_file;
@@ -20,10 +20,10 @@ pub fn create(
     source: FileSource,
     dir: InstallDir,
     dest_rel_path: []const u8,
-) *InstallFileStep {
+) *InstallFile {
     assert(dest_rel_path.len != 0);
     owner.pushInstalledFile(dir, dest_rel_path);
-    const self = owner.allocator.create(InstallFileStep) catch @panic("OOM");
+    const self = owner.allocator.create(InstallFile) catch @panic("OOM");
     self.* = .{
         .step = Step.init(.{
             .id = base_id,
@@ -43,7 +43,7 @@ pub fn create(
 fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     _ = prog_node;
     const src_builder = step.owner;
-    const self = @fieldParentPtr(InstallFileStep, "step", step);
+    const self = @fieldParentPtr(InstallFile, "step", step);
     const dest_builder = self.dest_builder;
     const full_src_path = self.source.getPath2(src_builder, step);
     const full_dest_path = dest_builder.getInstallPath(self.dir, self.dest_rel_path);
diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig
index 608c56591f..a8a0dafaef 100644
--- a/lib/std/Build/Step/ObjCopy.zig
+++ b/lib/std/Build/Step/ObjCopy.zig
@@ -1,12 +1,11 @@
 const std = @import("std");
-const ObjCopyStep = @This();
+const ObjCopy = @This();
 
 const Allocator = std.mem.Allocator;
 const ArenaAllocator = std.heap.ArenaAllocator;
 const ArrayListUnmanaged = std.ArrayListUnmanaged;
 const File = std.fs.File;
 const InstallDir = std.Build.InstallDir;
-const CompileStep = std.Build.CompileStep;
 const Step = std.Build.Step;
 const elf = std.elf;
 const fs = std.fs;
@@ -40,9 +39,9 @@ pub fn create(
     owner: *std.Build,
     file_source: std.Build.FileSource,
     options: Options,
-) *ObjCopyStep {
-    const self = owner.allocator.create(ObjCopyStep) catch @panic("OOM");
-    self.* = ObjCopyStep{
+) *ObjCopy {
+    const self = owner.allocator.create(ObjCopy) catch @panic("OOM");
+    self.* = ObjCopy{
         .step = Step.init(.{
             .id = base_id,
             .name = owner.fmt("objcopy {s}", .{file_source.getDisplayName()}),
@@ -61,19 +60,19 @@ pub fn create(
     return self;
 }
 
-pub fn getOutputSource(self: *const ObjCopyStep) std.Build.FileSource {
+pub fn getOutputSource(self: *const ObjCopy) std.Build.FileSource {
     return .{ .generated = &self.output_file };
 }
 
 fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     const b = step.owner;
-    const self = @fieldParentPtr(ObjCopyStep, "step", step);
+    const self = @fieldParentPtr(ObjCopy, "step", step);
 
     var man = b.cache.obtain();
     defer man.deinit();
 
-    // Random bytes to make ObjCopyStep unique. Refresh this with new random
-    // bytes when ObjCopyStep implementation is modified incompatibly.
+    // Random bytes to make ObjCopy unique. Refresh this with new random
+    // bytes when ObjCopy implementation is modified incompatibly.
     man.hash.add(@as(u32, 0xe18b7baf));
 
     const full_src_path = self.file_source.getPath(b);
diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig
index 101c284cf0..cc7152a81e 100644
--- a/lib/std/Build/Step/Options.zig
+++ b/lib/std/Build/Step/Options.zig
@@ -3,10 +3,9 @@ const builtin = @import("builtin");
 const fs = std.fs;
 const Step = std.Build.Step;
 const GeneratedFile = std.Build.GeneratedFile;
-const CompileStep = std.Build.CompileStep;
 const FileSource = std.Build.FileSource;
 
-const OptionsStep = @This();
+const Options = @This();
 
 pub const base_id = .options;
 
@@ -17,8 +16,8 @@ contents: std.ArrayList(u8),
 artifact_args: std.ArrayList(OptionArtifactArg),
 file_source_args: std.ArrayList(OptionFileSourceArg),
 
-pub fn create(owner: *std.Build) *OptionsStep {
-    const self = owner.allocator.create(OptionsStep) catch @panic("OOM");
+pub fn create(owner: *std.Build) *Options {
+    const self = owner.allocator.create(Options) catch @panic("OOM");
     self.* = .{
         .step = Step.init(.{
             .id = base_id,
@@ -36,11 +35,11 @@ pub fn create(owner: *std.Build) *OptionsStep {
     return self;
 }
 
-pub fn addOption(self: *OptionsStep, comptime T: type, name: []const u8, value: T) void {
+pub fn addOption(self: *Options, comptime T: type, name: []const u8, value: T) void {
     return addOptionFallible(self, T, name, value) catch @panic("unhandled error");
 }
 
-fn addOptionFallible(self: *OptionsStep, comptime T: type, name: []const u8, value: T) !void {
+fn addOptionFallible(self: *Options, comptime T: type, name: []const u8, value: T) !void {
     const out = self.contents.writer();
     switch (T) {
         []const []const u8 => {
@@ -189,7 +188,7 @@ fn printLiteral(out: anytype, val: anytype, indent: u8) !void {
 /// The value is the path in the cache dir.
 /// Adds a dependency automatically.
 pub fn addOptionFileSource(
-    self: *OptionsStep,
+    self: *Options,
     name: []const u8,
     source: FileSource,
 ) void {
@@ -202,19 +201,19 @@ pub fn addOptionFileSource(
 
 /// The value is the path in the cache dir.
 /// Adds a dependency automatically.
-pub fn addOptionArtifact(self: *OptionsStep, name: []const u8, artifact: *CompileStep) void {
+pub fn addOptionArtifact(self: *Options, name: []const u8, artifact: *Step.Compile) void {
     self.artifact_args.append(.{ .name = self.step.owner.dupe(name), .artifact = artifact }) catch @panic("OOM");
     self.step.dependOn(&artifact.step);
 }
 
-pub fn createModule(self: *OptionsStep) *std.Build.Module {
+pub fn createModule(self: *Options) *std.Build.Module {
     return self.step.owner.createModule(.{
         .source_file = self.getSource(),
         .dependencies = &.{},
     });
 }
 
-pub fn getSource(self: *OptionsStep) FileSource {
+pub fn getSource(self: *Options) FileSource {
     return .{ .generated = &self.generated_file };
 }
 
@@ -223,7 +222,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     _ = prog_node;
 
     const b = step.owner;
-    const self = @fieldParentPtr(OptionsStep, "step", step);
+    const self = @fieldParentPtr(Options, "step", step);
 
     for (self.artifact_args.items) |item| {
         self.addOption(
@@ -314,7 +313,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
 
 const OptionArtifactArg = struct {
     name: []const u8,
-    artifact: *CompileStep,
+    artifact: *Step.Compile,
 };
 
 const OptionFileSourceArg = struct {
@@ -322,7 +321,7 @@ const OptionFileSourceArg = struct {
     source: FileSource,
 };
 
-test "OptionsStep" {
+test Options {
     if (builtin.os.tag == .wasi) return error.SkipZigTest;
 
     var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
diff --git a/lib/std/Build/Step/RemoveDir.zig b/lib/std/Build/Step/RemoveDir.zig
index 59025a7e91..7666dd2a7d 100644
--- a/lib/std/Build/Step/RemoveDir.zig
+++ b/lib/std/Build/Step/RemoveDir.zig
@@ -1,15 +1,15 @@
 const std = @import("std");
 const fs = std.fs;
 const Step = std.Build.Step;
-const RemoveDirStep = @This();
+const RemoveDir = @This();
 
 pub const base_id = .remove_dir;
 
 step: Step,
 dir_path: []const u8,
 
-pub fn init(owner: *std.Build, dir_path: []const u8) RemoveDirStep {
-    return RemoveDirStep{
+pub fn init(owner: *std.Build, dir_path: []const u8) RemoveDir {
+    return RemoveDir{
         .step = Step.init(.{
             .id = .remove_dir,
             .name = owner.fmt("RemoveDir {s}", .{dir_path}),
@@ -26,7 +26,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     _ = prog_node;
 
     const b = step.owner;
-    const self = @fieldParentPtr(RemoveDirStep, "step", step);
+    const self = @fieldParentPtr(RemoveDir, "step", step);
 
     b.build_root.handle.deleteTree(self.dir_path) catch |err| {
         if (b.build_root.path) |base| {
diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig
index 4e973cfd98..c506e23f90 100644
--- a/lib/std/Build/Step/Run.zig
+++ b/lib/std/Build/Step/Run.zig
@@ -1,8 +1,6 @@
 const std = @import("std");
 const builtin = @import("builtin");
 const Step = std.Build.Step;
-const CompileStep = std.Build.CompileStep;
-const WriteFileStep = std.Build.WriteFileStep;
 const fs = std.fs;
 const mem = std.mem;
 const process = std.process;
@@ -12,7 +10,7 @@ const Allocator = mem.Allocator;
 const ExecError = std.Build.ExecError;
 const assert = std.debug.assert;
 
-const RunStep = @This();
+const Run = @This();
 
 pub const base_id: Step.Id = .run;
 
@@ -29,12 +27,12 @@ cwd: ?[]const u8,
 /// Override this field to modify the environment, or use setEnvironmentVariable
 env_map: ?*EnvMap,
 
-/// Configures whether the RunStep is considered to have side-effects, and also
-/// whether the RunStep will inherit stdio streams, forwarding them to the
+/// Configures whether the Run step is considered to have side-effects, and also
+/// whether the Run step will inherit stdio streams, forwarding them to the
 /// parent process, in which case will require a global lock to prevent other
 /// steps from interfering with stdio while the subprocess associated with this
-/// RunStep is running.
-/// If the RunStep is determined to not have side-effects, then execution will
+/// Run step is running.
+/// If the Run step is determined to not have side-effects, then execution will
 /// be skipped if all output files are up-to-date and input files are
 /// unchanged.
 stdio: StdIo = .infer_from_args,
@@ -42,9 +40,9 @@ stdio: StdIo = .infer_from_args,
 stdin: ?[]const u8 = null,
 
 /// Additional file paths relative to build.zig that, when modified, indicate
-/// that the RunStep should be re-executed.
-/// If the RunStep is determined to have side-effects, this field is ignored
-/// and the RunStep is always executed when it appears in the build graph.
+/// that the Run step should be re-executed.
+/// If the Run step is determined to have side-effects, this field is ignored
+/// and the Run step is always executed when it appears in the build graph.
 extra_file_dependencies: []const []const u8 = &.{},
 
 /// After adding an output argument, this step will by default rename itself
@@ -52,14 +50,14 @@ extra_file_dependencies: []const []const u8 = &.{},
 /// This can be disabled by setting this to false.
 rename_step_with_output_arg: bool = true,
 
-/// If this is true, a RunStep which is configured to check the output of the
+/// If this is true, a Run step which is configured to check the output of the
 /// executed binary will not fail the build if the binary cannot be executed
 /// due to being for a foreign binary to the host system which is running the
 /// build graph.
 /// Command-line arguments such as -fqemu and -fwasmtime may affect whether a
 /// binary is detected as foreign, as well as system configuration such as
 /// Rosetta (macOS) and binfmt_misc (Linux).
-/// If this RunStep is considered to have side-effects, then this flag does
+/// If this Run step is considered to have side-effects, then this flag does
 /// nothing.
 skip_foreign_checks: bool = false,
 
@@ -73,18 +71,18 @@ captured_stderr: ?*Output = null,
 has_side_effects: bool = false,
 
 pub const StdIo = union(enum) {
-    /// Whether the RunStep has side-effects will be determined by whether or not one
+    /// Whether the Run step has side-effects will be determined by whether or not one
     /// of the args is an output file (added with `addOutputFileArg`).
-    /// If the RunStep is determined to have side-effects, this is the same as `inherit`.
+    /// If the Run step is determined to have side-effects, this is the same as `inherit`.
     /// The step will fail if the subprocess crashes or returns a non-zero exit code.
     infer_from_args,
-    /// Causes the RunStep to be considered to have side-effects, and therefore
+    /// Causes the Run step to be considered to have side-effects, and therefore
     /// always execute when it appears in the build graph.
     /// It also means that this step will obtain a global lock to prevent other
     /// steps from running in the meantime.
     /// The step will fail if the subprocess crashes or returns a non-zero exit code.
     inherit,
-    /// Causes the RunStep to be considered to *not* have side-effects. The
+    /// Causes the Run step to be considered to *not* have side-effects. The
     /// process will be re-executed if any of the input dependencies are
     /// modified. The exit code and standard I/O streams will be checked for
     /// certain conditions, and the step will succeed or fail based on these
@@ -92,7 +90,7 @@ pub const StdIo = union(enum) {
     /// Note that an explicit check for exit code 0 needs to be added to this
     /// list if such a check is desirable.
     check: std.ArrayList(Check),
-    /// This RunStep is running a zig unit test binary and will communicate
+    /// This Run step is running a zig unit test binary and will communicate
     /// extra metadata over the IPC protocol.
     zig_test,
 
@@ -106,7 +104,7 @@ pub const StdIo = union(enum) {
 };
 
 pub const Arg = union(enum) {
-    artifact: *CompileStep,
+    artifact: *Step.Compile,
     file_source: std.Build.FileSource,
     directory_source: std.Build.FileSource,
     bytes: []u8,
@@ -119,8 +117,8 @@ pub const Output = struct {
     basename: []const u8,
 };
 
-pub fn create(owner: *std.Build, name: []const u8) *RunStep {
-    const self = owner.allocator.create(RunStep) catch @panic("OOM");
+pub fn create(owner: *std.Build, name: []const u8) *Run {
+    const self = owner.allocator.create(Run) catch @panic("OOM");
     self.* = .{
         .step = Step.init(.{
             .id = base_id,
@@ -135,17 +133,17 @@ pub fn create(owner: *std.Build, name: []const u8) *RunStep {
     return self;
 }
 
-pub fn setName(self: *RunStep, name: []const u8) void {
+pub fn setName(self: *Run, name: []const u8) void {
     self.step.name = name;
     self.rename_step_with_output_arg = false;
 }
 
-pub fn enableTestRunnerMode(rs: *RunStep) void {
+pub fn enableTestRunnerMode(rs: *Run) void {
     rs.stdio = .zig_test;
     rs.addArgs(&.{"--listen=-"});
 }
 
-pub fn addArtifactArg(self: *RunStep, artifact: *CompileStep) void {
+pub fn addArtifactArg(self: *Run, artifact: *Step.Compile) void {
     self.argv.append(Arg{ .artifact = artifact }) catch @panic("OOM");
     self.step.dependOn(&artifact.step);
 }
@@ -153,12 +151,12 @@ pub fn addArtifactArg(self: *RunStep, artifact: *CompileStep) void {
 /// This provides file path as a command line argument to the command being
 /// run, and returns a FileSource which can be used as inputs to other APIs
 /// throughout the build system.
-pub fn addOutputFileArg(rs: *RunStep, basename: []const u8) std.Build.FileSource {
+pub fn addOutputFileArg(rs: *Run, basename: []const u8) std.Build.FileSource {
     return addPrefixedOutputFileArg(rs, "", basename);
 }
 
 pub fn addPrefixedOutputFileArg(
-    rs: *RunStep,
+    rs: *Run,
     prefix: []const u8,
     basename: []const u8,
 ) std.Build.FileSource {
@@ -179,38 +177,38 @@ pub fn addPrefixedOutputFileArg(
     return .{ .generated = &output.generated_file };
 }
 
-pub fn addFileSourceArg(self: *RunStep, file_source: std.Build.FileSource) void {
+pub fn addFileSourceArg(self: *Run, file_source: std.Build.FileSource) void {
     self.argv.append(.{
         .file_source = file_source.dupe(self.step.owner),
     }) catch @panic("OOM");
     file_source.addStepDependencies(&self.step);
 }
 
-pub fn addDirectorySourceArg(self: *RunStep, directory_source: std.Build.FileSource) void {
+pub fn addDirectorySourceArg(self: *Run, directory_source: std.Build.FileSource) void {
     self.argv.append(.{
         .directory_source = directory_source.dupe(self.step.owner),
     }) catch @panic("OOM");
     directory_source.addStepDependencies(&self.step);
 }
 
-pub fn addArg(self: *RunStep, arg: []const u8) void {
+pub fn addArg(self: *Run, arg: []const u8) void {
     self.argv.append(.{ .bytes = self.step.owner.dupe(arg) }) catch @panic("OOM");
 }
 
-pub fn addArgs(self: *RunStep, args: []const []const u8) void {
+pub fn addArgs(self: *Run, args: []const []const u8) void {
     for (args) |arg| {
         self.addArg(arg);
     }
 }
 
-pub fn clearEnvironment(self: *RunStep) void {
+pub fn clearEnvironment(self: *Run) void {
     const b = self.step.owner;
     const new_env_map = b.allocator.create(EnvMap) catch @panic("OOM");
     new_env_map.* = EnvMap.init(b.allocator);
     self.env_map = new_env_map;
 }
 
-pub fn addPathDir(self: *RunStep, search_path: []const u8) void {
+pub fn addPathDir(self: *Run, search_path: []const u8) void {
     const b = self.step.owner;
     const env_map = getEnvMapInternal(self);
 
@@ -225,11 +223,11 @@ pub fn addPathDir(self: *RunStep, search_path: []const u8) void {
     }
 }
 
-pub fn getEnvMap(self: *RunStep) *EnvMap {
+pub fn getEnvMap(self: *Run) *EnvMap {
     return getEnvMapInternal(self);
 }
 
-fn getEnvMapInternal(self: *RunStep) *EnvMap {
+fn getEnvMapInternal(self: *Run) *EnvMap {
     const arena = self.step.owner.allocator;
     return self.env_map orelse {
         const env_map = arena.create(EnvMap) catch @panic("OOM");
@@ -239,25 +237,25 @@ fn getEnvMapInternal(self: *RunStep) *EnvMap {
     };
 }
 
-pub fn setEnvironmentVariable(self: *RunStep, key: []const u8, value: []const u8) void {
+pub fn setEnvironmentVariable(self: *Run, key: []const u8, value: []const u8) void {
     const b = self.step.owner;
     const env_map = self.getEnvMap();
     env_map.put(b.dupe(key), b.dupe(value)) catch @panic("unhandled error");
 }
 
-pub fn removeEnvironmentVariable(self: *RunStep, key: []const u8) void {
+pub fn removeEnvironmentVariable(self: *Run, key: []const u8) void {
     self.getEnvMap().remove(key);
 }
 
 /// Adds a check for exact stderr match. Does not add any other checks.
-pub fn expectStdErrEqual(self: *RunStep, bytes: []const u8) void {
+pub fn expectStdErrEqual(self: *Run, bytes: []const u8) void {
     const new_check: StdIo.Check = .{ .expect_stderr_exact = self.step.owner.dupe(bytes) };
     self.addCheck(new_check);
 }
 
 /// Adds a check for exact stdout match as well as a check for exit code 0, if
 /// there is not already an expected termination check.
-pub fn expectStdOutEqual(self: *RunStep, bytes: []const u8) void {
+pub fn expectStdOutEqual(self: *Run, bytes: []const u8) void {
     const new_check: StdIo.Check = .{ .expect_stdout_exact = self.step.owner.dupe(bytes) };
     self.addCheck(new_check);
     if (!self.hasTermCheck()) {
@@ -265,12 +263,12 @@ pub fn expectStdOutEqual(self: *RunStep, bytes: []const u8) void {
     }
 }
 
-pub fn expectExitCode(self: *RunStep, code: u8) void {
+pub fn expectExitCode(self: *Run, code: u8) void {
     const new_check: StdIo.Check = .{ .expect_term = .{ .Exited = code } };
     self.addCheck(new_check);
 }
 
-pub fn hasTermCheck(self: RunStep) bool {
+pub fn hasTermCheck(self: Run) bool {
     for (self.stdio.check.items) |check| switch (check) {
         .expect_term => return true,
         else => continue,
@@ -278,18 +276,18 @@ pub fn hasTermCheck(self: RunStep) bool {
     return false;
 }
 
-pub fn addCheck(self: *RunStep, new_check: StdIo.Check) void {
+pub fn addCheck(self: *Run, new_check: StdIo.Check) void {
     switch (self.stdio) {
         .infer_from_args => {
             self.stdio = .{ .check = std.ArrayList(StdIo.Check).init(self.step.owner.allocator) };
             self.stdio.check.append(new_check) catch @panic("OOM");
         },
         .check => |*checks| checks.append(new_check) catch @panic("OOM"),
-        else => @panic("illegal call to addCheck: conflicting helper method calls. Suggest to directly set stdio field of RunStep instead"),
+        else => @panic("illegal call to addCheck: conflicting helper method calls. Suggest to directly set stdio field of Run instead"),
     }
 }
 
-pub fn captureStdErr(self: *RunStep) std.Build.FileSource {
+pub fn captureStdErr(self: *Run) std.Build.FileSource {
     assert(self.stdio != .inherit);
 
     if (self.captured_stderr) |output| return .{ .generated = &output.generated_file };
@@ -304,7 +302,7 @@ pub fn captureStdErr(self: *RunStep) std.Build.FileSource {
     return .{ .generated = &output.generated_file };
 }
 
-pub fn captureStdOut(self: *RunStep) std.Build.FileSource {
+pub fn captureStdOut(self: *Run) std.Build.FileSource {
     assert(self.stdio != .inherit);
 
     if (self.captured_stdout) |output| return .{ .generated = &output.generated_file };
@@ -319,8 +317,8 @@ pub fn captureStdOut(self: *RunStep) std.Build.FileSource {
     return .{ .generated = &output.generated_file };
 }
 
-/// Returns whether the RunStep has side effects *other than* updating the output arguments.
-fn hasSideEffects(self: RunStep) bool {
+/// Returns whether the Run step has side effects *other than* updating the output arguments.
+fn hasSideEffects(self: Run) bool {
     if (self.has_side_effects) return true;
     return switch (self.stdio) {
         .infer_from_args => !self.hasAnyOutputArgs(),
@@ -330,7 +328,7 @@ fn hasSideEffects(self: RunStep) bool {
     };
 }
 
-fn hasAnyOutputArgs(self: RunStep) bool {
+fn hasAnyOutputArgs(self: Run) bool {
     if (self.captured_stdout != null) return true;
     if (self.captured_stderr != null) return true;
     for (self.argv.items) |arg| switch (arg) {
@@ -371,7 +369,7 @@ fn checksContainStderr(checks: []const StdIo.Check) bool {
 fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     const b = step.owner;
     const arena = b.allocator;
-    const self = @fieldParentPtr(RunStep, "step", step);
+    const self = @fieldParentPtr(Run, "step", step);
     const has_side_effects = self.hasSideEffects();
 
     var argv_list = ArrayList([]const u8).init(arena);
@@ -541,7 +539,7 @@ fn termMatches(expected: ?std.process.Child.Term, actual: std.process.Child.Term
 }
 
 fn runCommand(
-    self: *RunStep,
+    self: *Run,
     argv: []const []const u8,
     has_side_effects: bool,
     digest: ?*const [std.Build.Cache.hex_digest_len]u8,
@@ -567,7 +565,7 @@ fn runCommand(
         // FileNotFound: can happen with a wrong dynamic linker path
         if (err == error.InvalidExe or err == error.FileNotFound) interpret: {
             // TODO: learn the target from the binary directly rather than from
-            // relying on it being a CompileStep. This will make this logic
+            // relying on it being a Compile step. This will make this logic
             // work even for the edge case that the binary was produced by a
             // third party.
             const exe = switch (self.argv.items[0]) {
@@ -862,7 +860,7 @@ const ChildProcResult = struct {
 };
 
 fn spawnChildAndCollect(
-    self: *RunStep,
+    self: *Run,
     argv: []const []const u8,
     has_side_effects: bool,
     prog_node: *std.Progress.Node,
@@ -936,7 +934,7 @@ const StdIoResult = struct {
 };
 
 fn evalZigTest(
-    self: *RunStep,
+    self: *Run,
     child: *std.process.Child,
     prog_node: *std.Progress.Node,
 ) !StdIoResult {
@@ -1121,7 +1119,7 @@ fn sendRunTestMessage(file: std.fs.File, index: u32) !void {
     try file.writeAll(full_msg);
 }
 
-fn evalGeneric(self: *RunStep, child: *std.process.Child) !StdIoResult {
+fn evalGeneric(self: *Run, child: *std.process.Child) !StdIoResult {
     const arena = self.step.owner.allocator;
 
     if (self.stdin) |stdin| {
@@ -1188,7 +1186,7 @@ fn evalGeneric(self: *RunStep, child: *std.process.Child) !StdIoResult {
     };
 }
 
-fn addPathForDynLibs(self: *RunStep, artifact: *CompileStep) void {
+fn addPathForDynLibs(self: *Run, artifact: *Step.Compile) void {
     const b = self.step.owner;
     for (artifact.link_objects.items) |link_object| {
         switch (link_object) {
@@ -1204,10 +1202,10 @@ fn addPathForDynLibs(self: *RunStep, artifact: *CompileStep) void {
 }
 
 fn failForeign(
-    self: *RunStep,
+    self: *Run,
     suggested_flag: []const u8,
     argv0: []const u8,
-    exe: *CompileStep,
+    exe: *Step.Compile,
 ) error{ MakeFailed, MakeSkipped, OutOfMemory } {
     switch (self.stdio) {
         .check, .zig_test => {
diff --git a/lib/std/Build/Step/TranslateC.zig b/lib/std/Build/Step/TranslateC.zig
index 86727ea2f0..0c7ddc4720 100644
--- a/lib/std/Build/Step/TranslateC.zig
+++ b/lib/std/Build/Step/TranslateC.zig
@@ -1,12 +1,10 @@
 const std = @import("std");
 const Step = std.Build.Step;
-const CompileStep = std.Build.CompileStep;
-const CheckFileStep = std.Build.CheckFileStep;
 const fs = std.fs;
 const mem = std.mem;
 const CrossTarget = std.zig.CrossTarget;
 
-const TranslateCStep = @This();
+const TranslateC = @This();
 
 pub const base_id = .translate_c;
 
@@ -25,10 +23,10 @@ pub const Options = struct {
     optimize: std.builtin.OptimizeMode,
 };
 
-pub fn create(owner: *std.Build, options: Options) *TranslateCStep {
-    const self = owner.allocator.create(TranslateCStep) catch @panic("OOM");
+pub fn create(owner: *std.Build, options: Options) *TranslateC {
+    const self = owner.allocator.create(TranslateC) catch @panic("OOM");
     const source = options.source_file.dupe(owner);
-    self.* = TranslateCStep{
+    self.* = TranslateC{
         .step = Step.init(.{
             .id = .translate_c,
             .name = "translate-c",
@@ -52,11 +50,11 @@ pub const AddExecutableOptions = struct {
     version: ?std.builtin.Version = null,
     target: ?CrossTarget = null,
     optimize: ?std.builtin.Mode = null,
-    linkage: ?CompileStep.Linkage = null,
+    linkage: ?Step.Compile.Linkage = null,
 };
 
 /// Creates a step to build an executable from the translated source.
-pub fn addExecutable(self: *TranslateCStep, options: AddExecutableOptions) *CompileStep {
+pub fn addExecutable(self: *TranslateC, options: AddExecutableOptions) *Step.Compile {
     return self.step.owner.addExecutable(.{
         .root_source_file = .{ .generated = &self.output_file },
         .name = options.name orelse "translated_c",
@@ -67,12 +65,12 @@ pub fn addExecutable(self: *TranslateCStep, options: AddExecutableOptions) *Comp
     });
 }
 
-pub fn addIncludeDir(self: *TranslateCStep, include_dir: []const u8) void {
+pub fn addIncludeDir(self: *TranslateC, include_dir: []const u8) void {
     self.include_dirs.append(self.step.owner.dupePath(include_dir)) catch @panic("OOM");
 }
 
-pub fn addCheckFile(self: *TranslateCStep, expected_matches: []const []const u8) *CheckFileStep {
-    return CheckFileStep.create(
+pub fn addCheckFile(self: *TranslateC, expected_matches: []const []const u8) *Step.CheckFile {
+    return Step.CheckFile.create(
         self.step.owner,
         .{ .generated = &self.output_file },
         .{ .expected_matches = expected_matches },
@@ -81,19 +79,19 @@ pub fn addCheckFile(self: *TranslateCStep, expected_matches: []const []const u8)
 
 /// If the value is omitted, it is set to 1.
 /// `name` and `value` need not live longer than the function call.
-pub fn defineCMacro(self: *TranslateCStep, name: []const u8, value: ?[]const u8) void {
+pub fn defineCMacro(self: *TranslateC, name: []const u8, value: ?[]const u8) void {
     const macro = std.Build.constructCMacro(self.step.owner.allocator, name, value);
     self.c_macros.append(macro) catch @panic("OOM");
 }
 
 /// name_and_value looks like [name]=[value]. If the value is omitted, it is set to 1.
-pub fn defineCMacroRaw(self: *TranslateCStep, name_and_value: []const u8) void {
+pub fn defineCMacroRaw(self: *TranslateC, name_and_value: []const u8) void {
     self.c_macros.append(self.step.owner.dupe(name_and_value)) catch @panic("OOM");
 }
 
 fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     const b = step.owner;
-    const self = @fieldParentPtr(TranslateCStep, "step", step);
+    const self = @fieldParentPtr(TranslateC, "step", step);
 
     var argv_list = std.ArrayList([]const u8).init(b.allocator);
     try argv_list.append(b.zig_exe);
diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig
index 68f7c37c6c..0d817e7430 100644
--- a/lib/std/Build/Step/WriteFile.zig
+++ b/lib/std/Build/Step/WriteFile.zig
@@ -1,4 +1,4 @@
-//! WriteFileStep is primarily used to create a directory in an appropriate
+//! WriteFile is primarily used to create a directory in an appropriate
 //! location inside the local cache which has a set of files that have either
 //! been generated during the build, or are copied from the source package.
 //!
@@ -12,7 +12,7 @@ const std = @import("std");
 const Step = std.Build.Step;
 const fs = std.fs;
 const ArrayList = std.ArrayList;
-const WriteFileStep = @This();
+const WriteFile = @This();
 
 step: Step,
 /// The elements here are pointers because we need stable pointers for the
@@ -39,8 +39,8 @@ pub const Contents = union(enum) {
     copy: std.Build.FileSource,
 };
 
-pub fn create(owner: *std.Build) *WriteFileStep {
-    const wf = owner.allocator.create(WriteFileStep) catch @panic("OOM");
+pub fn create(owner: *std.Build) *WriteFile {
+    const wf = owner.allocator.create(WriteFile) catch @panic("OOM");
     wf.* = .{
         .step = Step.init(.{
             .id = .write_file,
@@ -55,7 +55,7 @@ pub fn create(owner: *std.Build) *WriteFileStep {
     return wf;
 }
 
-pub fn add(wf: *WriteFileStep, sub_path: []const u8, bytes: []const u8) void {
+pub fn add(wf: *WriteFile, sub_path: []const u8, bytes: []const u8) void {
     const b = wf.step.owner;
     const gpa = b.allocator;
     const file = gpa.create(File) catch @panic("OOM");
@@ -72,11 +72,11 @@ pub fn add(wf: *WriteFileStep, sub_path: []const u8, bytes: []const u8) void {
 /// Place the file into the generated directory within the local cache,
 /// along with all the rest of the files added to this step. The parameter
 /// here is the destination path relative to the local cache directory
-/// associated with this WriteFileStep. It may be a basename, or it may
+/// associated with this WriteFile. It may be a basename, or it may
 /// include sub-directories, in which case this step will ensure the
 /// required sub-path exists.
 /// This is the option expected to be used most commonly with `addCopyFile`.
-pub fn addCopyFile(wf: *WriteFileStep, source: std.Build.FileSource, sub_path: []const u8) void {
+pub fn addCopyFile(wf: *WriteFile, source: std.Build.FileSource, sub_path: []const u8) void {
     const b = wf.step.owner;
     const gpa = b.allocator;
     const file = gpa.create(File) catch @panic("OOM");
@@ -97,7 +97,7 @@ pub fn addCopyFile(wf: *WriteFileStep, source: std.Build.FileSource, sub_path: [
 /// run by a developer with intent to modify source files and then commit
 /// those changes to version control.
 /// A file added this way is not available with `getFileSource`.
-pub fn addCopyFileToSource(wf: *WriteFileStep, source: std.Build.FileSource, sub_path: []const u8) void {
+pub fn addCopyFileToSource(wf: *WriteFile, source: std.Build.FileSource, sub_path: []const u8) void {
     const b = wf.step.owner;
     wf.output_source_files.append(b.allocator, .{
         .contents = .{ .copy = source },
@@ -112,7 +112,7 @@ pub fn addCopyFileToSource(wf: *WriteFileStep, source: std.Build.FileSource, sub
 /// run by a developer with intent to modify source files and then commit
 /// those changes to version control.
 /// A file added this way is not available with `getFileSource`.
-pub fn addBytesToSource(wf: *WriteFileStep, bytes: []const u8, sub_path: []const u8) void {
+pub fn addBytesToSource(wf: *WriteFile, bytes: []const u8, sub_path: []const u8) void {
     const b = wf.step.owner;
     wf.output_source_files.append(b.allocator, .{
         .contents = .{ .bytes = bytes },
@@ -121,7 +121,7 @@ pub fn addBytesToSource(wf: *WriteFileStep, bytes: []const u8, sub_path: []const
 }
 
 /// Gets a file source for the given sub_path. If the file does not exist, returns `null`.
-pub fn getFileSource(wf: *WriteFileStep, sub_path: []const u8) ?std.Build.FileSource {
+pub fn getFileSource(wf: *WriteFile, sub_path: []const u8) ?std.Build.FileSource {
     for (wf.files.items) |file| {
         if (std.mem.eql(u8, file.sub_path, sub_path)) {
             return .{ .generated = &file.generated_file };
@@ -131,12 +131,12 @@ pub fn getFileSource(wf: *WriteFileStep, sub_path: []const u8) ?std.Build.FileSo
 }
 
 /// Returns a `FileSource` representing the base directory that contains all the
-/// files from this `WriteFileStep`.
-pub fn getDirectorySource(wf: *WriteFileStep) std.Build.FileSource {
+/// files from this `WriteFile`.
+pub fn getDirectorySource(wf: *WriteFile) std.Build.FileSource {
     return .{ .generated = &wf.generated_directory };
 }
 
-fn maybeUpdateName(wf: *WriteFileStep) void {
+fn maybeUpdateName(wf: *WriteFile) void {
     if (wf.files.items.len == 1) {
         // First time adding a file; update name.
         if (std.mem.eql(u8, wf.step.name, "WriteFile")) {
@@ -148,10 +148,10 @@ fn maybeUpdateName(wf: *WriteFileStep) void {
 fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     _ = prog_node;
     const b = step.owner;
-    const wf = @fieldParentPtr(WriteFileStep, "step", step);
+    const wf = @fieldParentPtr(WriteFile, "step", step);
 
     // Writing to source files is kind of an extra capability of this
-    // WriteFileStep - arguably it should be a different step. But anyway here
+    // WriteFile - arguably it should be a different step. But anyway here
     // it is, it happens unconditionally and does not interact with the other
     // files here.
     var any_miss = false;
@@ -194,14 +194,14 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     // the data to a file would probably be very fast - but as a way to find a canonical
     // location to put build artifacts.
 
-    // If, for example, a hard-coded path was used as the location to put WriteFileStep
-    // files, then two WriteFileSteps executing in parallel might clobber each other.
+    // If, for example, a hard-coded path was used as the location to put WriteFile
+    // files, then two WriteFiles executing in parallel might clobber each other.
 
     var man = b.cache.obtain();
     defer man.deinit();
 
-    // Random bytes to make WriteFileStep unique. Refresh this with
-    // new random bytes when WriteFileStep implementation is modified
+    // Random bytes to make WriteFile unique. Refresh this with
+    // new random bytes when WriteFile implementation is modified
     // in a non-backwards-compatible way.
     man.hash.add(@as(u32, 0xd767ee59));
 
diff --git a/test/link/macho/dead_strip/build.zig b/test/link/macho/dead_strip/build.zig
index 4c739b3d8c..9d00bad9e0 100644
--- a/test/link/macho/dead_strip/build.zig
+++ b/test/link/macho/dead_strip/build.zig
@@ -42,7 +42,7 @@ fn createScenario(
     optimize: std.builtin.OptimizeMode,
     target: std.zig.CrossTarget,
     name: []const u8,
-) *std.Build.CompileStep {
+) *std.Build.Step.Compile {
     const exe = b.addExecutable(.{
         .name = name,
         .optimize = optimize,
diff --git a/test/link/macho/dead_strip_dylibs/build.zig b/test/link/macho/dead_strip_dylibs/build.zig
index 47e53f853e..ec073e183a 100644
--- a/test/link/macho/dead_strip_dylibs/build.zig
+++ b/test/link/macho/dead_strip_dylibs/build.zig
@@ -46,7 +46,7 @@ fn createScenario(
     b: *std.Build,
     optimize: std.builtin.OptimizeMode,
     name: []const u8,
-) *std.Build.CompileStep {
+) *std.Build.Step.Compile {
     const exe = b.addExecutable(.{
         .name = name,
         .optimize = optimize,
diff --git a/test/link/macho/headerpad/build.zig b/test/link/macho/headerpad/build.zig
index 22cfcc90ec..99edfe72fa 100644
--- a/test/link/macho/headerpad/build.zig
+++ b/test/link/macho/headerpad/build.zig
@@ -104,7 +104,7 @@ fn simpleExe(
     b: *std.Build,
     optimize: std.builtin.OptimizeMode,
     name: []const u8,
-) *std.Build.CompileStep {
+) *std.Build.Step.Compile {
     const exe = b.addExecutable(.{
         .name = name,
         .optimize = optimize,
diff --git a/test/link/macho/search_strategy/build.zig b/test/link/macho/search_strategy/build.zig
index 4777629c8b..853c471969 100644
--- a/test/link/macho/search_strategy/build.zig
+++ b/test/link/macho/search_strategy/build.zig
@@ -46,7 +46,7 @@ fn createScenario(
     optimize: std.builtin.OptimizeMode,
     target: std.zig.CrossTarget,
     name: []const u8,
-) *std.Build.CompileStep {
+) *std.Build.Step.Compile {
     const static = b.addStaticLibrary(.{
         .name = name,
         .optimize = optimize,
diff --git a/test/link/macho/unwind_info/build.zig b/test/link/macho/unwind_info/build.zig
index 4ace2a4e96..96b5f6cacc 100644
--- a/test/link/macho/unwind_info/build.zig
+++ b/test/link/macho/unwind_info/build.zig
@@ -65,7 +65,7 @@ fn createScenario(
     optimize: std.builtin.OptimizeMode,
     target: std.zig.CrossTarget,
     name: []const u8,
-) *std.Build.CompileStep {
+) *std.Build.Step.Compile {
     const exe = b.addExecutable(.{
         .name = name,
         .optimize = optimize,
diff --git a/test/link/macho/uuid/build.zig b/test/link/macho/uuid/build.zig
index df58aeacb7..0072825f46 100644
--- a/test/link/macho/uuid/build.zig
+++ b/test/link/macho/uuid/build.zig
@@ -1,5 +1,4 @@
 const std = @import("std");
-const CompileStep = std.Build.CompileStep;
 const FileSource = std.Build.FileSource;
 const Step = std.Build.Step;
 
@@ -60,7 +59,7 @@ fn simpleDylib(
     b: *std.Build,
     optimize: std.builtin.OptimizeMode,
     target: std.zig.CrossTarget,
-) *std.Build.CompileStep {
+) *std.Build.Step.Compile {
     const dylib = b.addSharedLibrary(.{
         .name = "test",
         .version = .{ .major = 1, .minor = 0 },
diff --git a/test/src/Cases.zig b/test/src/Cases.zig
index 4b023f45b0..0451079a0e 100644
--- a/test/src/Cases.zig
+++ b/test/src/Cases.zig
@@ -465,7 +465,7 @@ pub fn lowerToBuildSteps(
     parent_step: *std.Build.Step,
     opt_test_filter: ?[]const u8,
     cases_dir_path: []const u8,
-    incremental_exe: *std.Build.CompileStep,
+    incremental_exe: *std.Build.Step.Compile,
 ) void {
     for (self.incremental_cases.items) |incr_case| {
         if (opt_test_filter) |test_filter| {
diff --git a/test/src/StackTrace.zig b/test/src/StackTrace.zig
index c32720a210..0d0b7155e6 100644
--- a/test/src/StackTrace.zig
+++ b/test/src/StackTrace.zig
@@ -3,7 +3,7 @@ step: *Step,
 test_index: usize,
 test_filter: ?[]const u8,
 optimize_modes: []const OptimizeMode,
-check_exe: *std.Build.CompileStep,
+check_exe: *std.Build.Step.Compile,
 
 const Expect = [@typeInfo(OptimizeMode).Enum.fields.len][]const u8;
 
diff --git a/test/standalone/install_raw_hex/build.zig b/test/standalone/install_raw_hex/build.zig
index b34bb01378..c05490a3e5 100644
--- a/test/standalone/install_raw_hex/build.zig
+++ b/test/standalone/install_raw_hex/build.zig
@@ -1,6 +1,5 @@
 const builtin = @import("builtin");
 const std = @import("std");
-const CheckFileStep = std.Build.CheckFileStep;
 
 pub fn build(b: *std.Build) void {
     const test_step = b.step("test", "Test it");
diff --git a/test/tests.zig b/test/tests.zig
index 7ec1aaaa65..641914aabe 100644
--- a/test/tests.zig
+++ b/test/tests.zig
@@ -1132,7 +1132,7 @@ pub fn addCases(
     b: *std.Build,
     parent_step: *Step,
     opt_test_filter: ?[]const u8,
-    check_case_exe: *std.Build.CompileStep,
+    check_case_exe: *std.Build.Step.Compile,
 ) !void {
     const arena = b.allocator;
     const gpa = b.allocator;

From 012f9a97eb74c63921ce7f3a036c71816fb59a24 Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Wed, 3 May 2023 20:42:12 +0100
Subject: [PATCH 075/725] std.c: add os_proc_available_memory for darwin

---
 lib/std/c/darwin.zig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/std/c/darwin.zig b/lib/std/c/darwin.zig
index 6e4ef59d38..2aa05e1cc7 100644
--- a/lib/std/c/darwin.zig
+++ b/lib/std/c/darwin.zig
@@ -3867,3 +3867,4 @@ pub const MIN = struct {
 };
 
 pub extern "c" fn mincore(addr: *align(std.mem.page_size) const anyopaque, length: usize, vec: [*]u8) c_int;
+pub extern "c" fn os_proc_available_memory() usize;

From 3eaca9bbc6fee4b6be0f22b844f7d3214222f434 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= 
Date: Thu, 4 May 2023 10:36:05 +0300
Subject: [PATCH 076/725] glibc hacks: also add a few dn_* functions

---
 lib/libc/include/generic-glibc/resolv.h | 25 ++++++++++++++++---------
 test/link/glibc_compat/main.c           |  3 +++
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/lib/libc/include/generic-glibc/resolv.h b/lib/libc/include/generic-glibc/resolv.h
index 406727e8fb..d0f9955f94 100644
--- a/lib/libc/include/generic-glibc/resolv.h
+++ b/lib/libc/include/generic-glibc/resolv.h
@@ -170,16 +170,20 @@ __END_DECLS
 #define res_isourserver		__res_isourserver
 
 /* In glibc 2.33 and earlier res_search, res_nsearch, res_query, res_nquery,
- * res_querydomain, res_nquerydomain were #define'd to __res_search,
- * __res_nsearch, etc. glibc 2.34 onwards removes the macros and exposes the
- * symbols directly. New glibc exposes compat symbols with underscores for
- * backwards compatibility. Applications linked to glibc 2.34+ are expected
- * to use the non-underscored symbols.
+ * res_querydomain, res_nquerydomain, dn_skipname, dn_comp, dn_expand were
+ * #define'd to __res_search, __res_nsearch, etc. glibc 2.34 onwards removes
+ * the macros and exposes the symbols directly. New glibc exposes compat
+ * symbols with underscores for backwards compatibility. Applications linked to
+ * glibc 2.34+ are expected to use the non-underscored symbols.
  *
- * It will be enough to bring the macros back when compiling against the older
- * glibc versions.
+ * It is enough to bring the macros back when compiling against the older glibc
+ * versions.
  *
- * See glibc commit ea9878ec271c791880fcbbe519d70c42f8113750.
+ * See glibc commits:
+ * - ea9878ec271c791880fcbbe519d70c42f8113750 res_*
+ * - 391e02236b931132c0e8b5ba4c3b087c2aaa1044 dn_skipname
+ * - fd8a87c0c1932de591e7ad108ff6288a4b6b18c9 dn_comp
+ * - 640bbdf71c6f10ac26252ac67a22902e26657bd8 dn_expand
  */
 #if (__GLIBC__ == 2 && __GLIBC_MINOR__ < 34)
 #define res_search        __res_search
@@ -188,6 +192,9 @@ __END_DECLS
 #define res_nquery        __res_nquery
 #define res_querydomain   __res_querydomain
 #define res_nquerydomain  __res_nquerydomain
+#define dn_skipname       __dn_skipname
+#define dn_comp           __dn_comp
+#define dn_expand         __dn_expand
 #endif
 /* end glibc compat hacks */
 
@@ -327,4 +334,4 @@ void		res_nclose (res_state) __THROW;
 
 __END_DECLS
 
-#endif /* !_RESOLV_H_ */
\ No newline at end of file
+#endif /* !_RESOLV_H_ */
diff --git a/test/link/glibc_compat/main.c b/test/link/glibc_compat/main.c
index d9059eb8a6..29ec920a58 100644
--- a/test/link/glibc_compat/main.c
+++ b/test/link/glibc_compat/main.c
@@ -34,4 +34,7 @@ int main() {
     printf("address to res_nquery: %p\n", res_nquery);
     printf("address to res_querydomain: %p\n", res_querydomain);
     printf("address to res_nquerydomain: %p\n", res_nquerydomain);
+    printf("address to dn_skipname: %p\n", dn_skipname);
+    printf("address to dn_comp: %p\n", dn_comp);
+    printf("address to dn_expand: %p\n", dn_expand);
 }

From 892d4e13027b4469f89f9b0d75991639a87cd03f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= 
Date: Thu, 4 May 2023 10:36:37 +0300
Subject: [PATCH 077/725] glibc hacks: add another commit to our patches

---
 lib/libc/glibc/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/libc/glibc/README.md b/lib/libc/glibc/README.md
index 399d3beb2d..e5fe99a532 100644
--- a/lib/libc/glibc/README.md
+++ b/lib/libc/glibc/README.md
@@ -6,3 +6,4 @@ need to be cherry-picked in the future glibc header upgrades:
 
 - 39083c31a550ed80f369f60d35791e98904b8096
 - a89813ef282c092a9caf699731c7faaf485acabe
+- 3eaca9bbc6fee4b6be0f22b844f7d3214222f434

From e363ffad903b0f66283fe25de98f3e181d371fc7 Mon Sep 17 00:00:00 2001
From: Krzysztof Wolicki 
Date: Fri, 5 May 2023 01:46:19 +0200
Subject: [PATCH 078/725] autodoc: Better comptimeExpr code for blocks

---
 lib/docs/main.js | 29 ++++++++++------
 src/Autodoc.zig  | 89 ++++++++++++++++++++++++++++--------------------
 2 files changed, 70 insertions(+), 48 deletions(-)

diff --git a/lib/docs/main.js b/lib/docs/main.js
index 14bcf0f544..1e25d0c944 100644
--- a/lib/docs/main.js
+++ b/lib/docs/main.js
@@ -649,16 +649,16 @@ const NAV_MODES = {
     let lines = code.split("\n");
     let result = "";
     let indent_level = 0;
-    for(let i = 0; i < lines.length; i += 1) {
+    for (let i = 0; i < lines.length; i += 1) {
       let line = lines[i].trim();
-      if(line[0] == "}") indent_level -= 1;
-      for(let j = 0; j < indent_level; j += 1) {
+      if (line[0] == "}") indent_level -= 1;
+      for (let j = 0; j < indent_level; j += 1) {
         result += "    ";
       }
       if (line.startsWith("\\\\")) result += "    "
       result += line;
-      result += "\n";
-      if(line[line.length - 1] == "{") indent_level += 1;
+      if (i != lines.length - 1) result += "\n";
+      if (line[line.length - 1] == "{") indent_level += 1;
     }
     return result;
   }
@@ -1730,7 +1730,7 @@ const NAV_MODES = {
         return payloadHtml + "}";
       }
       case "comptimeExpr": {
-        return zigAnalysis.comptimeExprs[expr.comptimeExpr].code;
+        return renderZigSource(zigAnalysis.comptimeExprs[expr.comptimeExpr].code);
       }
       case "call": {
         let call = zigAnalysis.calls[expr.call];
@@ -2571,13 +2571,13 @@ const NAV_MODES = {
     } else if (
       resolvedValue.expr.string !== undefined ||
       resolvedValue.expr.call !== undefined ||
-      resolvedValue.expr.comptimeExpr
+      (resolvedValue.expr.comptimeExpr !== undefined && resolvedValue.typeRef !== null)
     ) {
       domFnProtoCode.innerHTML =
         'const ' +
         escapeHtml(decl.name) +
         ": " +
-        exprName(resolvedValue.expr, { wantHtml: true, wantLink: true }) +
+        exprName(resolvedValue.typeRef !== null ? resolvedValue.typeRef : resolvedValue.expr, { wantHtml: true, wantLink: true }) +
         " = " +
         exprName(decl.value.expr, { wantHtml: true, wantLink: true }) +
         ";";
@@ -2628,13 +2628,13 @@ const NAV_MODES = {
     } else if (
       resolvedVar.expr.string !== undefined ||
       resolvedVar.expr.call !== undefined ||
-      resolvedVar.expr.comptimeExpr
+      resolvedVar.expr.comptimeExpr !== undefined
     ) {
       domFnProtoCode.innerHTML =
         'var ' +
         escapeHtml(decl.name) +
         ": " +
-        exprName(resolvedVar.expr, { wantHtml: true, wantLink: true }) +
+        exprName(resolvedVar.typeRef !== null ? resolvedVar.typeRef : resolvedVar.expr, { wantHtml: true, wantLink: true }) +
         " = " +
         exprName(decl.value.expr, { wantHtml: true, wantLink: true }) +
         ";";
@@ -4850,4 +4850,11 @@ function RadixTree() {
 
 // BUT!
 
-// We want to be able to search "Hash", for example!
\ No newline at end of file
+// We want to be able to search "Hash", for example!
+
+function findDeclByName(name) {
+  for (let i = 0; i < zigAnalysis.decls.length; i += 1) {
+    let decl = zigAnalysis.decls[i];
+    if(decl[0] == name) return decl;
+  }
+}
\ No newline at end of file
diff --git a/src/Autodoc.zig b/src/Autodoc.zig
index f73e718a4d..20f2c6a23e 100644
--- a/src/Autodoc.zig
+++ b/src/Autodoc.zig
@@ -2153,11 +2153,13 @@ fn walkInstruction(
             };
         },
         .block => {
-            const res = DocData.WalkResult{ .expr = .{
-                .comptimeExpr = self.comptime_exprs.items.len,
-            } };
+            const res = DocData.WalkResult{
+                .typeRef = .{ .type = @enumToInt(Ref.type_type) },
+                .expr = .{ .comptimeExpr = self.comptime_exprs.items.len },
+            };
+            const block_expr = try self.getBlockSource(file, parent_src, inst_index);
             try self.comptime_exprs.append(self.arena, .{
-                .code = "if (...) { ... }",
+                .code = block_expr,
             });
             return res;
         },
@@ -2167,11 +2169,13 @@ fn walkInstruction(
                 parent_scope,
                 parent_src,
                 getBlockInlineBreak(file.zir, inst_index) orelse {
-                    const res = DocData.WalkResult{ .expr = .{
-                        .comptimeExpr = self.comptime_exprs.items.len,
-                    } };
+                    const res = DocData.WalkResult{
+                        .typeRef = .{ .type = @enumToInt(Ref.type_type) },
+                        .expr = .{ .comptimeExpr = self.comptime_exprs.items.len },
+                    };
+                    const block_inline_expr = try self.getBlockSource(file, parent_src, inst_index);
                     try self.comptime_exprs.append(self.arena, .{
-                        .code = "if (...) { ... }",
+                        .code = block_inline_expr,
                     });
                     return res;
                 },
@@ -3201,29 +3205,7 @@ fn analyzeDecl(
     const decl_src = try self.srcLocInfo(file, value_pl_node.src_node, parent_src);
 
     const name: []const u8 = switch (decl_name_index) {
-        0, 1 => unreachable, // comptime or usingnamespace decl
-        2 => {
-            unreachable;
-            // decl test
-            // const decl_status = scope.resolveDeclName(doc_comment_index);
-            // const decl_being_tested = decl_status.Analyzed;
-            // const func_index = getBlockInlineBreak(file.zir, value_index).?;
-
-            // const pl_node = data[Zir.refToIndex(func_index).?].pl_node;
-            // const fn_src = try self.srcLocInfo(file, pl_node.src_node, decl_src);
-            // const tree = try file.getTree(self.comp_module.gpa);
-            // const test_source_code = tree.getNodeSource(fn_src.src_node);
-
-            // const ast_node_index = self.ast_nodes.items.len;
-            // try self.ast_nodes.append(self.arena, .{
-            //     .file = 0,
-            //     .line = 0,
-            //     .col = 0,
-            //     .code = test_source_code,
-            // });
-            // self.decls.items[decl_being_tested].decltest = ast_node_index;
-            // continue;
-        },
+        0, 1, 2 => unreachable, // comptime or usingnamespace decl, decltest
         else => blk: {
             if (file.zir.string_bytes[decl_name_index] == 0) {
                 // test decl
@@ -3360,15 +3342,11 @@ fn analyzeDecltest(
     const decl_name_index = file.zir.extra[d.sub_index + 7];
 
     // This is known to work because decl values are always block_inlines
+    const test_source_code = try self.getBlockSource(file, parent_src, value_index);
+
     const value_pl_node = data[value_index].pl_node;
     const decl_src = try self.srcLocInfo(file, value_pl_node.src_node, parent_src);
 
-    const func_index = getBlockInlineBreak(file.zir, value_index).?;
-    const pl_node = data[Zir.refToIndex(func_index).?].pl_node;
-    const fn_src = try self.srcLocInfo(file, pl_node.src_node, decl_src);
-    const tree = try file.getTree(self.comp_module.gpa);
-    const test_source_code = tree.getNodeSource(fn_src.src_node);
-
     const decl_name: ?[]const u8 = if (decl_name_index != 0)
         file.zir.nullTerminatedString(decl_name_index)
     else
@@ -4814,6 +4792,43 @@ fn declIsVar(
     return (tags[tok_idx] == .keyword_var);
 }
 
+fn getBlockSource(
+    self: Autodoc,
+    file: *File,
+    parent_src: SrcLocInfo,
+    value_index: usize,
+) AutodocErrors![]const u8 {
+    const data = file.zir.instructions.items(.data);
+    const tags = file.zir.instructions.items(.tag);
+    const tree = try file.getTree(self.comp_module.gpa);
+
+    // This is known to work because decl values are always block_inlines
+    const block_pl_node = data[value_index].pl_node;
+    const block_src = try self.srcLocInfo(file, block_pl_node.src_node, parent_src);
+
+    const blk_index = idx: {
+        const pl_node = data[value_index].pl_node;
+        const extra = file.zir.extraData(Zir.Inst.Block, pl_node.payload_index);
+        const end_index = file.zir.extra[extra.end..][extra.data.body_len - 1];
+
+        if (tags[end_index] == .break_inline) {
+            break :idx @as(usize, Zir.refToIndex(data[end_index].@"break".operand).?);
+        } else {
+            break :idx null;
+        }
+    };
+
+    const blk_source = src: {
+        if (blk_index) |idx| {
+            const pl_node = data[idx].pl_node;
+            break :src try self.srcLocInfo(file, pl_node.src_node, block_src);
+        } else {
+            break :src block_src;
+        }
+    };
+    return tree.getNodeSource(blk_source.src_node);
+}
+
 fn getTLDocComment(self: *Autodoc, file: *File) ![]const u8 {
     const source = (try file.getSource(self.comp_module.gpa)).bytes;
     var tokenizer = Tokenizer.init(source);

From c63338712e0267678d400e58b61853954d0e40e4 Mon Sep 17 00:00:00 2001
From: Krzysztof Wolicki 
Date: Fri, 5 May 2023 02:41:23 +0200
Subject: [PATCH 079/725] autodoc: Save switches as comptimeExpr with the code
 instead of analyzing it deeply; simplified getBlockSource;

---
 src/Autodoc.zig | 71 +++++++++++++++++++------------------------------
 1 file changed, 27 insertions(+), 44 deletions(-)

diff --git a/src/Autodoc.zig b/src/Autodoc.zig
index 20f2c6a23e..42813f7614 100644
--- a/src/Autodoc.zig
+++ b/src/Autodoc.zig
@@ -1876,12 +1876,14 @@ fn walkInstruction(
             // WIP
             const pl_node = data[inst_index].pl_node;
             const extra = file.zir.extraData(Zir.Inst.SwitchBlock, pl_node.payload_index);
+
+            const switch_cond = try self.walkRef(file, parent_scope, parent_src, extra.data.operand, false);
             const cond_index = self.exprs.items.len;
+            try self.exprs.append(self.arena, switch_cond.expr);
+            _ = cond_index;
 
-            _ = try self.walkRef(file, parent_scope, parent_src, extra.data.operand, false);
-
-            const ast_index = self.ast_nodes.items.len;
-            const type_index = self.types.items.len - 1;
+            // const ast_index = self.ast_nodes.items.len;
+            // const type_index = self.types.items.len - 1;
 
             // const ast_line = self.ast_nodes.items[ast_index - 1];
 
@@ -1894,12 +1896,18 @@ fn walkInstruction(
             // log.debug("{s}", .{sep});
 
             const switch_index = self.exprs.items.len;
-            try self.exprs.append(self.arena, .{ .switchOp = .{
-                .cond_index = cond_index,
-                .file_name = file.sub_file_path,
-                .src = ast_index,
-                .outer_decl = type_index,
-            } });
+
+            // const src_loc = try self.srcLocInfo(file, pl_node.src_node, parent_src);
+
+            const switch_expr = try self.getBlockSource(file, parent_src, pl_node.src_node);
+            try self.exprs.append(self.arena, .{ .comptimeExpr = self.comptime_exprs.items.len });
+            try self.comptime_exprs.append(self.arena, .{ .code = switch_expr });
+            // try self.exprs.append(self.arena, .{ .switchOp = .{
+            //     .cond_index = cond_index,
+            //     .file_name = file.sub_file_path,
+            //     .src = ast_index,
+            //     .outer_decl = type_index,
+            // } });
 
             return DocData.WalkResult{
                 .typeRef = .{ .type = @enumToInt(Ref.type_type) },
@@ -2157,7 +2165,8 @@ fn walkInstruction(
                 .typeRef = .{ .type = @enumToInt(Ref.type_type) },
                 .expr = .{ .comptimeExpr = self.comptime_exprs.items.len },
             };
-            const block_expr = try self.getBlockSource(file, parent_src, inst_index);
+            const pl_node = data[inst_index].pl_node;
+            const block_expr = try self.getBlockSource(file, parent_src, pl_node.src_node);
             try self.comptime_exprs.append(self.arena, .{
                 .code = block_expr,
             });
@@ -2173,7 +2182,8 @@ fn walkInstruction(
                         .typeRef = .{ .type = @enumToInt(Ref.type_type) },
                         .expr = .{ .comptimeExpr = self.comptime_exprs.items.len },
                     };
-                    const block_inline_expr = try self.getBlockSource(file, parent_src, inst_index);
+                    const pl_node = data[inst_index].pl_node;
+                    const block_inline_expr = try self.getBlockSource(file, parent_src, pl_node.src_node);
                     try self.comptime_exprs.append(self.arena, .{
                         .code = block_inline_expr,
                     });
@@ -3341,12 +3351,11 @@ fn analyzeDecltest(
     const value_index = file.zir.extra[d.sub_index + 6];
     const decl_name_index = file.zir.extra[d.sub_index + 7];
 
-    // This is known to work because decl values are always block_inlines
-    const test_source_code = try self.getBlockSource(file, parent_src, value_index);
-
     const value_pl_node = data[value_index].pl_node;
     const decl_src = try self.srcLocInfo(file, value_pl_node.src_node, parent_src);
 
+    const test_source_code = try self.getBlockSource(file, parent_src, value_pl_node.src_node);
+
     const decl_name: ?[]const u8 = if (decl_name_index != 0)
         file.zir.nullTerminatedString(decl_name_index)
     else
@@ -4796,37 +4805,11 @@ fn getBlockSource(
     self: Autodoc,
     file: *File,
     parent_src: SrcLocInfo,
-    value_index: usize,
+    block_src_node: i32,
 ) AutodocErrors![]const u8 {
-    const data = file.zir.instructions.items(.data);
-    const tags = file.zir.instructions.items(.tag);
     const tree = try file.getTree(self.comp_module.gpa);
-
-    // This is known to work because decl values are always block_inlines
-    const block_pl_node = data[value_index].pl_node;
-    const block_src = try self.srcLocInfo(file, block_pl_node.src_node, parent_src);
-
-    const blk_index = idx: {
-        const pl_node = data[value_index].pl_node;
-        const extra = file.zir.extraData(Zir.Inst.Block, pl_node.payload_index);
-        const end_index = file.zir.extra[extra.end..][extra.data.body_len - 1];
-
-        if (tags[end_index] == .break_inline) {
-            break :idx @as(usize, Zir.refToIndex(data[end_index].@"break".operand).?);
-        } else {
-            break :idx null;
-        }
-    };
-
-    const blk_source = src: {
-        if (blk_index) |idx| {
-            const pl_node = data[idx].pl_node;
-            break :src try self.srcLocInfo(file, pl_node.src_node, block_src);
-        } else {
-            break :src block_src;
-        }
-    };
-    return tree.getNodeSource(blk_source.src_node);
+    const block_src = try self.srcLocInfo(file, block_src_node, parent_src);
+    return tree.getNodeSource(block_src.src_node);
 }
 
 fn getTLDocComment(self: *Autodoc, file: *File) ![]const u8 {

From e0c65ce3f2eb9f6f689ec937564471477a02f018 Mon Sep 17 00:00:00 2001
From: Krzysztof Wolicki 
Date: Fri, 5 May 2023 02:46:04 +0200
Subject: [PATCH 080/725] autodoc: main.js exprName cleanup

---
 lib/docs/main.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/docs/main.js b/lib/docs/main.js
index 1e25d0c944..ceffd2abe7 100644
--- a/lib/docs/main.js
+++ b/lib/docs/main.js
@@ -2571,7 +2571,7 @@ const NAV_MODES = {
     } else if (
       resolvedValue.expr.string !== undefined ||
       resolvedValue.expr.call !== undefined ||
-      (resolvedValue.expr.comptimeExpr !== undefined && resolvedValue.typeRef !== null)
+      resolvedValue.expr.comptimeExpr !== undefined
     ) {
       domFnProtoCode.innerHTML =
         'const ' +

From 538c8258c724d31efd631b670d4a7d781009a1e1 Mon Sep 17 00:00:00 2001
From: Krzysztof Wolicki 
Date: Fri, 5 May 2023 12:01:00 +0200
Subject: [PATCH 081/725] autodoc: Remove debug code

---
 lib/docs/main.js | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/lib/docs/main.js b/lib/docs/main.js
index ceffd2abe7..9b650643e9 100644
--- a/lib/docs/main.js
+++ b/lib/docs/main.js
@@ -4850,11 +4850,4 @@ function RadixTree() {
 
 // BUT!
 
-// We want to be able to search "Hash", for example!
-
-function findDeclByName(name) {
-  for (let i = 0; i < zigAnalysis.decls.length; i += 1) {
-    let decl = zigAnalysis.decls[i];
-    if(decl[0] == name) return decl;
-  }
-}
\ No newline at end of file
+// We want to be able to search "Hash", for example!
\ No newline at end of file

From d70853ba39cb1694d91a763ea8492f3aed1494a7 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Fri, 5 May 2023 03:04:44 -0400
Subject: [PATCH 082/725] main: add debug dump-zir command

---
 src/Module.zig | 143 ++++++++++++++++++++++++++-----------------------
 src/main.zig   |  62 +++++++++++++++++++++
 2 files changed, 138 insertions(+), 67 deletions(-)

diff --git a/src/Module.zig b/src/Module.zig
index 93e247f7a3..538c716bc3 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -3671,74 +3671,13 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
                 file.sub_file_path, header.instructions_len,
             });
 
-            var instructions: std.MultiArrayList(Zir.Inst) = .{};
-            defer instructions.deinit(gpa);
-
-            try instructions.setCapacity(gpa, header.instructions_len);
-            instructions.len = header.instructions_len;
-
-            var zir: Zir = .{
-                .instructions = instructions.toOwnedSlice(),
-                .string_bytes = &.{},
-                .extra = &.{},
+            file.zir = loadZirCacheBody(gpa, header, cache_file) catch |err| switch (err) {
+                error.UnexpectedFileSize => {
+                    log.warn("unexpected EOF reading cached ZIR for {s}", .{file.sub_file_path});
+                    break :update;
+                },
+                else => |e| return e,
             };
-            var keep_zir = false;
-            defer if (!keep_zir) zir.deinit(gpa);
-
-            zir.string_bytes = try gpa.alloc(u8, header.string_bytes_len);
-            zir.extra = try gpa.alloc(u32, header.extra_len);
-
-            const safety_buffer = if (data_has_safety_tag)
-                try gpa.alloc([8]u8, header.instructions_len)
-            else
-                undefined;
-            defer if (data_has_safety_tag) gpa.free(safety_buffer);
-
-            const data_ptr = if (data_has_safety_tag)
-                @ptrCast([*]u8, safety_buffer.ptr)
-            else
-                @ptrCast([*]u8, zir.instructions.items(.data).ptr);
-
-            var iovecs = [_]std.os.iovec{
-                .{
-                    .iov_base = @ptrCast([*]u8, zir.instructions.items(.tag).ptr),
-                    .iov_len = header.instructions_len,
-                },
-                .{
-                    .iov_base = data_ptr,
-                    .iov_len = header.instructions_len * 8,
-                },
-                .{
-                    .iov_base = zir.string_bytes.ptr,
-                    .iov_len = header.string_bytes_len,
-                },
-                .{
-                    .iov_base = @ptrCast([*]u8, zir.extra.ptr),
-                    .iov_len = header.extra_len * 4,
-                },
-            };
-            const amt_read = try cache_file.readvAll(&iovecs);
-            const amt_expected = zir.instructions.len * 9 +
-                zir.string_bytes.len +
-                zir.extra.len * 4;
-            if (amt_read != amt_expected) {
-                log.warn("unexpected EOF reading cached ZIR for {s}", .{file.sub_file_path});
-                break :update;
-            }
-            if (data_has_safety_tag) {
-                const tags = zir.instructions.items(.tag);
-                for (zir.instructions.items(.data), 0..) |*data, i| {
-                    const union_tag = Zir.Inst.Tag.data_tags[@enumToInt(tags[i])];
-                    const as_struct = @ptrCast(*HackDataLayout, data);
-                    as_struct.* = .{
-                        .safety_tag = @enumToInt(union_tag),
-                        .data = safety_buffer[i],
-                    };
-                }
-            }
-
-            keep_zir = true;
-            file.zir = zir;
             file.zir_loaded = true;
             file.stat = .{
                 .size = header.stat_size,
@@ -3916,6 +3855,76 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
     }
 }
 
+pub fn loadZirCache(gpa: Allocator, cache_file: std.fs.File) !Zir {
+    return loadZirCacheBody(gpa, try cache_file.reader().readStruct(Zir.Header), cache_file);
+}
+
+fn loadZirCacheBody(gpa: Allocator, header: Zir.Header, cache_file: std.fs.File) !Zir {
+    var instructions: std.MultiArrayList(Zir.Inst) = .{};
+    errdefer instructions.deinit(gpa);
+
+    try instructions.setCapacity(gpa, header.instructions_len);
+    instructions.len = header.instructions_len;
+
+    var zir: Zir = .{
+        .instructions = instructions.toOwnedSlice(),
+        .string_bytes = &.{},
+        .extra = &.{},
+    };
+    errdefer zir.deinit(gpa);
+
+    zir.string_bytes = try gpa.alloc(u8, header.string_bytes_len);
+    zir.extra = try gpa.alloc(u32, header.extra_len);
+
+    const safety_buffer = if (data_has_safety_tag)
+        try gpa.alloc([8]u8, header.instructions_len)
+    else
+        undefined;
+    defer if (data_has_safety_tag) gpa.free(safety_buffer);
+
+    const data_ptr = if (data_has_safety_tag)
+        @ptrCast([*]u8, safety_buffer.ptr)
+    else
+        @ptrCast([*]u8, zir.instructions.items(.data).ptr);
+
+    var iovecs = [_]std.os.iovec{
+        .{
+            .iov_base = @ptrCast([*]u8, zir.instructions.items(.tag).ptr),
+            .iov_len = header.instructions_len,
+        },
+        .{
+            .iov_base = data_ptr,
+            .iov_len = header.instructions_len * 8,
+        },
+        .{
+            .iov_base = zir.string_bytes.ptr,
+            .iov_len = header.string_bytes_len,
+        },
+        .{
+            .iov_base = @ptrCast([*]u8, zir.extra.ptr),
+            .iov_len = header.extra_len * 4,
+        },
+    };
+    const amt_read = try cache_file.readvAll(&iovecs);
+    const amt_expected = zir.instructions.len * 9 +
+        zir.string_bytes.len +
+        zir.extra.len * 4;
+    if (amt_read != amt_expected) return error.UnexpectedFileSize;
+    if (data_has_safety_tag) {
+        const tags = zir.instructions.items(.tag);
+        for (zir.instructions.items(.data), 0..) |*data, i| {
+            const union_tag = Zir.Inst.Tag.data_tags[@enumToInt(tags[i])];
+            const as_struct = @ptrCast(*HackDataLayout, data);
+            as_struct.* = .{
+                .safety_tag = @enumToInt(union_tag),
+                .data = safety_buffer[i],
+            };
+        }
+    }
+
+    return zir;
+}
+
 /// Patch ups:
 /// * Struct.zir_index
 /// * Decl.zir_index
diff --git a/src/main.zig b/src/main.zig
index 45b0cf30b8..a680a5d89e 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -122,6 +122,7 @@ const debug_usage = normal_usage ++
     \\Debug Commands:
     \\
     \\  changelist       Compute mappings from old ZIR to new ZIR
+    \\  dump-zir         Dump a file containing cached ZIR
     \\
 ;
 
@@ -326,6 +327,8 @@ pub fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
         return cmdAstCheck(gpa, arena, cmd_args);
     } else if (debug_extensions_enabled and mem.eql(u8, cmd, "changelist")) {
         return cmdChangelist(gpa, arena, cmd_args);
+    } else if (debug_extensions_enabled and mem.eql(u8, cmd, "dump-zir")) {
+        return cmdDumpZir(gpa, arena, cmd_args);
     } else {
         std.log.info("{s}", .{usage});
         fatal("unknown command: {s}", .{args[1]});
@@ -5579,6 +5582,65 @@ pub fn cmdAstCheck(
     return @import("print_zir.zig").renderAsTextToFile(gpa, &file, io.getStdOut());
 }
 
+/// This is only enabled for debug builds.
+pub fn cmdDumpZir(
+    gpa: Allocator,
+    arena: Allocator,
+    args: []const []const u8,
+) !void {
+    _ = arena;
+    const Zir = @import("Zir.zig");
+
+    const cache_file = args[0];
+
+    var f = fs.cwd().openFile(cache_file, .{}) catch |err| {
+        fatal("unable to open zir cache file for dumping '{s}': {s}", .{ cache_file, @errorName(err) });
+    };
+    defer f.close();
+
+    var file: Module.File = .{
+        .status = .never_loaded,
+        .source_loaded = false,
+        .tree_loaded = false,
+        .zir_loaded = true,
+        .sub_file_path = undefined,
+        .source = undefined,
+        .stat = undefined,
+        .tree = undefined,
+        .zir = try Module.loadZirCache(gpa, f),
+        .pkg = undefined,
+        .root_decl = .none,
+    };
+
+    {
+        const instruction_bytes = file.zir.instructions.len *
+            // Here we don't use @sizeOf(Zir.Inst.Data) because it would include
+            // the debug safety tag but we want to measure release size.
+            (@sizeOf(Zir.Inst.Tag) + 8);
+        const extra_bytes = file.zir.extra.len * @sizeOf(u32);
+        const total_bytes = @sizeOf(Zir) + instruction_bytes + extra_bytes +
+            file.zir.string_bytes.len * @sizeOf(u8);
+        const stdout = io.getStdOut();
+        const fmtIntSizeBin = std.fmt.fmtIntSizeBin;
+        // zig fmt: off
+        try stdout.writer().print(
+            \\# Total ZIR bytes:    {}
+            \\# Instructions:       {d} ({})
+            \\# String Table Bytes: {}
+            \\# Extra Data Items:   {d} ({})
+            \\
+        , .{
+            fmtIntSizeBin(total_bytes),
+            file.zir.instructions.len, fmtIntSizeBin(instruction_bytes),
+            fmtIntSizeBin(file.zir.string_bytes.len),
+            file.zir.extra.len, fmtIntSizeBin(extra_bytes),
+        });
+        // zig fmt: on
+    }
+
+    return @import("print_zir.zig").renderAsTextToFile(gpa, &file, io.getStdOut());
+}
+
 /// This is only enabled for debug builds.
 pub fn cmdChangelist(
     gpa: Allocator,

From da878dc0776b56c1e2c251857a40415a35e9535a Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Fri, 5 May 2023 14:56:10 -0400
Subject: [PATCH 083/725] AstGen: fix branch on undefined

`isAlwaysVoid` was being called with the undefined tag added by
`addOne`, causing non-deterministic behavior failures with release
builds of the compiler.  Prevents the following random failure:

    test/behavior/defer.zig:120:40: error: expected type 'error{One}', found 'void'
---
 src/AstGen.zig | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/AstGen.zig b/src/AstGen.zig
index 2568b89980..000151a2cc 100644
--- a/src/AstGen.zig
+++ b/src/AstGen.zig
@@ -2905,7 +2905,10 @@ fn deferStmt(
     const sub_scope = if (!have_err_code) &defer_gen.base else blk: {
         try gz.addDbgBlockBegin();
         const ident_name = try gz.astgen.identAsString(payload_token);
-        remapped_err_code = @intCast(u32, try gz.astgen.instructions.addOne(gz.astgen.gpa));
+        remapped_err_code = @intCast(Zir.Inst.Index, gz.astgen.instructions.len);
+        // Use a placeholder tag of .as to allow querying things that depend on the tag,
+        // but undefined data to prevent querying of data.bin.
+        try gz.astgen.instructions.append(gz.astgen.gpa, .{ .tag = .as, .data = undefined });
         const remapped_err_code_ref = Zir.indexToRef(remapped_err_code);
         local_val_scope = .{
             .parent = &defer_gen.base,

From df4849c4f5aafc7eb80fd7b2299b49162fa2aa79 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Fri, 5 May 2023 16:07:27 -0400
Subject: [PATCH 084/725] AstGen: cleanup previous fix

Allocating an extended tag is much cleaner and easier to reason about
than reusing an existing tag.  The previous `.data = undefined` was a
clear indication that we don't have any data to store, and so might as
well store an extended tag in that space almost for free.
---
 src/AstGen.zig    | 11 ++++++++---
 src/Sema.zig      |  1 +
 src/Zir.zig       |  7 +++++--
 src/print_zir.zig |  1 +
 4 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/AstGen.zig b/src/AstGen.zig
index 000151a2cc..749e3d28c4 100644
--- a/src/AstGen.zig
+++ b/src/AstGen.zig
@@ -2906,9 +2906,14 @@ fn deferStmt(
         try gz.addDbgBlockBegin();
         const ident_name = try gz.astgen.identAsString(payload_token);
         remapped_err_code = @intCast(Zir.Inst.Index, gz.astgen.instructions.len);
-        // Use a placeholder tag of .as to allow querying things that depend on the tag,
-        // but undefined data to prevent querying of data.bin.
-        try gz.astgen.instructions.append(gz.astgen.gpa, .{ .tag = .as, .data = undefined });
+        try gz.astgen.instructions.append(gz.astgen.gpa, .{
+            .tag = .extended,
+            .data = .{ .extended = .{
+                .opcode = .errdefer_err_code,
+                .small = undefined,
+                .operand = undefined,
+            } },
+        });
         const remapped_err_code_ref = Zir.indexToRef(remapped_err_code);
         local_val_scope = .{
             .parent = &defer_gen.base,
diff --git a/src/Sema.zig b/src/Sema.zig
index 13f4d684a1..b2387dfe12 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -1198,6 +1198,7 @@ fn analyzeBodyInner(
                         i += 1;
                         continue;
                     },
+                    .errdefer_err_code => unreachable, // never appears in a body
                 };
             },
 
diff --git a/src/Zir.zig b/src/Zir.zig
index 51c90c61cb..7f9681e533 100644
--- a/src/Zir.zig
+++ b/src/Zir.zig
@@ -1979,7 +1979,7 @@ pub const Inst = struct {
         /// `operand` is `src_node: i32`.
         breakpoint,
         /// Implements the `@select` builtin.
-        /// operand` is payload index to `Select`.
+        /// `operand` is payload index to `Select`.
         select,
         /// Implement builtin `@errToInt`.
         /// `operand` is payload index to `UnNode`.
@@ -1999,7 +1999,7 @@ pub const Inst = struct {
         /// `operand` is payload index to `Cmpxchg`.
         cmpxchg,
         /// Implement the builtin `@addrSpaceCast`
-        /// `Operand` is payload index to `BinNode`. `lhs` is dest type, `rhs` is operand.
+        /// `operand` is payload index to `BinNode`. `lhs` is dest type, `rhs` is operand.
         addrspace_cast,
         /// Implement builtin `@cVaArg`.
         /// `operand` is payload index to `BinNode`.
@@ -2031,6 +2031,9 @@ pub const Inst = struct {
         /// Implements the `@inComptime` builtin.
         /// `operand` is `src_node: i32`.
         in_comptime,
+        /// Used as a placeholder for the capture of an `errdefer`.
+        /// This is replaced by Sema with the captured value.
+        errdefer_err_code,
 
         pub const InstData = struct {
             opcode: Extended,
diff --git a/src/print_zir.zig b/src/print_zir.zig
index 4e4f13641c..f228adc7ea 100644
--- a/src/print_zir.zig
+++ b/src/print_zir.zig
@@ -467,6 +467,7 @@ const Writer = struct {
             .breakpoint,
             .c_va_start,
             .in_comptime,
+            .errdefer_err_code,
             => try self.writeExtNode(stream, extended),
 
             .builtin_src => {

From 4bfd37ddb48af4dbfd2e892aec7a90a822c8cbe7 Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Fri, 5 May 2023 17:39:44 +0100
Subject: [PATCH 085/725] std.c: adding cpu affinity api for macOs (mainly
 x86_64)

---
 lib/std/c/darwin.zig | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/lib/std/c/darwin.zig b/lib/std/c/darwin.zig
index ab352844db..7879eddc4b 100644
--- a/lib/std/c/darwin.zig
+++ b/lib/std/c/darwin.zig
@@ -3896,3 +3896,23 @@ pub const MIN = struct {
 
 pub extern "c" fn mincore(addr: *align(std.mem.page_size) const anyopaque, length: usize, vec: [*]u8) c_int;
 pub extern "c" fn os_proc_available_memory() usize;
+
+pub const thread_affinity_policy = extern struct {
+    tag: integer_t,
+};
+
+pub const thread_policy_flavor_t = natural_t;
+pub const thread_policy_t = [*]integer_t;
+pub const thread_affinity_policy_data_t = thread_affinity_policy;
+pub const thread_affinity_policy_t = [*]thread_affinity_policy;
+
+pub const THREAD_AFFINITY = struct {
+    pub const POLICY = 0;
+    pub const POLICY_COUNT = @intCast(mach_msg_type_number_t, @sizeOf(thread_affinity_policy_data_t) / @sizeOf(integer_t));
+};
+
+/// cpu affinity api
+/// albeit it is also available on arm64, it always fails as in this architecture there is no sense of
+/// individual cpus (high performance cpus group and low consumption one), thus the pthread QOS api is more appropriate in this case.
+pub extern "c" fn thread_affinity_get(thread: thread_act_t, flavor: thread_policy_flavor_t, info: thread_policy_t, infocnt: [*]mach_msg_type_number_t, default: *boolean_t) kern_return_t;
+pub extern "c" fn thread_affinity_set(thread: thread_act_t, flavor: thread_policy_flavor_t, info: thread_policy_t, infocnt: mach_msg_type_number_t) kern_return_t;

From c40a1ee224a0f253578b170ab97661b2dbf5119d Mon Sep 17 00:00:00 2001
From: Krzysztof Wolicki 
Date: Sat, 6 May 2023 02:51:53 +0200
Subject: [PATCH 086/725] autodoc: Add gathering of top-level doc comments for
 imported files

---
 src/Autodoc.zig | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/Autodoc.zig b/src/Autodoc.zig
index 42813f7614..00fef3c768 100644
--- a/src/Autodoc.zig
+++ b/src/Autodoc.zig
@@ -996,6 +996,12 @@ fn walkInstruction(
                 };
             }
 
+            const maybe_tldoc_comment = try self.getTLDocComment(new_file.file);
+            try self.ast_nodes.append(self.arena, .{
+                .name = path,
+                .docs = maybe_tldoc_comment,
+            });
+
             result.value_ptr.* = self.types.items.len;
 
             var new_scope = Scope{

From 743976ef48f5efa89ada76dc5b8cea08a4626035 Mon Sep 17 00:00:00 2001
From: r00ster91 
Date: Sat, 29 Apr 2023 16:00:22 +0200
Subject: [PATCH 087/725] Sema: add some missing apostrophes to error messages

---
 src/Sema.zig                                                | 6 +++---
 test/cases/compile_errors/compile_time_null_ptr_cast.zig    | 2 +-
 .../compile_errors/incorrect_type_to_memset_memcpy.zig      | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/Sema.zig b/src/Sema.zig
index b2387dfe12..0288ff8132 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -19910,7 +19910,7 @@ fn zirPtrCast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air
             return sema.failWithUseOfUndef(block, operand_src);
         }
         if (!dest_ty.ptrAllowsZero() and operand_val.isNull()) {
-            return sema.fail(block, operand_src, "null pointer casted to type {}", .{dest_ty.fmt(sema.mod)});
+            return sema.fail(block, operand_src, "null pointer casted to type '{}'", .{dest_ty.fmt(sema.mod)});
         }
         if (dest_ty.zigTypeTag() == .Optional and sema.typeOf(ptr).zigTypeTag() != .Optional) {
             return sema.addConstant(dest_ty, try Value.Tag.opt_payload.create(sema.arena, operand_val));
@@ -22013,10 +22013,10 @@ fn zirMemcpy(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void
         const msg = msg: {
             const msg = try sema.errMsg(block, src, "unknown @memcpy length", .{});
             errdefer msg.destroy(sema.gpa);
-            try sema.errNote(block, dest_src, msg, "destination type {} provides no length", .{
+            try sema.errNote(block, dest_src, msg, "destination type '{}' provides no length", .{
                 sema.typeOf(dest_ptr).fmt(sema.mod),
             });
-            try sema.errNote(block, src_src, msg, "source type {} provides no length", .{
+            try sema.errNote(block, src_src, msg, "source type '{}' provides no length", .{
                 sema.typeOf(src_ptr).fmt(sema.mod),
             });
             break :msg msg;
diff --git a/test/cases/compile_errors/compile_time_null_ptr_cast.zig b/test/cases/compile_errors/compile_time_null_ptr_cast.zig
index d3750c8654..25805e9f35 100644
--- a/test/cases/compile_errors/compile_time_null_ptr_cast.zig
+++ b/test/cases/compile_errors/compile_time_null_ptr_cast.zig
@@ -8,4 +8,4 @@ comptime {
 // backend=llvm
 // target=native
 //
-// :3:32: error: null pointer casted to type *i32
+// :3:32: error: null pointer casted to type '*i32'
diff --git a/test/cases/compile_errors/incorrect_type_to_memset_memcpy.zig b/test/cases/compile_errors/incorrect_type_to_memset_memcpy.zig
index 910ec807a8..9243e414a4 100644
--- a/test/cases/compile_errors/incorrect_type_to_memset_memcpy.zig
+++ b/test/cases/compile_errors/incorrect_type_to_memset_memcpy.zig
@@ -25,8 +25,8 @@ pub export fn non_matching_lengths() void {
 // target=native
 //
 // :5:5: error: unknown @memcpy length
-// :5:18: note: destination type [*]u8 provides no length
-// :5:24: note: source type [*]align(4) const u8 provides no length
+// :5:18: note: destination type '[*]u8' provides no length
+// :5:24: note: source type '[*]align(4) const u8' provides no length
 // :10:13: error: type 'u8' does not support indexing
 // :10:13: note: for loop operand must be an array, slice, tuple, or vector
 // :15:13: error: type '*u8' does not support indexing

From 3485a0e7fbff913e6c934fbc843a02ac08f024f2 Mon Sep 17 00:00:00 2001
From: r00ster91 
Date: Sat, 29 Apr 2023 17:07:54 +0200
Subject: [PATCH 088/725] Sema: fix and improve errors for `for` loop objects
 and non-indexables

Operands to @memcpy and @memset were being called "for loop operands" in
the error.
---
 src/Sema.zig                                  | 54 ++++++++-----------
 .../array_access_of_non_array.zig             | 10 ++--
 .../compile_errors/array_access_of_type.zig   |  3 +-
 test/cases/compile_errors/for.zig             |  4 +-
 .../incorrect_type_to_memset_memcpy.zig       |  4 +-
 .../indexing_non-tuple_struct.zig             |  3 +-
 .../indexing_single-item_pointer.zig          |  3 +-
 7 files changed, 39 insertions(+), 42 deletions(-)

diff --git a/src/Sema.zig b/src/Sema.zig
index 0288ff8132..36538ba8f8 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -3401,8 +3401,8 @@ fn indexablePtrLen(
 ) CompileError!Air.Inst.Ref {
     const object_ty = sema.typeOf(object);
     const is_pointer_to = object_ty.isSinglePointer();
-    const array_ty = if (is_pointer_to) object_ty.childType() else object_ty;
-    try checkIndexable(sema, block, src, array_ty);
+    const indexable_ty = if (is_pointer_to) object_ty.childType() else object_ty;
+    try checkIndexable(sema, block, src, indexable_ty);
     return sema.fieldVal(block, src, object, "len", src);
 }
 
@@ -3413,7 +3413,7 @@ fn indexablePtrLenOrNone(
     object: Air.Inst.Ref,
 ) CompileError!Air.Inst.Ref {
     const object_ty = sema.typeOf(object);
-    const array_ty = t: {
+    const indexable_ty = t: {
         const ptr_size = object_ty.ptrSizeOrNull() orelse break :t object_ty;
         break :t switch (ptr_size) {
             .Many => return .none,
@@ -3421,7 +3421,7 @@ fn indexablePtrLenOrNone(
             else => object_ty,
         };
     };
-    try checkIndexable(sema, block, src, array_ty);
+    try checkIndexable(sema, block, src, indexable_ty);
     return sema.fieldVal(block, src, object, "len", src);
 }
 
@@ -3991,7 +3991,16 @@ fn zirForLen(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.
             .input_index = i,
         } };
         const arg_len_uncoerced = if (is_int) object else l: {
-            try checkIndexable(sema, block, arg_src, object_ty);
+            if (!object_ty.isIndexable()) {
+                // Instead of using checkIndexable we customize this error.
+                const msg = msg: {
+                    const msg = try sema.errMsg(block, arg_src, "type '{}' is not indexable and not a range", .{object_ty.fmt(sema.mod)});
+                    errdefer msg.destroy(sema.gpa);
+                    try sema.errNote(block, arg_src, msg, "for loop operand must be a range, array, slice, tuple, or vector", .{});
+                    break :msg msg;
+                };
+                return sema.failWithOwnedErrorMsg(msg);
+            }
             if (!object_ty.indexableHasLen()) continue;
 
             break :l try sema.fieldVal(block, arg_src, object, "len", arg_src);
@@ -24767,9 +24776,7 @@ fn elemPtr(
         .Pointer => indexable_ptr_ty.elemType(),
         else => return sema.fail(block, indexable_ptr_src, "expected pointer, found '{}'", .{indexable_ptr_ty.fmt(sema.mod)}),
     };
-    if (!indexable_ty.isIndexable()) {
-        return sema.fail(block, src, "element access of non-indexable type '{}'", .{indexable_ty.fmt(sema.mod)});
-    }
+    try checkIndexable(sema, block, src, indexable_ty);
 
     switch (indexable_ty.zigTypeTag()) {
         .Array, .Vector => return sema.elemPtrArray(block, src, indexable_ptr_src, indexable_ptr, elem_index_src, elem_index, init, oob_safety),
@@ -24801,9 +24808,7 @@ fn elemPtrOneLayerOnly(
     const indexable_ty = sema.typeOf(indexable);
     const target = sema.mod.getTarget();
 
-    if (!indexable_ty.isIndexable()) {
-        return sema.fail(block, src, "element access of non-indexable type '{}'", .{indexable_ty.fmt(sema.mod)});
-    }
+    try checkIndexable(sema, block, src, indexable_ty);
 
     switch (indexable_ty.ptrSize()) {
         .Slice => return sema.elemPtrSlice(block, src, indexable_src, indexable, elem_index_src, elem_index, oob_safety),
@@ -24824,7 +24829,7 @@ fn elemPtrOneLayerOnly(
             return block.addPtrElemPtr(indexable, elem_index, result_ty);
         },
         .One => {
-            assert(indexable_ty.childType().zigTypeTag() == .Array); // Guaranteed by isIndexable
+            assert(indexable_ty.childType().zigTypeTag() == .Array); // Guaranteed by checkIndexable
             return sema.elemPtrArray(block, src, indexable_src, indexable, elem_index_src, elem_index, init, oob_safety);
         },
     }
@@ -24843,9 +24848,7 @@ fn elemVal(
     const indexable_ty = sema.typeOf(indexable);
     const target = sema.mod.getTarget();
 
-    if (!indexable_ty.isIndexable()) {
-        return sema.fail(block, src, "element access of non-indexable type '{}'", .{indexable_ty.fmt(sema.mod)});
-    }
+    try checkIndexable(sema, block, src, indexable_ty);
 
     // TODO in case of a vector of pointers, we need to detect whether the element
     // index is a scalar or vector instead of unconditionally casting to usize.
@@ -24873,7 +24876,7 @@ fn elemVal(
                 return block.addBinOp(.ptr_elem_val, indexable, elem_index);
             },
             .One => {
-                assert(indexable_ty.childType().zigTypeTag() == .Array); // Guaranteed by isIndexable
+                assert(indexable_ty.childType().zigTypeTag() == .Array); // Guaranteed by checkIndexable
                 const elem_ptr = try sema.elemPtr(block, indexable_src, indexable, elem_index, elem_index_src, false, oob_safety);
                 return sema.analyzeLoad(block, indexable_src, elem_ptr, elem_index_src);
             },
@@ -30997,23 +31000,12 @@ fn checkBackingIntType(sema: *Sema, block: *Block, src: LazySrcLoc, backing_int_
     }
 }
 
-fn checkIndexable(sema: *Sema, block: *Block, src: LazySrcLoc, array_ty: Type) !void {
-    if (!array_ty.isIndexable()) {
+fn checkIndexable(sema: *Sema, block: *Block, src: LazySrcLoc, ty: Type) !void {
+    if (!ty.isIndexable()) {
         const msg = msg: {
-            const msg = try sema.errMsg(
-                block,
-                src,
-                "type '{}' does not support indexing",
-                .{array_ty.fmt(sema.mod)},
-            );
+            const msg = try sema.errMsg(block, src, "type '{}' does not support indexing", .{ty.fmt(sema.mod)});
             errdefer msg.destroy(sema.gpa);
-            try sema.errNote(
-                block,
-                src,
-                msg,
-                "for loop operand must be an array, slice, tuple, or vector",
-                .{},
-            );
+            try sema.errNote(block, src, msg, "operand must be an array, slice, tuple, or vector", .{});
             break :msg msg;
         };
         return sema.failWithOwnedErrorMsg(msg);
diff --git a/test/cases/compile_errors/array_access_of_non_array.zig b/test/cases/compile_errors/array_access_of_non_array.zig
index 06fa1569e6..0f06bae1dd 100644
--- a/test/cases/compile_errors/array_access_of_non_array.zig
+++ b/test/cases/compile_errors/array_access_of_non_array.zig
@@ -1,9 +1,9 @@
 export fn f() void {
-    var bad : bool = undefined;
+    var bad: bool = undefined;
     bad[0] = bad[0];
 }
 export fn g() void {
-    var bad : bool = undefined;
+    var bad: bool = undefined;
     _ = bad[0];
 }
 
@@ -11,5 +11,7 @@ export fn g() void {
 // backend=stage2
 // target=native
 //
-// :3:8: error: element access of non-indexable type 'bool'
-// :7:12: error: element access of non-indexable type 'bool'
+// :3:8: error: type 'bool' does not support indexing
+// :3:8: note: operand must be an array, slice, tuple, or vector
+// :7:12: error: type 'bool' does not support indexing
+// :7:12: note: operand must be an array, slice, tuple, or vector
diff --git a/test/cases/compile_errors/array_access_of_type.zig b/test/cases/compile_errors/array_access_of_type.zig
index d38a13b2fa..1e66ca3776 100644
--- a/test/cases/compile_errors/array_access_of_type.zig
+++ b/test/cases/compile_errors/array_access_of_type.zig
@@ -7,4 +7,5 @@ export fn foo() void {
 // backend=stage2
 // target=native
 //
-// :2:14: error: element access of non-indexable type 'type'
+// :2:14: error: type 'type' does not support indexing
+// :2:14: note: operand must be an array, slice, tuple, or vector
diff --git a/test/cases/compile_errors/for.zig b/test/cases/compile_errors/for.zig
index 5bd3aa0c64..435bb68607 100644
--- a/test/cases/compile_errors/for.zig
+++ b/test/cases/compile_errors/for.zig
@@ -31,8 +31,8 @@ export fn d() void {
 // :2:5: error: non-matching for loop lengths
 // :2:11: note: length 10 here
 // :2:19: note: length 11 here
-// :9:14: error: type 'bool' does not support indexing
-// :9:14: note: for loop operand must be an array, slice, tuple, or vector
+// :9:14: error: type 'bool' is not indexable and not a range
+// :9:14: note: for loop operand must be a range, array, slice, tuple, or vector
 // :15:16: error: pointer capture of non pointer type '[10]u8'
 // :15:10: note: consider using '&' here
 // :22:5: error: unbounded for loop
diff --git a/test/cases/compile_errors/incorrect_type_to_memset_memcpy.zig b/test/cases/compile_errors/incorrect_type_to_memset_memcpy.zig
index 9243e414a4..2a4990b1ec 100644
--- a/test/cases/compile_errors/incorrect_type_to_memset_memcpy.zig
+++ b/test/cases/compile_errors/incorrect_type_to_memset_memcpy.zig
@@ -28,9 +28,9 @@ pub export fn non_matching_lengths() void {
 // :5:18: note: destination type '[*]u8' provides no length
 // :5:24: note: source type '[*]align(4) const u8' provides no length
 // :10:13: error: type 'u8' does not support indexing
-// :10:13: note: for loop operand must be an array, slice, tuple, or vector
+// :10:13: note: operand must be an array, slice, tuple, or vector
 // :15:13: error: type '*u8' does not support indexing
-// :15:13: note: for loop operand must be an array, slice, tuple, or vector
+// :15:13: note: operand must be an array, slice, tuple, or vector
 // :20:5: error: non-matching @memcpy lengths
 // :20:13: note: length 6 here
 // :20:20: note: length 5 here
diff --git a/test/cases/compile_errors/indexing_non-tuple_struct.zig b/test/cases/compile_errors/indexing_non-tuple_struct.zig
index 912ee689d1..1691faaae0 100644
--- a/test/cases/compile_errors/indexing_non-tuple_struct.zig
+++ b/test/cases/compile_errors/indexing_non-tuple_struct.zig
@@ -8,4 +8,5 @@ export fn a() void {
 // backend=stage2
 // target=native
 //
-// :4:6: error: element access of non-indexable type 'tmp.a.S'
+// :4:6: error: type 'tmp.a.S' does not support indexing
+// :4:6: note: operand must be an array, slice, tuple, or vector
diff --git a/test/cases/compile_errors/indexing_single-item_pointer.zig b/test/cases/compile_errors/indexing_single-item_pointer.zig
index efd063817c..41b94c8f26 100644
--- a/test/cases/compile_errors/indexing_single-item_pointer.zig
+++ b/test/cases/compile_errors/indexing_single-item_pointer.zig
@@ -6,4 +6,5 @@ export fn entry(ptr: *i32) i32 {
 // backend=stage2
 // target=native
 //
-// :2:15: error: element access of non-indexable type '*i32'
+// :2:15: error: type '*i32' does not support indexing
+// :2:15: note: operand must be an array, slice, tuple, or vector

From 9dde710ea15f53939c3819fed58d3b1cddefd508 Mon Sep 17 00:00:00 2001
From: r00ster91 
Date: Sat, 29 Apr 2023 18:27:48 +0200
Subject: [PATCH 089/725] Sema: fix @extern empty lib name error message

---
 src/Sema.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Sema.zig b/src/Sema.zig
index 36538ba8f8..225a6c5bff 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -22755,7 +22755,7 @@ fn resolveExternOptions(
         const payload = library_name_val.castTag(.opt_payload).?.data;
         const library_name = try payload.toAllocatedBytes(Type.initTag(.const_slice_u8), sema.arena, mod);
         if (library_name.len == 0) {
-            return sema.fail(block, library_src, "library name name cannot be empty", .{});
+            return sema.fail(block, library_src, "library name cannot be empty", .{});
         }
         break :blk try sema.handleExternLibName(block, library_src, library_name);
     } else null;

From d71a43ec2c28a53a3e1d9bcb538707eca00a6fc0 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sat, 6 May 2023 06:41:59 -0400
Subject: [PATCH 090/725] cbe: fix typos

---
 src/codegen/c.zig | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/codegen/c.zig b/src/codegen/c.zig
index f69cec960e..7da99de5c1 100644
--- a/src/codegen/c.zig
+++ b/src/codegen/c.zig
@@ -159,7 +159,7 @@ const reserved_idents = std.ComptimeStringMap(void, .{
     .{ "double", {} },
     .{ "else", {} },
     .{ "enum", {} },
-    .{ "extern ", {} },
+    .{ "extern", {} },
     .{ "float", {} },
     .{ "for", {} },
     .{ "fortran", {} },
@@ -198,7 +198,7 @@ const reserved_idents = std.ComptimeStringMap(void, .{
     .{ "unsigned", {} },
     .{ "void", {} },
     .{ "volatile", {} },
-    .{ "while ", {} },
+    .{ "while", {} },
 
     // stdarg.h
     .{ "va_start", {} },

From 71c228fe6572b9f3b30e82035bf8fd7e2b1dd29d Mon Sep 17 00:00:00 2001
From: Nameless 
Date: Tue, 25 Apr 2023 09:22:20 -0500
Subject: [PATCH 091/725] std.http: add simple standalone http tests, add state
 check for http server

---
 lib/std/Uri.zig          |   9 +-
 lib/std/http.zig         |   1 -
 lib/std/http/Client.zig  |  16 +-
 lib/std/http/Headers.zig |  11 -
 lib/std/http/Server.zig  | 135 ++++++++----
 lib/std/http/test.zig    |  72 -------
 test/standalone.zig      |   4 +
 test/standalone/http.zig | 432 +++++++++++++++++++++++++++++++++++++++
 8 files changed, 546 insertions(+), 134 deletions(-)
 delete mode 100644 lib/std/http/test.zig
 create mode 100644 test/standalone/http.zig

diff --git a/lib/std/Uri.zig b/lib/std/Uri.zig
index b0bb3047cb..2cf0f7a46b 100644
--- a/lib/std/Uri.zig
+++ b/lib/std/Uri.zig
@@ -216,6 +216,7 @@ pub fn format(
 
     const needs_absolute = comptime std.mem.indexOf(u8, fmt, "+") != null;
     const needs_path = comptime std.mem.indexOf(u8, fmt, "/") != null or fmt.len == 0;
+    const needs_fragment = comptime std.mem.indexOf(u8, fmt, "#") != null;
 
     if (needs_absolute) {
         try writer.writeAll(uri.scheme);
@@ -253,9 +254,11 @@ pub fn format(
             try Uri.writeEscapedQuery(writer, q);
         }
 
-        if (uri.fragment) |f| {
-            try writer.writeAll("#");
-            try Uri.writeEscapedQuery(writer, f);
+        if (needs_fragment) {
+            if (uri.fragment) |f| {
+                try writer.writeAll("#");
+                try Uri.writeEscapedQuery(writer, f);
+            }
         }
     }
 }
diff --git a/lib/std/http.zig b/lib/std/http.zig
index 744615d7d7..e9c62705b5 100644
--- a/lib/std/http.zig
+++ b/lib/std/http.zig
@@ -275,5 +275,4 @@ test {
     _ = Client;
     _ = Method;
     _ = Status;
-    _ = @import("http/test.zig");
 }
diff --git a/lib/std/http/Client.zig b/lib/std/http/Client.zig
index 42ef766bd3..74321492c8 100644
--- a/lib/std/http/Client.zig
+++ b/lib/std/http/Client.zig
@@ -264,7 +264,7 @@ pub const BufferedConnection = struct {
         const nread = try bconn.conn.read(bconn.buf[0..]);
         if (nread == 0) return error.EndOfStream;
         bconn.start = 0;
-        bconn.end = @truncate(u16, nread);
+        bconn.end = @intCast(u16, nread);
     }
 
     pub fn peek(bconn: *BufferedConnection) []const u8 {
@@ -282,7 +282,7 @@ pub const BufferedConnection = struct {
             const left = buffer.len - out_index;
 
             if (available > 0) {
-                const can_read = @truncate(u16, @min(available, left));
+                const can_read = @intCast(u16, @min(available, left));
 
                 @memcpy(buffer[out_index..][0..can_read], bconn.buf[bconn.start..][0..can_read]);
                 out_index += can_read;
@@ -355,8 +355,6 @@ pub const Compression = union(enum) {
 /// A HTTP response originating from a server.
 pub const Response = struct {
     pub const ParseError = Allocator.Error || error{
-        ShortHttpStatusLine,
-        BadHttpVersion,
         HttpHeadersInvalid,
         HttpHeaderContinuationsUnsupported,
         HttpTransferEncodingUnsupported,
@@ -370,12 +368,12 @@ pub const Response = struct {
 
         const first_line = it.next() orelse return error.HttpHeadersInvalid;
         if (first_line.len < 12)
-            return error.ShortHttpStatusLine;
+            return error.HttpHeadersInvalid;
 
         const version: http.Version = switch (int64(first_line[0..8])) {
             int64("HTTP/1.0") => .@"HTTP/1.0",
             int64("HTTP/1.1") => .@"HTTP/1.1",
-            else => return error.BadHttpVersion,
+            else => return error.HttpHeadersInvalid,
         };
         if (first_line[8] != ' ') return error.HttpHeadersInvalid;
         const status = @intToEnum(http.Status, parseInt3(first_line[9..12].*));
@@ -695,7 +693,6 @@ pub const Request = struct {
 
             if (req.method == .CONNECT and req.response.status == .ok) {
                 req.connection.data.closing = false;
-                req.connection.data.proxied = true;
                 req.response.parser.done = true;
             }
 
@@ -725,6 +722,11 @@ pub const Request = struct {
                 req.response.parser.done = true;
             }
 
+            // HEAD requests have no body
+            if (req.method == .HEAD) {
+                req.response.parser.done = true;
+            }
+
             if (req.transfer_encoding == .none and req.response.status.class() == .redirect and req.handle_redirects) {
                 req.response.skip = true;
 
diff --git a/lib/std/http/Headers.zig b/lib/std/http/Headers.zig
index 376fd60b61..429df9368a 100644
--- a/lib/std/http/Headers.zig
+++ b/lib/std/http/Headers.zig
@@ -36,17 +36,6 @@ pub const Field = struct {
     name: []const u8,
     value: []const u8,
 
-    pub fn modify(entry: *Field, allocator: Allocator, new_value: []const u8) !void {
-        if (entry.value.len <= new_value.len) {
-            // TODO: eliminate this use of `@constCast`.
-            @memcpy(@constCast(entry.value)[0..new_value.len], new_value);
-        } else {
-            allocator.free(entry.value);
-
-            entry.value = try allocator.dupe(u8, new_value);
-        }
-    }
-
     fn lessThan(ctx: void, a: Field, b: Field) bool {
         _ = ctx;
         if (a.name.ptr == b.name.ptr) return false;
diff --git a/lib/std/http/Server.zig b/lib/std/http/Server.zig
index c7f2a86c27..1b5fd045fa 100644
--- a/lib/std/http/Server.zig
+++ b/lib/std/http/Server.zig
@@ -108,7 +108,7 @@ pub const BufferedConnection = struct {
         const nread = try bconn.conn.read(bconn.buf[0..]);
         if (nread == 0) return error.EndOfStream;
         bconn.start = 0;
-        bconn.end = @truncate(u16, nread);
+        bconn.end = @intCast(u16, nread);
     }
 
     pub fn peek(bconn: *BufferedConnection) []const u8 {
@@ -126,7 +126,7 @@ pub const BufferedConnection = struct {
             const left = buffer.len - out_index;
 
             if (available > 0) {
-                const can_read = @truncate(u16, @min(available, left));
+                const can_read = @intCast(u16, @min(available, left));
 
                 @memcpy(buffer[out_index..][0..can_read], bconn.buf[bconn.start..][0..can_read]);
                 out_index += can_read;
@@ -199,8 +199,6 @@ pub const Compression = union(enum) {
 /// A HTTP request originating from a client.
 pub const Request = struct {
     pub const ParseError = Allocator.Error || error{
-        ShortHttpStatusLine,
-        BadHttpVersion,
         UnknownHttpMethod,
         HttpHeadersInvalid,
         HttpHeaderContinuationsUnsupported,
@@ -215,7 +213,7 @@ pub const Request = struct {
 
         const first_line = it.next() orelse return error.HttpHeadersInvalid;
         if (first_line.len < 10)
-            return error.ShortHttpStatusLine;
+            return error.HttpHeadersInvalid;
 
         const method_end = mem.indexOfScalar(u8, first_line, ' ') orelse return error.HttpHeadersInvalid;
         const method_str = first_line[0..method_end];
@@ -229,7 +227,7 @@ pub const Request = struct {
         const version: http.Version = switch (int64(version_str[0..8])) {
             int64("HTTP/1.0") => .@"HTTP/1.0",
             int64("HTTP/1.1") => .@"HTTP/1.1",
-            else => return error.BadHttpVersion,
+            else => return error.HttpHeadersInvalid,
         };
 
         const target = first_line[method_end + 1 .. version_start];
@@ -312,7 +310,7 @@ pub const Request = struct {
     transfer_encoding: ?http.TransferEncoding = null,
     transfer_compression: ?http.ContentEncoding = null,
 
-    headers: http.Headers = undefined,
+    headers: http.Headers,
     parser: proto.HeadersParser,
     compression: Compression = .none,
 };
@@ -336,14 +334,52 @@ pub const Response = struct {
     headers: http.Headers,
     request: Request,
 
+    state: State = .first,
+
+    const State = enum {
+        first,
+        start,
+        waited,
+        responded,
+        finished,
+    };
+
     pub fn deinit(res: *Response) void {
-        res.server.allocator.destroy(res);
+        res.connection.close();
+
+        res.headers.deinit();
+        res.request.headers.deinit();
+
+        if (res.request.parser.header_bytes_owned) {
+            res.request.parser.header_bytes.deinit(res.server.allocator);
+        }
     }
 
     /// Reset this response to its initial state. This must be called before handling a second request on the same connection.
-    pub fn reset(res: *Response) void {
-        res.request.headers.deinit();
-        res.headers.deinit();
+    pub fn reset(res: *Response) bool {
+        if (res.state == .first) {
+            res.state = .start;
+            return true;
+        }
+
+        if (!res.request.parser.done) {
+            // If the response wasn't fully read, then we need to close the connection.
+            res.connection.conn.closing = true;
+            return false;
+        }
+
+        // A connection is only keep-alive if the Connection header is present and it's value is not "close".
+        // The server and client must both agree
+        const res_connection = res.headers.getFirstValue("connection");
+        const res_keepalive = res_connection != null and !std.ascii.eqlIgnoreCase("close", res_connection.?);
+
+        const req_connection = res.request.headers.getFirstValue("connection");
+        const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
+        if (res_keepalive and req_keepalive) {
+            res.connection.conn.closing = false;
+        } else {
+            res.connection.conn.closing = true;
+        }
 
         switch (res.request.compression) {
             .none => {},
@@ -352,26 +388,38 @@ pub const Response = struct {
             .zstd => |*zstd| zstd.deinit(),
         }
 
-        if (!res.request.parser.done) {
-            // If the response wasn't fully read, then we need to close the connection.
-            res.connection.conn.closing = true;
-        }
+        res.state = .start;
+        res.version = .@"HTTP/1.1";
+        res.status = .ok;
+        res.reason = null;
 
-        if (res.connection.conn.closing) {
-            res.connection.close();
+        res.transfer_encoding = .none;
 
-            if (res.request.parser.header_bytes_owned) {
-                res.request.parser.header_bytes.deinit(res.server.allocator);
-            }
-        } else {
-            res.request.parser.reset();
-        }
+        res.headers.clearRetainingCapacity();
+
+        res.request.headers.clearRetainingCapacity();
+        res.request.parser.reset();
+
+        res.request = Request{
+            .version = undefined,
+            .method = undefined,
+            .target = undefined,
+            .headers = res.request.headers,
+            .parser = res.request.parser,
+        };
+
+        return !res.connection.conn.closing;
     }
 
     pub const DoError = BufferedConnection.WriteError || error{ UnsupportedTransferEncoding, InvalidContentLength };
 
     /// Send the response headers.
     pub fn do(res: *Response) !void {
+        switch (res.state) {
+            .waited => res.state = .responded,
+            .first, .start, .responded, .finished => unreachable,
+        }
+
         var buffered = std.io.bufferedWriter(res.connection.writer());
         const w = buffered.writer();
 
@@ -452,6 +500,11 @@ pub const Response = struct {
 
     /// Wait for the client to send a complete request head.
     pub fn wait(res: *Response) WaitError!void {
+        switch (res.state) {
+            .first, .start => res.state = .waited,
+            .waited, .responded, .finished => unreachable,
+        }
+
         while (true) {
             try res.connection.fill();
 
@@ -464,17 +517,6 @@ pub const Response = struct {
         res.request.headers = .{ .allocator = res.server.allocator, .owned = true };
         try res.request.parse(res.request.parser.header_bytes.items);
 
-        const res_connection = res.headers.getFirstValue("connection");
-        const res_keepalive = res_connection != null and !std.ascii.eqlIgnoreCase("close", res_connection.?);
-
-        const req_connection = res.request.headers.getFirstValue("connection");
-        const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
-        if (res_keepalive and req_keepalive) {
-            res.connection.conn.closing = false;
-        } else {
-            res.connection.conn.closing = true;
-        }
-
         if (res.request.transfer_encoding) |te| {
             switch (te) {
                 .chunked => {
@@ -515,6 +557,11 @@ pub const Response = struct {
     }
 
     pub fn read(res: *Response, buffer: []u8) ReadError!usize {
+        switch (res.state) {
+            .waited, .responded, .finished => {},
+            .first, .start => unreachable,
+        }
+
         const out_index = switch (res.request.compression) {
             .deflate => |*deflate| deflate.read(buffer) catch return error.DecompressionFailure,
             .gzip => |*gzip| gzip.read(buffer) catch return error.DecompressionFailure,
@@ -564,6 +611,11 @@ pub const Response = struct {
 
     /// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.
     pub fn write(res: *Response, bytes: []const u8) WriteError!usize {
+        switch (res.state) {
+            .responded => {},
+            .first, .waited, .start, .finished => unreachable,
+        }
+
         switch (res.transfer_encoding) {
             .chunked => {
                 try res.connection.writer().print("{x}\r\n", .{bytes.len});
@@ -583,7 +635,7 @@ pub const Response = struct {
         }
     }
 
-    pub fn writeAll(req: *Request, bytes: []const u8) WriteError!void {
+    pub fn writeAll(req: *Response, bytes: []const u8) WriteError!void {
         var index: usize = 0;
         while (index < bytes.len) {
             index += try write(req, bytes[index..]);
@@ -594,6 +646,11 @@ pub const Response = struct {
 
     /// Finish the body of a request. This notifies the server that you have no more data to send.
     pub fn finish(res: *Response) FinishError!void {
+        switch (res.state) {
+            .responded => res.state = .finished,
+            .first, .waited, .start, .finished => unreachable,
+        }
+
         switch (res.transfer_encoding) {
             .chunked => try res.connection.writeAll("0\r\n\r\n"),
             .content_length => |len| if (len != 0) return error.MessageNotCompleted,
@@ -636,11 +693,10 @@ pub const HeaderStrategy = union(enum) {
 };
 
 /// Accept a new connection and allocate a Response for it.
-pub fn accept(server: *Server, options: HeaderStrategy) AcceptError!*Response {
+pub fn accept(server: *Server, options: HeaderStrategy) AcceptError!Response {
     const in = try server.socket.accept();
 
-    const res = try server.allocator.create(Response);
-    res.* = .{
+    return Response{
         .server = server,
         .address = in.address,
         .connection = .{ .conn = .{
@@ -652,14 +708,13 @@ pub fn accept(server: *Server, options: HeaderStrategy) AcceptError!*Response {
             .version = undefined,
             .method = undefined,
             .target = undefined,
+            .headers = .{ .allocator = server.allocator, .owned = false },
             .parser = switch (options) {
                 .dynamic => |max| proto.HeadersParser.initDynamic(max),
                 .static => |buf| proto.HeadersParser.initStatic(buf),
             },
         },
     };
-
-    return res;
 }
 
 test "HTTP server handles a chunked transfer coding request" {
diff --git a/lib/std/http/test.zig b/lib/std/http/test.zig
deleted file mode 100644
index ee164c297a..0000000000
--- a/lib/std/http/test.zig
+++ /dev/null
@@ -1,72 +0,0 @@
-const std = @import("std");
-const expect = std.testing.expect;
-
-test "client requests server" {
-    const builtin = @import("builtin");
-
-    // This test requires spawning threads.
-    if (builtin.single_threaded) {
-        return error.SkipZigTest;
-    }
-
-    const native_endian = comptime builtin.cpu.arch.endian();
-    if (builtin.zig_backend == .stage2_llvm and native_endian == .Big) {
-        // https://github.com/ziglang/zig/issues/13782
-        return error.SkipZigTest;
-    }
-
-    if (builtin.os.tag == .wasi) return error.SkipZigTest;
-
-    const allocator = std.testing.allocator;
-
-    const max_header_size = 8192;
-    var server = std.http.Server.init(allocator, .{ .reuse_address = true });
-    defer server.deinit();
-
-    const address = try std.net.Address.parseIp("127.0.0.1", 0);
-    try server.listen(address);
-    const server_port = server.socket.listen_address.in.getPort();
-
-    const server_thread = try std.Thread.spawn(.{}, (struct {
-        fn apply(s: *std.http.Server) !void {
-            const res = try s.accept(.{ .dynamic = max_header_size });
-            defer res.deinit();
-            defer res.reset();
-            try res.wait();
-
-            const server_body: []const u8 = "message from server!\n";
-            res.transfer_encoding = .{ .content_length = server_body.len };
-            try res.headers.append("content-type", "text/plain");
-            try res.headers.append("connection", "close");
-            try res.do();
-
-            var buf: [128]u8 = undefined;
-            const n = try res.readAll(&buf);
-            try expect(std.mem.eql(u8, buf[0..n], "Hello, World!\n"));
-            _ = try res.writer().writeAll(server_body);
-            try res.finish();
-        }
-    }).apply, .{&server});
-
-    var uri_buf: [22]u8 = undefined;
-    const uri = try std.Uri.parse(try std.fmt.bufPrint(&uri_buf, "http://127.0.0.1:{d}", .{server_port}));
-    var client = std.http.Client{ .allocator = allocator };
-    defer client.deinit();
-    var client_headers = std.http.Headers{ .allocator = allocator };
-    defer client_headers.deinit();
-    var client_req = try client.request(.POST, uri, client_headers, .{});
-    defer client_req.deinit();
-
-    client_req.transfer_encoding = .{ .content_length = 14 }; // this will be checked to ensure you sent exactly 14 bytes
-    try client_req.start(); // this sends the request
-    try client_req.writeAll("Hello, ");
-    try client_req.writeAll("World!\n");
-    try client_req.finish();
-    try client_req.wait(); // this waits for a response
-
-    const body = try client_req.reader().readAllAlloc(allocator, 8192 * 1024);
-    defer allocator.free(body);
-    try expect(std.mem.eql(u8, body, "message from server!\n"));
-
-    server_thread.join();
-}
diff --git a/test/standalone.zig b/test/standalone.zig
index 1f4d7cfded..a055da9761 100644
--- a/test/standalone.zig
+++ b/test/standalone.zig
@@ -55,6 +55,10 @@ pub const simple_cases = [_]SimpleCase{
         .os_filter = .windows,
         .link_libc = true,
     },
+    .{
+        .src_path = "test/standalone/http.zig",
+        .all_modes = true,
+    },
 
     // Ensure the development tools are buildable. Alphabetically sorted.
     // No need to build `tools/spirv/grammar.zig`.
diff --git a/test/standalone/http.zig b/test/standalone/http.zig
new file mode 100644
index 0000000000..1ed4b1b279
--- /dev/null
+++ b/test/standalone/http.zig
@@ -0,0 +1,432 @@
+const std = @import("std");
+
+const http = std.http;
+const Server = http.Server;
+const Client = http.Client;
+
+const mem = std.mem;
+const testing = std.testing;
+
+const max_header_size = 8192;
+
+var gpa_server = std.heap.GeneralPurposeAllocator(.{}){};
+var gpa_client = std.heap.GeneralPurposeAllocator(.{}){};
+
+const salloc = gpa_server.allocator();
+const calloc = gpa_client.allocator();
+
+fn handleRequest(res: *Server.Response) !void {
+    const log = std.log.scoped(.server);
+
+    log.info("{s} {s} {s}", .{ @tagName(res.request.method), @tagName(res.request.version), res.request.target });
+
+    const body = try res.reader().readAllAlloc(salloc, 8192);
+    defer salloc.free(body);
+
+    if (res.request.headers.contains("connection")) {
+        try res.headers.append("connection", "keep-alive");
+    }
+
+    if (mem.startsWith(u8, res.request.target, "/get")) {
+        if (std.mem.indexOf(u8, res.request.target, "?chunked") != null) {
+            res.transfer_encoding = .chunked;
+        } else {
+            res.transfer_encoding = .{ .content_length = 14 };
+        }
+
+        try res.headers.append("content-type", "text/plain");
+
+        try res.do();
+        if (res.request.method != .HEAD) {
+            try res.writeAll("Hello, ");
+            try res.writeAll("World!\n");
+            try res.finish();
+        }
+    } else if (mem.eql(u8, res.request.target, "/echo-content")) {
+        try testing.expectEqualStrings("Hello, World!\n", body);
+        try testing.expectEqualStrings("text/plain", res.request.headers.getFirstValue("content-type").?);
+
+        if (res.request.headers.contains("transfer-encoding")) {
+            try testing.expectEqualStrings("chunked", res.request.headers.getFirstValue("transfer-encoding").?);
+            res.transfer_encoding = .chunked;
+        } else {
+            res.transfer_encoding = .{ .content_length = 14 };
+            try testing.expectEqualStrings("14", res.request.headers.getFirstValue("content-length").?);
+        }
+
+        try res.do();
+        try res.writeAll("Hello, ");
+        try res.writeAll("World!\n");
+        try res.finish();
+    } else if (mem.eql(u8, res.request.target, "/trailer")) {
+        res.transfer_encoding = .chunked;
+
+        try res.do();
+        try res.writeAll("Hello, ");
+        try res.writeAll("World!\n");
+        // try res.finish();
+        try res.connection.writeAll("0\r\nX-Checksum: aaaa\r\n\r\n");
+    } else if (mem.eql(u8, res.request.target, "/redirect/1")) {
+        res.transfer_encoding = .chunked;
+
+        res.status = .found;
+        try res.headers.append("location", "../../get");
+
+        try res.do();
+        try res.writeAll("Hello, ");
+        try res.writeAll("Redirected!\n");
+        try res.finish();
+    } else if (mem.eql(u8, res.request.target, "/redirect/2")) {
+        res.transfer_encoding = .chunked;
+
+        res.status = .found;
+        try res.headers.append("location", "/redirect/1");
+
+        try res.do();
+        try res.writeAll("Hello, ");
+        try res.writeAll("Redirected!\n");
+        try res.finish();
+    } else if (mem.eql(u8, res.request.target, "/redirect/3")) {
+        res.transfer_encoding = .chunked;
+
+        const location = try std.fmt.allocPrint(salloc, "http://127.0.0.1:{d}/redirect/2", .{res.server.socket.listen_address.getPort()});
+        defer salloc.free(location);
+
+        res.status = .found;
+        try res.headers.append("location", location);
+
+        try res.do();
+        try res.writeAll("Hello, ");
+        try res.writeAll("Redirected!\n");
+        try res.finish();
+    } else if (mem.eql(u8, res.request.target, "/redirect/4")) {
+        res.transfer_encoding = .chunked;
+
+        res.status = .found;
+        try res.headers.append("location", "/redirect/3");
+
+        try res.do();
+        try res.writeAll("Hello, ");
+        try res.writeAll("Redirected!\n");
+        try res.finish();
+    } else {
+        res.status = .not_found;
+        try res.do();
+    }
+}
+
+var handle_new_requests = true;
+
+fn runServer(srv: *Server) !void {
+    outer: while (handle_new_requests) {
+        var res = try srv.accept(.{ .dynamic = max_header_size });
+        defer res.deinit();
+
+        while (res.reset()) {
+            res.wait() catch |err| switch (err) {
+                error.HttpHeadersInvalid => continue :outer,
+                error.EndOfStream => continue,
+                else => return err,
+            };
+
+            try handleRequest(&res);
+        }
+    }
+}
+
+fn serverThread(srv: *Server) void {
+    defer srv.deinit();
+    defer _ = gpa_server.deinit();
+
+    runServer(srv) catch |err| {
+        std.debug.print("server error: {}\n", .{err});
+
+        if (@errorReturnTrace()) |trace| {
+            std.debug.dumpStackTrace(trace.*);
+        }
+
+        _ = gpa_server.deinit();
+        std.os.exit(1);
+    };
+}
+
+fn killServer(addr: std.net.Address) void {
+    handle_new_requests = false;
+
+    const conn = std.net.tcpConnectToAddress(addr) catch return;
+    conn.close();
+}
+
+pub fn main() !void {
+    const log = std.log.scoped(.client);
+
+    defer _ = gpa_client.deinit();
+
+    var server = Server.init(salloc, .{ .reuse_address = true });
+
+    const addr = std.net.Address.parseIp("127.0.0.1", 0) catch unreachable;
+    try server.listen(addr);
+
+    const port = server.socket.listen_address.getPort();
+
+    const server_thread = try std.Thread.spawn(.{}, serverThread, .{&server});
+
+    var client = Client{ .allocator = calloc };
+
+    defer client.deinit();
+
+    { // read content-length response
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.GET, uri, h, .{});
+        defer req.deinit();
+
+        try req.start();
+        try req.wait();
+
+        const body = try req.reader().readAllAlloc(calloc, 8192);
+        defer calloc.free(body);
+
+        try testing.expectEqualStrings("Hello, World!\n", body);
+        try testing.expectEqualStrings("text/plain", req.response.headers.getFirstValue("content-type").?);
+    }
+
+    { // send head request and not read chunked
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.HEAD, uri, h, .{});
+        defer req.deinit();
+
+        try req.start();
+        try req.wait();
+
+        const body = try req.reader().readAllAlloc(calloc, 8192);
+        defer calloc.free(body);
+
+        try testing.expectEqualStrings("", body);
+        try testing.expectEqualStrings("text/plain", req.response.headers.getFirstValue("content-type").?);
+        try testing.expectEqualStrings("14", req.response.headers.getFirstValue("content-length").?);
+    }
+
+    { // read chunked response
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get?chunked", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.GET, uri, h, .{});
+        defer req.deinit();
+
+        try req.start();
+        try req.wait();
+
+        const body = try req.reader().readAllAlloc(calloc, 8192);
+        defer calloc.free(body);
+
+        try testing.expectEqualStrings("Hello, World!\n", body);
+        try testing.expectEqualStrings("text/plain", req.response.headers.getFirstValue("content-type").?);
+    }
+
+    { // send head request and not read chunked
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get?chunked", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.HEAD, uri, h, .{});
+        defer req.deinit();
+
+        try req.start();
+        try req.wait();
+
+        const body = try req.reader().readAllAlloc(calloc, 8192);
+        defer calloc.free(body);
+
+        try testing.expectEqualStrings("", body);
+        try testing.expectEqualStrings("text/plain", req.response.headers.getFirstValue("content-type").?);
+        try testing.expectEqualStrings("chunked", req.response.headers.getFirstValue("transfer-encoding").?);
+    }
+
+    { // check trailing headers
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/trailer", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.GET, uri, h, .{});
+        defer req.deinit();
+
+        try req.start();
+        try req.wait();
+
+        const body = try req.reader().readAllAlloc(calloc, 8192);
+        defer calloc.free(body);
+
+        try testing.expectEqualStrings("Hello, World!\n", body);
+        try testing.expectEqualStrings("aaaa", req.response.headers.getFirstValue("x-checksum").?);
+    }
+
+    { // send content-length request
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        try h.append("content-type", "text/plain");
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.POST, uri, h, .{});
+        defer req.deinit();
+
+        req.transfer_encoding = .{ .content_length = 14 };
+
+        try req.start();
+        try req.writeAll("Hello, ");
+        try req.writeAll("World!\n");
+        try req.finish();
+
+        try req.wait();
+
+        const body = try req.reader().readAllAlloc(calloc, 8192);
+        defer calloc.free(body);
+
+        try testing.expectEqualStrings("Hello, World!\n", body);
+    }
+
+    { // send chunked request
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        try h.append("content-type", "text/plain");
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.POST, uri, h, .{});
+        defer req.deinit();
+
+        req.transfer_encoding = .chunked;
+
+        try req.start();
+        try req.writeAll("Hello, ");
+        try req.writeAll("World!\n");
+        try req.finish();
+
+        try req.wait();
+
+        const body = try req.reader().readAllAlloc(calloc, 8192);
+        defer calloc.free(body);
+
+        try testing.expectEqualStrings("Hello, World!\n", body);
+    }
+
+    { // relative redirect
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/redirect/1", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.GET, uri, h, .{});
+        defer req.deinit();
+
+        try req.start();
+        try req.wait();
+
+        const body = try req.reader().readAllAlloc(calloc, 8192);
+        defer calloc.free(body);
+
+        try testing.expectEqualStrings("Hello, World!\n", body);
+    }
+
+    { // redirect from root
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/redirect/2", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.GET, uri, h, .{});
+        defer req.deinit();
+
+        try req.start();
+        try req.wait();
+
+        const body = try req.reader().readAllAlloc(calloc, 8192);
+        defer calloc.free(body);
+
+        try testing.expectEqualStrings("Hello, World!\n", body);
+    }
+
+    { // absolute redirect
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/redirect/3", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.GET, uri, h, .{});
+        defer req.deinit();
+
+        try req.start();
+        try req.wait();
+
+        const body = try req.reader().readAllAlloc(calloc, 8192);
+        defer calloc.free(body);
+
+        try testing.expectEqualStrings("Hello, World!\n", body);
+    }
+
+    { // too many redirects
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/redirect/4", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.GET, uri, h, .{});
+        defer req.deinit();
+
+        try req.start();
+        req.wait() catch |err| switch (err) {
+            error.TooManyHttpRedirects => {},
+            else => return err,
+        };
+    }
+
+    killServer(server.socket.listen_address);
+    server_thread.join();
+}

From 6513eb4696551a91e322fe2d9879335cd73c92db Mon Sep 17 00:00:00 2001
From: Nameless 
Date: Thu, 27 Apr 2023 20:25:46 -0500
Subject: [PATCH 092/725] std.http.Server: use client recommendation for
 keepalive

---
 lib/std/http/Server.zig | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/std/http/Server.zig b/lib/std/http/Server.zig
index 1b5fd045fa..71a7a351ad 100644
--- a/lib/std/http/Server.zig
+++ b/lib/std/http/Server.zig
@@ -439,7 +439,14 @@ pub const Response = struct {
         }
 
         if (!res.headers.contains("connection")) {
-            try w.writeAll("Connection: keep-alive\r\n");
+            const req_connection = res.request.headers.getFirstValue("connection");
+            const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
+
+            if (req_keepalive) {
+                try w.writeAll("Connection: keep-alive\r\n");
+            } else {
+                try w.writeAll("Connection: close\r\n");
+            }
         }
 
         const has_transfer_encoding = res.headers.contains("transfer-encoding");

From 533049fdd80a4c7bc3098512b4033a60daea745e Mon Sep 17 00:00:00 2001
From: Nameless 
Date: Fri, 28 Apr 2023 09:55:23 -0500
Subject: [PATCH 093/725] std.http.Server: use enum for reset state instead of
 bool

---
 lib/std/http/Server.zig  | 16 +++++++++++-----
 test/standalone/http.zig |  2 +-
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/lib/std/http/Server.zig b/lib/std/http/Server.zig
index 71a7a351ad..2e7d985e4c 100644
--- a/lib/std/http/Server.zig
+++ b/lib/std/http/Server.zig
@@ -355,17 +355,19 @@ pub const Response = struct {
         }
     }
 
+    pub const ResetState = enum { reset, closing };
+
     /// Reset this response to its initial state. This must be called before handling a second request on the same connection.
-    pub fn reset(res: *Response) bool {
+    pub fn reset(res: *Response) ResetState {
         if (res.state == .first) {
             res.state = .start;
-            return true;
+            return .reset;
         }
 
         if (!res.request.parser.done) {
             // If the response wasn't fully read, then we need to close the connection.
             res.connection.conn.closing = true;
-            return false;
+            return .closing;
         }
 
         // A connection is only keep-alive if the Connection header is present and it's value is not "close".
@@ -408,7 +410,11 @@ pub const Response = struct {
             .parser = res.request.parser,
         };
 
-        return !res.connection.conn.closing;
+        if (res.connection.conn.closing) {
+            return .closing;
+        } else {
+            return .reset;
+        }
     }
 
     pub const DoError = BufferedConnection.WriteError || error{ UnsupportedTransferEncoding, InvalidContentLength };
@@ -699,7 +705,7 @@ pub const HeaderStrategy = union(enum) {
     static: []u8,
 };
 
-/// Accept a new connection and allocate a Response for it.
+/// Accept a new connection.
 pub fn accept(server: *Server, options: HeaderStrategy) AcceptError!Response {
     const in = try server.socket.accept();
 
diff --git a/test/standalone/http.zig b/test/standalone/http.zig
index 1ed4b1b279..493931a2ea 100644
--- a/test/standalone/http.zig
+++ b/test/standalone/http.zig
@@ -122,7 +122,7 @@ fn runServer(srv: *Server) !void {
         var res = try srv.accept(.{ .dynamic = max_header_size });
         defer res.deinit();
 
-        while (res.reset()) {
+        while (res.reset() != .closing) {
             res.wait() catch |err| switch (err) {
                 error.HttpHeadersInvalid => continue :outer,
                 error.EndOfStream => continue,

From 7b0962938859a955fa8e057ef34f4abd925bb1ca Mon Sep 17 00:00:00 2001
From: Nameless 
Date: Mon, 1 May 2023 12:13:33 -0500
Subject: [PATCH 094/725] std.http: buffer writes

---
 lib/std/http/Client.zig | 62 +++++++++++++++++++++++++++++------------
 lib/std/http/Server.zig | 62 +++++++++++++++++++++++++++++------------
 2 files changed, 88 insertions(+), 36 deletions(-)

diff --git a/lib/std/http/Client.zig b/lib/std/http/Client.zig
index 74321492c8..72f16b0e76 100644
--- a/lib/std/http/Client.zig
+++ b/lib/std/http/Client.zig
@@ -254,44 +254,47 @@ pub const BufferedConnection = struct {
     pub const buffer_size = 0x2000;
 
     conn: Connection,
-    buf: [buffer_size]u8 = undefined,
-    start: u16 = 0,
-    end: u16 = 0,
+    read_buf: [buffer_size]u8 = undefined,
+    read_start: u16 = 0,
+    read_end: u16 = 0,
+
+    write_buf: [buffer_size]u8 = undefined,
+    write_end: u16 = 0,
 
     pub fn fill(bconn: *BufferedConnection) ReadError!void {
-        if (bconn.end != bconn.start) return;
+        if (bconn.read_end != bconn.read_start) return;
 
-        const nread = try bconn.conn.read(bconn.buf[0..]);
+        const nread = try bconn.conn.read(bconn.read_buf[0..]);
         if (nread == 0) return error.EndOfStream;
-        bconn.start = 0;
-        bconn.end = @intCast(u16, nread);
+        bconn.read_start = 0;
+        bconn.read_end = @intCast(u16, nread);
     }
 
     pub fn peek(bconn: *BufferedConnection) []const u8 {
-        return bconn.buf[bconn.start..bconn.end];
+        return bconn.read_buf[bconn.read_start..bconn.read_end];
     }
 
     pub fn clear(bconn: *BufferedConnection, num: u16) void {
-        bconn.start += num;
+        bconn.read_start += num;
     }
 
     pub fn readAtLeast(bconn: *BufferedConnection, buffer: []u8, len: usize) ReadError!usize {
         var out_index: u16 = 0;
         while (out_index < len) {
-            const available = bconn.end - bconn.start;
+            const available = bconn.read_end - bconn.read_start;
             const left = buffer.len - out_index;
 
             if (available > 0) {
                 const can_read = @intCast(u16, @min(available, left));
 
-                @memcpy(buffer[out_index..][0..can_read], bconn.buf[bconn.start..][0..can_read]);
+                @memcpy(buffer[out_index..][0..can_read], bconn.read_buf[bconn.read_start..][0..can_read]);
                 out_index += can_read;
-                bconn.start += can_read;
+                bconn.read_start += can_read;
 
                 continue;
             }
 
-            if (left > bconn.buf.len) {
+            if (left > bconn.read_buf.len) {
                 // skip the buffer if the output is large enough
                 return bconn.conn.read(buffer[out_index..]);
             }
@@ -314,11 +317,33 @@ pub const BufferedConnection = struct {
     }
 
     pub fn writeAll(bconn: *BufferedConnection, buffer: []const u8) WriteError!void {
-        return bconn.conn.writeAll(buffer);
+        if (bconn.write_buf.len - bconn.write_end <= buffer.len) {
+            @memcpy(bconn.write_buf[bconn.write_end..], buffer);
+            bconn.write_end += @intCast(u16, buffer.len);
+        } else {
+            try bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
+            bconn.write_end = 0;
+
+            try bconn.conn.writeAll(buffer);
+        }
     }
 
     pub fn write(bconn: *BufferedConnection, buffer: []const u8) WriteError!usize {
-        return bconn.conn.write(buffer);
+        if (bconn.write_buf.len - bconn.write_end <= buffer.len) {
+            @memcpy(bconn.write_buf[bconn.write_end..], buffer);
+            bconn.write_end += @intCast(u16, buffer.len);
+
+            return buffer.len;
+        } else {
+            try bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
+            bconn.write_end = 0;
+
+            return try bconn.conn.write(buffer);
+        }
+    }
+
+    pub fn flush(bconn: *BufferedConnection) WriteError!void {
+        return bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
     }
 
     pub const WriteError = Connection.WriteError;
@@ -567,8 +592,7 @@ pub const Request = struct {
 
     /// Send the request to the server.
     pub fn start(req: *Request) StartError!void {
-        var buffered = std.io.bufferedWriter(req.connection.data.buffered.writer());
-        const w = buffered.writer();
+        const w = req.connection.data.buffered.writer();
 
         try w.writeAll(@tagName(req.method));
         try w.writeByte(' ');
@@ -642,7 +666,7 @@ pub const Request = struct {
 
         try w.writeAll("\r\n");
 
-        try buffered.flush();
+        try req.connection.data.buffered.flush();
     }
 
     pub const TransferReadError = BufferedConnection.ReadError || proto.HeadersParser.ReadError;
@@ -868,6 +892,8 @@ pub const Request = struct {
             .content_length => |len| if (len != 0) return error.MessageNotCompleted,
             .none => {},
         }
+
+        try req.connection.data.buffered.flush();
     }
 };
 
diff --git a/lib/std/http/Server.zig b/lib/std/http/Server.zig
index 2e7d985e4c..46c689ba6d 100644
--- a/lib/std/http/Server.zig
+++ b/lib/std/http/Server.zig
@@ -98,44 +98,47 @@ pub const BufferedConnection = struct {
     pub const buffer_size = 0x2000;
 
     conn: Connection,
-    buf: [buffer_size]u8 = undefined,
-    start: u16 = 0,
-    end: u16 = 0,
+    read_buf: [buffer_size]u8 = undefined,
+    read_start: u16 = 0,
+    read_end: u16 = 0,
+
+    write_buf: [buffer_size]u8 = undefined,
+    write_end: u16 = 0,
 
     pub fn fill(bconn: *BufferedConnection) ReadError!void {
-        if (bconn.end != bconn.start) return;
+        if (bconn.read_end != bconn.read_start) return;
 
-        const nread = try bconn.conn.read(bconn.buf[0..]);
+        const nread = try bconn.conn.read(bconn.read_buf[0..]);
         if (nread == 0) return error.EndOfStream;
-        bconn.start = 0;
-        bconn.end = @intCast(u16, nread);
+        bconn.read_start = 0;
+        bconn.read_end = @intCast(u16, nread);
     }
 
     pub fn peek(bconn: *BufferedConnection) []const u8 {
-        return bconn.buf[bconn.start..bconn.end];
+        return bconn.read_buf[bconn.read_start..bconn.read_end];
     }
 
     pub fn clear(bconn: *BufferedConnection, num: u16) void {
-        bconn.start += num;
+        bconn.read_start += num;
     }
 
     pub fn readAtLeast(bconn: *BufferedConnection, buffer: []u8, len: usize) ReadError!usize {
         var out_index: u16 = 0;
         while (out_index < len) {
-            const available = bconn.end - bconn.start;
+            const available = bconn.read_end - bconn.read_start;
             const left = buffer.len - out_index;
 
             if (available > 0) {
                 const can_read = @intCast(u16, @min(available, left));
 
-                @memcpy(buffer[out_index..][0..can_read], bconn.buf[bconn.start..][0..can_read]);
+                @memcpy(buffer[out_index..][0..can_read], bconn.read_buf[bconn.read_start..][0..can_read]);
                 out_index += can_read;
-                bconn.start += can_read;
+                bconn.read_start += can_read;
 
                 continue;
             }
 
-            if (left > bconn.buf.len) {
+            if (left > bconn.read_buf.len) {
                 // skip the buffer if the output is large enough
                 return bconn.conn.read(buffer[out_index..]);
             }
@@ -158,11 +161,33 @@ pub const BufferedConnection = struct {
     }
 
     pub fn writeAll(bconn: *BufferedConnection, buffer: []const u8) WriteError!void {
-        return bconn.conn.writeAll(buffer);
+        if (bconn.write_buf.len - bconn.write_end <= buffer.len) {
+            @memcpy(bconn.write_buf[bconn.write_end..], buffer);
+            bconn.write_end += @intCast(u16, buffer.len);
+        } else {
+            try bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
+            bconn.write_end = 0;
+
+            try bconn.conn.writeAll(buffer);
+        }
     }
 
     pub fn write(bconn: *BufferedConnection, buffer: []const u8) WriteError!usize {
-        return bconn.conn.write(buffer);
+        if (bconn.write_buf.len - bconn.write_end <= buffer.len) {
+            @memcpy(bconn.write_buf[bconn.write_end..], buffer);
+            bconn.write_end += @intCast(u16, buffer.len);
+
+            return buffer.len;
+        } else {
+            try bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
+            bconn.write_end = 0;
+
+            return try bconn.conn.write(buffer);
+        }
+    }
+
+    pub fn flush(bconn: *BufferedConnection) WriteError!void {
+        return bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
     }
 
     pub const WriteError = Connection.WriteError;
@@ -426,8 +451,7 @@ pub const Response = struct {
             .first, .start, .responded, .finished => unreachable,
         }
 
-        var buffered = std.io.bufferedWriter(res.connection.writer());
-        const w = buffered.writer();
+        const w = res.connection.writer();
 
         try w.writeAll(@tagName(res.version));
         try w.writeByte(' ');
@@ -485,7 +509,7 @@ pub const Response = struct {
 
         try w.writeAll("\r\n");
 
-        try buffered.flush();
+        try res.connection.flush();
     }
 
     pub const TransferReadError = BufferedConnection.ReadError || proto.HeadersParser.ReadError;
@@ -669,6 +693,8 @@ pub const Response = struct {
             .content_length => |len| if (len != 0) return error.MessageNotCompleted,
             .none => {},
         }
+
+        try res.connection.flush();
     }
 };
 

From 5f219a2d118cac1410888fb2c0abc0cc91d092de Mon Sep 17 00:00:00 2001
From: Nameless 
Date: Mon, 1 May 2023 17:49:34 -0500
Subject: [PATCH 095/725] std.http.Server: give Response access to their own
 allocator

* This makes it easier for threaded servers to use a different allocator
  for each request.
---
 lib/std/http/Server.zig  | 33 +++++++++++++++++++--------------
 test/standalone/http.zig | 11 ++++++++---
 2 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/lib/std/http/Server.zig b/lib/std/http/Server.zig
index 46c689ba6d..d473366092 100644
--- a/lib/std/http/Server.zig
+++ b/lib/std/http/Server.zig
@@ -352,7 +352,7 @@ pub const Response = struct {
 
     transfer_encoding: ResponseTransfer = .none,
 
-    server: *Server,
+    allocator: Allocator,
     address: net.Address,
     connection: BufferedConnection,
 
@@ -376,7 +376,7 @@ pub const Response = struct {
         res.request.headers.deinit();
 
         if (res.request.parser.header_bytes_owned) {
-            res.request.parser.header_bytes.deinit(res.server.allocator);
+            res.request.parser.header_bytes.deinit(res.allocator);
         }
     }
 
@@ -545,13 +545,13 @@ pub const Response = struct {
         while (true) {
             try res.connection.fill();
 
-            const nchecked = try res.request.parser.checkCompleteHead(res.server.allocator, res.connection.peek());
+            const nchecked = try res.request.parser.checkCompleteHead(res.allocator, res.connection.peek());
             res.connection.clear(@intCast(u16, nchecked));
 
             if (res.request.parser.state.isContent()) break;
         }
 
-        res.request.headers = .{ .allocator = res.server.allocator, .owned = true };
+        res.request.headers = .{ .allocator = res.allocator, .owned = true };
         try res.request.parse(res.request.parser.header_bytes.items);
 
         if (res.request.transfer_encoding) |te| {
@@ -573,13 +573,13 @@ pub const Response = struct {
             if (res.request.transfer_compression) |tc| switch (tc) {
                 .compress => return error.CompressionNotSupported,
                 .deflate => res.request.compression = .{
-                    .deflate = std.compress.zlib.zlibStream(res.server.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,
+                    .deflate = std.compress.zlib.zlibStream(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,
                 },
                 .gzip => res.request.compression = .{
-                    .gzip = std.compress.gzip.decompress(res.server.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,
+                    .gzip = std.compress.gzip.decompress(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,
                 },
                 .zstd => res.request.compression = .{
-                    .zstd = std.compress.zstd.decompressStream(res.server.allocator, res.transferReader()),
+                    .zstd = std.compress.zstd.decompressStream(res.allocator, res.transferReader()),
                 },
             };
         }
@@ -612,12 +612,12 @@ pub const Response = struct {
             while (!res.request.parser.state.isContent()) { // read trailing headers
                 try res.connection.fill();
 
-                const nchecked = try res.request.parser.checkCompleteHead(res.server.allocator, res.connection.peek());
+                const nchecked = try res.request.parser.checkCompleteHead(res.allocator, res.connection.peek());
                 res.connection.clear(@intCast(u16, nchecked));
             }
 
             if (has_trail) {
-                res.request.headers = http.Headers{ .allocator = res.server.allocator, .owned = false };
+                res.request.headers = http.Headers{ .allocator = res.allocator, .owned = false };
 
                 // The response headers before the trailers are already guaranteed to be valid, so they will always be parsed again and cannot return an error.
                 // This will *only* fail for a malformed trailer.
@@ -731,24 +731,29 @@ pub const HeaderStrategy = union(enum) {
     static: []u8,
 };
 
+pub const AcceptOptions = struct {
+    allocator: Allocator,
+    header_strategy: HeaderStrategy = .{ .dynamic = 8192 },
+};
+
 /// Accept a new connection.
-pub fn accept(server: *Server, options: HeaderStrategy) AcceptError!Response {
+pub fn accept(server: *Server, options: AcceptOptions) AcceptError!Response {
     const in = try server.socket.accept();
 
     return Response{
-        .server = server,
+        .allocator = options.allocator,
         .address = in.address,
         .connection = .{ .conn = .{
             .stream = in.stream,
             .protocol = .plain,
         } },
-        .headers = .{ .allocator = server.allocator },
+        .headers = .{ .allocator = options.allocator },
         .request = .{
             .version = undefined,
             .method = undefined,
             .target = undefined,
-            .headers = .{ .allocator = server.allocator, .owned = false },
-            .parser = switch (options) {
+            .headers = .{ .allocator = options.allocator, .owned = false },
+            .parser = switch (options.header_strategy) {
                 .dynamic => |max| proto.HeadersParser.initDynamic(max),
                 .static => |buf| proto.HeadersParser.initStatic(buf),
             },
diff --git a/test/standalone/http.zig b/test/standalone/http.zig
index 493931a2ea..180e863bba 100644
--- a/test/standalone/http.zig
+++ b/test/standalone/http.zig
@@ -15,6 +15,8 @@ var gpa_client = std.heap.GeneralPurposeAllocator(.{}){};
 const salloc = gpa_server.allocator();
 const calloc = gpa_client.allocator();
 
+var server: Server = undefined;
+
 fn handleRequest(res: *Server.Response) !void {
     const log = std.log.scoped(.server);
 
@@ -89,7 +91,7 @@ fn handleRequest(res: *Server.Response) !void {
     } else if (mem.eql(u8, res.request.target, "/redirect/3")) {
         res.transfer_encoding = .chunked;
 
-        const location = try std.fmt.allocPrint(salloc, "http://127.0.0.1:{d}/redirect/2", .{res.server.socket.listen_address.getPort()});
+        const location = try std.fmt.allocPrint(salloc, "http://127.0.0.1:{d}/redirect/2", .{server.socket.listen_address.getPort()});
         defer salloc.free(location);
 
         res.status = .found;
@@ -119,7 +121,10 @@ var handle_new_requests = true;
 
 fn runServer(srv: *Server) !void {
     outer: while (handle_new_requests) {
-        var res = try srv.accept(.{ .dynamic = max_header_size });
+        var res = try srv.accept(.{
+            .allocator = salloc,
+            .header_strategy = .{ .dynamic = max_header_size },
+        });
         defer res.deinit();
 
         while (res.reset() != .closing) {
@@ -162,7 +167,7 @@ pub fn main() !void {
 
     defer _ = gpa_client.deinit();
 
-    var server = Server.init(salloc, .{ .reuse_address = true });
+    server = Server.init(salloc, .{ .reuse_address = true });
 
     const addr = std.net.Address.parseIp("127.0.0.1", 0) catch unreachable;
     try server.listen(addr);

From 1b3ebfefd8d8fd05152d55c431f741880d1ce2a7 Mon Sep 17 00:00:00 2001
From: Nameless 
Date: Wed, 3 May 2023 14:34:10 -0500
Subject: [PATCH 096/725] fix keepalive and large buffered writes

---
 lib/std/http/Client.zig  |  22 ++++----
 lib/std/http/Server.zig  |  23 ++++----
 test/standalone/http.zig | 112 +++++++++++++++++++++++++++++++++++++--
 3 files changed, 129 insertions(+), 28 deletions(-)

diff --git a/lib/std/http/Client.zig b/lib/std/http/Client.zig
index 72f16b0e76..77f4c6c6c3 100644
--- a/lib/std/http/Client.zig
+++ b/lib/std/http/Client.zig
@@ -71,7 +71,7 @@ pub const ConnectionPool = struct {
         while (next) |node| : (next = node.prev) {
             if ((node.data.buffered.conn.protocol == .tls) != criteria.is_tls) continue;
             if (node.data.port != criteria.port) continue;
-            if (mem.eql(u8, node.data.host, criteria.host)) continue;
+            if (!mem.eql(u8, node.data.host, criteria.host)) continue;
 
             pool.acquireUnsafe(node);
             return node;
@@ -317,32 +317,29 @@ pub const BufferedConnection = struct {
     }
 
     pub fn writeAll(bconn: *BufferedConnection, buffer: []const u8) WriteError!void {
-        if (bconn.write_buf.len - bconn.write_end <= buffer.len) {
-            @memcpy(bconn.write_buf[bconn.write_end..], buffer);
+        if (bconn.write_buf.len - bconn.write_end >= buffer.len) {
+            @memcpy(bconn.write_buf[bconn.write_end..][0..buffer.len], buffer);
             bconn.write_end += @intCast(u16, buffer.len);
         } else {
-            try bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
-            bconn.write_end = 0;
-
+            try bconn.flush();
             try bconn.conn.writeAll(buffer);
         }
     }
 
     pub fn write(bconn: *BufferedConnection, buffer: []const u8) WriteError!usize {
-        if (bconn.write_buf.len - bconn.write_end <= buffer.len) {
-            @memcpy(bconn.write_buf[bconn.write_end..], buffer);
+        if (bconn.write_buf.len - bconn.write_end >= buffer.len) {
+            @memcpy(bconn.write_buf[bconn.write_end..][0..buffer.len], buffer);
             bconn.write_end += @intCast(u16, buffer.len);
 
             return buffer.len;
         } else {
-            try bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
-            bconn.write_end = 0;
-
+            try bconn.flush();
             return try bconn.conn.write(buffer);
         }
     }
 
     pub fn flush(bconn: *BufferedConnection) WriteError!void {
+        defer bconn.write_end = 0;
         return bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
     }
 
@@ -720,12 +717,13 @@ pub const Request = struct {
                 req.response.parser.done = true;
             }
 
+            // we default to using keep-alive if not provided
             const req_connection = req.headers.getFirstValue("connection");
             const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
 
             const res_connection = req.response.headers.getFirstValue("connection");
             const res_keepalive = res_connection != null and !std.ascii.eqlIgnoreCase("close", res_connection.?);
-            if (req_keepalive and res_keepalive) {
+            if (res_keepalive and (req_keepalive or req_connection == null)) {
                 req.connection.data.closing = false;
             } else {
                 req.connection.data.closing = true;
diff --git a/lib/std/http/Server.zig b/lib/std/http/Server.zig
index d473366092..2d3032bcdb 100644
--- a/lib/std/http/Server.zig
+++ b/lib/std/http/Server.zig
@@ -161,32 +161,29 @@ pub const BufferedConnection = struct {
     }
 
     pub fn writeAll(bconn: *BufferedConnection, buffer: []const u8) WriteError!void {
-        if (bconn.write_buf.len - bconn.write_end <= buffer.len) {
-            @memcpy(bconn.write_buf[bconn.write_end..], buffer);
+        if (bconn.write_buf.len - bconn.write_end >= buffer.len) {
+            @memcpy(bconn.write_buf[bconn.write_end..][0..buffer.len], buffer);
             bconn.write_end += @intCast(u16, buffer.len);
         } else {
-            try bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
-            bconn.write_end = 0;
-
+            try bconn.flush();
             try bconn.conn.writeAll(buffer);
         }
     }
 
     pub fn write(bconn: *BufferedConnection, buffer: []const u8) WriteError!usize {
-        if (bconn.write_buf.len - bconn.write_end <= buffer.len) {
-            @memcpy(bconn.write_buf[bconn.write_end..], buffer);
+        if (bconn.write_buf.len - bconn.write_end >= buffer.len) {
+            @memcpy(bconn.write_buf[bconn.write_end..][0..buffer.len], buffer);
             bconn.write_end += @intCast(u16, buffer.len);
 
             return buffer.len;
         } else {
-            try bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
-            bconn.write_end = 0;
-
+            try bconn.flush();
             return try bconn.conn.write(buffer);
         }
     }
 
     pub fn flush(bconn: *BufferedConnection) WriteError!void {
+        defer bconn.write_end = 0;
         return bconn.conn.writeAll(bconn.write_buf[0..bconn.write_end]);
     }
 
@@ -397,12 +394,14 @@ pub const Response = struct {
 
         // A connection is only keep-alive if the Connection header is present and it's value is not "close".
         // The server and client must both agree
+        //
+        // do() defaults to using keep-alive if the client requests it.
         const res_connection = res.headers.getFirstValue("connection");
         const res_keepalive = res_connection != null and !std.ascii.eqlIgnoreCase("close", res_connection.?);
 
         const req_connection = res.request.headers.getFirstValue("connection");
         const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
-        if (res_keepalive and req_keepalive) {
+        if (req_keepalive and (res_keepalive or res_connection == null)) {
             res.connection.conn.closing = false;
         } else {
             res.connection.conn.closing = true;
@@ -424,7 +423,7 @@ pub const Response = struct {
 
         res.headers.clearRetainingCapacity();
 
-        res.request.headers.clearRetainingCapacity();
+        res.request.headers.clearAndFree(); // FIXME: figure out why `clearRetainingCapacity` causes a leak in hash_map here
         res.request.parser.reset();
 
         res.request = Request{
diff --git a/test/standalone/http.zig b/test/standalone/http.zig
index 180e863bba..13dc278b6d 100644
--- a/test/standalone/http.zig
+++ b/test/standalone/http.zig
@@ -9,8 +9,8 @@ const testing = std.testing;
 
 const max_header_size = 8192;
 
-var gpa_server = std.heap.GeneralPurposeAllocator(.{}){};
-var gpa_client = std.heap.GeneralPurposeAllocator(.{}){};
+var gpa_server = std.heap.GeneralPurposeAllocator(.{ .stack_trace_frames = 12 }){};
+var gpa_client = std.heap.GeneralPurposeAllocator(.{ .stack_trace_frames = 12 }){};
 
 const salloc = gpa_server.allocator();
 const calloc = gpa_client.allocator();
@@ -44,6 +44,24 @@ fn handleRequest(res: *Server.Response) !void {
             try res.writeAll("World!\n");
             try res.finish();
         }
+    } else if (mem.startsWith(u8, res.request.target, "/large")) {
+        res.transfer_encoding = .{ .content_length = 14 * 1024 + 14 * 10 };
+
+        try res.do();
+
+        var i: u32 = 0;
+        while (i < 5) : (i += 1) {
+            try res.writeAll("Hello, World!\n");
+        }
+
+        try res.writeAll("Hello, World!\n" ** 1024);
+
+        i = 0;
+        while (i < 5) : (i += 1) {
+            try res.writeAll("Hello, World!\n");
+        }
+
+        try res.finish();
     } else if (mem.eql(u8, res.request.target, "/echo-content")) {
         try testing.expectEqualStrings("Hello, World!\n", body);
         try testing.expectEqualStrings("text/plain", res.request.headers.getFirstValue("content-type").?);
@@ -68,6 +86,7 @@ fn handleRequest(res: *Server.Response) !void {
         try res.writeAll("World!\n");
         // try res.finish();
         try res.connection.writeAll("0\r\nX-Checksum: aaaa\r\n\r\n");
+        try res.connection.flush();
     } else if (mem.eql(u8, res.request.target, "/redirect/1")) {
         res.transfer_encoding = .chunked;
 
@@ -177,8 +196,7 @@ pub fn main() !void {
     const server_thread = try std.Thread.spawn(.{}, serverThread, .{&server});
 
     var client = Client{ .allocator = calloc };
-
-    defer client.deinit();
+    // defer client.deinit(); handled below
 
     { // read content-length response
         var h = http.Headers{ .allocator = calloc };
@@ -202,6 +220,33 @@ pub fn main() !void {
         try testing.expectEqualStrings("text/plain", req.response.headers.getFirstValue("content-type").?);
     }
 
+    // connection has been kept alive
+    try testing.expect(client.connection_pool.free_len == 1);
+
+    { // read large content-length response
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/large", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.GET, uri, h, .{});
+        defer req.deinit();
+
+        try req.start();
+        try req.wait();
+
+        const body = try req.reader().readAllAlloc(calloc, 8192 * 1024);
+        defer calloc.free(body);
+
+        try testing.expectEqual(@as(usize, 14 * 1024 + 14 * 10), body.len);
+    }
+
+    // connection has been kept alive
+    try testing.expect(client.connection_pool.free_len == 1);
+
     { // send head request and not read chunked
         var h = http.Headers{ .allocator = calloc };
         defer h.deinit();
@@ -225,6 +270,9 @@ pub fn main() !void {
         try testing.expectEqualStrings("14", req.response.headers.getFirstValue("content-length").?);
     }
 
+    // connection has been kept alive
+    try testing.expect(client.connection_pool.free_len == 1);
+
     { // read chunked response
         var h = http.Headers{ .allocator = calloc };
         defer h.deinit();
@@ -247,6 +295,9 @@ pub fn main() !void {
         try testing.expectEqualStrings("text/plain", req.response.headers.getFirstValue("content-type").?);
     }
 
+    // connection has been kept alive
+    try testing.expect(client.connection_pool.free_len == 1);
+
     { // send head request and not read chunked
         var h = http.Headers{ .allocator = calloc };
         defer h.deinit();
@@ -270,6 +321,9 @@ pub fn main() !void {
         try testing.expectEqualStrings("chunked", req.response.headers.getFirstValue("transfer-encoding").?);
     }
 
+    // connection has been kept alive
+    try testing.expect(client.connection_pool.free_len == 1);
+
     { // check trailing headers
         var h = http.Headers{ .allocator = calloc };
         defer h.deinit();
@@ -292,6 +346,9 @@ pub fn main() !void {
         try testing.expectEqualStrings("aaaa", req.response.headers.getFirstValue("x-checksum").?);
     }
 
+    // connection has been kept alive
+    try testing.expect(client.connection_pool.free_len == 1);
+
     { // send content-length request
         var h = http.Headers{ .allocator = calloc };
         defer h.deinit();
@@ -321,6 +378,36 @@ pub fn main() !void {
         try testing.expectEqualStrings("Hello, World!\n", body);
     }
 
+    // connection has been kept alive
+    try testing.expect(client.connection_pool.free_len == 1);
+
+    { // read content-length response with connection close
+        var h = http.Headers{ .allocator = calloc };
+        defer h.deinit();
+
+        try h.append("connection", "close");
+
+        const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});
+        defer calloc.free(location);
+        const uri = try std.Uri.parse(location);
+
+        log.info("{s}", .{location});
+        var req = try client.request(.GET, uri, h, .{});
+        defer req.deinit();
+
+        try req.start();
+        try req.wait();
+
+        const body = try req.reader().readAllAlloc(calloc, 8192);
+        defer calloc.free(body);
+
+        try testing.expectEqualStrings("Hello, World!\n", body);
+        try testing.expectEqualStrings("text/plain", req.response.headers.getFirstValue("content-type").?);
+    }
+
+    // connection has been closed
+    try testing.expect(client.connection_pool.free_len == 0);
+
     { // send chunked request
         var h = http.Headers{ .allocator = calloc };
         defer h.deinit();
@@ -350,6 +437,9 @@ pub fn main() !void {
         try testing.expectEqualStrings("Hello, World!\n", body);
     }
 
+    // connection has been kept alive
+    try testing.expect(client.connection_pool.free_len == 1);
+
     { // relative redirect
         var h = http.Headers{ .allocator = calloc };
         defer h.deinit();
@@ -371,6 +461,9 @@ pub fn main() !void {
         try testing.expectEqualStrings("Hello, World!\n", body);
     }
 
+    // connection has been kept alive
+    try testing.expect(client.connection_pool.free_len == 1);
+
     { // redirect from root
         var h = http.Headers{ .allocator = calloc };
         defer h.deinit();
@@ -392,6 +485,9 @@ pub fn main() !void {
         try testing.expectEqualStrings("Hello, World!\n", body);
     }
 
+    // connection has been kept alive
+    try testing.expect(client.connection_pool.free_len == 1);
+
     { // absolute redirect
         var h = http.Headers{ .allocator = calloc };
         defer h.deinit();
@@ -413,6 +509,9 @@ pub fn main() !void {
         try testing.expectEqualStrings("Hello, World!\n", body);
     }
 
+    // connection has been kept alive
+    try testing.expect(client.connection_pool.free_len == 1);
+
     { // too many redirects
         var h = http.Headers{ .allocator = calloc };
         defer h.deinit();
@@ -432,6 +531,11 @@ pub fn main() !void {
         };
     }
 
+    // connection has been kept alive
+    try testing.expect(client.connection_pool.free_len == 1);
+
+    client.deinit();
+
     killServer(server.socket.listen_address);
     server_thread.join();
 }

From 9017d758b96c1f296249670dffb774a598bc8598 Mon Sep 17 00:00:00 2001
From: Nameless 
Date: Sat, 6 May 2023 21:35:04 -0500
Subject: [PATCH 097/725] std.http: use larger read buffer to hit faster tls
 code

---
 lib/std/http/Client.zig | 2 +-
 lib/std/http/Server.zig | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/std/http/Client.zig b/lib/std/http/Client.zig
index 77f4c6c6c3..023bdd28bc 100644
--- a/lib/std/http/Client.zig
+++ b/lib/std/http/Client.zig
@@ -251,7 +251,7 @@ pub const Connection = struct {
 
 /// A buffered (and peekable) Connection.
 pub const BufferedConnection = struct {
-    pub const buffer_size = 0x2000;
+    pub const buffer_size = std.crypto.tls.max_ciphertext_record_len;
 
     conn: Connection,
     read_buf: [buffer_size]u8 = undefined,
diff --git a/lib/std/http/Server.zig b/lib/std/http/Server.zig
index 2d3032bcdb..6b5db6725f 100644
--- a/lib/std/http/Server.zig
+++ b/lib/std/http/Server.zig
@@ -95,7 +95,7 @@ pub const Connection = struct {
 
 /// A buffered (and peekable) Connection.
 pub const BufferedConnection = struct {
-    pub const buffer_size = 0x2000;
+    pub const buffer_size = std.crypto.tls.max_ciphertext_record_len;
 
     conn: Connection,
     read_buf: [buffer_size]u8 = undefined,

From a62b5d84d834757682be9b5a0606ff8168c1d594 Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Sun, 30 Apr 2023 12:25:13 +1000
Subject: [PATCH 098/725] zir: add slice_length tag

---
 src/AstGen.zig    |  1 +
 src/Autodoc.zig   |  1 +
 src/Sema.zig      | 10 ++++++++++
 src/Zir.zig       | 14 ++++++++++++++
 src/print_zir.zig | 13 +++++++++++++
 5 files changed, 39 insertions(+)

diff --git a/src/AstGen.zig b/src/AstGen.zig
index 749e3d28c4..0233a9819d 100644
--- a/src/AstGen.zig
+++ b/src/AstGen.zig
@@ -2557,6 +2557,7 @@ fn addEnsureResult(gz: *GenZir, maybe_unused_result: Zir.Inst.Ref, statement: As
             .slice_start,
             .slice_end,
             .slice_sentinel,
+            .slice_length,
             .import,
             .switch_block,
             .switch_cond,
diff --git a/src/Autodoc.zig b/src/Autodoc.zig
index 42813f7614..db24f747f7 100644
--- a/src/Autodoc.zig
+++ b/src/Autodoc.zig
@@ -1287,6 +1287,7 @@ fn walkInstruction(
                 .expr = .{ .sliceIndex = slice_index },
             };
         },
+        .slice_length => @panic("TODO: implement walkInstruction for .slice_length"),
 
         // @check array_cat and array_mul
         .add,
diff --git a/src/Sema.zig b/src/Sema.zig
index b2387dfe12..d225a8f7b3 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -985,6 +985,7 @@ fn analyzeBodyInner(
             .slice_end                    => try sema.zirSliceEnd(block, inst),
             .slice_sentinel               => try sema.zirSliceSentinel(block, inst),
             .slice_start                  => try sema.zirSliceStart(block, inst),
+            .slice_length                 => try sema.zirSliceLength(block, inst),
             .str                          => try sema.zirStr(block, inst),
             .switch_block                 => try sema.zirSwitchBlock(block, inst),
             .switch_cond                  => try sema.zirSwitchCond(block, inst, false),
@@ -9961,6 +9962,15 @@ fn zirSliceSentinel(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileErr
     return sema.analyzeSlice(block, src, array_ptr, start, end, sentinel, sentinel_src, ptr_src, start_src, end_src);
 }
 
+fn zirSliceLength(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
+    const tracy = trace(@src());
+    defer tracy.end();
+
+    const inst_data = sema.code.instructions.items(.data)[inst].pl_node;
+    const src = inst_data.src();
+    return sema.fail(block, src, "TODO: implement .slice_length", .{});
+}
+
 fn zirSwitchCapture(
     sema: *Sema,
     block: *Block,
diff --git a/src/Zir.zig b/src/Zir.zig
index 7f9681e533..27ea02e642 100644
--- a/src/Zir.zig
+++ b/src/Zir.zig
@@ -570,6 +570,10 @@ pub const Inst = struct {
         /// Returns a pointer to the subslice.
         /// Uses the `pl_node` field. AST node is the slice syntax. Payload is `SliceSentinel`.
         slice_sentinel,
+        /// Slice operation `array_ptr[start..][0..len]`. No sentinel.
+        /// Returns a pointer to the subslice.
+        /// Uses the `pl_node` field. AST node is the slice syntax. Payload is `SliceLength`.
+        slice_length,
         /// Write a value to a pointer. For loading, see `load`.
         /// Source location is assumed to be same as previous instruction.
         /// Uses the `bin` union field.
@@ -1135,6 +1139,7 @@ pub const Inst = struct {
                 .slice_start,
                 .slice_end,
                 .slice_sentinel,
+                .slice_length,
                 .import,
                 .typeof_log2_int_type,
                 .resolve_inferred_alloc,
@@ -1430,6 +1435,7 @@ pub const Inst = struct {
                 .slice_start,
                 .slice_end,
                 .slice_sentinel,
+                .slice_length,
                 .import,
                 .typeof_log2_int_type,
                 .switch_capture,
@@ -1667,6 +1673,7 @@ pub const Inst = struct {
                 .slice_start = .pl_node,
                 .slice_end = .pl_node,
                 .slice_sentinel = .pl_node,
+                .slice_length = .pl_node,
                 .store = .bin,
                 .store_node = .pl_node,
                 .store_to_block_ptr = .bin,
@@ -2980,6 +2987,13 @@ pub const Inst = struct {
         sentinel: Ref,
     };
 
+    pub const SliceLength = struct {
+        lhs: Ref,
+        start: Ref,
+        len: Ref,
+        start_src_node_offset: i32,
+    };
+
     /// The meaning of these operands depends on the corresponding `Tag`.
     pub const Bin = struct {
         lhs: Ref,
diff --git a/src/print_zir.zig b/src/print_zir.zig
index f228adc7ea..6d1a716d3c 100644
--- a/src/print_zir.zig
+++ b/src/print_zir.zig
@@ -267,6 +267,7 @@ const Writer = struct {
             .slice_start => try self.writeSliceStart(stream, inst),
             .slice_end => try self.writeSliceEnd(stream, inst),
             .slice_sentinel => try self.writeSliceSentinel(stream, inst),
+            .slice_length => try self.writeSliceLength(stream, inst),
 
             .union_init => try self.writeUnionInit(stream, inst),
 
@@ -756,6 +757,18 @@ const Writer = struct {
         try self.writeSrc(stream, inst_data.src());
     }
 
+    fn writeSliceLength(self: *Writer, stream: anytype, inst: Zir.Inst.Index) !void {
+        const inst_data = self.code.instructions.items(.data)[inst].pl_node;
+        const extra = self.code.extraData(Zir.Inst.SliceLength, inst_data.payload_index).data;
+        try self.writeInstRef(stream, extra.lhs);
+        try stream.writeAll(", ");
+        try self.writeInstRef(stream, extra.start);
+        try stream.writeAll(", ");
+        try self.writeInstRef(stream, extra.len);
+        try stream.writeAll(") ");
+        try self.writeSrc(stream, inst_data.src());
+    }
+
     fn writeUnionInit(self: *Writer, stream: anytype, inst: Zir.Inst.Index) !void {
         const inst_data = self.code.instructions.items(.data)[inst].pl_node;
         const extra = self.code.extraData(Zir.Inst.UnionInit, inst_data.payload_index).data;

From e58a0c5e9e9a6eab7f5efdc60b0f3d5b9c106786 Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Sun, 30 Apr 2023 15:00:28 +1000
Subject: [PATCH 099/725] sema: implement slice_length ZIR instruction

---
 src/Sema.zig | 133 +++++++++++++++++++++++++++++----------------------
 1 file changed, 76 insertions(+), 57 deletions(-)

diff --git a/src/Sema.zig b/src/Sema.zig
index d225a8f7b3..bcb13ec6f0 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -9923,7 +9923,7 @@ fn zirSliceStart(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!
     const start_src: LazySrcLoc = .{ .node_offset_slice_start = inst_data.src_node };
     const end_src: LazySrcLoc = .{ .node_offset_slice_end = inst_data.src_node };
 
-    return sema.analyzeSlice(block, src, array_ptr, start, .none, .none, .unneeded, ptr_src, start_src, end_src);
+    return sema.analyzeSlice(block, src, array_ptr, start, .none, .none, .unneeded, ptr_src, start_src, end_src, false);
 }
 
 fn zirSliceEnd(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
@@ -9940,7 +9940,7 @@ fn zirSliceEnd(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai
     const start_src: LazySrcLoc = .{ .node_offset_slice_start = inst_data.src_node };
     const end_src: LazySrcLoc = .{ .node_offset_slice_end = inst_data.src_node };
 
-    return sema.analyzeSlice(block, src, array_ptr, start, end, .none, .unneeded, ptr_src, start_src, end_src);
+    return sema.analyzeSlice(block, src, array_ptr, start, end, .none, .unneeded, ptr_src, start_src, end_src, false);
 }
 
 fn zirSliceSentinel(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
@@ -9959,7 +9959,7 @@ fn zirSliceSentinel(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileErr
     const start_src: LazySrcLoc = .{ .node_offset_slice_start = inst_data.src_node };
     const end_src: LazySrcLoc = .{ .node_offset_slice_end = inst_data.src_node };
 
-    return sema.analyzeSlice(block, src, array_ptr, start, end, sentinel, sentinel_src, ptr_src, start_src, end_src);
+    return sema.analyzeSlice(block, src, array_ptr, start, end, sentinel, sentinel_src, ptr_src, start_src, end_src, false);
 }
 
 fn zirSliceLength(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
@@ -9968,7 +9968,15 @@ fn zirSliceLength(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
 
     const inst_data = sema.code.instructions.items(.data)[inst].pl_node;
     const src = inst_data.src();
-    return sema.fail(block, src, "TODO: implement .slice_length", .{});
+    const extra = sema.code.extraData(Zir.Inst.SliceLength, inst_data.payload_index).data;
+    const array_ptr = try sema.resolveInst(extra.lhs);
+    const start = try sema.resolveInst(extra.start);
+    const len = try sema.resolveInst(extra.len);
+    const ptr_src: LazySrcLoc = .{ .node_offset_slice_ptr = inst_data.src_node };
+    const start_src: LazySrcLoc = .{ .node_offset_slice_start = extra.start_src_node_offset };
+    const end_src: LazySrcLoc = .{ .node_offset_slice_end = inst_data.src_node };
+
+    return sema.analyzeSlice(block, src, array_ptr, start, len, .none, .unneeded, ptr_src, start_src, end_src, true);
 }
 
 fn zirSwitchCapture(
@@ -29200,6 +29208,7 @@ fn analyzeSlice(
     ptr_src: LazySrcLoc,
     start_src: LazySrcLoc,
     end_src: LazySrcLoc,
+    by_length: bool,
 ) CompileError!Air.Inst.Ref {
     // Slice expressions can operate on a variable whose type is an array. This requires
     // the slice operand to be a pointer. In the case of a non-array, it will be a double pointer.
@@ -29274,7 +29283,12 @@ fn analyzeSlice(
     // we might learn of the length because it is a comptime-known slice value.
     var end_is_len = uncasted_end_opt == .none;
     const end = e: {
-        if (array_ty.zigTypeTag() == .Array) {
+        if (by_length and !end_is_len) {
+            const len = try sema.coerce(block, Type.usize, uncasted_end_opt, end_src);
+            const uncasted_end = try sema.analyzeArithmetic(block, .add, start, len, src, start_src, end_src, false);
+            const end = try sema.coerce(block, Type.usize, uncasted_end, end_src);
+            break :e end;
+        } else if (array_ty.zigTypeTag() == .Array) {
             const len_val = try Value.Tag.int_u64.create(sema.arena, array_ty.arrayLen());
 
             if (!end_is_len) {
@@ -29384,66 +29398,71 @@ fn analyzeSlice(
     const slice_sentinel = if (sentinel_opt != .none) sentinel else null;
 
     // requirement: start <= end
-    if (try sema.resolveDefinedValue(block, end_src, end)) |end_val| {
-        if (try sema.resolveDefinedValue(block, start_src, start)) |start_val| {
-            if (!(try sema.compareAll(start_val, .lte, end_val, Type.usize))) {
-                return sema.fail(
-                    block,
-                    start_src,
-                    "start index {} is larger than end index {}",
-                    .{
-                        start_val.fmtValue(Type.usize, mod),
-                        end_val.fmtValue(Type.usize, mod),
-                    },
-                );
-            }
-            if (try sema.resolveMaybeUndefVal(new_ptr)) |ptr_val| sentinel_check: {
-                const expected_sentinel = sentinel orelse break :sentinel_check;
-                const start_int = start_val.getUnsignedInt(sema.mod.getTarget()).?;
-                const end_int = end_val.getUnsignedInt(sema.mod.getTarget()).?;
-                const sentinel_index = try sema.usizeCast(block, end_src, end_int - start_int);
-
-                const elem_ptr = try ptr_val.elemPtr(sema.typeOf(new_ptr), sema.arena, sentinel_index, sema.mod);
-                const res = try sema.pointerDerefExtra(block, src, elem_ptr, elem_ty, false);
-                const actual_sentinel = switch (res) {
-                    .runtime_load => break :sentinel_check,
-                    .val => |v| v,
-                    .needed_well_defined => |ty| return sema.fail(
+    if (!by_length) {
+        if (try sema.resolveDefinedValue(block, end_src, end)) |end_val| {
+            if (try sema.resolveDefinedValue(block, start_src, start)) |start_val| {
+                if (!(try sema.compareAll(start_val, .lte, end_val, Type.usize))) {
+                    return sema.fail(
                         block,
-                        src,
-                        "comptime dereference requires '{}' to have a well-defined layout, but it does not.",
-                        .{ty.fmt(sema.mod)},
-                    ),
-                    .out_of_bounds => |ty| return sema.fail(
-                        block,
-                        end_src,
-                        "slice end index {d} exceeds bounds of containing decl of type '{}'",
-                        .{ end_int, ty.fmt(sema.mod) },
-                    ),
-                };
+                        start_src,
+                        "start index {} is larger than end index {}",
+                        .{
+                            start_val.fmtValue(Type.usize, mod),
+                            end_val.fmtValue(Type.usize, mod),
+                        },
+                    );
+                }
+                if (try sema.resolveMaybeUndefVal(new_ptr)) |ptr_val| sentinel_check: {
+                    const expected_sentinel = sentinel orelse break :sentinel_check;
+                    const start_int = start_val.getUnsignedInt(sema.mod.getTarget()).?;
+                    const end_int = end_val.getUnsignedInt(sema.mod.getTarget()).?;
+                    const sentinel_index = try sema.usizeCast(block, end_src, end_int - start_int);
 
-                if (!actual_sentinel.eql(expected_sentinel, elem_ty, sema.mod)) {
-                    const msg = msg: {
-                        const msg = try sema.errMsg(block, src, "value in memory does not match slice sentinel", .{});
-                        errdefer msg.destroy(sema.gpa);
-                        try sema.errNote(block, src, msg, "expected '{}', found '{}'", .{
-                            expected_sentinel.fmtValue(elem_ty, sema.mod),
-                            actual_sentinel.fmtValue(elem_ty, sema.mod),
-                        });
-
-                        break :msg msg;
+                    const elem_ptr = try ptr_val.elemPtr(sema.typeOf(new_ptr), sema.arena, sentinel_index, sema.mod);
+                    const res = try sema.pointerDerefExtra(block, src, elem_ptr, elem_ty, false);
+                    const actual_sentinel = switch (res) {
+                        .runtime_load => break :sentinel_check,
+                        .val => |v| v,
+                        .needed_well_defined => |ty| return sema.fail(
+                            block,
+                            src,
+                            "comptime dereference requires '{}' to have a well-defined layout, but it does not.",
+                            .{ty.fmt(sema.mod)},
+                        ),
+                        .out_of_bounds => |ty| return sema.fail(
+                            block,
+                            end_src,
+                            "slice end index {d} exceeds bounds of containing decl of type '{}'",
+                            .{ end_int, ty.fmt(sema.mod) },
+                        ),
                     };
-                    return sema.failWithOwnedErrorMsg(msg);
+
+                    if (!actual_sentinel.eql(expected_sentinel, elem_ty, sema.mod)) {
+                        const msg = msg: {
+                            const msg = try sema.errMsg(block, src, "value in memory does not match slice sentinel", .{});
+                            errdefer msg.destroy(sema.gpa);
+                            try sema.errNote(block, src, msg, "expected '{}', found '{}'", .{
+                                expected_sentinel.fmtValue(elem_ty, sema.mod),
+                                actual_sentinel.fmtValue(elem_ty, sema.mod),
+                            });
+
+                            break :msg msg;
+                        };
+                        return sema.failWithOwnedErrorMsg(msg);
+                    }
                 }
             }
         }
-    }
 
-    if (block.wantSafety() and !block.is_comptime) {
-        // requirement: start <= end
-        try sema.panicStartLargerThanEnd(block, start, end);
+        if (block.wantSafety() and !block.is_comptime) {
+            // requirement: start <= end
+            try sema.panicStartLargerThanEnd(block, start, end);
+        }
     }
-    const new_len = try sema.analyzeArithmetic(block, .sub, end, start, src, end_src, start_src, false);
+    const new_len = if (by_length)
+        try sema.coerce(block, Type.usize, uncasted_end_opt, end_src)
+    else
+        try sema.analyzeArithmetic(block, .sub, end, start, src, end_src, start_src, false);
     const opt_new_len_val = try sema.resolveDefinedValue(block, src, new_len);
 
     const new_ptr_ty_info = sema.typeOf(new_ptr).ptrInfo().data;

From 9fb5b047e97896d7b25e897dc3737769d6b35a78 Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Sun, 30 Apr 2023 15:01:21 +1000
Subject: [PATCH 100/725] astgen: lower s[start..][0..len] to slice_length ZIR

---
 src/AstGen.zig | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/AstGen.zig b/src/AstGen.zig
index 0233a9819d..c8d079f1e1 100644
--- a/src/AstGen.zig
+++ b/src/AstGen.zig
@@ -849,10 +849,25 @@ fn expr(gz: *GenZir, scope: *Scope, ri: ResultInfo, node: Ast.Node.Index) InnerE
             return rvalue(gz, ri, result, node);
         },
         .slice => {
+            const extra = tree.extraData(node_datas[node].rhs, Ast.Node.Slice);
+            const lhs_node = node_datas[node].lhs;
+            if (node_tags[lhs_node] == .slice_open and nodeIsTriviallyZero(tree, extra.start)) {
+                const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[lhs_node].lhs);
+                const start = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[lhs_node].rhs);
+                const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
+                const len = if (extra.end != 0) try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end) else .none;
+                try emitDbgStmt(gz, cursor);
+                const result = try gz.addPlNode(.slice_length, node, Zir.Inst.SliceLength{
+                    .lhs = lhs,
+                    .start = start,
+                    .len = len,
+                    .start_src_node_offset = gz.nodeIndexToRelative(lhs_node),
+                });
+                return rvalue(gz, ri, result, node);
+            }
             const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[node].lhs);
 
             const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
-            const extra = tree.extraData(node_datas[node].rhs, Ast.Node.Slice);
             const start = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.start);
             const end = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end);
             try emitDbgStmt(gz, cursor);

From 8aa70cf6cd88b6103200d558c60295550c7b911d Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Sun, 30 Apr 2023 14:38:11 +1000
Subject: [PATCH 101/725] sema: omit extraneous addition when safety is
 unwanted

---
 src/Sema.zig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Sema.zig b/src/Sema.zig
index bcb13ec6f0..17ba89a6e3 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -29284,6 +29284,7 @@ fn analyzeSlice(
     var end_is_len = uncasted_end_opt == .none;
     const end = e: {
         if (by_length and !end_is_len) {
+            if (!block.wantSafety()) break :e undefined;
             const len = try sema.coerce(block, Type.usize, uncasted_end_opt, end_src);
             const uncasted_end = try sema.analyzeArithmetic(block, .add, start, len, src, start_src, end_src, false);
             const end = try sema.coerce(block, Type.usize, uncasted_end, end_src);

From 2c2a0402c65986674d427390292f2fe38474c90f Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Sun, 30 Apr 2023 16:54:25 +1000
Subject: [PATCH 102/725] autodoc: implement slice_length case in
 walkInstruction

---
 src/Autodoc.zig | 43 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/src/Autodoc.zig b/src/Autodoc.zig
index db24f747f7..b068c260ef 100644
--- a/src/Autodoc.zig
+++ b/src/Autodoc.zig
@@ -1287,7 +1287,48 @@ fn walkInstruction(
                 .expr = .{ .sliceIndex = slice_index },
             };
         },
-        .slice_length => @panic("TODO: implement walkInstruction for .slice_length"),
+        .slice_length => {
+            const pl_node = data[inst_index].pl_node;
+            const extra = file.zir.extraData(Zir.Inst.SliceLength, pl_node.payload_index);
+
+            const slice_index = self.exprs.items.len;
+            try self.exprs.append(self.arena, .{ .slice = .{ .lhs = 0, .start = 0 } });
+
+            var lhs: DocData.WalkResult = try self.walkRef(
+                file,
+                parent_scope,
+                parent_src,
+                extra.data.lhs,
+                false,
+            );
+            var start: DocData.WalkResult = try self.walkRef(
+                file,
+                parent_scope,
+                parent_src,
+                extra.data.start,
+                false,
+            );
+            var len: DocData.WalkResult = try self.walkRef(
+                file,
+                parent_scope,
+                parent_src,
+                extra.data.len,
+                false,
+            );
+
+            const lhs_index = self.exprs.items.len;
+            try self.exprs.append(self.arena, lhs.expr);
+            const start_index = self.exprs.items.len;
+            try self.exprs.append(self.arena, start.expr);
+            const len_index = self.exprs.items.len;
+            try self.exprs.append(self.arena, len.expr);
+            self.exprs.items[slice_index] = .{ .slice = .{ .lhs = lhs_index, .start = start_index, .end = len_index } };
+
+            return DocData.WalkResult{
+                .typeRef = self.decls.items[lhs.expr.declRef.Analyzed].value.typeRef,
+                .expr = .{ .sliceIndex = slice_index },
+            };
+        },
 
         // @check array_cat and array_mul
         .add,

From 7c8d60e814b985d3aab43c7c467f424596942ef9 Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Mon, 1 May 2023 15:37:25 +1000
Subject: [PATCH 103/725] test: add behavior tests for slice-by-length

---
 test/behavior/slice.zig | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/test/behavior/slice.zig b/test/behavior/slice.zig
index 237ddd195b..51538d9b10 100644
--- a/test/behavior/slice.zig
+++ b/test/behavior/slice.zig
@@ -355,6 +355,10 @@ test "slice syntax resulting in pointer-to-array" {
             try testSlice();
             try testSliceOpt();
             try testSliceAlign();
+            try testSliceLength();
+            try testSliceLengthZ();
+            try testArrayLength();
+            try testArrayLengthZ();
         }
 
         fn testArray() !void {
@@ -465,6 +469,40 @@ test "slice syntax resulting in pointer-to-array" {
             try expectEqualSlices("a"[0..] ++ "b"[0..], "ab");
             try expectEqualSlices("a"[0.. :0] ++ "b"[0.. :0], "ab");
         }
+
+        fn testSliceLength() !void {
+            var array = [5]u8{ 1, 2, 3, 4, 5 };
+            var slice: []u8 = &array;
+            comptime try expect(@TypeOf(slice[1..][0..2]) == *[2]u8);
+            comptime try expect(@TypeOf(slice[1..][0..4]) == *[4]u8);
+            comptime try expect(@TypeOf(slice[1..][0..2 :4]) == *[2:4]u8);
+        }
+
+        fn testSliceLengthZ() !void {
+            var array = [5:0]u8{ 1, 2, 3, 4, 5 };
+            var slice: [:0]u8 = &array;
+            comptime try expect(@TypeOf(slice[1..][0..2]) == *[2]u8);
+            comptime try expect(@TypeOf(slice[1..][0..2 :4]) == *[2:4]u8);
+            comptime try expect(@TypeOf(slice[1.. :0][0..2]) == *[2]u8);
+            comptime try expect(@TypeOf(slice[1.. :0][0..2 :4]) == *[2:4]u8);
+        }
+
+        fn testArrayLength() !void {
+            var array = [5]u8{ 1, 2, 3, 4, 5 };
+            comptime try expect(@TypeOf(array[1..][0..2]) == *[2]u8);
+            comptime try expect(@TypeOf(array[1..][0..4]) == *[4]u8);
+            comptime try expect(@TypeOf(array[1..][0..2 :4]) == *[2:4]u8);
+        }
+
+        fn testArrayLengthZ() !void {
+            var array = [5:0]u8{ 1, 2, 3, 4, 5 };
+            comptime try expect(@TypeOf(array[1..][0..2]) == *[2]u8);
+            comptime try expect(@TypeOf(array[1..][0..4]) == *[4:0]u8);
+            comptime try expect(@TypeOf(array[1..][0..2 :4]) == *[2:4]u8);
+            comptime try expect(@TypeOf(array[1.. :0][0..2]) == *[2]u8);
+            comptime try expect(@TypeOf(array[1.. :0][0..4]) == *[4:0]u8);
+            comptime try expect(@TypeOf(array[1.. :0][0..2 :4]) == *[2:4]u8);
+        }
     };
 
     try S.doTheTest();

From 64e319f5555a25dbf4c4d7e65289f7f96d6588b4 Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Sun, 30 Apr 2023 23:50:08 +1000
Subject: [PATCH 104/725] add optional sentinel to slice_length ZIR

---
 src/AstGen.zig    |  42 +++++++++++++--
 src/Autodoc.zig   |  17 +++++-
 src/Sema.zig      | 131 ++++++++++++++++++++++++----------------------
 src/Zir.zig       |   3 +-
 src/print_zir.zig |   4 ++
 5 files changed, 129 insertions(+), 68 deletions(-)

diff --git a/src/AstGen.zig b/src/AstGen.zig
index c8d079f1e1..8df8f0e1cf 100644
--- a/src/AstGen.zig
+++ b/src/AstGen.zig
@@ -851,9 +851,18 @@ fn expr(gz: *GenZir, scope: *Scope, ri: ResultInfo, node: Ast.Node.Index) InnerE
         .slice => {
             const extra = tree.extraData(node_datas[node].rhs, Ast.Node.Slice);
             const lhs_node = node_datas[node].lhs;
-            if (node_tags[lhs_node] == .slice_open and nodeIsTriviallyZero(tree, extra.start)) {
+            const lhs_tag = node_tags[lhs_node];
+            const lhs_is_slice_sentinel = lhs_tag == .slice_sentinel;
+            const lhs_is_open_slice = lhs_tag == .slice_open or
+                (lhs_is_slice_sentinel and tree.extraData(node_datas[lhs_node].rhs, Ast.Node.SliceSentinel).end == 0);
+            if (lhs_is_open_slice and nodeIsTriviallyZero(tree, extra.start)) {
                 const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[lhs_node].lhs);
-                const start = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[lhs_node].rhs);
+
+                const start = if (lhs_is_slice_sentinel) start: {
+                    const lhs_extra = tree.extraData(node_datas[lhs_node].rhs, Ast.Node.SliceSentinel);
+                    break :start try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, lhs_extra.start);
+                } else try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[lhs_node].rhs);
+
                 const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
                 const len = if (extra.end != 0) try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end) else .none;
                 try emitDbgStmt(gz, cursor);
@@ -862,6 +871,7 @@ fn expr(gz: *GenZir, scope: *Scope, ri: ResultInfo, node: Ast.Node.Index) InnerE
                     .start = start,
                     .len = len,
                     .start_src_node_offset = gz.nodeIndexToRelative(lhs_node),
+                    .sentinel = .none,
                 });
                 return rvalue(gz, ri, result, node);
             }
@@ -879,10 +889,36 @@ fn expr(gz: *GenZir, scope: *Scope, ri: ResultInfo, node: Ast.Node.Index) InnerE
             return rvalue(gz, ri, result, node);
         },
         .slice_sentinel => {
+            const extra = tree.extraData(node_datas[node].rhs, Ast.Node.SliceSentinel);
+            const lhs_node = node_datas[node].lhs;
+            const lhs_tag = node_tags[lhs_node];
+            const lhs_is_slice_sentinel = lhs_tag == .slice_sentinel;
+            const lhs_is_open_slice = lhs_tag == .slice_open or
+                (lhs_is_slice_sentinel and tree.extraData(node_datas[lhs_node].rhs, Ast.Node.SliceSentinel).end == 0);
+            if (lhs_is_open_slice and nodeIsTriviallyZero(tree, extra.start)) {
+                const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[lhs_node].lhs);
+
+                const start = if (lhs_is_slice_sentinel) start: {
+                    const lhs_extra = tree.extraData(node_datas[lhs_node].rhs, Ast.Node.SliceSentinel);
+                    break :start try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, lhs_extra.start);
+                } else try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[lhs_node].rhs);
+
+                const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
+                const len = if (extra.end != 0) try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end) else .none;
+                const sentinel = try expr(gz, scope, .{ .rl = .none }, extra.sentinel);
+                try emitDbgStmt(gz, cursor);
+                const result = try gz.addPlNode(.slice_length, node, Zir.Inst.SliceLength{
+                    .lhs = lhs,
+                    .start = start,
+                    .len = len,
+                    .start_src_node_offset = gz.nodeIndexToRelative(lhs_node),
+                    .sentinel = sentinel,
+                });
+                return rvalue(gz, ri, result, node);
+            }
             const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[node].lhs);
 
             const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
-            const extra = tree.extraData(node_datas[node].rhs, Ast.Node.SliceSentinel);
             const start = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.start);
             const end = if (extra.end != 0) try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end) else .none;
             const sentinel = try expr(gz, scope, .{ .rl = .none }, extra.sentinel);
diff --git a/src/Autodoc.zig b/src/Autodoc.zig
index b068c260ef..eba53b7e17 100644
--- a/src/Autodoc.zig
+++ b/src/Autodoc.zig
@@ -1315,6 +1315,16 @@ fn walkInstruction(
                 extra.data.len,
                 false,
             );
+            var sentinel_opt: ?DocData.WalkResult = if (extra.data.sentinel != .none)
+                try self.walkRef(
+                    file,
+                    parent_scope,
+                    parent_src,
+                    extra.data.sentinel,
+                    false,
+                )
+            else
+                null;
 
             const lhs_index = self.exprs.items.len;
             try self.exprs.append(self.arena, lhs.expr);
@@ -1322,7 +1332,12 @@ fn walkInstruction(
             try self.exprs.append(self.arena, start.expr);
             const len_index = self.exprs.items.len;
             try self.exprs.append(self.arena, len.expr);
-            self.exprs.items[slice_index] = .{ .slice = .{ .lhs = lhs_index, .start = start_index, .end = len_index } };
+            const sentinel_index = if (sentinel_opt) |sentinel| sentinel_index: {
+                const index = self.exprs.items.len;
+                try self.exprs.append(self.arena, sentinel.expr);
+                break :sentinel_index index;
+            } else null;
+            self.exprs.items[slice_index] = .{ .slice = .{ .lhs = lhs_index, .start = start_index, .end = len_index, .sentinel = sentinel_index } };
 
             return DocData.WalkResult{
                 .typeRef = self.decls.items[lhs.expr.declRef.Analyzed].value.typeRef,
diff --git a/src/Sema.zig b/src/Sema.zig
index 17ba89a6e3..8deca97505 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -9972,11 +9972,16 @@ fn zirSliceLength(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
     const array_ptr = try sema.resolveInst(extra.lhs);
     const start = try sema.resolveInst(extra.start);
     const len = try sema.resolveInst(extra.len);
+    const sentinel = try sema.resolveInst(extra.sentinel);
     const ptr_src: LazySrcLoc = .{ .node_offset_slice_ptr = inst_data.src_node };
     const start_src: LazySrcLoc = .{ .node_offset_slice_start = extra.start_src_node_offset };
     const end_src: LazySrcLoc = .{ .node_offset_slice_end = inst_data.src_node };
+    const sentinel_src: LazySrcLoc = if (sentinel == .none)
+        .unneeded
+    else
+        .{ .node_offset_slice_sentinel = inst_data.src_node };
 
-    return sema.analyzeSlice(block, src, array_ptr, start, len, .none, .unneeded, ptr_src, start_src, end_src, true);
+    return sema.analyzeSlice(block, src, array_ptr, start, len, sentinel, sentinel_src, ptr_src, start_src, end_src, true);
 }
 
 fn zirSwitchCapture(
@@ -29283,17 +29288,15 @@ fn analyzeSlice(
     // we might learn of the length because it is a comptime-known slice value.
     var end_is_len = uncasted_end_opt == .none;
     const end = e: {
-        if (by_length and !end_is_len) {
-            if (!block.wantSafety()) break :e undefined;
-            const len = try sema.coerce(block, Type.usize, uncasted_end_opt, end_src);
-            const uncasted_end = try sema.analyzeArithmetic(block, .add, start, len, src, start_src, end_src, false);
-            const end = try sema.coerce(block, Type.usize, uncasted_end, end_src);
-            break :e end;
-        } else if (array_ty.zigTypeTag() == .Array) {
+        if (array_ty.zigTypeTag() == .Array) {
             const len_val = try Value.Tag.int_u64.create(sema.arena, array_ty.arrayLen());
 
             if (!end_is_len) {
-                const end = try sema.coerce(block, Type.usize, uncasted_end_opt, end_src);
+                const end = if (by_length) end: {
+                    const len = try sema.coerce(block, Type.usize, uncasted_end_opt, end_src);
+                    const uncasted_end = try sema.analyzeArithmetic(block, .add, start, len, src, start_src, end_src, false);
+                    break :end try sema.coerce(block, Type.usize, uncasted_end, end_src);
+                } else try sema.coerce(block, Type.usize, uncasted_end_opt, end_src);
                 if (try sema.resolveMaybeUndefVal(end)) |end_val| {
                     const len_s_val = try Value.Tag.int_u64.create(
                         sema.arena,
@@ -29330,7 +29333,11 @@ fn analyzeSlice(
             break :e try sema.addConstant(Type.usize, len_val);
         } else if (slice_ty.isSlice()) {
             if (!end_is_len) {
-                const end = try sema.coerce(block, Type.usize, uncasted_end_opt, end_src);
+                const end = if (by_length) end: {
+                    const len = try sema.coerce(block, Type.usize, uncasted_end_opt, end_src);
+                    const uncasted_end = try sema.analyzeArithmetic(block, .add, start, len, src, start_src, end_src, false);
+                    break :end try sema.coerce(block, Type.usize, uncasted_end, end_src);
+                } else try sema.coerce(block, Type.usize, uncasted_end_opt, end_src);
                 if (try sema.resolveDefinedValue(block, end_src, end)) |end_val| {
                     if (try sema.resolveMaybeUndefVal(ptr_or_slice)) |slice_val| {
                         if (slice_val.isUndef()) {
@@ -29399,66 +29406,64 @@ fn analyzeSlice(
     const slice_sentinel = if (sentinel_opt != .none) sentinel else null;
 
     // requirement: start <= end
-    if (!by_length) {
-        if (try sema.resolveDefinedValue(block, end_src, end)) |end_val| {
-            if (try sema.resolveDefinedValue(block, start_src, start)) |start_val| {
-                if (!(try sema.compareAll(start_val, .lte, end_val, Type.usize))) {
-                    return sema.fail(
+    if (try sema.resolveDefinedValue(block, end_src, end)) |end_val| {
+        if (try sema.resolveDefinedValue(block, start_src, start)) |start_val| {
+            if (!by_length and !(try sema.compareAll(start_val, .lte, end_val, Type.usize))) {
+                return sema.fail(
+                    block,
+                    start_src,
+                    "start index {} is larger than end index {}",
+                    .{
+                        start_val.fmtValue(Type.usize, mod),
+                        end_val.fmtValue(Type.usize, mod),
+                    },
+                );
+            }
+            if (try sema.resolveMaybeUndefVal(new_ptr)) |ptr_val| sentinel_check: {
+                const expected_sentinel = sentinel orelse break :sentinel_check;
+                const start_int = start_val.getUnsignedInt(sema.mod.getTarget()).?;
+                const end_int = end_val.getUnsignedInt(sema.mod.getTarget()).?;
+                const sentinel_index = try sema.usizeCast(block, end_src, end_int - start_int);
+
+                const elem_ptr = try ptr_val.elemPtr(sema.typeOf(new_ptr), sema.arena, sentinel_index, sema.mod);
+                const res = try sema.pointerDerefExtra(block, src, elem_ptr, elem_ty, false);
+                const actual_sentinel = switch (res) {
+                    .runtime_load => break :sentinel_check,
+                    .val => |v| v,
+                    .needed_well_defined => |ty| return sema.fail(
                         block,
-                        start_src,
-                        "start index {} is larger than end index {}",
-                        .{
-                            start_val.fmtValue(Type.usize, mod),
-                            end_val.fmtValue(Type.usize, mod),
-                        },
-                    );
-                }
-                if (try sema.resolveMaybeUndefVal(new_ptr)) |ptr_val| sentinel_check: {
-                    const expected_sentinel = sentinel orelse break :sentinel_check;
-                    const start_int = start_val.getUnsignedInt(sema.mod.getTarget()).?;
-                    const end_int = end_val.getUnsignedInt(sema.mod.getTarget()).?;
-                    const sentinel_index = try sema.usizeCast(block, end_src, end_int - start_int);
+                        src,
+                        "comptime dereference requires '{}' to have a well-defined layout, but it does not.",
+                        .{ty.fmt(sema.mod)},
+                    ),
+                    .out_of_bounds => |ty| return sema.fail(
+                        block,
+                        end_src,
+                        "slice end index {d} exceeds bounds of containing decl of type '{}'",
+                        .{ end_int, ty.fmt(sema.mod) },
+                    ),
+                };
 
-                    const elem_ptr = try ptr_val.elemPtr(sema.typeOf(new_ptr), sema.arena, sentinel_index, sema.mod);
-                    const res = try sema.pointerDerefExtra(block, src, elem_ptr, elem_ty, false);
-                    const actual_sentinel = switch (res) {
-                        .runtime_load => break :sentinel_check,
-                        .val => |v| v,
-                        .needed_well_defined => |ty| return sema.fail(
-                            block,
-                            src,
-                            "comptime dereference requires '{}' to have a well-defined layout, but it does not.",
-                            .{ty.fmt(sema.mod)},
-                        ),
-                        .out_of_bounds => |ty| return sema.fail(
-                            block,
-                            end_src,
-                            "slice end index {d} exceeds bounds of containing decl of type '{}'",
-                            .{ end_int, ty.fmt(sema.mod) },
-                        ),
+                if (!actual_sentinel.eql(expected_sentinel, elem_ty, sema.mod)) {
+                    const msg = msg: {
+                        const msg = try sema.errMsg(block, src, "value in memory does not match slice sentinel", .{});
+                        errdefer msg.destroy(sema.gpa);
+                        try sema.errNote(block, src, msg, "expected '{}', found '{}'", .{
+                            expected_sentinel.fmtValue(elem_ty, sema.mod),
+                            actual_sentinel.fmtValue(elem_ty, sema.mod),
+                        });
+
+                        break :msg msg;
                     };
-
-                    if (!actual_sentinel.eql(expected_sentinel, elem_ty, sema.mod)) {
-                        const msg = msg: {
-                            const msg = try sema.errMsg(block, src, "value in memory does not match slice sentinel", .{});
-                            errdefer msg.destroy(sema.gpa);
-                            try sema.errNote(block, src, msg, "expected '{}', found '{}'", .{
-                                expected_sentinel.fmtValue(elem_ty, sema.mod),
-                                actual_sentinel.fmtValue(elem_ty, sema.mod),
-                            });
-
-                            break :msg msg;
-                        };
-                        return sema.failWithOwnedErrorMsg(msg);
-                    }
+                    return sema.failWithOwnedErrorMsg(msg);
                 }
             }
         }
+    }
 
-        if (block.wantSafety() and !block.is_comptime) {
-            // requirement: start <= end
-            try sema.panicStartLargerThanEnd(block, start, end);
-        }
+    if (!by_length and block.wantSafety() and !block.is_comptime) {
+        // requirement: start <= end
+        try sema.panicStartLargerThanEnd(block, start, end);
     }
     const new_len = if (by_length)
         try sema.coerce(block, Type.usize, uncasted_end_opt, end_src)
diff --git a/src/Zir.zig b/src/Zir.zig
index 27ea02e642..ccff32c435 100644
--- a/src/Zir.zig
+++ b/src/Zir.zig
@@ -570,7 +570,7 @@ pub const Inst = struct {
         /// Returns a pointer to the subslice.
         /// Uses the `pl_node` field. AST node is the slice syntax. Payload is `SliceSentinel`.
         slice_sentinel,
-        /// Slice operation `array_ptr[start..][0..len]`. No sentinel.
+        /// Slice operation `array_ptr[start..][0..len]`. Optional sentinel.
         /// Returns a pointer to the subslice.
         /// Uses the `pl_node` field. AST node is the slice syntax. Payload is `SliceLength`.
         slice_length,
@@ -2991,6 +2991,7 @@ pub const Inst = struct {
         lhs: Ref,
         start: Ref,
         len: Ref,
+        sentinel: Ref,
         start_src_node_offset: i32,
     };
 
diff --git a/src/print_zir.zig b/src/print_zir.zig
index 6d1a716d3c..f5e84fcf5b 100644
--- a/src/print_zir.zig
+++ b/src/print_zir.zig
@@ -765,6 +765,10 @@ const Writer = struct {
         try self.writeInstRef(stream, extra.start);
         try stream.writeAll(", ");
         try self.writeInstRef(stream, extra.len);
+        if (extra.sentinel != .none) {
+            try stream.writeAll(", ");
+            try self.writeInstRef(stream, extra.sentinel);
+        }
         try stream.writeAll(") ");
         try self.writeSrc(stream, inst_data.src());
     }

From 5bb8e9cd975ea68c924d04be6608f96b41204b7f Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Tue, 2 May 2023 12:51:58 +1000
Subject: [PATCH 105/725] langref: mention slice-by-length pattern

---
 doc/langref.html.in | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/langref.html.in b/doc/langref.html.in
index 69ad0624c4..5876fd79ba 100644
--- a/doc/langref.html.in
+++ b/doc/langref.html.in
@@ -2953,6 +2953,14 @@ test "basic slices" {
     const array_ptr = array[0..array.len];
     try expect(@TypeOf(array_ptr) == *[array.len]i32);
 
+    // You can perform a slice-by-length by slicing twice. This allows the compiler
+    // to perform some optimisations like recognising a comptime-known length when
+    // the start position is only known at runtime.
+    var runtime_start: usize = 1;
+    const length = 2;
+    const array_ptr_len = array[runtime_start..][0..length];
+    try expect(@TypeOf(array_ptr_len) == *[length]i32);
+
     // Using the address-of operator on a slice gives a single-item pointer,
     // while using the `ptr` field gives a many-item pointer.
     try expect(@TypeOf(slice.ptr) == [*]i32);

From 2c5924c59a148335e5025b72a5ba98d765ed771d Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Tue, 2 May 2023 22:04:15 +1000
Subject: [PATCH 106/725] autodoc: fix support for slice_length ZIR instruction

---
 lib/docs/main.js | 19 ++++++++++++++++++-
 src/Autodoc.zig  | 14 +++++++++++++-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/lib/docs/main.js b/lib/docs/main.js
index 9b650643e9..f803fddb81 100644
--- a/lib/docs/main.js
+++ b/lib/docs/main.js
@@ -1093,6 +1093,23 @@ const NAV_MODES = {
         payloadHtml += decl + "[" + start + ".." + end + sentinel + "]";
         return payloadHtml;
       }
+      case "sliceLength": {
+        let payloadHtml = "";
+        const lhsExpr = zigAnalysis.exprs[expr.sliceLength.lhs];
+        const startExpr = zigAnalysis.exprs[expr.sliceLength.start];
+        const lenExpr = zigAnalysis.exprs[expr.sliceLength.len];
+        let decl = exprName(lhsExpr, opts);
+        let start = exprName(startExpr, opts);
+        let len = exprName(lenExpr, opts);
+        let sentinel = "";
+        if (expr.sliceLength["sentinel"]) {
+            const sentinelExpr = zigAnalysis.exprs[expr.sliceLength.sentinel];
+            let sentinel_ = exprName(sentinelExpr, options);
+            sentinel += " :" + sentinel_;
+        }
+        payloadHtml += decl + "[" + start + "..][0.." + len + sentinel + "]";
+        return payloadHtml;
+      }
       case "sliceIndex": {
         const sliceIndex = zigAnalysis.exprs[expr.sliceIndex];
         return exprName(sliceIndex, opts, opts);
@@ -4850,4 +4867,4 @@ function RadixTree() {
 
 // BUT!
 
-// We want to be able to search "Hash", for example!
\ No newline at end of file
+// We want to be able to search "Hash", for example!
diff --git a/src/Autodoc.zig b/src/Autodoc.zig
index eba53b7e17..1dfe036fd6 100644
--- a/src/Autodoc.zig
+++ b/src/Autodoc.zig
@@ -755,6 +755,7 @@ const DocData = struct {
         string: []const u8, // direct value
         sliceIndex: usize,
         slice: Slice,
+        sliceLength: SliceLength,
         cmpxchgIndex: usize,
         cmpxchg: Cmpxchg,
         builtin: Builtin,
@@ -791,6 +792,12 @@ const DocData = struct {
             end: ?usize = null,
             sentinel: ?usize = null, // index in `exprs`
         };
+        const SliceLength = struct {
+            lhs: usize,
+            start: usize,
+            len: usize,
+            sentinel: ?usize = null,
+        };
         const Cmpxchg = struct {
             name: []const u8,
             type: usize,
@@ -1337,7 +1344,12 @@ fn walkInstruction(
                 try self.exprs.append(self.arena, sentinel.expr);
                 break :sentinel_index index;
             } else null;
-            self.exprs.items[slice_index] = .{ .slice = .{ .lhs = lhs_index, .start = start_index, .end = len_index, .sentinel = sentinel_index } };
+            self.exprs.items[slice_index] = .{ .sliceLength = .{
+                .lhs = lhs_index,
+                .start = start_index,
+                .len = len_index,
+                .sentinel = sentinel_index,
+            } };
 
             return DocData.WalkResult{
                 .typeRef = self.decls.items[lhs.expr.declRef.Analyzed].value.typeRef,

From bd3360e03d89fe947e3728ccacd4274653926376 Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Tue, 2 May 2023 22:08:54 +1000
Subject: [PATCH 107/725] convert s[start..start+len] to s[start..][0..len]

---
 lib/std/Build/Cache/DepTokenizer.zig            | 2 +-
 lib/std/array_list.zig                          | 4 ++--
 lib/std/bounded_array.zig                       | 2 +-
 lib/std/compress/deflate/decompressor.zig       | 2 +-
 lib/std/compress/deflate/huffman_bit_writer.zig | 6 +++---
 lib/std/hash/crc.zig                            | 2 +-
 lib/std/hash/wyhash.zig                         | 2 +-
 lib/std/json.zig                                | 6 +++---
 lib/std/math/big/int.zig                        | 2 +-
 lib/std/net.zig                                 | 2 +-
 lib/std/os/windows.zig                          | 4 ++--
 lib/std/testing.zig                             | 2 +-
 lib/std/unicode.zig                             | 2 +-
 src/link/Plan9/aout.zig                         | 2 +-
 14 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/lib/std/Build/Cache/DepTokenizer.zig b/lib/std/Build/Cache/DepTokenizer.zig
index c640fa4adc..1a4e2ddb74 100644
--- a/lib/std/Build/Cache/DepTokenizer.zig
+++ b/lib/std/Build/Cache/DepTokenizer.zig
@@ -974,7 +974,7 @@ fn hexDump(out: anytype, bytes: []const u8) !void {
     var line: usize = 0;
     var offset: usize = 0;
     while (line < n16) : (line += 1) {
-        try hexDump16(out, offset, bytes[offset .. offset + 16]);
+        try hexDump16(out, offset, bytes[offset..][0..16]);
         offset += 16;
     }
 
diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig
index cbca601b82..bbfa588d6d 100644
--- a/lib/std/array_list.zig
+++ b/lib/std/array_list.zig
@@ -173,7 +173,7 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
             @memcpy(self.items[i..][0..items.len], items);
         }
 
-        /// Replace range of elements `list[start..start+len]` with `new_items`.
+        /// Replace range of elements `list[start..][0..len]` with `new_items`.
         /// Grows list if `len < new_items.len`.
         /// Shrinks list if `len > new_items.len`.
         /// Invalidates pointers if this ArrayList is resized.
@@ -654,7 +654,7 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
             @memcpy(self.items[i..][0..items.len], items);
         }
 
-        /// Replace range of elements `list[start..start+len]` with `new_items`
+        /// Replace range of elements `list[start..][0..len]` with `new_items`
         /// Grows list if `len < new_items.len`.
         /// Shrinks list if `len > new_items.len`
         /// Invalidates pointers if this ArrayList is resized.
diff --git a/lib/std/bounded_array.zig b/lib/std/bounded_array.zig
index be32093fe7..0e0b601af6 100644
--- a/lib/std/bounded_array.zig
+++ b/lib/std/bounded_array.zig
@@ -168,7 +168,7 @@ pub fn BoundedArrayAligned(
             @memcpy(self.slice()[i..][0..items.len], items);
         }
 
-        /// Replace range of elements `slice[start..start+len]` with `new_items`.
+        /// Replace range of elements `slice[start..][0..len]` with `new_items`.
         /// Grows slice if `len < new_items.len`.
         /// Shrinks slice if `len > new_items.len`.
         pub fn replaceRange(
diff --git a/lib/std/compress/deflate/decompressor.zig b/lib/std/compress/deflate/decompressor.zig
index 6c232c598e..40bde67326 100644
--- a/lib/std/compress/deflate/decompressor.zig
+++ b/lib/std/compress/deflate/decompressor.zig
@@ -591,7 +591,7 @@ pub fn Decompressor(comptime ReaderType: type) type {
             }
 
             if (!try self.hd1.init(self.allocator, self.bits[0..nlit]) or
-                !try self.hd2.init(self.allocator, self.bits[nlit .. nlit + ndist]))
+                !try self.hd2.init(self.allocator, self.bits[nlit..][0..ndist]))
             {
                 corrupt_input_error_offset = self.roffset;
                 self.err = InflateError.CorruptInput;
diff --git a/lib/std/compress/deflate/huffman_bit_writer.zig b/lib/std/compress/deflate/huffman_bit_writer.zig
index a42aae467b..0b76d9d89d 100644
--- a/lib/std/compress/deflate/huffman_bit_writer.zig
+++ b/lib/std/compress/deflate/huffman_bit_writer.zig
@@ -139,7 +139,7 @@ pub fn HuffmanBitWriter(comptime WriterType: type) type {
                 self.bits >>= 48;
                 self.nbits -= 48;
                 var n = self.nbytes;
-                var bytes = self.bytes[n .. n + 6];
+                var bytes = self.bytes[n..][0..6];
                 bytes[0] = @truncate(u8, bits);
                 bytes[1] = @truncate(u8, bits >> 8);
                 bytes[2] = @truncate(u8, bits >> 16);
@@ -344,7 +344,7 @@ pub fn HuffmanBitWriter(comptime WriterType: type) type {
                 self.bits >>= 48;
                 self.nbits -= 48;
                 var n = self.nbytes;
-                var bytes = self.bytes[n .. n + 6];
+                var bytes = self.bytes[n..][0..6];
                 bytes[0] = @truncate(u8, bits);
                 bytes[1] = @truncate(u8, bits >> 8);
                 bytes[2] = @truncate(u8, bits >> 16);
@@ -751,7 +751,7 @@ pub fn HuffmanBitWriter(comptime WriterType: type) type {
                 var bits = self.bits;
                 self.bits >>= 48;
                 self.nbits -= 48;
-                var bytes = self.bytes[n .. n + 6];
+                var bytes = self.bytes[n..][0..6];
                 bytes[0] = @truncate(u8, bits);
                 bytes[1] = @truncate(u8, bits >> 8);
                 bytes[2] = @truncate(u8, bits >> 16);
diff --git a/lib/std/hash/crc.zig b/lib/std/hash/crc.zig
index 271b4f93da..0ad154abef 100644
--- a/lib/std/hash/crc.zig
+++ b/lib/std/hash/crc.zig
@@ -160,7 +160,7 @@ pub fn Crc32WithPoly(comptime poly: Polynomial) type {
         pub fn update(self: *Self, input: []const u8) void {
             var i: usize = 0;
             while (i + 8 <= input.len) : (i += 8) {
-                const p = input[i .. i + 8];
+                const p = input[i..][0..8];
 
                 // Unrolling this way gives ~50Mb/s increase
                 self.crc ^= std.mem.readIntLittle(u32, p[0..4]);
diff --git a/lib/std/hash/wyhash.zig b/lib/std/hash/wyhash.zig
index 772395eab9..934c4f96ad 100644
--- a/lib/std/hash/wyhash.zig
+++ b/lib/std/hash/wyhash.zig
@@ -65,7 +65,7 @@ const WyhashStateless = struct {
 
         var off: usize = 0;
         while (off < b.len) : (off += 32) {
-            @call(.always_inline, self.round, .{b[off .. off + 32]});
+            @call(.always_inline, self.round, .{b[off..][0..32]});
         }
 
         self.msg_len += b.len;
diff --git a/lib/std/json.zig b/lib/std/json.zig
index af928bc564..011463faef 100644
--- a/lib/std/json.zig
+++ b/lib/std/json.zig
@@ -63,7 +63,7 @@ fn encodesTo(decoded: []const u8, encoded: []const u8) bool {
                 var buf: [4]u8 = undefined;
                 const len = std.unicode.utf8Encode(codepoint, &buf) catch unreachable;
                 if (i + len > decoded.len) return false;
-                if (!mem.eql(u8, decoded[i .. i + len], buf[0..len])) return false;
+                if (!mem.eql(u8, decoded[i..][0..len], buf[0..len])) return false;
                 i += len;
             }
         }
@@ -2285,10 +2285,10 @@ pub fn encodeJsonStringChars(chars: []const u8, options: StringifyOptions, write
                 const ulen = std.unicode.utf8ByteSequenceLength(chars[i]) catch unreachable;
                 // control characters (only things left with 1 byte length) should always be printed as unicode escapes
                 if (ulen == 1 or options.string.String.escape_unicode) {
-                    const codepoint = std.unicode.utf8Decode(chars[i .. i + ulen]) catch unreachable;
+                    const codepoint = std.unicode.utf8Decode(chars[i..][0..ulen]) catch unreachable;
                     try outputUnicodeEscape(codepoint, writer);
                 } else {
-                    try writer.writeAll(chars[i .. i + ulen]);
+                    try writer.writeAll(chars[i..][0..ulen]);
                 }
                 i += ulen - 1;
             },
diff --git a/lib/std/math/big/int.zig b/lib/std/math/big/int.zig
index b01d9b04ff..5683d8681c 100644
--- a/lib/std/math/big/int.zig
+++ b/lib/std/math/big/int.zig
@@ -4035,7 +4035,7 @@ fn llsquareBasecase(r: []Limb, x: []const Limb) void {
 
     for (x_norm, 0..) |v, i| {
         // Compute and add the squares
-        const overflow = llmulLimb(.add, r[2 * i ..], x[i .. i + 1], v);
+        const overflow = llmulLimb(.add, r[2 * i ..], x[i..][0..1], v);
         assert(!overflow);
     }
 }
diff --git a/lib/std/net.zig b/lib/std/net.zig
index 15a9e01da8..57e50a7349 100644
--- a/lib/std/net.zig
+++ b/lib/std/net.zig
@@ -1701,7 +1701,7 @@ fn dnsParse(
         p += @as(usize, 1) + @boolToInt(p[0] != 0);
         const len = p[8] * @as(usize, 256) + p[9];
         if (@ptrToInt(p) + len > @ptrToInt(r.ptr) + r.len) return error.InvalidDnsPacket;
-        try callback(ctx, p[1], p[10 .. 10 + len], r);
+        try callback(ctx, p[1], p[10..][0..len], r);
         p += 10 + len;
     }
 }
diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig
index 22ffc850e6..6755e7adf2 100644
--- a/lib/std/os/windows.zig
+++ b/lib/std/os/windows.zig
@@ -835,14 +835,14 @@ pub fn ReadLink(dir: ?HANDLE, sub_path_w: []const u16, out_buffer: []u8) ReadLin
             const len = buf.SubstituteNameLength >> 1;
             const path_buf = @as([*]const u16, &buf.PathBuffer);
             const is_relative = buf.Flags & SYMLINK_FLAG_RELATIVE != 0;
-            return parseReadlinkPath(path_buf[offset .. offset + len], is_relative, out_buffer);
+            return parseReadlinkPath(path_buf[offset..][0..len], is_relative, out_buffer);
         },
         IO_REPARSE_TAG_MOUNT_POINT => {
             const buf = @ptrCast(*const MOUNT_POINT_REPARSE_BUFFER, @alignCast(@alignOf(MOUNT_POINT_REPARSE_BUFFER), &reparse_struct.DataBuffer[0]));
             const offset = buf.SubstituteNameOffset >> 1;
             const len = buf.SubstituteNameLength >> 1;
             const path_buf = @as([*]const u16, &buf.PathBuffer);
-            return parseReadlinkPath(path_buf[offset .. offset + len], false, out_buffer);
+            return parseReadlinkPath(path_buf[offset..][0..len], false, out_buffer);
         },
         else => |value| {
             std.debug.print("unsupported symlink type: {}", .{value});
diff --git a/lib/std/testing.zig b/lib/std/testing.zig
index 37e15ff08b..2857ebdbd3 100644
--- a/lib/std/testing.zig
+++ b/lib/std/testing.zig
@@ -941,7 +941,7 @@ fn printIndicatorLine(source: []const u8, indicator_index: usize) void {
 fn printWithVisibleNewlines(source: []const u8) void {
     var i: usize = 0;
     while (std.mem.indexOfScalar(u8, source[i..], '\n')) |nl| : (i += nl + 1) {
-        printLine(source[i .. i + nl]);
+        printLine(source[i..][0..nl]);
     }
     print("{s}␃\n", .{source[i..]}); // End of Text symbol (ETX)
 }
diff --git a/lib/std/unicode.zig b/lib/std/unicode.zig
index 378b216ef3..9d334218c1 100644
--- a/lib/std/unicode.zig
+++ b/lib/std/unicode.zig
@@ -185,7 +185,7 @@ pub fn utf8CountCodepoints(s: []const u8) !usize {
 
             switch (n) {
                 1 => {}, // ASCII, no validation needed
-                else => _ = try utf8Decode(s[i .. i + n]),
+                else => _ = try utf8Decode(s[i..][0..n]),
             }
 
             i += n;
diff --git a/src/link/Plan9/aout.zig b/src/link/Plan9/aout.zig
index b39cae5166..d6fbb47d35 100644
--- a/src/link/Plan9/aout.zig
+++ b/src/link/Plan9/aout.zig
@@ -21,7 +21,7 @@ pub const ExecHdr = extern struct {
         var buf: [40]u8 = undefined;
         var i: u8 = 0;
         inline for (std.meta.fields(@This())) |f| {
-            std.mem.writeIntSliceBig(u32, buf[i .. i + 4], @field(self, f.name));
+            std.mem.writeIntSliceBig(u32, buf[i..][0..4], @field(self, f.name));
             i += 4;
         }
         return buf;

From cba9077cc12b953dd00369f9c0340d7667af1611 Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Wed, 3 May 2023 11:30:30 +1000
Subject: [PATCH 108/725] sema: fix slice by length non-array, non-slice case

---
 src/Sema.zig | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/Sema.zig b/src/Sema.zig
index 8deca97505..68fbfe08d1 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -29384,7 +29384,11 @@ fn analyzeSlice(
             break :e try sema.analyzeSliceLen(block, src, ptr_or_slice);
         }
         if (!end_is_len) {
-            break :e try sema.coerce(block, Type.usize, uncasted_end_opt, end_src);
+            if (by_length) {
+                const len = try sema.coerce(block, Type.usize, uncasted_end_opt, end_src);
+                const uncasted_end = try sema.analyzeArithmetic(block, .add, start, len, src, start_src, end_src, false);
+                break :e try sema.coerce(block, Type.usize, uncasted_end, end_src);
+            } else break :e try sema.coerce(block, Type.usize, uncasted_end_opt, end_src);
         }
         return sema.fail(block, src, "slice of pointer must include end value", .{});
     };

From e507f0c0aa2267f003ba2afb30448fc723d54418 Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Wed, 3 May 2023 12:51:40 +1000
Subject: [PATCH 109/725] test: add behavior tests for pointer slice-by-length

---
 test/behavior/slice.zig | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/test/behavior/slice.zig b/test/behavior/slice.zig
index 51538d9b10..575665cf7f 100644
--- a/test/behavior/slice.zig
+++ b/test/behavior/slice.zig
@@ -180,6 +180,17 @@ test "slicing zero length array" {
     try expect(mem.eql(u32, s2, &[_]u32{}));
 }
 
+test "slicing pointer by length" {
+    const array = [_]u8{ 1, 2, 3, 4, 5, 6, 7, 8 };
+    const ptr: [*]const u8 = @ptrCast([*]const u8, &array);
+    const slice = ptr[1..][0..5];
+    try expect(slice.len == 5);
+    var i: usize = 0;
+    while (i < slice.len) : (i += 1) {
+        try expect(slice[i] == i + 2);
+    }
+}
+
 const x = @intToPtr([*]i32, 0x1000)[0..0x500];
 const y = x[0x100..];
 test "compile time slice of pointer to hard coded address" {
@@ -503,6 +514,33 @@ test "slice syntax resulting in pointer-to-array" {
             comptime try expect(@TypeOf(array[1.. :0][0..4]) == *[4:0]u8);
             comptime try expect(@TypeOf(array[1.. :0][0..2 :4]) == *[2:4]u8);
         }
+
+        fn testMultiPointer() !void {
+            var array = [5]u8{ 1, 2, 3, 4, 5 };
+            var ptr: [*]u8 = &array;
+            comptime try expect(@TypeOf(ptr[1..][0..2]) == *[2]u8);
+            comptime try expect(@TypeOf(ptr[1..][0..4]) == *[4]u8);
+            comptime try expect(@TypeOf(ptr[1..][0..2 :4]) == *[2:4]u8);
+        }
+
+        fn testMultiPointerLengthZ() !void {
+            var array = [5:0]u8{ 1, 2, 3, 4, 5 };
+            var ptr: [*]u8 = &array;
+            comptime try expect(@TypeOf(ptr[1..][0..2]) == *[2]u8);
+            comptime try expect(@TypeOf(ptr[1..][0..4]) == *[4:0]u8);
+            comptime try expect(@TypeOf(ptr[1..][0..2 :4]) == *[2:4]u8);
+            comptime try expect(@TypeOf(ptr[1.. :0][0..2]) == *[2]u8);
+            comptime try expect(@TypeOf(ptr[1.. :0][0..4]) == *[4:0]u8);
+            comptime try expect(@TypeOf(ptr[1.. :0][0..2 :4]) == *[2:4]u8);
+
+            var ptr_z: [*:0]u8 = &array;
+            comptime try expect(@TypeOf(ptr_z[1..][0..2]) == *[2]u8);
+            comptime try expect(@TypeOf(ptr_z[1..][0..4]) == *[4:0]u8);
+            comptime try expect(@TypeOf(ptr_z[1..][0..2 :4]) == *[2:4]u8);
+            comptime try expect(@TypeOf(ptr_z[1.. :0][0..2]) == *[2]u8);
+            comptime try expect(@TypeOf(ptr_z[1.. :0][0..4]) == *[4:0]u8);
+            comptime try expect(@TypeOf(ptr_z[1.. :0][0..2 :4]) == *[2:4]u8);
+        }
     };
 
     try S.doTheTest();

From 4d296debefccbd80f2007685a27386b7434464dd Mon Sep 17 00:00:00 2001
From: dweiller <4678790+dweiller@users.noreplay.github.com>
Date: Thu, 4 May 2023 10:58:30 +1000
Subject: [PATCH 110/725] test: disable by-length slice test on wasm backend

---
 test/behavior/slice.zig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/behavior/slice.zig b/test/behavior/slice.zig
index 575665cf7f..29ceb061c1 100644
--- a/test/behavior/slice.zig
+++ b/test/behavior/slice.zig
@@ -181,6 +181,7 @@ test "slicing zero length array" {
 }
 
 test "slicing pointer by length" {
+    if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
     const array = [_]u8{ 1, 2, 3, 4, 5, 6, 7, 8 };
     const ptr: [*]const u8 = @ptrCast([*]const u8, &array);
     const slice = ptr[1..][0..5];

From 66ff7d9161d5152589341563990d364754238fbf Mon Sep 17 00:00:00 2001
From: Jakub Konka 
Date: Sun, 7 May 2023 09:05:18 +0200
Subject: [PATCH 111/725] elf: add more missing defs for SHT_* and SHF_*

---
 lib/std/elf.zig | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/std/elf.zig b/lib/std/elf.zig
index e2cad5640e..46a5dd2f39 100644
--- a/lib/std/elf.zig
+++ b/lib/std/elf.zig
@@ -300,10 +300,14 @@ pub const SHT_LOOS = 0x60000000;
 pub const SHT_HIOS = 0x6fffffff;
 /// Start of processor-specific
 pub const SHT_LOPROC = 0x70000000;
+/// Unwind information
+pub const SHT_X86_64_UNWIND = 0x70000001;
 /// End of processor-specific
 pub const SHT_HIPROC = 0x7fffffff;
 /// Start of application-specific
 pub const SHT_LOUSER = 0x80000000;
+/// LLVM address-significance table
+pub const SHT_LLVM_ADDRSIG = 0xfff4c03;
 /// End of application-specific
 pub const SHT_HIUSER = 0xffffffff;
 
@@ -1632,6 +1636,9 @@ pub const SHF_TLS = 0x400;
 /// Identifies a section containing compressed data.
 pub const SHF_COMPRESSED = 0x800;
 
+/// Not to be GCed by the linker
+pub const SHF_GNU_RETAIN = 0x200000;
+
 /// This section is excluded from the final executable or shared library.
 pub const SHF_EXCLUDE = 0x80000000;
 

From 49c1384bac75a908c347a74e75e9b9e0110b0700 Mon Sep 17 00:00:00 2001
From: Jakub Konka 
Date: Sun, 7 May 2023 09:09:19 +0200
Subject: [PATCH 112/725] elf: fix typo in def of SHT_LLVM_ADDRSIG

---
 lib/std/elf.zig | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/std/elf.zig b/lib/std/elf.zig
index 46a5dd2f39..751f82a9ea 100644
--- a/lib/std/elf.zig
+++ b/lib/std/elf.zig
@@ -296,6 +296,8 @@ pub const SHT_GROUP = 17;
 pub const SHT_SYMTAB_SHNDX = 18;
 /// Start of OS-specific
 pub const SHT_LOOS = 0x60000000;
+/// LLVM address-significance table
+pub const SHT_LLVM_ADDRSIG = 0x6fff4c03;
 /// End of OS-specific
 pub const SHT_HIOS = 0x6fffffff;
 /// Start of processor-specific
@@ -306,8 +308,6 @@ pub const SHT_X86_64_UNWIND = 0x70000001;
 pub const SHT_HIPROC = 0x7fffffff;
 /// Start of application-specific
 pub const SHT_LOUSER = 0x80000000;
-/// LLVM address-significance table
-pub const SHT_LLVM_ADDRSIG = 0xfff4c03;
 /// End of application-specific
 pub const SHT_HIUSER = 0xffffffff;
 

From e1f5ad3cc875d255946cafc92e0b415a285919c9 Mon Sep 17 00:00:00 2001
From: Dominic <4678790+dweiller@users.noreply.github.com>
Date: Sun, 7 May 2023 18:05:53 +1000
Subject: [PATCH 113/725] Fix parsing of hexadecimal literals

---
 lib/std/zig/number_literal.zig                      | 10 +++-------
 src/AstGen.zig                                      |  6 ++++--
 .../compile_errors/number_literal_bad_exponent.zig  | 13 +++++++++++++
 3 files changed, 20 insertions(+), 9 deletions(-)
 create mode 100644 test/cases/compile_errors/number_literal_bad_exponent.zig

diff --git a/lib/std/zig/number_literal.zig b/lib/std/zig/number_literal.zig
index 1b41908371..b021190ad9 100644
--- a/lib/std/zig/number_literal.zig
+++ b/lib/std/zig/number_literal.zig
@@ -44,8 +44,6 @@ pub const Error = union(enum) {
     duplicate_period,
     /// Float literal has multiple exponents.
     duplicate_exponent: usize,
-    /// Decimal float has hexadecimal exponent.
-    invalid_hex_exponent: usize,
     /// Exponent comes directly after '_' digit separator.
     exponent_after_underscore: usize,
     /// Special character (+-.) comes directly after exponent.
@@ -103,7 +101,6 @@ pub fn parseNumberLiteral(bytes: []const u8) Result {
             },
             'e', 'E' => if (base == 10) {
                 float = true;
-                if (base != 10 and base != 16) return .{ .failure = .{ .invalid_float_base = 2 } };
                 if (exponent) return .{ .failure = .{ .duplicate_exponent = i } };
                 if (underscore) return .{ .failure = .{ .exponent_after_underscore = i } };
                 special = c;
@@ -112,10 +109,8 @@ pub fn parseNumberLiteral(bytes: []const u8) Result {
             },
             'p', 'P' => if (base == 16) {
                 float = true;
-                if (base != 10 and base != 16) return .{ .failure = .{ .invalid_float_base = 2 } };
                 if (exponent) return .{ .failure = .{ .duplicate_exponent = i } };
                 if (underscore) return .{ .failure = .{ .exponent_after_underscore = i } };
-                if (base != 16) return .{ .failure = .{ .invalid_hex_exponent = i } };
                 special = c;
                 exponent = true;
                 continue;
@@ -123,7 +118,7 @@ pub fn parseNumberLiteral(bytes: []const u8) Result {
             '.' => {
                 float = true;
                 if (base != 10 and base != 16) return .{ .failure = .{ .invalid_float_base = 2 } };
-                if (period) return .{ .failure = .{ .duplicate_exponent = i } };
+                if (period) return .{ .failure = .duplicate_period };
                 period = true;
                 if (underscore) return .{ .failure = .{ .special_after_underscore = i } };
                 special = c;
@@ -131,7 +126,8 @@ pub fn parseNumberLiteral(bytes: []const u8) Result {
             },
             '+', '-' => {
                 switch (special) {
-                    'p', 'P', 'e', 'E' => {},
+                    'p', 'P' => {},
+                    'e', 'E' => if (base != 10) return .{ .failure = .{ .invalid_exponent_sign = i } },
                     else => return .{ .failure = .{ .invalid_exponent_sign = i } },
                 }
                 special = c;
diff --git a/src/AstGen.zig b/src/AstGen.zig
index 749e3d28c4..300aae8d77 100644
--- a/src/AstGen.zig
+++ b/src/AstGen.zig
@@ -7622,14 +7622,16 @@ fn failWithNumberError(astgen: *AstGen, err: std.zig.number_literal.Error, token
         .invalid_digit => |info| return astgen.failOff(token, @intCast(u32, info.i), "invalid digit '{c}' for {s} base", .{ bytes[info.i], @tagName(info.base) }),
         .invalid_digit_exponent => |i| return astgen.failOff(token, @intCast(u32, i), "invalid digit '{c}' in exponent", .{bytes[i]}),
         .duplicate_exponent => |i| return astgen.failOff(token, @intCast(u32, i), "duplicate exponent", .{}),
-        .invalid_hex_exponent => |i| return astgen.failOff(token, @intCast(u32, i), "hex exponent in decimal float", .{}),
         .exponent_after_underscore => |i| return astgen.failOff(token, @intCast(u32, i), "expected digit before exponent", .{}),
         .special_after_underscore => |i| return astgen.failOff(token, @intCast(u32, i), "expected digit before '{c}'", .{bytes[i]}),
         .trailing_special => |i| return astgen.failOff(token, @intCast(u32, i), "expected digit after '{c}'", .{bytes[i - 1]}),
         .trailing_underscore => |i| return astgen.failOff(token, @intCast(u32, i), "trailing digit separator", .{}),
         .duplicate_period => unreachable, // Validated by tokenizer
         .invalid_character => unreachable, // Validated by tokenizer
-        .invalid_exponent_sign => unreachable, // Validated by tokenizer
+        .invalid_exponent_sign => |i| {
+            assert(bytes.len >= 2 and bytes[0] == '0' and bytes[1] == 'x'); // Validated by tokenizer
+            return astgen.failOff(token, @intCast(u32, i), "sign '{c}' cannot follow digit '{c}' in hex base", .{ bytes[i], bytes[i - 1] });
+        },
     }
 }
 
diff --git a/test/cases/compile_errors/number_literal_bad_exponent.zig b/test/cases/compile_errors/number_literal_bad_exponent.zig
new file mode 100644
index 0000000000..158c205018
--- /dev/null
+++ b/test/cases/compile_errors/number_literal_bad_exponent.zig
@@ -0,0 +1,13 @@
+const a = 0x1e-4;
+const b = 0x1e+4;
+const c = 0x1E-4;
+const d = 0x1E+4;
+
+// error
+// backend=stage2
+// target=native
+//
+// :1:15: error: sign '-' cannot follow digit 'e' in hex base
+// :2:15: error: sign '+' cannot follow digit 'e' in hex base
+// :3:15: error: sign '-' cannot follow digit 'E' in hex base
+// :4:15: error: sign '+' cannot follow digit 'E' in hex base

From ff59c4584041517a2ee07cccf923ef5460032d68 Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Sun, 7 May 2023 08:19:59 +0100
Subject: [PATCH 114/725] std.c: darwin add host_info based data.

---
 lib/std/c/darwin.zig | 86 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)

diff --git a/lib/std/c/darwin.zig b/lib/std/c/darwin.zig
index 7879eddc4b..e5ee8a432a 100644
--- a/lib/std/c/darwin.zig
+++ b/lib/std/c/darwin.zig
@@ -3916,3 +3916,89 @@ pub const THREAD_AFFINITY = struct {
 /// individual cpus (high performance cpus group and low consumption one), thus the pthread QOS api is more appropriate in this case.
 pub extern "c" fn thread_affinity_get(thread: thread_act_t, flavor: thread_policy_flavor_t, info: thread_policy_t, infocnt: [*]mach_msg_type_number_t, default: *boolean_t) kern_return_t;
 pub extern "c" fn thread_affinity_set(thread: thread_act_t, flavor: thread_policy_flavor_t, info: thread_policy_t, infocnt: mach_msg_type_number_t) kern_return_t;
+
+pub const cpu_type_t = integer_t;
+pub const cpu_subtype_t = integer_t;
+pub const cpu_threadtype_t = integer_t;
+pub const host_flavor_t = integer_t;
+pub const host_info_t = *integer_t;
+pub const host_can_has_debugger_info = extern struct {
+    can_has_debugger: boolean_t,
+};
+pub const host_can_has_debugger_info_data_t = host_can_has_debugger_info;
+pub const host_can_has_debugger_info_t = *host_can_has_debugger_info;
+
+pub const host_sched_info = extern struct {
+    min_timeout: integer_t,
+    min_quantum: integer_t,
+};
+pub const host_sched_info_data_t = host_sched_info;
+pub const host_sched_info_t = *host_sched_info;
+
+pub const kernel_resource_sizes = extern struct {
+    task: natural_t,
+    thread: natural_t,
+    port: natural_t,
+    memory_region: natural_t,
+    memory_object: natural_t,
+};
+
+pub const kernel_resource_sizes_data_t = kernel_resource_sizes;
+pub const kernel_resource_sizes_t = *kernel_resource_sizes;
+
+pub const host_priority_info = extern struct {
+    kernel_priority: integer_t,
+    system_priority: integer_t,
+    server_priority: integer_t,
+    user_priority: integer_t,
+    depress_priority: integer_t,
+    idle_priority: integer_t,
+    minimum_priority: integer_t,
+    maximum_priority: integer_t,
+};
+
+pub const host_priority_info_data_t = host_priority_info;
+pub const host_priority_info_t = *host_priority_info;
+
+pub const CPU_STATE_MAX = 4;
+
+pub const host_cpu_load_info = extern struct {
+    cpu_ticks: [CPU_STATE_MAX]natural_t,
+};
+
+pub const host_cpu_load_info_data_t = host_cpu_load_info;
+pub const host_cpu_load_info_t = *host_cpu_load_info;
+
+pub const HOST = struct {
+    pub const BASIC_INFO = 1;
+    pub const SCHED_INFO = 3;
+    pub const RESOURCE_SIZES = 4;
+    pub const PRIORITY_INFO = 5;
+    pub const SEMAPHORE_TRAPS = 7;
+    pub const MACH_MSG_TRAPS = 8;
+    pub const VM_PURGEABLE = 9;
+    pub const DEBUG_INFO_INTERNAL = 10;
+    pub const CAN_HAS_DEBUGGER = 11;
+    pub const PREFERRED_USER_ARCH = 12;
+    pub const CAN_HAS_DEBUGGER_COUNT = @intCast(mach_msg_type_number_t, @sizeOf(host_can_has_debugger_info_data_t) / @sizeOf(integer_t));
+    pub const SCHED_INFO_COUNT = @intCast(mach_msg_type_number_t, @sizeOf(host_sched_info_data_t) / @sizeOf(integer_t));
+    pub const RESOURCES_SIZES_COUNT = @intCast(mach_msg_type_number_t, @sizeOf(kernel_resource_sizes_data_t) / @sizeOf(integer_t));
+    pub const PRIORITY_INFO_COUNT = @intCast(mach_msg_type_number_t, @sizeOf(host_priority_info_data_t) / @sizeOf(integer_t));
+    pub const CPU_LOAD_INFO_COUNT = @intCast(mach_msg_type_number_t, @sizeOf(host_cpu_load_info_data_t) / @sizeOf(integer_t));
+};
+
+pub const host_basic_info = packed struct(u32) {
+    max_cpus: integer_t,
+    avail_cpus: integer_t,
+    memory_size: natural_t,
+    cpu_type: cpu_type_t,
+    cpu_subtype: cpu_subtype_t,
+    cpu_threadtype: cpu_threadtype_t,
+    physical_cpu: integer_t,
+    physical_cpu_max: integer_t,
+    logical_cpu: integer_t,
+    logical_cpu_max: integer_t,
+    max_mem: u64,
+};
+
+pub extern "c" fn host_info(host: host_t, flavor: host_flavor_t, info_out: host_info_t, info_outCnt: [*]mach_msg_type_number_t) kern_return_t;

From bac3a28214d07f761e25fcc12a6708f70c8ccc01 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sun, 7 May 2023 21:53:17 -0400
Subject: [PATCH 115/725] fmt: avoid canonicalizing enum fields named `@"_"` to
 `_`

This can be used to escape the usual meaning of `_` to indicate a
non-exhaustive enum and create an enum tag that is a literal underscore,
so zig fmt should allow this syntax.

Before, zig fmt changes

    const E = enum { @"_" };

to the semantically different

    const E = enum { _ };

After, it remains the same.
---
 lib/std/zig/Ast.zig         |  4 +++
 lib/std/zig/parser_test.zig | 56 +++++++++++++++++++++++++++++
 lib/std/zig/render.zig      | 71 +++++++++++++++++++++----------------
 3 files changed, 100 insertions(+), 31 deletions(-)

diff --git a/lib/std/zig/Ast.zig b/lib/std/zig/Ast.zig
index cb86696e13..7bc78c17da 100644
--- a/lib/std/zig/Ast.zig
+++ b/lib/std/zig/Ast.zig
@@ -3511,3 +3511,7 @@ const Token = std.zig.Token;
 const Ast = @This();
 const Allocator = std.mem.Allocator;
 const Parse = @import("Parse.zig");
+
+test {
+    testing.refAllDecls(@This());
+}
diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig
index 9176e14480..ced57ebf5c 100644
--- a/lib/std/zig/parser_test.zig
+++ b/lib/std/zig/parser_test.zig
@@ -5732,6 +5732,62 @@ test "zig fmt: canonicalize symbols (asm)" {
     );
 }
 
+test "zig fmt: don't canonicalize _ in enums" {
+    try testTransform(
+        \\const A = enum {
+        \\    first,
+        \\    second,
+        \\    third,
+        \\    _,
+        \\};
+        \\const B = enum {
+        \\    @"_",
+        \\    @"__",
+        \\    @"___",
+        \\    @"____",
+        \\};
+        \\const C = struct {
+        \\    @"_": u8,
+        \\    @"__": u8,
+        \\    @"___": u8,
+        \\    @"____": u8,
+        \\};
+        \\const D = union {
+        \\    @"_": u8,
+        \\    @"__": u8,
+        \\    @"___": u8,
+        \\    @"____": u8,
+        \\};
+        \\
+    ,
+        \\const A = enum {
+        \\    first,
+        \\    second,
+        \\    third,
+        \\    _,
+        \\};
+        \\const B = enum {
+        \\    @"_",
+        \\    __,
+        \\    ___,
+        \\    ____,
+        \\};
+        \\const C = struct {
+        \\    _: u8,
+        \\    __: u8,
+        \\    ___: u8,
+        \\    ____: u8,
+        \\};
+        \\const D = union {
+        \\    _: u8,
+        \\    __: u8,
+        \\    ___: u8,
+        \\    ____: u8,
+        \\};
+        \\
+    );
+}
+
 test "zig fmt: error for missing sentinel value in sentinel slice" {
     try testError(
         \\const foo = foo[0..:];
diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig
index 367d06f7c6..e1ccc8e0e8 100644
--- a/lib/std/zig/render.zig
+++ b/lib/std/zig/render.zig
@@ -40,27 +40,28 @@ pub fn renderTree(buffer: *std.ArrayList(u8), tree: Ast) Error!void {
 /// Render all members in the given slice, keeping empty lines where appropriate
 fn renderMembers(gpa: Allocator, ais: *Ais, tree: Ast, members: []const Ast.Node.Index) Error!void {
     if (members.len == 0) return;
-    var is_tuple = true;
-    for (members) |member| {
-        const container_field = tree.fullContainerField(member) orelse continue;
-        if (!container_field.ast.tuple_like) {
-            is_tuple = false;
-            break;
-        }
-    }
-    try renderMember(gpa, ais, tree, members[0], is_tuple, .newline);
+    const container: Container = for (members) |member| {
+        if (tree.fullContainerField(member)) |field| if (!field.ast.tuple_like) break .other;
+    } else .tuple;
+    try renderMember(gpa, ais, tree, container, members[0], .newline);
     for (members[1..]) |member| {
         try renderExtraNewline(ais, tree, member);
-        try renderMember(gpa, ais, tree, member, is_tuple, .newline);
+        try renderMember(gpa, ais, tree, container, member, .newline);
     }
 }
 
+const Container = enum {
+    @"enum",
+    tuple,
+    other,
+};
+
 fn renderMember(
     gpa: Allocator,
     ais: *Ais,
     tree: Ast,
+    container: Container,
     decl: Ast.Node.Index,
-    is_tuple: bool,
     space: Space,
 ) Error!void {
     const token_tags = tree.tokens.items(.tag);
@@ -180,7 +181,7 @@ fn renderMember(
         .container_field_init,
         .container_field_align,
         .container_field,
-        => return renderContainerField(gpa, ais, tree, tree.fullContainerField(decl).?, is_tuple, space),
+        => return renderContainerField(gpa, ais, tree, container, tree.fullContainerField(decl).?, space),
 
         .@"comptime" => return renderExpression(gpa, ais, tree, decl, space),
 
@@ -1279,19 +1280,23 @@ fn renderContainerField(
     gpa: Allocator,
     ais: *Ais,
     tree: Ast,
+    container: Container,
     field_param: Ast.full.ContainerField,
-    is_tuple: bool,
     space: Space,
 ) Error!void {
     var field = field_param;
-    if (!is_tuple) field.convertToNonTupleLike(tree.nodes);
+    if (container != .tuple) field.convertToNonTupleLike(tree.nodes);
+    const quote: QuoteBehavior = switch (container) {
+        .@"enum" => .eagerly_unquote_except_underscore,
+        .tuple, .other => .eagerly_unquote,
+    };
 
     if (field.comptime_token) |t| {
         try renderToken(ais, tree, t, .space); // comptime
     }
     if (field.ast.type_expr == 0 and field.ast.value_expr == 0) {
         if (field.ast.align_expr != 0) {
-            try renderIdentifier(ais, tree, field.ast.main_token, .space, .eagerly_unquote); // name
+            try renderIdentifier(ais, tree, field.ast.main_token, .space, quote); // name
             const lparen_token = tree.firstToken(field.ast.align_expr) - 1;
             const align_kw = lparen_token - 1;
             const rparen_token = tree.lastToken(field.ast.align_expr) + 1;
@@ -1300,11 +1305,11 @@ fn renderContainerField(
             try renderExpression(gpa, ais, tree, field.ast.align_expr, .none); // alignment
             return renderToken(ais, tree, rparen_token, .space); // )
         }
-        return renderIdentifierComma(ais, tree, field.ast.main_token, space, .eagerly_unquote); // name
+        return renderIdentifierComma(ais, tree, field.ast.main_token, space, quote); // name
     }
     if (field.ast.type_expr != 0 and field.ast.value_expr == 0) {
         if (!field.ast.tuple_like) {
-            try renderIdentifier(ais, tree, field.ast.main_token, .none, .eagerly_unquote); // name
+            try renderIdentifier(ais, tree, field.ast.main_token, .none, quote); // name
             try renderToken(ais, tree, field.ast.main_token + 1, .space); // :
         }
 
@@ -1321,7 +1326,7 @@ fn renderContainerField(
         }
     }
     if (field.ast.type_expr == 0 and field.ast.value_expr != 0) {
-        try renderIdentifier(ais, tree, field.ast.main_token, .space, .eagerly_unquote); // name
+        try renderIdentifier(ais, tree, field.ast.main_token, .space, quote); // name
         if (field.ast.align_expr != 0) {
             const lparen_token = tree.firstToken(field.ast.align_expr) - 1;
             const align_kw = lparen_token - 1;
@@ -1335,7 +1340,7 @@ fn renderContainerField(
         return renderExpressionComma(gpa, ais, tree, field.ast.value_expr, space); // value
     }
     if (!field.ast.tuple_like) {
-        try renderIdentifier(ais, tree, field.ast.main_token, .none, .eagerly_unquote); // name
+        try renderIdentifier(ais, tree, field.ast.main_token, .none, quote); // name
         try renderToken(ais, tree, field.ast.main_token + 1, .space); // :
     }
     try renderExpression(gpa, ais, tree, field.ast.type_expr, .space); // type
@@ -2054,13 +2059,12 @@ fn renderContainerDecl(
         try renderToken(ais, tree, layout_token, .space);
     }
 
-    var is_tuple = token_tags[container_decl.ast.main_token] == .keyword_struct;
-    if (is_tuple) for (container_decl.ast.members) |member| {
-        const container_field = tree.fullContainerField(member) orelse continue;
-        if (!container_field.ast.tuple_like) {
-            is_tuple = false;
-            break;
-        }
+    const container: Container = switch (token_tags[container_decl.ast.main_token]) {
+        .keyword_enum => .@"enum",
+        .keyword_struct => for (container_decl.ast.members) |member| {
+            if (tree.fullContainerField(member)) |field| if (!field.ast.tuple_like) break .other;
+        } else .tuple,
+        else => .other,
     };
 
     var lbrace: Ast.TokenIndex = undefined;
@@ -2129,7 +2133,7 @@ fn renderContainerDecl(
         // Print all the declarations on the same line.
         try renderToken(ais, tree, lbrace, .space); // lbrace
         for (container_decl.ast.members) |member| {
-            try renderMember(gpa, ais, tree, member, is_tuple, .space);
+            try renderMember(gpa, ais, tree, container, member, .space);
         }
         return renderToken(ais, tree, rbrace, space); // rbrace
     }
@@ -2147,9 +2151,9 @@ fn renderContainerDecl(
             .container_field_init,
             .container_field_align,
             .container_field,
-            => try renderMember(gpa, ais, tree, member, is_tuple, .comma),
+            => try renderMember(gpa, ais, tree, container, member, .comma),
 
-            else => try renderMember(gpa, ais, tree, member, is_tuple, .newline),
+            else => try renderMember(gpa, ais, tree, container, member, .newline),
         }
     }
     ais.popIndent();
@@ -2565,6 +2569,7 @@ fn renderSpace(ais: *Ais, tree: Ast, token_index: Ast.TokenIndex, lexeme_len: us
 const QuoteBehavior = enum {
     preserve_when_shadowing,
     eagerly_unquote,
+    eagerly_unquote_except_underscore,
 };
 
 fn renderIdentifier(ais: *Ais, tree: Ast, token_index: Ast.TokenIndex, space: Space, quote: QuoteBehavior) Error!void {
@@ -2589,7 +2594,9 @@ fn renderIdentifier(ais: *Ais, tree: Ast, token_index: Ast.TokenIndex, space: Sp
     // Special case for _ which would incorrectly be rejected by isValidId below.
     if (contents.len == 1 and contents[0] == '_') switch (quote) {
         .eagerly_unquote => return renderQuotedIdentifier(ais, tree, token_index, space, true),
-        .preserve_when_shadowing => return renderQuotedIdentifier(ais, tree, token_index, space, false),
+        .eagerly_unquote_except_underscore,
+        .preserve_when_shadowing,
+        => return renderQuotedIdentifier(ais, tree, token_index, space, false),
     };
 
     // Scan the entire name for characters that would (after un-escaping) be illegal in a symbol,
@@ -2653,7 +2660,9 @@ fn renderIdentifier(ais: *Ais, tree: Ast, token_index: Ast.TokenIndex, space: Sp
             return renderQuotedIdentifier(ais, tree, token_index, space, false);
         }
         if (primitives.isPrimitive(buf[0..buf_i])) switch (quote) {
-            .eagerly_unquote => return renderQuotedIdentifier(ais, tree, token_index, space, true),
+            .eagerly_unquote,
+            .eagerly_unquote_except_underscore,
+            => return renderQuotedIdentifier(ais, tree, token_index, space, true),
             .preserve_when_shadowing => return renderQuotedIdentifier(ais, tree, token_index, space, false),
         };
     }

From 5a3eca5d4ca42f92abe60040e21ffd8e307d8466 Mon Sep 17 00:00:00 2001
From: Dominic <4678790+dweiller@users.noreply.github.com>
Date: Mon, 8 May 2023 17:59:06 +1000
Subject: [PATCH 116/725] Disallow named test decls with duplicate names

---
 lib/std/fmt.zig                               | 75 +++++++++----------
 lib/std/hash_map.zig                          | 34 ++++-----
 lib/std/io/reader.zig                         | 23 +++---
 lib/std/math/big/int_test.zig                 | 13 +---
 lib/std/math/big/rational.zig                 | 62 +++++++--------
 lib/std/net/test.zig                          |  2 +-
 lib/std/priority_dequeue.zig                  |  4 +-
 lib/std/zig/parser_test.zig                   |  8 +-
 src/Module.zig                                | 26 ++++++-
 test/behavior.zig                             |  1 +
 test/behavior/basic.zig                       |  2 +-
 test/behavior/comptime_memory.zig             |  2 +-
 test/behavior/duplicated_test_names.zig       | 17 +++++
 test/behavior/vector.zig                      | 14 ----
 .../invalid_duplicate_test_decl_name.zig      | 10 +++
 15 files changed, 163 insertions(+), 130 deletions(-)
 create mode 100644 test/behavior/duplicated_test_names.zig
 create mode 100644 test/cases/compile_errors/invalid_duplicate_test_decl_name.zig

diff --git a/lib/std/fmt.zig b/lib/std/fmt.zig
index cf791df1a6..c3ccd75d27 100644
--- a/lib/std/fmt.zig
+++ b/lib/std/fmt.zig
@@ -2244,8 +2244,8 @@ test "struct" {
             field: u8,
         };
         const value = Struct{ .field = 42 };
-        try expectFmt("struct: Struct{ .field = 42 }\n", "struct: {}\n", .{value});
-        try expectFmt("struct: Struct{ .field = 42 }\n", "struct: {}\n", .{&value});
+        try expectFmt("struct: fmt.test.struct.Struct{ .field = 42 }\n", "struct: {}\n", .{value});
+        try expectFmt("struct: fmt.test.struct.Struct{ .field = 42 }\n", "struct: {}\n", .{&value});
     }
     {
         const Struct = struct {
@@ -2253,8 +2253,24 @@ test "struct" {
             b: u1,
         };
         const value = Struct{ .a = 0, .b = 1 };
-        try expectFmt("struct: Struct{ .a = 0, .b = 1 }\n", "struct: {}\n", .{value});
+        try expectFmt("struct: fmt.test.struct.Struct{ .a = 0, .b = 1 }\n", "struct: {}\n", .{value});
     }
+
+    const S = struct {
+        a: u32,
+        b: anyerror,
+    };
+
+    const inst = S{
+        .a = 456,
+        .b = error.Unused,
+    };
+
+    try expectFmt("fmt.test.struct.S{ .a = 456, .b = error.Unused }", "{}", .{inst});
+    // Tuples
+    try expectFmt("{ }", "{}", .{.{}});
+    try expectFmt("{ -1 }", "{}", .{.{-1}});
+    try expectFmt("{ -1, 42, 2.5e+04 }", "{}", .{.{ -1, 42, 0.25e5 }});
 }
 
 test "enum" {
@@ -2263,13 +2279,26 @@ test "enum" {
         Two,
     };
     const value = Enum.Two;
-    try expectFmt("enum: Enum.Two\n", "enum: {}\n", .{value});
-    try expectFmt("enum: Enum.Two\n", "enum: {}\n", .{&value});
-    try expectFmt("enum: Enum.One\n", "enum: {}\n", .{Enum.One});
-    try expectFmt("enum: Enum.Two\n", "enum: {}\n", .{Enum.Two});
+    try expectFmt("enum: fmt.test.enum.Enum.Two\n", "enum: {}\n", .{value});
+    try expectFmt("enum: fmt.test.enum.Enum.Two\n", "enum: {}\n", .{&value});
+    try expectFmt("enum: fmt.test.enum.Enum.One\n", "enum: {}\n", .{Enum.One});
+    try expectFmt("enum: fmt.test.enum.Enum.Two\n", "enum: {}\n", .{Enum.Two});
 
     // test very large enum to verify ct branch quota is large enough
-    try expectFmt("enum: os.windows.win32error.Win32Error.INVALID_FUNCTION\n", "enum: {}\n", .{std.os.windows.Win32Error.INVALID_FUNCTION});
+    // TODO: https://github.com/ziglang/zig/issues/15609
+    if (!((builtin.cpu.arch == .wasm32) and builtin.mode == .Debug)) {
+        try expectFmt("enum: os.windows.win32error.Win32Error.INVALID_FUNCTION\n", "enum: {}\n", .{std.os.windows.Win32Error.INVALID_FUNCTION});
+    }
+
+    const E = enum {
+        One,
+        Two,
+        Three,
+    };
+
+    const inst = E.Two;
+
+    try expectFmt("fmt.test.enum.E.Two", "{}", .{inst});
 }
 
 test "non-exhaustive enum" {
@@ -2445,24 +2474,6 @@ test "custom" {
     try expectFmt("dim: 10.200x2.220\n", "dim: {d}\n", .{value});
 }
 
-test "struct" {
-    const S = struct {
-        a: u32,
-        b: anyerror,
-    };
-
-    const inst = S{
-        .a = 456,
-        .b = error.Unused,
-    };
-
-    try expectFmt("fmt.test.struct.S{ .a = 456, .b = error.Unused }", "{}", .{inst});
-    // Tuples
-    try expectFmt("{ }", "{}", .{.{}});
-    try expectFmt("{ -1 }", "{}", .{.{-1}});
-    try expectFmt("{ -1, 42, 2.5e+04 }", "{}", .{.{ -1, 42, 0.25e5 }});
-}
-
 test "union" {
     const TU = union(enum) {
         float: f32,
@@ -2493,18 +2504,6 @@ test "union" {
     try std.testing.expect(mem.eql(u8, eu_result[0..18], "fmt.test.union.EU@"));
 }
 
-test "enum" {
-    const E = enum {
-        One,
-        Two,
-        Three,
-    };
-
-    const inst = E.Two;
-
-    try expectFmt("fmt.test.enum.E.Two", "{}", .{inst});
-}
-
 test "struct.self-referential" {
     const S = struct {
         const SelfType = @This();
diff --git a/lib/std/hash_map.zig b/lib/std/hash_map.zig
index df3446a2c0..91f5682831 100644
--- a/lib/std/hash_map.zig
+++ b/lib/std/hash_map.zig
@@ -1741,6 +1741,22 @@ test "std.hash_map clone" {
     try expectEqual(b.get(1).?, 1);
     try expectEqual(b.get(2).?, 2);
     try expectEqual(b.get(3).?, 3);
+
+    var original = AutoHashMap(i32, i32).init(std.testing.allocator);
+    defer original.deinit();
+
+    var i: u8 = 0;
+    while (i < 10) : (i += 1) {
+        try original.putNoClobber(i, i * 10);
+    }
+
+    var copy = try original.clone();
+    defer copy.deinit();
+
+    i = 0;
+    while (i < 10) : (i += 1) {
+        try testing.expect(copy.get(i).? == i * 10);
+    }
 }
 
 test "std.hash_map ensureTotalCapacity with existing elements" {
@@ -2072,24 +2088,6 @@ test "std.hash_map basic hash map usage" {
     try testing.expect(map.remove(3) == true);
 }
 
-test "std.hash_map clone" {
-    var original = AutoHashMap(i32, i32).init(std.testing.allocator);
-    defer original.deinit();
-
-    var i: u8 = 0;
-    while (i < 10) : (i += 1) {
-        try original.putNoClobber(i, i * 10);
-    }
-
-    var copy = try original.clone();
-    defer copy.deinit();
-
-    i = 0;
-    while (i < 10) : (i += 1) {
-        try testing.expect(copy.get(i).? == i * 10);
-    }
-}
-
 test "std.hash_map getOrPutAdapted" {
     const AdaptedContext = struct {
         fn eql(self: @This(), adapted_key: []const u8, test_key: u64) bool {
diff --git a/lib/std/io/reader.zig b/lib/std/io/reader.zig
index 65b7a086c5..0b88e6b31a 100644
--- a/lib/std/io/reader.zig
+++ b/lib/std/io/reader.zig
@@ -553,10 +553,18 @@ test "Reader.readUntilDelimiter returns StreamTooLong, then bytes read until the
 }
 
 test "Reader.readUntilDelimiter returns EndOfStream" {
-    var buf: [5]u8 = undefined;
-    var fis = std.io.fixedBufferStream("");
-    const reader = fis.reader();
-    try std.testing.expectError(error.EndOfStream, reader.readUntilDelimiter(&buf, '\n'));
+    {
+        var buf: [5]u8 = undefined;
+        var fis = std.io.fixedBufferStream("");
+        const reader = fis.reader();
+        try std.testing.expectError(error.EndOfStream, reader.readUntilDelimiter(&buf, '\n'));
+    }
+    {
+        var buf: [5]u8 = undefined;
+        var fis = std.io.fixedBufferStream("1234");
+        const reader = fis.reader();
+        try std.testing.expectError(error.EndOfStream, reader.readUntilDelimiter(&buf, '\n'));
+    }
 }
 
 test "Reader.readUntilDelimiter returns bytes read until delimiter, then EndOfStream" {
@@ -567,13 +575,6 @@ test "Reader.readUntilDelimiter returns bytes read until delimiter, then EndOfSt
     try std.testing.expectError(error.EndOfStream, reader.readUntilDelimiter(&buf, '\n'));
 }
 
-test "Reader.readUntilDelimiter returns EndOfStream" {
-    var buf: [5]u8 = undefined;
-    var fis = std.io.fixedBufferStream("1234");
-    const reader = fis.reader();
-    try std.testing.expectError(error.EndOfStream, reader.readUntilDelimiter(&buf, '\n'));
-}
-
 test "Reader.readUntilDelimiter returns StreamTooLong, then EndOfStream" {
     var buf: [5]u8 = undefined;
     var fis = std.io.fixedBufferStream("12345");
diff --git a/lib/std/math/big/int_test.zig b/lib/std/math/big/int_test.zig
index 0066ce9940..25f9815f9d 100644
--- a/lib/std/math/big/int_test.zig
+++ b/lib/std/math/big/int_test.zig
@@ -2012,15 +2012,10 @@ test "big.int shift-right negative" {
     defer arg2.deinit();
     try a.shiftRight(&arg2, 10);
     try testing.expect((try a.to(i32)) == -1); // -5 >> 10 == -1
-}
 
-test "big.int shift-right negative" {
-    var a = try Managed.init(testing.allocator);
-    defer a.deinit();
-
-    var arg = try Managed.initSet(testing.allocator, -10);
-    defer arg.deinit();
-    try a.shiftRight(&arg, 1232);
+    var arg3 = try Managed.initSet(testing.allocator, -10);
+    defer arg3.deinit();
+    try a.shiftRight(&arg3, 1232);
     try testing.expect((try a.to(i32)) == -1); // -10 >> 1232 == -1
 }
 
@@ -2483,7 +2478,7 @@ test "big.int gcd non-one small" {
     try testing.expect((try r.to(u32)) == 1);
 }
 
-test "big.int gcd non-one small" {
+test "big.int gcd non-one medium" {
     var a = try Managed.initSet(testing.allocator, 4864);
     defer a.deinit();
     var b = try Managed.initSet(testing.allocator, 3458);
diff --git a/lib/std/math/big/rational.zig b/lib/std/math/big/rational.zig
index c3609a6fa2..cdc33e351d 100644
--- a/lib/std/math/big/rational.zig
+++ b/lib/std/math/big/rational.zig
@@ -782,36 +782,38 @@ test "big.rational mul" {
 }
 
 test "big.rational div" {
-    var a = try Rational.init(testing.allocator);
-    defer a.deinit();
-    var b = try Rational.init(testing.allocator);
-    defer b.deinit();
-    var r = try Rational.init(testing.allocator);
-    defer r.deinit();
+    {
+        var a = try Rational.init(testing.allocator);
+        defer a.deinit();
+        var b = try Rational.init(testing.allocator);
+        defer b.deinit();
+        var r = try Rational.init(testing.allocator);
+        defer r.deinit();
 
-    try a.setRatio(78923, 23341);
-    try b.setRatio(123097, 12441414);
-    try a.div(a, b);
+        try a.setRatio(78923, 23341);
+        try b.setRatio(123097, 12441414);
+        try a.div(a, b);
 
-    try r.setRatio(75531824394, 221015929);
-    try testing.expect((try a.order(r)) == .eq);
-}
-
-test "big.rational div" {
-    var a = try Rational.init(testing.allocator);
-    defer a.deinit();
-    var r = try Rational.init(testing.allocator);
-    defer r.deinit();
-
-    try a.setRatio(78923, 23341);
-    a.invert();
-
-    try r.setRatio(23341, 78923);
-    try testing.expect((try a.order(r)) == .eq);
-
-    try a.setRatio(-78923, 23341);
-    a.invert();
-
-    try r.setRatio(-23341, 78923);
-    try testing.expect((try a.order(r)) == .eq);
+        try r.setRatio(75531824394, 221015929);
+        try testing.expect((try a.order(r)) == .eq);
+    }
+
+    {
+        var a = try Rational.init(testing.allocator);
+        defer a.deinit();
+        var r = try Rational.init(testing.allocator);
+        defer r.deinit();
+
+        try a.setRatio(78923, 23341);
+        a.invert();
+
+        try r.setRatio(23341, 78923);
+        try testing.expect((try a.order(r)) == .eq);
+
+        try a.setRatio(-78923, 23341);
+        a.invert();
+
+        try r.setRatio(-23341, 78923);
+        try testing.expect((try a.order(r)) == .eq);
+    }
 }
diff --git a/lib/std/net/test.zig b/lib/std/net/test.zig
index 9923e4932b..817d6c2593 100644
--- a/lib/std/net/test.zig
+++ b/lib/std/net/test.zig
@@ -182,7 +182,7 @@ test "listen on a port, send bytes, receive bytes" {
     try testing.expectEqualSlices(u8, "Hello world!", buf[0..n]);
 }
 
-test "listen on a port, send bytes, receive bytes" {
+test "listen on a port, send bytes, receive bytes, async-only" {
     if (!std.io.is_async) return error.SkipZigTest;
 
     if (builtin.os.tag != .linux and !builtin.os.tag.isDarwin()) {
diff --git a/lib/std/priority_dequeue.zig b/lib/std/priority_dequeue.zig
index db55be3804..05e3d7e58b 100644
--- a/lib/std/priority_dequeue.zig
+++ b/lib/std/priority_dequeue.zig
@@ -633,7 +633,7 @@ test "std.PriorityDequeue: peekMax" {
     try expect(queue.peekMax().? == 9);
 }
 
-test "std.PriorityDequeue: sift up with odd indices" {
+test "std.PriorityDequeue: sift up with odd indices, removeMin" {
     var queue = PDQ.init(testing.allocator, {});
     defer queue.deinit();
     const items = [_]u32{ 15, 7, 21, 14, 13, 22, 12, 6, 7, 25, 5, 24, 11, 16, 15, 24, 2, 1 };
@@ -647,7 +647,7 @@ test "std.PriorityDequeue: sift up with odd indices" {
     }
 }
 
-test "std.PriorityDequeue: sift up with odd indices" {
+test "std.PriorityDequeue: sift up with odd indices, removeMax" {
     var queue = PDQ.init(testing.allocator, {});
     defer queue.deinit();
     const items = [_]u32{ 15, 7, 21, 14, 13, 22, 12, 6, 7, 25, 5, 24, 11, 16, 15, 24, 2, 1 };
diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig
index ced57ebf5c..21785278ec 100644
--- a/lib/std/zig/parser_test.zig
+++ b/lib/std/zig/parser_test.zig
@@ -1240,7 +1240,7 @@ test "zig fmt: infix operator and then multiline string literal" {
     );
 }
 
-test "zig fmt: infix operator and then multiline string literal" {
+test "zig fmt: infix operator and then multiline string literal over multiple lines" {
     try testCanonical(
         \\const x = "" ++
         \\    \\ hi0
@@ -4310,7 +4310,7 @@ test "zig fmt: comptime before comptime field" {
     });
 }
 
-test "zig fmt: invalid else branch statement" {
+test "zig fmt: invalid doc comments on comptime and test blocks" {
     try testError(
         \\/// This is a doc comment for a comptime block.
         \\comptime {}
@@ -5191,7 +5191,7 @@ test "zig fmt: preserve container doc comment in container without trailing comm
     );
 }
 
-test "zig fmt: make single-line if no trailing comma" {
+test "zig fmt: make single-line if no trailing comma, fmt: off" {
     try testCanonical(
         \\// Test trailing comma syntax
         \\// zig fmt: off
@@ -5270,7 +5270,7 @@ test "zig fmt: variable initialized with ==" {
     , &.{.wrong_equal_var_decl});
 }
 
-test "zig fmt: missing const/var before local variable" {
+test "zig fmt: missing const/var before local variable in comptime block" {
     try testError(
         \\comptime {
         \\    z: u32;
diff --git a/src/Module.zig b/src/Module.zig
index 538c716bc3..6a33990463 100644
--- a/src/Module.zig
+++ b/src/Module.zig
@@ -5280,6 +5280,9 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) Allocator.Err
             }
         },
     };
+    var must_free_decl_name = true;
+    defer if (must_free_decl_name) gpa.free(decl_name);
+
     const is_exported = export_bit and decl_name_index != 0;
     if (kind == .@"usingnamespace") try namespace.usingnamespace_set.ensureUnusedCapacity(gpa, 1);
 
@@ -5296,6 +5299,7 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) Allocator.Err
         const new_decl = mod.declPtr(new_decl_index);
         new_decl.kind = kind;
         new_decl.name = decl_name;
+        must_free_decl_name = false;
         if (kind == .@"usingnamespace") {
             namespace.usingnamespace_set.putAssumeCapacity(new_decl_index, is_pub);
         }
@@ -5339,9 +5343,29 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) Allocator.Err
         new_decl.alive = true; // This Decl corresponds to an AST node and therefore always alive.
         return;
     }
-    gpa.free(decl_name);
     const decl_index = gop.key_ptr.*;
     const decl = mod.declPtr(decl_index);
+    if (kind == .@"test") {
+        const src_loc = SrcLoc{
+            .file_scope = decl.getFileScope(),
+            .parent_decl_node = decl.src_node,
+            .lazy = .{ .token_offset = 1 },
+        };
+        const msg = try ErrorMsg.create(
+            gpa,
+            src_loc,
+            "found test declaration with duplicate name: {s}",
+            .{decl_name},
+        );
+        errdefer msg.destroy(gpa);
+        try mod.failed_decls.putNoClobber(gpa, decl_index, msg);
+        const other_src_loc = SrcLoc{
+            .file_scope = namespace.file_scope,
+            .parent_decl_node = decl_node,
+            .lazy = .{ .token_offset = 1 },
+        };
+        try mod.errNoteNonLazy(other_src_loc, msg, "other test here", .{});
+    }
     log.debug("scan existing {*} ({s}) of {*}", .{ decl, decl.name, namespace });
     // Update the AST node of the decl; even if its contents are unchanged, it may
     // have been re-ordered.
diff --git a/test/behavior.zig b/test/behavior.zig
index 70293bf45d..24652b6612 100644
--- a/test/behavior.zig
+++ b/test/behavior.zig
@@ -150,6 +150,7 @@ test {
     _ = @import("behavior/comptime_memory.zig");
     _ = @import("behavior/const_slice_child.zig");
     _ = @import("behavior/decltest.zig");
+    _ = @import("behavior/duplicated_test_names.zig");
     _ = @import("behavior/defer.zig");
     _ = @import("behavior/empty_tuple_fields.zig");
     _ = @import("behavior/empty_union.zig");
diff --git a/test/behavior/basic.zig b/test/behavior/basic.zig
index 6fdd309371..8727d0dd41 100644
--- a/test/behavior/basic.zig
+++ b/test/behavior/basic.zig
@@ -203,7 +203,7 @@ test "multiline string comments at multiple places" {
     try expect(mem.eql(u8, s1, s2));
 }
 
-test "string concatenation" {
+test "string concatenation simple" {
     try expect(mem.eql(u8, "OK" ++ " IT " ++ "WORKED", "OK IT WORKED"));
 }
 
diff --git a/test/behavior/comptime_memory.zig b/test/behavior/comptime_memory.zig
index 5c3012d1dc..ea7a67651c 100644
--- a/test/behavior/comptime_memory.zig
+++ b/test/behavior/comptime_memory.zig
@@ -45,7 +45,7 @@ test "type pun signed and unsigned as offset many pointer" {
     }
 }
 
-test "type pun signed and unsigned as array pointer" {
+test "type pun signed and unsigned as array pointer with pointer arithemtic" {
     if (true) {
         // TODO https://github.com/ziglang/zig/issues/9646
         return error.SkipZigTest;
diff --git a/test/behavior/duplicated_test_names.zig b/test/behavior/duplicated_test_names.zig
new file mode 100644
index 0000000000..d59945ce30
--- /dev/null
+++ b/test/behavior/duplicated_test_names.zig
@@ -0,0 +1,17 @@
+const Namespace = struct {
+    test "thingy" {}
+};
+
+fn thingy(a: usize, b: usize) usize {
+    return a + b;
+}
+
+comptime {
+    _ = Namespace;
+}
+
+test "thingy" {}
+
+test thingy {
+    if (thingy(1, 2) != 3) unreachable;
+}
diff --git a/test/behavior/vector.zig b/test/behavior/vector.zig
index 2c55af5f85..b0e717d131 100644
--- a/test/behavior/vector.zig
+++ b/test/behavior/vector.zig
@@ -1129,20 +1129,6 @@ test "array of vectors is copied" {
     try std.testing.expectEqual(points2[6], Vec3{ -345, -311, 381 });
 }
 
-test "byte vector initialized in inline function" {
-    const S = struct {
-        inline fn boolx4(e0: bool, e1: bool, e2: bool, e3: bool) @Vector(4, bool) {
-            return .{ e0, e1, e2, e3 };
-        }
-
-        fn all(vb: @Vector(4, bool)) bool {
-            return @reduce(.And, vb);
-        }
-    };
-
-    try expect(S.all(S.boolx4(true, true, true, true)));
-}
-
 test "byte vector initialized in inline function" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
diff --git a/test/cases/compile_errors/invalid_duplicate_test_decl_name.zig b/test/cases/compile_errors/invalid_duplicate_test_decl_name.zig
new file mode 100644
index 0000000000..e27bbe7c1a
--- /dev/null
+++ b/test/cases/compile_errors/invalid_duplicate_test_decl_name.zig
@@ -0,0 +1,10 @@
+test "thingy" {}
+test "thingy" {}
+
+// error
+// backend=stage2
+// target=native
+// is_test=1
+//
+// :1:6: error: found test declaration with duplicate name: test.thingy
+// :2:6: note: other test here

From b18b4db7095f58b1e8b3f5ac4993bda38231c2af Mon Sep 17 00:00:00 2001
From: Jayden 
Date: Mon, 8 May 2023 06:42:24 -0400
Subject: [PATCH 117/725] parse_float: Error when a float is attempted to be
 parsed into an invalid type

Co-authored-by: Ryan Liptak 
---
 lib/std/fmt/parse_float/parse_float.zig | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/std/fmt/parse_float/parse_float.zig b/lib/std/fmt/parse_float/parse_float.zig
index 6d77346e9b..08d1c55862 100644
--- a/lib/std/fmt/parse_float/parse_float.zig
+++ b/lib/std/fmt/parse_float/parse_float.zig
@@ -12,6 +12,14 @@ pub const ParseFloatError = error{
 };
 
 pub fn parseFloat(comptime T: type, s: []const u8) ParseFloatError!T {
+    if (@typeInfo(T) != .Float) {
+        @compileError("Cannot parse a float into a non-floating point type.");
+    }
+
+    if (T == f80) {
+        @compileError("TODO support parsing float to f80");
+    }
+
     if (s.len == 0) {
         return error.InvalidCharacter;
     }

From 9bea854dc2af293cba1d000b31f6e82d9c431285 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Thu, 4 May 2023 03:36:28 -0400
Subject: [PATCH 118/725] x86_64: implement `@floor`, `@ceil`, and `@trunc`

---
 src/arch/x86_64/CodeGen.zig | 44 ++++++++++++++++++++++++++++++++++---
 1 file changed, 41 insertions(+), 3 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 97e672b71f..fbed0bcf96 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -1480,12 +1480,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .log,
             .log2,
             .log10,
-            .floor,
-            .ceil,
             .round,
-            .trunc_float,
             => try self.airUnaryMath(inst),
 
+            .floor => try self.airRound(inst, Immediate.u(0b1_0_01)),
+            .ceil => try self.airRound(inst, Immediate.u(0b1_0_10)),
+            .trunc_float => try self.airRound(inst, Immediate.u(0b1_0_11)),
             .sqrt => try self.airSqrt(inst),
             .neg, .fabs => try self.airFloatSign(inst),
 
@@ -4258,6 +4258,44 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
 }
 
+fn airRound(self: *Self, inst: Air.Inst.Index, mode: Immediate) !void {
+    const un_op = self.air.instructions.items(.data)[inst].un_op;
+    const ty = self.air.typeOf(un_op);
+
+    if (!Target.x86.featureSetHas(self.target.cpu.features, .sse4_1))
+        return self.fail("TODO implement airRound without sse4_1 feature", .{});
+
+    const src_mcv = try self.resolveInst(un_op);
+    const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
+        src_mcv
+    else
+        try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
+
+    const mir_tag: Mir.Inst.Tag = switch (ty.zigTypeTag()) {
+        .Float => switch (ty.floatBits(self.target.*)) {
+            32 => .roundss,
+            64 => .roundsd,
+            else => return self.fail("TODO implement airRound for {}", .{
+                ty.fmt(self.bin_file.options.module.?),
+            }),
+        },
+        else => return self.fail("TODO implement airRound for {}", .{
+            ty.fmt(self.bin_file.options.module.?),
+        }),
+    };
+    assert(dst_mcv.isRegister());
+    if (src_mcv.isRegister())
+        try self.asmRegisterRegisterImmediate(mir_tag, dst_mcv.getReg().?, src_mcv.getReg().?, mode)
+    else
+        try self.asmRegisterMemoryImmediate(
+            mir_tag,
+            dst_mcv.getReg().?,
+            src_mcv.mem(Memory.PtrSize.fromSize(@intCast(u32, ty.abiSize(self.target.*)))),
+            mode,
+        );
+    return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
+}
+
 fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     const ty = self.air.typeOf(un_op);

From 1a261917ce41efb49fe41ea0c6d9083212c17797 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Thu, 4 May 2023 03:36:04 -0400
Subject: [PATCH 119/725] x86_64: implement `@ctz` and `@clz` for `u128`

---
 src/arch/x86_64/CodeGen.zig | 93 ++++++++++++++++++++++++++++---------
 test/behavior/bugs/2114.zig |  3 +-
 test/behavior/math.zig      |  3 +-
 3 files changed, 74 insertions(+), 25 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index fbed0bcf96..55b18985da 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -3798,19 +3798,38 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
 
         const dst_reg = try self.register_manager.allocReg(inst, gp);
         const dst_mcv = MCValue{ .register = dst_reg };
-        const dst_lock = self.register_manager.lockReg(dst_reg);
-        defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+        const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
+        defer self.register_manager.unlockReg(dst_lock);
 
+        const src_bits = src_ty.bitSize(self.target.*);
         if (Target.x86.featureSetHas(self.target.cpu.features, .lzcnt)) {
-            try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
-            const extra_bits = self.regExtraBits(src_ty);
-            if (extra_bits > 0) {
-                try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });
-            }
+            if (src_bits <= 64) {
+                try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
+
+                const extra_bits = self.regExtraBits(src_ty);
+                if (extra_bits > 0) {
+                    try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });
+                }
+            } else if (src_bits <= 128) {
+                const tmp_reg = try self.register_manager.allocReg(null, gp);
+                const tmp_mcv = MCValue{ .register = tmp_reg };
+                const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
+                defer self.register_manager.unlockReg(tmp_lock);
+
+                try self.genBinOpMir(.lzcnt, Type.u64, dst_mcv, mat_src_mcv);
+                try self.genBinOpMir(.add, dst_ty, dst_mcv, .{ .immediate = 64 });
+                try self.genBinOpMir(.lzcnt, Type.u64, tmp_mcv, mat_src_mcv.address().offset(8).deref());
+                try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
+
+                if (src_bits < 128) {
+                    try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = 128 - src_bits });
+                }
+            } else return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
             break :result dst_mcv;
         }
 
-        const src_bits = src_ty.bitSize(self.target.*);
+        if (src_bits > 64)
+            return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
         if (math.isPowerOfTwo(src_bits)) {
             const imm_reg = try self.copyToTmpRegister(dst_ty, .{
                 .immediate = src_bits ^ (src_bits - 1),
@@ -3870,24 +3889,52 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
         defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
         if (Target.x86.featureSetHas(self.target.cpu.features, .bmi)) {
-            const extra_bits = self.regExtraBits(src_ty);
-            const masked_mcv = if (extra_bits > 0) masked: {
-                const mask_mcv = MCValue{
-                    .immediate = ((@as(u64, 1) << @intCast(u6, extra_bits)) - 1) <<
-                        @intCast(u6, src_bits),
-                };
-                const tmp_mcv = tmp: {
-                    if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) break :tmp src_mcv;
-                    try self.genSetReg(dst_reg, src_ty, src_mcv);
-                    break :tmp dst_mcv;
-                };
-                try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);
-                break :masked tmp_mcv;
-            } else mat_src_mcv;
-            try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);
+            if (src_bits <= 64) {
+                const extra_bits = self.regExtraBits(src_ty);
+                const masked_mcv = if (extra_bits > 0) masked: {
+                    const tmp_mcv = tmp: {
+                        if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0))
+                            break :tmp src_mcv;
+                        try self.genSetReg(dst_reg, src_ty, src_mcv);
+                        break :tmp dst_mcv;
+                    };
+                    try self.genBinOpMir(
+                        .@"or",
+                        src_ty,
+                        tmp_mcv,
+                        .{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - extra_bits)) <<
+                            @intCast(u6, src_bits) },
+                    );
+                    break :masked tmp_mcv;
+                } else mat_src_mcv;
+                try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);
+            } else if (src_bits <= 128) {
+                const tmp_reg = try self.register_manager.allocReg(null, gp);
+                const tmp_mcv = MCValue{ .register = tmp_reg };
+                const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
+                defer self.register_manager.unlockReg(tmp_lock);
+
+                const masked_mcv = if (src_bits < 128) masked: {
+                    try self.genCopy(Type.u64, dst_mcv, mat_src_mcv.address().offset(8).deref());
+                    try self.genBinOpMir(
+                        .@"or",
+                        Type.u64,
+                        dst_mcv,
+                        .{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(u6, src_bits - 64) },
+                    );
+                    break :masked dst_mcv;
+                } else mat_src_mcv.address().offset(8).deref();
+                try self.genBinOpMir(.tzcnt, Type.u64, dst_mcv, masked_mcv);
+                try self.genBinOpMir(.add, dst_ty, dst_mcv, .{ .immediate = 64 });
+                try self.genBinOpMir(.tzcnt, Type.u64, tmp_mcv, mat_src_mcv);
+                try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
+            } else return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
             break :result dst_mcv;
         }
 
+        if (src_bits > 64)
+            return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
+
         const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
         try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
 
diff --git a/test/behavior/bugs/2114.zig b/test/behavior/bugs/2114.zig
index f92728eff6..3ad4a97b80 100644
--- a/test/behavior/bugs/2114.zig
+++ b/test/behavior/bugs/2114.zig
@@ -9,7 +9,8 @@ fn ctz(x: anytype) usize {
 
 test "fixed" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .bmi)) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/math.zig b/test/behavior/math.zig
index f9c9f43927..0362bd3a2b 100644
--- a/test/behavior/math.zig
+++ b/test/behavior/math.zig
@@ -77,7 +77,8 @@ fn testClz() !void {
 }
 
 test "@clz big ints" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .lzcnt)) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From 32ab930f1d39c374265ae14f1de9d837dcd7f650 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Fri, 5 May 2023 01:32:39 -0400
Subject: [PATCH 120/725] x86_64: implement f16 conversions when supported

---
 src/arch/x86_64/CodeGen.zig   |   66 +-
 src/arch/x86_64/Encoding.zig  |   95 +-
 src/arch/x86_64/Lower.zig     |    3 +
 src/arch/x86_64/Mir.zig       |    5 +
 src/arch/x86_64/encoder.zig   |  160 +++-
 src/arch/x86_64/encodings.zig | 1606 +++++++++++++++++----------------
 test/behavior/vector.zig      |    3 +-
 7 files changed, 1081 insertions(+), 857 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 55b18985da..b7fd81db68 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -2172,12 +2172,9 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
 fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
     const dst_ty = self.air.typeOfIndex(inst);
+    const dst_bits = dst_ty.floatBits(self.target.*);
     const src_ty = self.air.typeOf(ty_op.operand);
-    if (dst_ty.floatBits(self.target.*) != 32 or src_ty.floatBits(self.target.*) != 64 or
-        !Target.x86.featureSetHas(self.target.cpu.features, .sse2))
-        return self.fail("TODO implement airFptrunc from {} to {}", .{
-            src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
-        });
+    const src_bits = src_ty.floatBits(self.target.*);
 
     const src_mcv = try self.resolveInst(ty_op.operand);
     const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
@@ -2187,19 +2184,32 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
     const dst_lock = self.register_manager.lockReg(dst_mcv.register);
     defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
-    try self.genBinOpMir(.cvtsd2ss, src_ty, dst_mcv, src_mcv);
+    if (src_bits == 32 and dst_bits == 16 and self.hasFeature(.f16c))
+        try self.asmRegisterRegisterImmediate(
+            .vcvtps2ph,
+            dst_mcv.register,
+            if (src_mcv.isRegister()) src_mcv.getReg().? else src_reg: {
+                const src_reg = dst_mcv.register;
+                try self.genSetReg(src_reg, src_ty, src_mcv);
+                break :src_reg src_reg;
+            },
+            Immediate.u(0b1_00),
+        )
+    else if (src_bits == 64 and dst_bits == 32)
+        try self.genBinOpMir(.cvtsd2ss, src_ty, dst_mcv, src_mcv)
+    else
+        return self.fail("TODO implement airFptrunc from {} to {}", .{
+            src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
+        });
     return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
 }
 
 fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
     const ty_op = self.air.instructions.items(.data)[inst].ty_op;
     const dst_ty = self.air.typeOfIndex(inst);
+    const dst_bits = dst_ty.floatBits(self.target.*);
     const src_ty = self.air.typeOf(ty_op.operand);
-    if (dst_ty.floatBits(self.target.*) != 64 or src_ty.floatBits(self.target.*) != 32 or
-        !Target.x86.featureSetHas(self.target.cpu.features, .sse2))
-        return self.fail("TODO implement airFpext from {} to {}", .{
-            src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
-        });
+    const src_bits = src_ty.floatBits(self.target.*);
 
     const src_mcv = try self.resolveInst(ty_op.operand);
     const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
@@ -2209,7 +2219,19 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
     const dst_lock = self.register_manager.lockReg(dst_mcv.register);
     defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
-    try self.genBinOpMir(.cvtss2sd, src_ty, dst_mcv, src_mcv);
+    try self.genBinOpMir(
+        if (src_bits == 16 and dst_bits == 32 and self.hasFeature(.f16c))
+            .vcvtph2ps
+        else if (src_bits == 32 and dst_bits == 64)
+            .cvtss2sd
+        else
+            return self.fail("TODO implement airFpext from {} to {}", .{
+                src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
+            }),
+        src_ty,
+        dst_mcv,
+        src_mcv,
+    );
     return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
 }
 
@@ -3802,7 +3824,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
         defer self.register_manager.unlockReg(dst_lock);
 
         const src_bits = src_ty.bitSize(self.target.*);
-        if (Target.x86.featureSetHas(self.target.cpu.features, .lzcnt)) {
+        if (self.hasFeature(.lzcnt)) {
             if (src_bits <= 64) {
                 try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
 
@@ -3888,7 +3910,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
         const dst_lock = self.register_manager.lockReg(dst_reg);
         defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
-        if (Target.x86.featureSetHas(self.target.cpu.features, .bmi)) {
+        if (self.hasFeature(.bmi)) {
             if (src_bits <= 64) {
                 const extra_bits = self.regExtraBits(src_ty);
                 const masked_mcv = if (extra_bits > 0) masked: {
@@ -3956,7 +3978,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
         const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
         const src_mcv = try self.resolveInst(ty_op.operand);
 
-        if (Target.x86.featureSetHas(self.target.cpu.features, .popcnt)) {
+        if (self.hasFeature(.popcnt)) {
             const mat_src_mcv = switch (src_mcv) {
                 .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
                 else => src_mcv,
@@ -4309,7 +4331,7 @@ fn airRound(self: *Self, inst: Air.Inst.Index, mode: Immediate) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     const ty = self.air.typeOf(un_op);
 
-    if (!Target.x86.featureSetHas(self.target.cpu.features, .sse4_1))
+    if (!self.hasFeature(.sse4_1))
         return self.fail("TODO implement airRound without sse4_1 feature", .{});
 
     const src_mcv = try self.resolveInst(un_op);
@@ -5712,7 +5734,7 @@ fn genBinOp(
                 => {},
                 .div_trunc,
                 .div_floor,
-                => if (Target.x86.featureSetHas(self.target.cpu.features, .sse4_1)) {
+                => if (self.hasFeature(.sse4_1)) {
                     const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
                     const dst_alias = registerAlias(dst_mcv.register, abi_size);
                     try self.asmRegisterRegisterImmediate(switch (lhs_ty.floatBits(self.target.*)) {
@@ -9593,3 +9615,13 @@ fn regBitSize(self: *Self, ty: Type) u64 {
 fn regExtraBits(self: *Self, ty: Type) u64 {
     return self.regBitSize(ty) - ty.bitSize(self.target.*);
 }
+
+fn hasFeature(self: *Self, feature: Target.x86.Feature) bool {
+    return Target.x86.featureSetHas(self.target.cpu.features, feature);
+}
+fn hasAnyFeatures(self: *Self, features: anytype) bool {
+    return Target.x86.featureSetHasAny(self.target.cpu.features, features);
+}
+fn hasAllFeatures(self: *Self, features: anytype) bool {
+    return Target.x86.featureSetHasAll(self.target.cpu.features, features);
+}
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index 944fe85458..05c48ecddf 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -23,6 +23,7 @@ const Data = struct {
     opc: [7]u8,
     modrm_ext: u3,
     mode: Mode,
+    feature: Feature,
 };
 
 pub fn findByMnemonic(
@@ -58,7 +59,7 @@ pub fn findByMnemonic(
     next: for (mnemonic_to_encodings_map[@enumToInt(mnemonic)]) |data| {
         switch (data.mode) {
             .rex => if (!rex_required) continue,
-            .long, .sse_long, .sse2_long => {},
+            .long => {},
             else => if (rex_required) continue,
         }
         for (input_ops, data.ops) |input_op, data_op|
@@ -136,22 +137,20 @@ pub fn modRmExt(encoding: Encoding) u3 {
 }
 
 pub fn operandBitSize(encoding: Encoding) u64 {
-    switch (encoding.data.mode) {
-        .short => return 16,
-        .long, .sse_long, .sse2_long => return 64,
-        else => {},
-    }
-    const bit_size: u64 = switch (encoding.data.op_en) {
-        .np => switch (encoding.data.ops[0]) {
-            .o16 => 16,
-            .o32 => 32,
-            .o64 => 64,
-            else => 32,
+    return switch (encoding.data.mode) {
+        .short => 16,
+        .long => 64,
+        else => switch (encoding.data.op_en) {
+            .np => switch (encoding.data.ops[0]) {
+                .o16 => 16,
+                .o32 => 32,
+                .o64 => 64,
+                else => 32,
+            },
+            .td => encoding.data.ops[1].bitSize(),
+            else => encoding.data.ops[0].bitSize(),
         },
-        .td => encoding.data.ops[1].bitSize(),
-        else => encoding.data.ops[0].bitSize(),
     };
-    return bit_size;
 }
 
 pub fn format(
@@ -162,12 +161,50 @@ pub fn format(
 ) !void {
     _ = options;
     _ = fmt;
+
+    var opc = encoding.opcode();
     switch (encoding.data.mode) {
-        .long, .sse_long, .sse2_long => try writer.writeAll("REX.W + "),
         else => {},
+        .long => try writer.writeAll("REX.W + "),
+        .vex_128, .vex_128_long, .vex_256, .vex_256_long => {
+            try writer.writeAll("VEX.");
+
+            switch (encoding.data.mode) {
+                .vex_128, .vex_128_long => try writer.writeAll("128"),
+                .vex_256, .vex_256_long => try writer.writeAll("256"),
+                else => unreachable,
+            }
+
+            switch (opc[0]) {
+                else => {},
+                0x66, 0xf3, 0xf2 => {
+                    try writer.print(".{X:0>2}", .{opc[0]});
+                    opc = opc[1..];
+                },
+            }
+
+            try writer.print(".{X:0>2}", .{opc[0]});
+            opc = opc[1..];
+
+            switch (opc[0]) {
+                else => {},
+                0x38, 0x3A => {
+                    try writer.print("{X:0>2}", .{opc[0]});
+                    opc = opc[1..];
+                },
+            }
+
+            try writer.writeByte('.');
+            try writer.writeAll(switch (encoding.data.mode) {
+                .vex_128, .vex_256 => "W0",
+                .vex_128_long, .vex_256_long => "W1",
+                else => unreachable,
+            });
+            try writer.writeByte(' ');
+        },
     }
 
-    for (encoding.opcode()) |byte| {
+    for (opc) |byte| {
         try writer.print("{x:0>2} ", .{byte});
     }
 
@@ -184,15 +221,16 @@ pub fn format(
             try writer.print("+{s} ", .{tag});
         },
         .m, .mi, .m1, .mc => try writer.print("/{d} ", .{encoding.modRmExt()}),
-        .mr, .rm, .rmi, .mri, .mrc => try writer.writeAll("/r "),
+        .mr, .rm, .rmi, .mri, .mrc, .rrm, .rrmi => try writer.writeAll("/r "),
     }
 
     switch (encoding.data.op_en) {
-        .i, .d, .zi, .oi, .mi, .rmi, .mri => {
+        .i, .d, .zi, .oi, .mi, .rmi, .mri, .rrmi => {
             const op = switch (encoding.data.op_en) {
                 .i, .d => encoding.data.ops[0],
                 .zi, .oi, .mi => encoding.data.ops[1],
                 .rmi, .mri => encoding.data.ops[2],
+                .rrmi => encoding.data.ops[3],
                 else => unreachable,
             };
             const tag = switch (op) {
@@ -207,7 +245,7 @@ pub fn format(
             };
             try writer.print("{s} ", .{tag});
         },
-        .np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc => {},
+        .np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc, .rrm => {},
     }
 
     try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
@@ -305,6 +343,8 @@ pub const Mnemonic = enum {
     // SSE4.1
     roundss,
     roundsd,
+    // F16C
+    vcvtph2ps, vcvtps2ph,
     // zig fmt: on
 };
 
@@ -317,6 +357,7 @@ pub const OpEn = enum {
     fd, td,
     m1, mc, mi, mr, rm,
     rmi, mri, mrc,
+    rrm, rrmi,
     // zig fmt: on
 };
 
@@ -549,14 +590,21 @@ pub const Op = enum {
 pub const Mode = enum {
     none,
     short,
-    fpu,
     rex,
     long,
+    vex_128,
+    vex_128_long,
+    vex_256,
+    vex_256_long,
+};
+
+pub const Feature = enum {
+    none,
+    f16c,
     sse,
-    sse_long,
     sse2,
-    sse2_long,
     sse4_1,
+    x87,
 };
 
 fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Operand) usize {
@@ -593,6 +641,7 @@ const mnemonic_to_encodings_map = init: {
             .opc = undefined,
             .modrm_ext = entry[4],
             .mode = entry[5],
+            .feature = entry[6],
         };
         // TODO: use `@memcpy` for these. When I did that, I got a false positive
         // compile error for this copy happening at compile time.
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index 4289cfaf2a..9571f50e7c 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -133,6 +133,9 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
         .subsd,
         .ucomisd,
         .xorpd,
+
+        .vcvtph2ps,
+        .vcvtps2ph,
         => try lower.mirGeneric(inst),
 
         .cmps,
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index 6b2db1b696..c4e19fdc0e 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -247,6 +247,11 @@ pub const Inst = struct {
         /// Bitwise logical xor of packed double precision floating-point values
         xorpd,
 
+        /// Convert 16-bit floating-point values to single-precision floating-point values
+        vcvtph2ps,
+        /// Convert single-precision floating-point values to 16-bit floating-point values
+        vcvtps2ph,
+
         /// Compare string operands
         cmps,
         /// Load string
diff --git a/src/arch/x86_64/encoder.zig b/src/arch/x86_64/encoder.zig
index 4c900697f5..94f4eb56d5 100644
--- a/src/arch/x86_64/encoder.zig
+++ b/src/arch/x86_64/encoder.zig
@@ -209,10 +209,19 @@ pub const Instruction = struct {
         const enc = inst.encoding;
         const data = enc.data;
 
-        try inst.encodeLegacyPrefixes(encoder);
-        try inst.encodeMandatoryPrefix(encoder);
-        try inst.encodeRexPrefix(encoder);
-        try inst.encodeOpcode(encoder);
+        switch (data.mode) {
+            .none, .short, .rex, .long => {
+                try inst.encodeLegacyPrefixes(encoder);
+                try inst.encodeMandatoryPrefix(encoder);
+                try inst.encodeRexPrefix(encoder);
+                try inst.encodeOpcode(encoder);
+            },
+            .vex_128, .vex_128_long, .vex_256, .vex_256_long => {
+                try inst.encodeVexPrefix(encoder);
+                const opc = inst.encoding.opcode();
+                try encoder.opcode_1byte(opc[opc.len - 1]);
+            },
+        }
 
         switch (data.op_en) {
             .np, .o => {},
@@ -309,6 +318,7 @@ pub const Instruction = struct {
                 }
             else
                 null,
+            .rrm, .rrmi => unreachable,
         };
         if (segment_override) |seg| {
             legacy.setSegmentOverride(seg);
@@ -322,10 +332,7 @@ pub const Instruction = struct {
 
         var rex = Rex{};
         rex.present = inst.encoding.data.mode == .rex;
-        switch (inst.encoding.data.mode) {
-            .long, .sse_long, .sse2_long => rex.w = true,
-            else => {},
-        }
+        rex.w = inst.encoding.data.mode == .long;
 
         switch (op_en) {
             .np, .i, .zi, .fd, .td, .d => {},
@@ -346,11 +353,76 @@ pub const Instruction = struct {
                 rex.b = b_x_op.isBaseExtended();
                 rex.x = b_x_op.isIndexExtended();
             },
+            .rrm, .rrmi => unreachable,
         }
 
         try encoder.rex(rex);
     }
 
+    fn encodeVexPrefix(inst: Instruction, encoder: anytype) !void {
+        const op_en = inst.encoding.data.op_en;
+        const opc = inst.encoding.opcode();
+        const mand_pre = inst.encoding.mandatoryPrefix();
+
+        var vex = Vex{};
+
+        vex.w = switch (inst.encoding.data.mode) {
+            .vex_128, .vex_256 => false,
+            .vex_128_long, .vex_256_long => true,
+            else => unreachable,
+        };
+
+        switch (op_en) {
+            .np, .i, .zi, .fd, .td, .d => {},
+            .o, .oi => vex.b = inst.ops[0].reg.isExtended(),
+            .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rrm, .rrmi => {
+                const r_op = switch (op_en) {
+                    .rm, .rmi, .rrm, .rrmi => inst.ops[0],
+                    .mr, .mri, .mrc => inst.ops[1],
+                    else => .none,
+                };
+                vex.r = r_op.isBaseExtended();
+
+                const b_x_op = switch (op_en) {
+                    .rm, .rmi => inst.ops[1],
+                    .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],
+                    .rrm, .rrmi => inst.ops[2],
+                    else => unreachable,
+                };
+                vex.b = b_x_op.isBaseExtended();
+                vex.x = b_x_op.isIndexExtended();
+            },
+        }
+
+        vex.l = switch (inst.encoding.data.mode) {
+            .vex_128, .vex_128_long => false,
+            .vex_256, .vex_256_long => true,
+            else => unreachable,
+        };
+
+        vex.p = if (mand_pre) |mand| switch (mand) {
+            0x66 => .@"66",
+            0xf2 => .f2,
+            0xf3 => .f3,
+            else => unreachable,
+        } else .none;
+
+        const leading: usize = if (mand_pre) |_| 1 else 0;
+        assert(opc[leading] == 0x0f);
+        vex.m = switch (opc[leading + 1]) {
+            else => .@"0f",
+            0x38 => .@"0f38",
+            0x3a => .@"0f3a",
+        };
+
+        switch (op_en) {
+            else => {},
+            .rrm, .rrmi => vex.v = inst.ops[1].reg,
+        }
+
+        try encoder.vex(vex);
+    }
+
     fn encodeMandatoryPrefix(inst: Instruction, encoder: anytype) !void {
         const prefix = inst.encoding.mandatoryPrefix() orelse return;
         try encoder.opcode_1byte(prefix);
@@ -562,17 +634,48 @@ fn Encoder(comptime T: type, comptime opts: Options) type {
         /// or one of reg, index, r/m, base, or opcode-reg might be extended.
         ///
         /// See struct `Rex` for a description of each field.
-        pub fn rex(self: Self, byte: Rex) !void {
-            if (!byte.present and !byte.isSet()) return;
+        pub fn rex(self: Self, fields: Rex) !void {
+            if (!fields.present and !fields.isSet()) return;
 
-            var value: u8 = 0b0100_0000;
+            var byte: u8 = 0b0100_0000;
 
-            if (byte.w) value |= 0b1000;
-            if (byte.r) value |= 0b0100;
-            if (byte.x) value |= 0b0010;
-            if (byte.b) value |= 0b0001;
+            if (fields.w) byte |= 0b1000;
+            if (fields.r) byte |= 0b0100;
+            if (fields.x) byte |= 0b0010;
+            if (fields.b) byte |= 0b0001;
 
-            try self.writer.writeByte(value);
+            try self.writer.writeByte(byte);
+        }
+
+        /// Encodes a VEX prefix given all the fields
+        ///
+        /// See struct `Vex` for a description of each field.
+        pub fn vex(self: Self, fields: Vex) !void {
+            if (fields.is3Byte()) {
+                try self.writer.writeByte(0b1100_0100);
+
+                try self.writer.writeByte(
+                    @as(u8, ~@boolToInt(fields.r)) << 7 |
+                        @as(u8, ~@boolToInt(fields.x)) << 6 |
+                        @as(u8, ~@boolToInt(fields.b)) << 5 |
+                        @as(u8, @enumToInt(fields.m)) << 0,
+                );
+
+                try self.writer.writeByte(
+                    @as(u8, @boolToInt(fields.w)) << 7 |
+                        @as(u8, ~fields.v.enc()) << 3 |
+                        @as(u8, @boolToInt(fields.l)) << 2 |
+                        @as(u8, @enumToInt(fields.p)) << 0,
+                );
+            } else {
+                try self.writer.writeByte(0b1100_0101);
+                try self.writer.writeByte(
+                    @as(u8, ~@boolToInt(fields.r)) << 7 |
+                        @as(u8, ~fields.v.enc()) << 3 |
+                        @as(u8, @boolToInt(fields.l)) << 2 |
+                        @as(u8, @enumToInt(fields.p)) << 0,
+                );
+            }
         }
 
         // ------
@@ -848,6 +951,31 @@ pub const Rex = struct {
     }
 };
 
+pub const Vex = struct {
+    w: bool = false,
+    r: bool = false,
+    x: bool = false,
+    b: bool = false,
+    l: bool = false,
+    p: enum(u2) {
+        none = 0b00,
+        @"66" = 0b01,
+        f3 = 0b10,
+        f2 = 0b11,
+    } = .none,
+    m: enum(u5) {
+        @"0f" = 0b0_0001,
+        @"0f38" = 0b0_0010,
+        @"0f3a" = 0b0_0011,
+        _,
+    } = .@"0f",
+    v: Register = .ymm0,
+
+    pub fn is3Byte(vex: Vex) bool {
+        return vex.w or vex.x or vex.b or vex.m != .@"0f";
+    }
+};
+
 // Tests
 fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
     assert(expected.len > 0);
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index f87a110e99..52b8cc29d6 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -3,933 +3,939 @@ const Mnemonic = Encoding.Mnemonic;
 const OpEn = Encoding.OpEn;
 const Op = Encoding.Op;
 const Mode = Encoding.Mode;
+const Feature = Encoding.Feature;
 
 const modrm_ext = u3;
 
-pub const Entry = struct { Mnemonic, OpEn, []const Op, []const u8, modrm_ext, Mode };
+pub const Entry = struct { Mnemonic, OpEn, []const Op, []const u8, modrm_ext, Mode, Feature };
 
 // TODO move this into a .zon file when Zig is capable of importing .zon files
 // zig fmt: off
 pub const table = [_]Entry{
     // General-purpose
-    .{ .adc, .zi, &.{ .al,   .imm8   }, &.{ 0x14 }, 0, .none },
-    .{ .adc, .zi, &.{ .ax,   .imm16  }, &.{ 0x15 }, 0, .none },
-    .{ .adc, .zi, &.{ .eax,  .imm32  }, &.{ 0x15 }, 0, .none },
-    .{ .adc, .zi, &.{ .rax,  .imm32s }, &.{ 0x15 }, 0, .long },
-    .{ .adc, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 2, .none },
-    .{ .adc, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 2, .rex  },
-    .{ .adc, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 2, .none },
-    .{ .adc, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 2, .none },
-    .{ .adc, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 2, .long },
-    .{ .adc, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 2, .none },
-    .{ .adc, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 2, .none },
-    .{ .adc, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 2, .long },
-    .{ .adc, .mr, &.{ .rm8,  .r8     }, &.{ 0x10 }, 0, .none },
-    .{ .adc, .mr, &.{ .rm8,  .r8     }, &.{ 0x10 }, 0, .rex  },
-    .{ .adc, .mr, &.{ .rm16, .r16    }, &.{ 0x11 }, 0, .none },
-    .{ .adc, .mr, &.{ .rm32, .r32    }, &.{ 0x11 }, 0, .none },
-    .{ .adc, .mr, &.{ .rm64, .r64    }, &.{ 0x11 }, 0, .long },
-    .{ .adc, .rm, &.{ .r8,   .rm8    }, &.{ 0x12 }, 0, .none },
-    .{ .adc, .rm, &.{ .r8,   .rm8    }, &.{ 0x12 }, 0, .rex  },
-    .{ .adc, .rm, &.{ .r16,  .rm16   }, &.{ 0x13 }, 0, .none },
-    .{ .adc, .rm, &.{ .r32,  .rm32   }, &.{ 0x13 }, 0, .none },
-    .{ .adc, .rm, &.{ .r64,  .rm64   }, &.{ 0x13 }, 0, .long },
+    .{ .adc, .zi, &.{ .al,   .imm8   }, &.{ 0x14 }, 0, .none, .none },
+    .{ .adc, .zi, &.{ .ax,   .imm16  }, &.{ 0x15 }, 0, .none, .none },
+    .{ .adc, .zi, &.{ .eax,  .imm32  }, &.{ 0x15 }, 0, .none, .none },
+    .{ .adc, .zi, &.{ .rax,  .imm32s }, &.{ 0x15 }, 0, .long, .none },
+    .{ .adc, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 2, .none, .none },
+    .{ .adc, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 2, .rex,  .none },
+    .{ .adc, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 2, .none, .none },
+    .{ .adc, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 2, .none, .none },
+    .{ .adc, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 2, .long, .none },
+    .{ .adc, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 2, .none, .none },
+    .{ .adc, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 2, .none, .none },
+    .{ .adc, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 2, .long, .none },
+    .{ .adc, .mr, &.{ .rm8,  .r8     }, &.{ 0x10 }, 0, .none, .none },
+    .{ .adc, .mr, &.{ .rm8,  .r8     }, &.{ 0x10 }, 0, .rex,  .none },
+    .{ .adc, .mr, &.{ .rm16, .r16    }, &.{ 0x11 }, 0, .none, .none },
+    .{ .adc, .mr, &.{ .rm32, .r32    }, &.{ 0x11 }, 0, .none, .none },
+    .{ .adc, .mr, &.{ .rm64, .r64    }, &.{ 0x11 }, 0, .long, .none },
+    .{ .adc, .rm, &.{ .r8,   .rm8    }, &.{ 0x12 }, 0, .none, .none },
+    .{ .adc, .rm, &.{ .r8,   .rm8    }, &.{ 0x12 }, 0, .rex,  .none },
+    .{ .adc, .rm, &.{ .r16,  .rm16   }, &.{ 0x13 }, 0, .none, .none },
+    .{ .adc, .rm, &.{ .r32,  .rm32   }, &.{ 0x13 }, 0, .none, .none },
+    .{ .adc, .rm, &.{ .r64,  .rm64   }, &.{ 0x13 }, 0, .long, .none },
 
-    .{ .add, .zi, &.{ .al,   .imm8   }, &.{ 0x04 }, 0, .none },
-    .{ .add, .zi, &.{ .ax,   .imm16  }, &.{ 0x05 }, 0, .none },
-    .{ .add, .zi, &.{ .eax,  .imm32  }, &.{ 0x05 }, 0, .none },
-    .{ .add, .zi, &.{ .rax,  .imm32s }, &.{ 0x05 }, 0, .long },
-    .{ .add, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 0, .none },
-    .{ .add, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 0, .rex  },
-    .{ .add, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 0, .none },
-    .{ .add, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 0, .none },
-    .{ .add, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 0, .long },
-    .{ .add, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 0, .none },
-    .{ .add, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 0, .none },
-    .{ .add, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 0, .long },
-    .{ .add, .mr, &.{ .rm8,  .r8     }, &.{ 0x00 }, 0, .none },
-    .{ .add, .mr, &.{ .rm8,  .r8     }, &.{ 0x00 }, 0, .rex  },
-    .{ .add, .mr, &.{ .rm16, .r16    }, &.{ 0x01 }, 0, .none },
-    .{ .add, .mr, &.{ .rm32, .r32    }, &.{ 0x01 }, 0, .none },
-    .{ .add, .mr, &.{ .rm64, .r64    }, &.{ 0x01 }, 0, .long },
-    .{ .add, .rm, &.{ .r8,   .rm8    }, &.{ 0x02 }, 0, .none },
-    .{ .add, .rm, &.{ .r8,   .rm8    }, &.{ 0x02 }, 0, .rex  },
-    .{ .add, .rm, &.{ .r16,  .rm16   }, &.{ 0x03 }, 0, .none },
-    .{ .add, .rm, &.{ .r32,  .rm32   }, &.{ 0x03 }, 0, .none },
-    .{ .add, .rm, &.{ .r64,  .rm64   }, &.{ 0x03 }, 0, .long },
+    .{ .add, .zi, &.{ .al,   .imm8   }, &.{ 0x04 }, 0, .none, .none },
+    .{ .add, .zi, &.{ .ax,   .imm16  }, &.{ 0x05 }, 0, .none, .none },
+    .{ .add, .zi, &.{ .eax,  .imm32  }, &.{ 0x05 }, 0, .none, .none },
+    .{ .add, .zi, &.{ .rax,  .imm32s }, &.{ 0x05 }, 0, .long, .none },
+    .{ .add, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 0, .none, .none },
+    .{ .add, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 0, .rex,  .none },
+    .{ .add, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 0, .none, .none },
+    .{ .add, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 0, .none, .none },
+    .{ .add, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 0, .long, .none },
+    .{ .add, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 0, .none, .none },
+    .{ .add, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 0, .none, .none },
+    .{ .add, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 0, .long, .none },
+    .{ .add, .mr, &.{ .rm8,  .r8     }, &.{ 0x00 }, 0, .none, .none },
+    .{ .add, .mr, &.{ .rm8,  .r8     }, &.{ 0x00 }, 0, .rex,  .none },
+    .{ .add, .mr, &.{ .rm16, .r16    }, &.{ 0x01 }, 0, .none, .none },
+    .{ .add, .mr, &.{ .rm32, .r32    }, &.{ 0x01 }, 0, .none, .none },
+    .{ .add, .mr, &.{ .rm64, .r64    }, &.{ 0x01 }, 0, .long, .none },
+    .{ .add, .rm, &.{ .r8,   .rm8    }, &.{ 0x02 }, 0, .none, .none },
+    .{ .add, .rm, &.{ .r8,   .rm8    }, &.{ 0x02 }, 0, .rex,  .none },
+    .{ .add, .rm, &.{ .r16,  .rm16   }, &.{ 0x03 }, 0, .none, .none },
+    .{ .add, .rm, &.{ .r32,  .rm32   }, &.{ 0x03 }, 0, .none, .none },
+    .{ .add, .rm, &.{ .r64,  .rm64   }, &.{ 0x03 }, 0, .long, .none },
 
-    .{ .@"and", .zi, &.{ .al,   .imm8   }, &.{ 0x24 }, 0, .none },
-    .{ .@"and", .zi, &.{ .ax,   .imm16  }, &.{ 0x25 }, 0, .none },
-    .{ .@"and", .zi, &.{ .eax,  .imm32  }, &.{ 0x25 }, 0, .none },
-    .{ .@"and", .zi, &.{ .rax,  .imm32s }, &.{ 0x25 }, 0, .long },
-    .{ .@"and", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 4, .none },
-    .{ .@"and", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 4, .rex  },
-    .{ .@"and", .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 4, .none },
-    .{ .@"and", .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 4, .none },
-    .{ .@"and", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 4, .long },
-    .{ .@"and", .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 4, .none },
-    .{ .@"and", .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 4, .none },
-    .{ .@"and", .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 4, .long },
-    .{ .@"and", .mr, &.{ .rm8,  .r8     }, &.{ 0x20 }, 0, .none },
-    .{ .@"and", .mr, &.{ .rm8,  .r8     }, &.{ 0x20 }, 0, .rex  },
-    .{ .@"and", .mr, &.{ .rm16, .r16    }, &.{ 0x21 }, 0, .none },
-    .{ .@"and", .mr, &.{ .rm32, .r32    }, &.{ 0x21 }, 0, .none },
-    .{ .@"and", .mr, &.{ .rm64, .r64    }, &.{ 0x21 }, 0, .long },
-    .{ .@"and", .rm, &.{ .r8,   .rm8    }, &.{ 0x22 }, 0, .none },
-    .{ .@"and", .rm, &.{ .r8,   .rm8    }, &.{ 0x22 }, 0, .rex  },
-    .{ .@"and", .rm, &.{ .r16,  .rm16   }, &.{ 0x23 }, 0, .none },
-    .{ .@"and", .rm, &.{ .r32,  .rm32   }, &.{ 0x23 }, 0, .none },
-    .{ .@"and", .rm, &.{ .r64,  .rm64   }, &.{ 0x23 }, 0, .long },
+    .{ .@"and", .zi, &.{ .al,   .imm8   }, &.{ 0x24 }, 0, .none, .none },
+    .{ .@"and", .zi, &.{ .ax,   .imm16  }, &.{ 0x25 }, 0, .none, .none },
+    .{ .@"and", .zi, &.{ .eax,  .imm32  }, &.{ 0x25 }, 0, .none, .none },
+    .{ .@"and", .zi, &.{ .rax,  .imm32s }, &.{ 0x25 }, 0, .long, .none },
+    .{ .@"and", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 4, .none, .none },
+    .{ .@"and", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 4, .rex,  .none },
+    .{ .@"and", .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 4, .none, .none },
+    .{ .@"and", .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 4, .none, .none },
+    .{ .@"and", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 4, .long, .none },
+    .{ .@"and", .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 4, .none, .none },
+    .{ .@"and", .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 4, .none, .none },
+    .{ .@"and", .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 4, .long, .none },
+    .{ .@"and", .mr, &.{ .rm8,  .r8     }, &.{ 0x20 }, 0, .none, .none },
+    .{ .@"and", .mr, &.{ .rm8,  .r8     }, &.{ 0x20 }, 0, .rex,  .none },
+    .{ .@"and", .mr, &.{ .rm16, .r16    }, &.{ 0x21 }, 0, .none, .none },
+    .{ .@"and", .mr, &.{ .rm32, .r32    }, &.{ 0x21 }, 0, .none, .none },
+    .{ .@"and", .mr, &.{ .rm64, .r64    }, &.{ 0x21 }, 0, .long, .none },
+    .{ .@"and", .rm, &.{ .r8,   .rm8    }, &.{ 0x22 }, 0, .none, .none },
+    .{ .@"and", .rm, &.{ .r8,   .rm8    }, &.{ 0x22 }, 0, .rex,  .none },
+    .{ .@"and", .rm, &.{ .r16,  .rm16   }, &.{ 0x23 }, 0, .none, .none },
+    .{ .@"and", .rm, &.{ .r32,  .rm32   }, &.{ 0x23 }, 0, .none, .none },
+    .{ .@"and", .rm, &.{ .r64,  .rm64   }, &.{ 0x23 }, 0, .long, .none },
 
-    .{ .bsf, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbc }, 0, .none },
-    .{ .bsf, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbc }, 0, .none },
-    .{ .bsf, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbc }, 0, .long },
+    .{ .bsf, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbc }, 0, .none, .none },
+    .{ .bsf, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbc }, 0, .none, .none },
+    .{ .bsf, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbc }, 0, .long, .none },
 
-    .{ .bsr, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbd }, 0, .none },
-    .{ .bsr, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbd }, 0, .none },
-    .{ .bsr, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbd }, 0, .long },
+    .{ .bsr, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbd }, 0, .none, .none },
+    .{ .bsr, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbd }, 0, .none, .none },
+    .{ .bsr, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbd }, 0, .long, .none },
 
-    .{ .bswap, .o, &.{ .r32 }, &.{ 0x0f, 0xc8 }, 0, .none },
-    .{ .bswap, .o, &.{ .r64 }, &.{ 0x0f, 0xc8 }, 0, .long },
+    .{ .bswap, .o, &.{ .r32 }, &.{ 0x0f, 0xc8 }, 0, .none, .none },
+    .{ .bswap, .o, &.{ .r64 }, &.{ 0x0f, 0xc8 }, 0, .long, .none },
 
-    .{ .bt, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xa3 }, 0, .none },
-    .{ .bt, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xa3 }, 0, .none },
-    .{ .bt, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xa3 }, 0, .long },
-    .{ .bt, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 4, .none },
-    .{ .bt, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 4, .none },
-    .{ .bt, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 4, .long },
+    .{ .bt, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xa3 }, 0, .none, .none },
+    .{ .bt, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xa3 }, 0, .none, .none },
+    .{ .bt, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xa3 }, 0, .long, .none },
+    .{ .bt, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 4, .none, .none },
+    .{ .bt, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 4, .none, .none },
+    .{ .bt, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 4, .long, .none },
 
-    .{ .btc, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xbb }, 0, .none },
-    .{ .btc, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xbb }, 0, .none },
-    .{ .btc, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xbb }, 0, .long },
-    .{ .btc, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 7, .none },
-    .{ .btc, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 7, .none },
-    .{ .btc, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 7, .long },
+    .{ .btc, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xbb }, 0, .none, .none },
+    .{ .btc, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xbb }, 0, .none, .none },
+    .{ .btc, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xbb }, 0, .long, .none },
+    .{ .btc, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 7, .none, .none },
+    .{ .btc, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 7, .none, .none },
+    .{ .btc, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 7, .long, .none },
 
-    .{ .btr, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xb3 }, 0, .none },
-    .{ .btr, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xb3 }, 0, .none },
-    .{ .btr, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xb3 }, 0, .long },
-    .{ .btr, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 6, .none },
-    .{ .btr, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 6, .none },
-    .{ .btr, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 6, .long },
+    .{ .btr, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xb3 }, 0, .none, .none },
+    .{ .btr, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xb3 }, 0, .none, .none },
+    .{ .btr, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xb3 }, 0, .long, .none },
+    .{ .btr, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 6, .none, .none },
+    .{ .btr, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 6, .none, .none },
+    .{ .btr, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 6, .long, .none },
 
-    .{ .bts, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xab }, 0, .none },
-    .{ .bts, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xab }, 0, .none },
-    .{ .bts, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xab }, 0, .long },
-    .{ .bts, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 5, .none },
-    .{ .bts, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 5, .none },
-    .{ .bts, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 5, .long },
+    .{ .bts, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xab }, 0, .none, .none },
+    .{ .bts, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xab }, 0, .none, .none },
+    .{ .bts, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xab }, 0, .long, .none },
+    .{ .bts, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 5, .none, .none },
+    .{ .bts, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 5, .none, .none },
+    .{ .bts, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 5, .long, .none },
 
     // This is M encoding according to Intel, but D makes more sense here.
-    .{ .call, .d, &.{ .rel32 }, &.{ 0xe8 }, 0, .none },
-    .{ .call, .m, &.{ .rm64  }, &.{ 0xff }, 2, .none },
+    .{ .call, .d, &.{ .rel32 }, &.{ 0xe8 }, 0, .none, .none },
+    .{ .call, .m, &.{ .rm64  }, &.{ 0xff }, 2, .none, .none },
 
-    .{ .cbw,  .np, &.{ .o16 }, &.{ 0x98 }, 0, .none },
-    .{ .cwde, .np, &.{ .o32 }, &.{ 0x98 }, 0, .none },
-    .{ .cdqe, .np, &.{ .o64 }, &.{ 0x98 }, 0, .long },
+    .{ .cbw,  .np, &.{ .o16 }, &.{ 0x98 }, 0, .none, .none },
+    .{ .cwde, .np, &.{ .o32 }, &.{ 0x98 }, 0, .none, .none },
+    .{ .cdqe, .np, &.{ .o64 }, &.{ 0x98 }, 0, .long, .none },
 
-    .{ .cwd, .np, &.{ .o16 }, &.{ 0x99 }, 0, .none },
-    .{ .cdq, .np, &.{ .o32 }, &.{ 0x99 }, 0, .none },
-    .{ .cqo, .np, &.{ .o64 }, &.{ 0x99 }, 0, .long },
+    .{ .cwd, .np, &.{ .o16 }, &.{ 0x99 }, 0, .none, .none },
+    .{ .cdq, .np, &.{ .o32 }, &.{ 0x99 }, 0, .none, .none },
+    .{ .cqo, .np, &.{ .o64 }, &.{ 0x99 }, 0, .long, .none },
 
-    .{ .cmova,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .none },
-    .{ .cmova,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none },
-    .{ .cmova,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long },
-    .{ .cmovae,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none },
-    .{ .cmovae,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none },
-    .{ .cmovae,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long },
-    .{ .cmovb,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none },
-    .{ .cmovb,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none },
-    .{ .cmovb,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long },
-    .{ .cmovbe,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .none },
-    .{ .cmovbe,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none },
-    .{ .cmovbe,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long },
-    .{ .cmovc,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none },
-    .{ .cmovc,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none },
-    .{ .cmovc,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long },
-    .{ .cmove,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .none },
-    .{ .cmove,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none },
-    .{ .cmove,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long },
-    .{ .cmovg,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .none },
-    .{ .cmovg,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none },
-    .{ .cmovg,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long },
-    .{ .cmovge,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .none },
-    .{ .cmovge,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none },
-    .{ .cmovge,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long },
-    .{ .cmovl,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .none },
-    .{ .cmovl,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none },
-    .{ .cmovl,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long },
-    .{ .cmovle,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .none },
-    .{ .cmovle,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none },
-    .{ .cmovle,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long },
-    .{ .cmovna,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .none },
-    .{ .cmovna,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none },
-    .{ .cmovna,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long },
-    .{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none },
-    .{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none },
-    .{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long },
-    .{ .cmovnb,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none },
-    .{ .cmovnb,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none },
-    .{ .cmovnb,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long },
-    .{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .none },
-    .{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none },
-    .{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long },
-    .{ .cmovnc,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none },
-    .{ .cmovnc,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none },
-    .{ .cmovnc,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long },
-    .{ .cmovne,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .none },
-    .{ .cmovne,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none },
-    .{ .cmovne,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long },
-    .{ .cmovng,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .none },
-    .{ .cmovng,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none },
-    .{ .cmovng,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long },
-    .{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .none },
-    .{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none },
-    .{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long },
-    .{ .cmovnl,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .none },
-    .{ .cmovnl,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none },
-    .{ .cmovnl,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long },
-    .{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .none },
-    .{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none },
-    .{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long },
-    .{ .cmovno,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .none },
-    .{ .cmovno,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none },
-    .{ .cmovno,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long },
-    .{ .cmovnp,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .none },
-    .{ .cmovnp,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none },
-    .{ .cmovnp,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long },
-    .{ .cmovns,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .none },
-    .{ .cmovns,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none },
-    .{ .cmovns,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long },
-    .{ .cmovnz,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .none },
-    .{ .cmovnz,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none },
-    .{ .cmovnz,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long },
-    .{ .cmovo,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .none },
-    .{ .cmovo,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none },
-    .{ .cmovo,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long },
-    .{ .cmovp,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .none },
-    .{ .cmovp,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none },
-    .{ .cmovp,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long },
-    .{ .cmovpe,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .none },
-    .{ .cmovpe,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none },
-    .{ .cmovpe,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long },
-    .{ .cmovpo,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .none },
-    .{ .cmovpo,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none },
-    .{ .cmovpo,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long },
-    .{ .cmovs,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .none },
-    .{ .cmovs,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none },
-    .{ .cmovs,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long },
-    .{ .cmovz,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .none },
-    .{ .cmovz,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none },
-    .{ .cmovz,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long },
+    .{ .cmova,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
+    .{ .cmova,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
+    .{ .cmova,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },
+    .{ .cmovae,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
+    .{ .cmovae,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
+    .{ .cmovae,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
+    .{ .cmovb,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
+    .{ .cmovb,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
+    .{ .cmovb,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
+    .{ .cmovbe,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
+    .{ .cmovbe,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
+    .{ .cmovbe,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },
+    .{ .cmovc,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
+    .{ .cmovc,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
+    .{ .cmovc,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
+    .{ .cmove,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
+    .{ .cmove,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
+    .{ .cmove,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },
+    .{ .cmovg,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
+    .{ .cmovg,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
+    .{ .cmovg,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },
+    .{ .cmovge,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
+    .{ .cmovge,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
+    .{ .cmovge,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },
+    .{ .cmovl,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
+    .{ .cmovl,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
+    .{ .cmovl,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },
+    .{ .cmovle,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
+    .{ .cmovle,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
+    .{ .cmovle,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },
+    .{ .cmovna,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
+    .{ .cmovna,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
+    .{ .cmovna,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },
+    .{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
+    .{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
+    .{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
+    .{ .cmovnb,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
+    .{ .cmovnb,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
+    .{ .cmovnb,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
+    .{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
+    .{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
+    .{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },
+    .{ .cmovnc,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
+    .{ .cmovnc,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
+    .{ .cmovnc,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
+    .{ .cmovne,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
+    .{ .cmovne,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
+    .{ .cmovne,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },
+    .{ .cmovng,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
+    .{ .cmovng,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
+    .{ .cmovng,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },
+    .{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
+    .{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
+    .{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },
+    .{ .cmovnl,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
+    .{ .cmovnl,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
+    .{ .cmovnl,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },
+    .{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
+    .{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
+    .{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },
+    .{ .cmovno,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .none, .none },
+    .{ .cmovno,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none, .none },
+    .{ .cmovno,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long, .none },
+    .{ .cmovnp,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
+    .{ .cmovnp,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
+    .{ .cmovnp,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },
+    .{ .cmovns,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .none, .none },
+    .{ .cmovns,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none, .none },
+    .{ .cmovns,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long, .none },
+    .{ .cmovnz,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
+    .{ .cmovnz,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
+    .{ .cmovnz,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },
+    .{ .cmovo,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .none, .none },
+    .{ .cmovo,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none, .none },
+    .{ .cmovo,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long, .none },
+    .{ .cmovp,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
+    .{ .cmovp,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
+    .{ .cmovp,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },
+    .{ .cmovpe,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
+    .{ .cmovpe,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
+    .{ .cmovpe,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },
+    .{ .cmovpo,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
+    .{ .cmovpo,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
+    .{ .cmovpo,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },
+    .{ .cmovs,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .none, .none },
+    .{ .cmovs,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none, .none },
+    .{ .cmovs,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long, .none },
+    .{ .cmovz,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
+    .{ .cmovz,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
+    .{ .cmovz,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },
 
-    .{ .cmp, .zi, &.{ .al,   .imm8   }, &.{ 0x3c }, 0, .none },
-    .{ .cmp, .zi, &.{ .ax,   .imm16  }, &.{ 0x3d }, 0, .none },
-    .{ .cmp, .zi, &.{ .eax,  .imm32  }, &.{ 0x3d }, 0, .none },
-    .{ .cmp, .zi, &.{ .rax,  .imm32s }, &.{ 0x3d }, 0, .long },
-    .{ .cmp, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 7, .none },
-    .{ .cmp, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 7, .rex  },
-    .{ .cmp, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 7, .none },
-    .{ .cmp, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 7, .none },
-    .{ .cmp, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 7, .long },
-    .{ .cmp, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 7, .none },
-    .{ .cmp, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 7, .none },
-    .{ .cmp, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 7, .long },
-    .{ .cmp, .mr, &.{ .rm8,  .r8     }, &.{ 0x38 }, 0, .none },
-    .{ .cmp, .mr, &.{ .rm8,  .r8     }, &.{ 0x38 }, 0, .rex  },
-    .{ .cmp, .mr, &.{ .rm16, .r16    }, &.{ 0x39 }, 0, .none },
-    .{ .cmp, .mr, &.{ .rm32, .r32    }, &.{ 0x39 }, 0, .none },
-    .{ .cmp, .mr, &.{ .rm64, .r64    }, &.{ 0x39 }, 0, .long },
-    .{ .cmp, .rm, &.{ .r8,   .rm8    }, &.{ 0x3a }, 0, .none },
-    .{ .cmp, .rm, &.{ .r8,   .rm8    }, &.{ 0x3a }, 0, .rex  },
-    .{ .cmp, .rm, &.{ .r16,  .rm16   }, &.{ 0x3b }, 0, .none },
-    .{ .cmp, .rm, &.{ .r32,  .rm32   }, &.{ 0x3b }, 0, .none },
-    .{ .cmp, .rm, &.{ .r64,  .rm64   }, &.{ 0x3b }, 0, .long },
+    .{ .cmp, .zi, &.{ .al,   .imm8   }, &.{ 0x3c }, 0, .none, .none },
+    .{ .cmp, .zi, &.{ .ax,   .imm16  }, &.{ 0x3d }, 0, .none, .none },
+    .{ .cmp, .zi, &.{ .eax,  .imm32  }, &.{ 0x3d }, 0, .none, .none },
+    .{ .cmp, .zi, &.{ .rax,  .imm32s }, &.{ 0x3d }, 0, .long, .none },
+    .{ .cmp, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 7, .none, .none },
+    .{ .cmp, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 7, .rex,  .none },
+    .{ .cmp, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 7, .none, .none },
+    .{ .cmp, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 7, .none, .none },
+    .{ .cmp, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 7, .long, .none },
+    .{ .cmp, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 7, .none, .none },
+    .{ .cmp, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 7, .none, .none },
+    .{ .cmp, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 7, .long, .none },
+    .{ .cmp, .mr, &.{ .rm8,  .r8     }, &.{ 0x38 }, 0, .none, .none },
+    .{ .cmp, .mr, &.{ .rm8,  .r8     }, &.{ 0x38 }, 0, .rex,  .none },
+    .{ .cmp, .mr, &.{ .rm16, .r16    }, &.{ 0x39 }, 0, .none, .none },
+    .{ .cmp, .mr, &.{ .rm32, .r32    }, &.{ 0x39 }, 0, .none, .none },
+    .{ .cmp, .mr, &.{ .rm64, .r64    }, &.{ 0x39 }, 0, .long, .none },
+    .{ .cmp, .rm, &.{ .r8,   .rm8    }, &.{ 0x3a }, 0, .none, .none },
+    .{ .cmp, .rm, &.{ .r8,   .rm8    }, &.{ 0x3a }, 0, .rex,  .none },
+    .{ .cmp, .rm, &.{ .r16,  .rm16   }, &.{ 0x3b }, 0, .none, .none },
+    .{ .cmp, .rm, &.{ .r32,  .rm32   }, &.{ 0x3b }, 0, .none, .none },
+    .{ .cmp, .rm, &.{ .r64,  .rm64   }, &.{ 0x3b }, 0, .long, .none },
 
-    .{ .cmps,  .np, &.{ .m8,   .m8   }, &.{ 0xa6 }, 0, .none  },
-    .{ .cmps,  .np, &.{ .m16,  .m16  }, &.{ 0xa7 }, 0, .none  },
-    .{ .cmps,  .np, &.{ .m32,  .m32  }, &.{ 0xa7 }, 0, .none  },
-    .{ .cmps,  .np, &.{ .m64,  .m64  }, &.{ 0xa7 }, 0, .long  },
+    .{ .cmps,  .np, &.{ .m8,   .m8   }, &.{ 0xa6 }, 0, .none, .none },
+    .{ .cmps,  .np, &.{ .m16,  .m16  }, &.{ 0xa7 }, 0, .none, .none },
+    .{ .cmps,  .np, &.{ .m32,  .m32  }, &.{ 0xa7 }, 0, .none, .none },
+    .{ .cmps,  .np, &.{ .m64,  .m64  }, &.{ 0xa7 }, 0, .long, .none },
 
-    .{ .cmpsb, .np, &.{}, &.{ 0xa6 }, 0, .none  },
-    .{ .cmpsw, .np, &.{}, &.{ 0xa7 }, 0, .short },
-    .{ .cmpsd, .np, &.{}, &.{ 0xa7 }, 0, .none  },
-    .{ .cmpsq, .np, &.{}, &.{ 0xa7 }, 0, .long  },
+    .{ .cmpsb, .np, &.{}, &.{ 0xa6 }, 0, .none,  .none },
+    .{ .cmpsw, .np, &.{}, &.{ 0xa7 }, 0, .short, .none },
+    .{ .cmpsd, .np, &.{}, &.{ 0xa7 }, 0, .none,  .none },
+    .{ .cmpsq, .np, &.{}, &.{ 0xa7 }, 0, .long,  .none },
 
-    .{ .cmpxchg, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xb0 }, 0, .none },
-    .{ .cmpxchg, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xb0 }, 0, .rex  },
-    .{ .cmpxchg, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xb1 }, 0, .none },
-    .{ .cmpxchg, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xb1 }, 0, .none },
-    .{ .cmpxchg, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xb1 }, 0, .long },
+    .{ .cmpxchg, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xb0 }, 0, .none, .none },
+    .{ .cmpxchg, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xb0 }, 0, .rex,  .none },
+    .{ .cmpxchg, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xb1 }, 0, .none, .none },
+    .{ .cmpxchg, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xb1 }, 0, .none, .none },
+    .{ .cmpxchg, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xb1 }, 0, .long, .none },
 
-    .{ .cmpxchg8b , .m, &.{ .m64  }, &.{ 0x0f, 0xc7 }, 1, .none },
-    .{ .cmpxchg16b, .m, &.{ .m128 }, &.{ 0x0f, 0xc7 }, 1, .long },
+    .{ .cmpxchg8b,  .m, &.{ .m64  }, &.{ 0x0f, 0xc7 }, 1, .none, .none },
+    .{ .cmpxchg16b, .m, &.{ .m128 }, &.{ 0x0f, 0xc7 }, 1, .long, .none },
 
-    .{ .div, .m, &.{ .rm8  }, &.{ 0xf6 }, 6, .none },
-    .{ .div, .m, &.{ .rm8  }, &.{ 0xf6 }, 6, .rex  },
-    .{ .div, .m, &.{ .rm16 }, &.{ 0xf7 }, 6, .none },
-    .{ .div, .m, &.{ .rm32 }, &.{ 0xf7 }, 6, .none },
-    .{ .div, .m, &.{ .rm64 }, &.{ 0xf7 }, 6, .long },
+    .{ .div, .m, &.{ .rm8  }, &.{ 0xf6 }, 6, .none, .none },
+    .{ .div, .m, &.{ .rm8  }, &.{ 0xf6 }, 6, .rex,  .none },
+    .{ .div, .m, &.{ .rm16 }, &.{ 0xf7 }, 6, .none, .none },
+    .{ .div, .m, &.{ .rm32 }, &.{ 0xf7 }, 6, .none, .none },
+    .{ .div, .m, &.{ .rm64 }, &.{ 0xf7 }, 6, .long, .none },
 
-    .{ .fisttp, .m, &.{ .m16 }, &.{ 0xdf }, 1, .fpu },
-    .{ .fisttp, .m, &.{ .m32 }, &.{ 0xdb }, 1, .fpu },
-    .{ .fisttp, .m, &.{ .m64 }, &.{ 0xdd }, 1, .fpu },
+    .{ .fisttp, .m, &.{ .m16 }, &.{ 0xdf }, 1, .none, .x87 },
+    .{ .fisttp, .m, &.{ .m32 }, &.{ 0xdb }, 1, .none, .x87 },
+    .{ .fisttp, .m, &.{ .m64 }, &.{ 0xdd }, 1, .none, .x87 },
 
-    .{ .fld, .m, &.{ .m32 }, &.{ 0xd9 }, 0, .fpu },
-    .{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .fpu },
-    .{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .fpu },
+    .{ .fld, .m, &.{ .m32 }, &.{ 0xd9 }, 0, .none, .x87 },
+    .{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .none, .x87 },
+    .{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .none, .x87 },
 
-    .{ .idiv, .m, &.{ .rm8  }, &.{ 0xf6 }, 7, .none },
-    .{ .idiv, .m, &.{ .rm8  }, &.{ 0xf6 }, 7, .rex  },
-    .{ .idiv, .m, &.{ .rm16 }, &.{ 0xf7 }, 7, .none },
-    .{ .idiv, .m, &.{ .rm32 }, &.{ 0xf7 }, 7, .none },
-    .{ .idiv, .m, &.{ .rm64 }, &.{ 0xf7 }, 7, .long },
+    .{ .idiv, .m, &.{ .rm8  }, &.{ 0xf6 }, 7, .none, .none },
+    .{ .idiv, .m, &.{ .rm8  }, &.{ 0xf6 }, 7, .rex,  .none },
+    .{ .idiv, .m, &.{ .rm16 }, &.{ 0xf7 }, 7, .none, .none },
+    .{ .idiv, .m, &.{ .rm32 }, &.{ 0xf7 }, 7, .none, .none },
+    .{ .idiv, .m, &.{ .rm64 }, &.{ 0xf7 }, 7, .long, .none },
 
-    .{ .imul, .m,   &.{ .rm8                 }, &.{ 0xf6       }, 5, .none },
-    .{ .imul, .m,   &.{ .rm8                 }, &.{ 0xf6       }, 5, .rex  },
-    .{ .imul, .m,   &.{ .rm16,               }, &.{ 0xf7       }, 5, .none },
-    .{ .imul, .m,   &.{ .rm32,               }, &.{ 0xf7       }, 5, .none },
-    .{ .imul, .m,   &.{ .rm64,               }, &.{ 0xf7       }, 5, .long },
-    .{ .imul, .rm,  &.{ .r16,  .rm16,        }, &.{ 0x0f, 0xaf }, 0, .none },
-    .{ .imul, .rm,  &.{ .r32,  .rm32,        }, &.{ 0x0f, 0xaf }, 0, .none },
-    .{ .imul, .rm,  &.{ .r64,  .rm64,        }, &.{ 0x0f, 0xaf }, 0, .long },
-    .{ .imul, .rmi, &.{ .r16,  .rm16, .imm8s }, &.{ 0x6b       }, 0, .none },
-    .{ .imul, .rmi, &.{ .r32,  .rm32, .imm8s }, &.{ 0x6b       }, 0, .none },
-    .{ .imul, .rmi, &.{ .r64,  .rm64, .imm8s }, &.{ 0x6b       }, 0, .long },
-    .{ .imul, .rmi, &.{ .r16,  .rm16, .imm16 }, &.{ 0x69       }, 0, .none },
-    .{ .imul, .rmi, &.{ .r32,  .rm32, .imm32 }, &.{ 0x69       }, 0, .none },
-    .{ .imul, .rmi, &.{ .r64,  .rm64, .imm32 }, &.{ 0x69       }, 0, .long },
+    .{ .imul, .m,   &.{ .rm8                 }, &.{ 0xf6       }, 5, .none, .none },
+    .{ .imul, .m,   &.{ .rm8                 }, &.{ 0xf6       }, 5, .rex,  .none },
+    .{ .imul, .m,   &.{ .rm16,               }, &.{ 0xf7       }, 5, .none, .none },
+    .{ .imul, .m,   &.{ .rm32,               }, &.{ 0xf7       }, 5, .none, .none },
+    .{ .imul, .m,   &.{ .rm64,               }, &.{ 0xf7       }, 5, .long, .none },
+    .{ .imul, .rm,  &.{ .r16,  .rm16,        }, &.{ 0x0f, 0xaf }, 0, .none, .none },
+    .{ .imul, .rm,  &.{ .r32,  .rm32,        }, &.{ 0x0f, 0xaf }, 0, .none, .none },
+    .{ .imul, .rm,  &.{ .r64,  .rm64,        }, &.{ 0x0f, 0xaf }, 0, .long, .none },
+    .{ .imul, .rmi, &.{ .r16,  .rm16, .imm8s }, &.{ 0x6b       }, 0, .none, .none },
+    .{ .imul, .rmi, &.{ .r32,  .rm32, .imm8s }, &.{ 0x6b       }, 0, .none, .none },
+    .{ .imul, .rmi, &.{ .r64,  .rm64, .imm8s }, &.{ 0x6b       }, 0, .long, .none },
+    .{ .imul, .rmi, &.{ .r16,  .rm16, .imm16 }, &.{ 0x69       }, 0, .none, .none },
+    .{ .imul, .rmi, &.{ .r32,  .rm32, .imm32 }, &.{ 0x69       }, 0, .none, .none },
+    .{ .imul, .rmi, &.{ .r64,  .rm64, .imm32 }, &.{ 0x69       }, 0, .long, .none },
 
-    .{ .int3, .np, &.{}, &.{ 0xcc }, 0, .none },
+    .{ .int3, .np, &.{}, &.{ 0xcc }, 0, .none, .none },
 
-    .{ .ja,    .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none },
-    .{ .jae,   .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none },
-    .{ .jb,    .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none },
-    .{ .jbe,   .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none },
-    .{ .jc,    .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none },
-    .{ .jrcxz, .d, &.{ .rel32 }, &.{ 0xe3       }, 0, .none },
-    .{ .je,    .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none },
-    .{ .jg,    .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none },
-    .{ .jge,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none },
-    .{ .jl,    .d, &.{ .rel32 }, &.{ 0x0f, 0x8c }, 0, .none },
-    .{ .jle,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8e }, 0, .none },
-    .{ .jna,   .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none },
-    .{ .jnae,  .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none },
-    .{ .jnb,   .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none },
-    .{ .jnbe,  .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none },
-    .{ .jnc,   .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none },
-    .{ .jne,   .d, &.{ .rel32 }, &.{ 0x0f, 0x85 }, 0, .none },
-    .{ .jng,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8e }, 0, .none },
-    .{ .jnge,  .d, &.{ .rel32 }, &.{ 0x0f, 0x8c }, 0, .none },
-    .{ .jnl,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none },
-    .{ .jnle,  .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none },
-    .{ .jno,   .d, &.{ .rel32 }, &.{ 0x0f, 0x81 }, 0, .none },
-    .{ .jnp,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8b }, 0, .none },
-    .{ .jns,   .d, &.{ .rel32 }, &.{ 0x0f, 0x89 }, 0, .none },
-    .{ .jnz,   .d, &.{ .rel32 }, &.{ 0x0f, 0x85 }, 0, .none },
-    .{ .jo,    .d, &.{ .rel32 }, &.{ 0x0f, 0x80 }, 0, .none },
-    .{ .jp,    .d, &.{ .rel32 }, &.{ 0x0f, 0x8a }, 0, .none },
-    .{ .jpe,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8a }, 0, .none },
-    .{ .jpo,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8b }, 0, .none },
-    .{ .js,    .d, &.{ .rel32 }, &.{ 0x0f, 0x88 }, 0, .none },
-    .{ .jz,    .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none },
+    .{ .ja,    .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none, .none },
+    .{ .jae,   .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },
+    .{ .jb,    .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none, .none },
+    .{ .jbe,   .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none, .none },
+    .{ .jc,    .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none, .none },
+    .{ .jrcxz, .d, &.{ .rel32 }, &.{ 0xe3       }, 0, .none, .none },
+    .{ .je,    .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none, .none },
+    .{ .jg,    .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none, .none },
+    .{ .jge,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none, .none },
+    .{ .jl,    .d, &.{ .rel32 }, &.{ 0x0f, 0x8c }, 0, .none, .none },
+    .{ .jle,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8e }, 0, .none, .none },
+    .{ .jna,   .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none, .none },
+    .{ .jnae,  .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none, .none },
+    .{ .jnb,   .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },
+    .{ .jnbe,  .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none, .none },
+    .{ .jnc,   .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },
+    .{ .jne,   .d, &.{ .rel32 }, &.{ 0x0f, 0x85 }, 0, .none, .none },
+    .{ .jng,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8e }, 0, .none, .none },
+    .{ .jnge,  .d, &.{ .rel32 }, &.{ 0x0f, 0x8c }, 0, .none, .none },
+    .{ .jnl,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none, .none },
+    .{ .jnle,  .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none, .none },
+    .{ .jno,   .d, &.{ .rel32 }, &.{ 0x0f, 0x81 }, 0, .none, .none },
+    .{ .jnp,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8b }, 0, .none, .none },
+    .{ .jns,   .d, &.{ .rel32 }, &.{ 0x0f, 0x89 }, 0, .none, .none },
+    .{ .jnz,   .d, &.{ .rel32 }, &.{ 0x0f, 0x85 }, 0, .none, .none },
+    .{ .jo,    .d, &.{ .rel32 }, &.{ 0x0f, 0x80 }, 0, .none, .none },
+    .{ .jp,    .d, &.{ .rel32 }, &.{ 0x0f, 0x8a }, 0, .none, .none },
+    .{ .jpe,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8a }, 0, .none, .none },
+    .{ .jpo,   .d, &.{ .rel32 }, &.{ 0x0f, 0x8b }, 0, .none, .none },
+    .{ .js,    .d, &.{ .rel32 }, &.{ 0x0f, 0x88 }, 0, .none, .none },
+    .{ .jz,    .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none, .none },
 
-    .{ .jmp, .d, &.{ .rel32 }, &.{ 0xe9 }, 0, .none },
-    .{ .jmp, .m, &.{ .rm64  }, &.{ 0xff }, 4, .none },
+    .{ .jmp, .d, &.{ .rel32 }, &.{ 0xe9 }, 0, .none, .none },
+    .{ .jmp, .m, &.{ .rm64  }, &.{ 0xff }, 4, .none, .none },
 
-    .{ .lea, .rm, &.{ .r16, .m }, &.{ 0x8d }, 0, .none },
-    .{ .lea, .rm, &.{ .r32, .m }, &.{ 0x8d }, 0, .none },
-    .{ .lea, .rm, &.{ .r64, .m }, &.{ 0x8d }, 0, .long },
+    .{ .lea, .rm, &.{ .r16, .m }, &.{ 0x8d }, 0, .none, .none },
+    .{ .lea, .rm, &.{ .r32, .m }, &.{ 0x8d }, 0, .none, .none },
+    .{ .lea, .rm, &.{ .r64, .m }, &.{ 0x8d }, 0, .long, .none },
 
-    .{ .lfence, .np, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none },
+    .{ .lfence, .np, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none, .none },
 
-    .{ .lods,  .np, &.{ .m8  }, &.{ 0xac }, 0, .none  },
-    .{ .lods,  .np, &.{ .m16 }, &.{ 0xad }, 0, .none  },
-    .{ .lods,  .np, &.{ .m32 }, &.{ 0xad }, 0, .none  },
-    .{ .lods,  .np, &.{ .m64 }, &.{ 0xad }, 0, .long  },
+    .{ .lods,  .np, &.{ .m8  }, &.{ 0xac }, 0, .none, .none },
+    .{ .lods,  .np, &.{ .m16 }, &.{ 0xad }, 0, .none, .none },
+    .{ .lods,  .np, &.{ .m32 }, &.{ 0xad }, 0, .none, .none },
+    .{ .lods,  .np, &.{ .m64 }, &.{ 0xad }, 0, .long, .none },
 
-    .{ .lodsb, .np, &.{}, &.{ 0xac }, 0, .none  },
-    .{ .lodsw, .np, &.{}, &.{ 0xad }, 0, .short },
-    .{ .lodsd, .np, &.{}, &.{ 0xad }, 0, .none  },
-    .{ .lodsq, .np, &.{}, &.{ 0xad }, 0, .long  },
+    .{ .lodsb, .np, &.{}, &.{ 0xac }, 0, .none,  .none },
+    .{ .lodsw, .np, &.{}, &.{ 0xad }, 0, .short, .none },
+    .{ .lodsd, .np, &.{}, &.{ 0xad }, 0, .none,  .none },
+    .{ .lodsq, .np, &.{}, &.{ 0xad }, 0, .long,  .none },
 
-    .{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
-    .{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
-    .{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long },
+    .{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .none },
+    .{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .none },
+    .{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long, .none },
 
-    .{ .mfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none },
+    .{ .mfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none, .none },
 
-    .{ .mov, .mr, &.{ .rm8,   .r8     }, &.{ 0x88 }, 0, .none },
-    .{ .mov, .mr, &.{ .rm8,   .r8     }, &.{ 0x88 }, 0, .rex  },
-    .{ .mov, .mr, &.{ .rm16,  .r16    }, &.{ 0x89 }, 0, .none },
-    .{ .mov, .mr, &.{ .rm32,  .r32    }, &.{ 0x89 }, 0, .none },
-    .{ .mov, .mr, &.{ .rm64,  .r64    }, &.{ 0x89 }, 0, .long },
-    .{ .mov, .rm, &.{ .r8,    .rm8    }, &.{ 0x8a }, 0, .none },
-    .{ .mov, .rm, &.{ .r8,    .rm8    }, &.{ 0x8a }, 0, .rex  },
-    .{ .mov, .rm, &.{ .r16,   .rm16   }, &.{ 0x8b }, 0, .none },
-    .{ .mov, .rm, &.{ .r32,   .rm32   }, &.{ 0x8b }, 0, .none },
-    .{ .mov, .rm, &.{ .r64,   .rm64   }, &.{ 0x8b }, 0, .long },
-    .{ .mov, .mr, &.{ .rm16,  .sreg   }, &.{ 0x8c }, 0, .none },
-    .{ .mov, .mr, &.{ .rm64,  .sreg   }, &.{ 0x8c }, 0, .long },
-    .{ .mov, .rm, &.{ .sreg,  .rm16   }, &.{ 0x8e }, 0, .none },
-    .{ .mov, .rm, &.{ .sreg,  .rm64   }, &.{ 0x8e }, 0, .long },
-    .{ .mov, .fd, &.{ .al,    .moffs  }, &.{ 0xa0 }, 0, .none },
-    .{ .mov, .fd, &.{ .ax,    .moffs  }, &.{ 0xa1 }, 0, .none },
-    .{ .mov, .fd, &.{ .eax,   .moffs  }, &.{ 0xa1 }, 0, .none },
-    .{ .mov, .fd, &.{ .rax,   .moffs  }, &.{ 0xa1 }, 0, .long },
-    .{ .mov, .td, &.{ .moffs, .al     }, &.{ 0xa2 }, 0, .none },
-    .{ .mov, .td, &.{ .moffs, .ax     }, &.{ 0xa3 }, 0, .none },
-    .{ .mov, .td, &.{ .moffs, .eax    }, &.{ 0xa3 }, 0, .none },
-    .{ .mov, .td, &.{ .moffs, .rax    }, &.{ 0xa3 }, 0, .long },
-    .{ .mov, .oi, &.{ .r8,    .imm8   }, &.{ 0xb0 }, 0, .none },
-    .{ .mov, .oi, &.{ .r8,    .imm8   }, &.{ 0xb0 }, 0, .rex  },
-    .{ .mov, .oi, &.{ .r16,   .imm16  }, &.{ 0xb8 }, 0, .none },
-    .{ .mov, .oi, &.{ .r32,   .imm32  }, &.{ 0xb8 }, 0, .none },
-    .{ .mov, .oi, &.{ .r64,   .imm64  }, &.{ 0xb8 }, 0, .long },
-    .{ .mov, .mi, &.{ .rm8,   .imm8   }, &.{ 0xc6 }, 0, .none },
-    .{ .mov, .mi, &.{ .rm8,   .imm8   }, &.{ 0xc6 }, 0, .rex  },
-    .{ .mov, .mi, &.{ .rm16,  .imm16  }, &.{ 0xc7 }, 0, .none },
-    .{ .mov, .mi, &.{ .rm32,  .imm32  }, &.{ 0xc7 }, 0, .none },
-    .{ .mov, .mi, &.{ .rm64,  .imm32s }, &.{ 0xc7 }, 0, .long },
+    .{ .mov, .mr, &.{ .rm8,   .r8     }, &.{ 0x88 }, 0, .none, .none },
+    .{ .mov, .mr, &.{ .rm8,   .r8     }, &.{ 0x88 }, 0, .rex,  .none },
+    .{ .mov, .mr, &.{ .rm16,  .r16    }, &.{ 0x89 }, 0, .none, .none },
+    .{ .mov, .mr, &.{ .rm32,  .r32    }, &.{ 0x89 }, 0, .none, .none },
+    .{ .mov, .mr, &.{ .rm64,  .r64    }, &.{ 0x89 }, 0, .long, .none },
+    .{ .mov, .rm, &.{ .r8,    .rm8    }, &.{ 0x8a }, 0, .none, .none },
+    .{ .mov, .rm, &.{ .r8,    .rm8    }, &.{ 0x8a }, 0, .rex,  .none },
+    .{ .mov, .rm, &.{ .r16,   .rm16   }, &.{ 0x8b }, 0, .none, .none },
+    .{ .mov, .rm, &.{ .r32,   .rm32   }, &.{ 0x8b }, 0, .none, .none },
+    .{ .mov, .rm, &.{ .r64,   .rm64   }, &.{ 0x8b }, 0, .long, .none },
+    .{ .mov, .mr, &.{ .rm16,  .sreg   }, &.{ 0x8c }, 0, .none, .none },
+    .{ .mov, .mr, &.{ .rm64,  .sreg   }, &.{ 0x8c }, 0, .long, .none },
+    .{ .mov, .rm, &.{ .sreg,  .rm16   }, &.{ 0x8e }, 0, .none, .none },
+    .{ .mov, .rm, &.{ .sreg,  .rm64   }, &.{ 0x8e }, 0, .long, .none },
+    .{ .mov, .fd, &.{ .al,    .moffs  }, &.{ 0xa0 }, 0, .none, .none },
+    .{ .mov, .fd, &.{ .ax,    .moffs  }, &.{ 0xa1 }, 0, .none, .none },
+    .{ .mov, .fd, &.{ .eax,   .moffs  }, &.{ 0xa1 }, 0, .none, .none },
+    .{ .mov, .fd, &.{ .rax,   .moffs  }, &.{ 0xa1 }, 0, .long, .none },
+    .{ .mov, .td, &.{ .moffs, .al     }, &.{ 0xa2 }, 0, .none, .none },
+    .{ .mov, .td, &.{ .moffs, .ax     }, &.{ 0xa3 }, 0, .none, .none },
+    .{ .mov, .td, &.{ .moffs, .eax    }, &.{ 0xa3 }, 0, .none, .none },
+    .{ .mov, .td, &.{ .moffs, .rax    }, &.{ 0xa3 }, 0, .long, .none },
+    .{ .mov, .oi, &.{ .r8,    .imm8   }, &.{ 0xb0 }, 0, .none, .none },
+    .{ .mov, .oi, &.{ .r8,    .imm8   }, &.{ 0xb0 }, 0, .rex,  .none },
+    .{ .mov, .oi, &.{ .r16,   .imm16  }, &.{ 0xb8 }, 0, .none, .none },
+    .{ .mov, .oi, &.{ .r32,   .imm32  }, &.{ 0xb8 }, 0, .none, .none },
+    .{ .mov, .oi, &.{ .r64,   .imm64  }, &.{ 0xb8 }, 0, .long, .none },
+    .{ .mov, .mi, &.{ .rm8,   .imm8   }, &.{ 0xc6 }, 0, .none, .none },
+    .{ .mov, .mi, &.{ .rm8,   .imm8   }, &.{ 0xc6 }, 0, .rex,  .none },
+    .{ .mov, .mi, &.{ .rm16,  .imm16  }, &.{ 0xc7 }, 0, .none, .none },
+    .{ .mov, .mi, &.{ .rm32,  .imm32  }, &.{ 0xc7 }, 0, .none, .none },
+    .{ .mov, .mi, &.{ .rm64,  .imm32s }, &.{ 0xc7 }, 0, .long, .none },
 
-    .{ .movbe, .rm, &.{ .r16, .m16 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none },
-    .{ .movbe, .rm, &.{ .r32, .m32 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none },
-    .{ .movbe, .rm, &.{ .r64, .m64 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .long },
-    .{ .movbe, .mr, &.{ .m16, .r16 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none },
-    .{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none },
-    .{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long },
+    .{ .movbe, .rm, &.{ .r16, .m16 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none, .none },
+    .{ .movbe, .rm, &.{ .r32, .m32 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none, .none },
+    .{ .movbe, .rm, &.{ .r64, .m64 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .long, .none },
+    .{ .movbe, .mr, &.{ .m16, .r16 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none, .none },
+    .{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none, .none },
+    .{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long, .none },
 
-    .{ .movs,  .np, &.{ .m8,  .m8  }, &.{ 0xa4 }, 0, .none  },
-    .{ .movs,  .np, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .none  },
-    .{ .movs,  .np, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none  },
-    .{ .movs,  .np, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long  },
+    .{ .movs,  .np, &.{ .m8,  .m8  }, &.{ 0xa4 }, 0, .none, .none },
+    .{ .movs,  .np, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .none, .none },
+    .{ .movs,  .np, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none, .none },
+    .{ .movs,  .np, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long, .none },
 
-    .{ .movsb, .np, &.{}, &.{ 0xa4 }, 0, .none  },
-    .{ .movsw, .np, &.{}, &.{ 0xa5 }, 0, .short },
-    .{ .movsd, .np, &.{}, &.{ 0xa5 }, 0, .none  },
-    .{ .movsq, .np, &.{}, &.{ 0xa5 }, 0, .long  },
+    .{ .movsb, .np, &.{}, &.{ 0xa4 }, 0, .none,  .none },
+    .{ .movsw, .np, &.{}, &.{ 0xa5 }, 0, .short, .none },
+    .{ .movsd, .np, &.{}, &.{ 0xa5 }, 0, .none,  .none },
+    .{ .movsq, .np, &.{}, &.{ 0xa5 }, 0, .long,  .none },
 
-    .{ .movsx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xbe }, 0, .none },
-    .{ .movsx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xbe }, 0, .rex  },
-    .{ .movsx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xbe }, 0, .none },
-    .{ .movsx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xbe }, 0, .rex  },
-    .{ .movsx, .rm, &.{ .r64, .rm8  }, &.{ 0x0f, 0xbe }, 0, .long },
-    .{ .movsx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xbf }, 0, .none },
-    .{ .movsx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xbf }, 0, .long },
+    .{ .movsx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xbe }, 0, .none, .none },
+    .{ .movsx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xbe }, 0, .rex,  .none },
+    .{ .movsx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xbe }, 0, .none, .none },
+    .{ .movsx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xbe }, 0, .rex,  .none },
+    .{ .movsx, .rm, &.{ .r64, .rm8  }, &.{ 0x0f, 0xbe }, 0, .long, .none },
+    .{ .movsx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xbf }, 0, .none, .none },
+    .{ .movsx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xbf }, 0, .long, .none },
 
     // This instruction is discouraged.
-    .{ .movsxd, .rm, &.{ .r32, .rm32 }, &.{ 0x63 }, 0, .none },
-    .{ .movsxd, .rm, &.{ .r64, .rm32 }, &.{ 0x63 }, 0, .long },
+    .{ .movsxd, .rm, &.{ .r32, .rm32 }, &.{ 0x63 }, 0, .none, .none },
+    .{ .movsxd, .rm, &.{ .r64, .rm32 }, &.{ 0x63 }, 0, .long, .none },
 
-    .{ .movzx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .none },
-    .{ .movzx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .none },
-    .{ .movzx, .rm, &.{ .r64, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .long },
-    .{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .none },
-    .{ .movzx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .long },
+    .{ .movzx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .none, .none },
+    .{ .movzx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .none, .none },
+    .{ .movzx, .rm, &.{ .r64, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .long, .none },
+    .{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .none, .none },
+    .{ .movzx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .long, .none },
 
-    .{ .mul, .m, &.{ .rm8  }, &.{ 0xf6 }, 4, .none },
-    .{ .mul, .m, &.{ .rm8  }, &.{ 0xf6 }, 4, .rex  },
-    .{ .mul, .m, &.{ .rm16 }, &.{ 0xf7 }, 4, .none },
-    .{ .mul, .m, &.{ .rm32 }, &.{ 0xf7 }, 4, .none },
-    .{ .mul, .m, &.{ .rm64 }, &.{ 0xf7 }, 4, .long },
+    .{ .mul, .m, &.{ .rm8  }, &.{ 0xf6 }, 4, .none, .none },
+    .{ .mul, .m, &.{ .rm8  }, &.{ 0xf6 }, 4, .rex,  .none },
+    .{ .mul, .m, &.{ .rm16 }, &.{ 0xf7 }, 4, .none, .none },
+    .{ .mul, .m, &.{ .rm32 }, &.{ 0xf7 }, 4, .none, .none },
+    .{ .mul, .m, &.{ .rm64 }, &.{ 0xf7 }, 4, .long, .none },
 
-    .{ .neg, .m, &.{ .rm8  }, &.{ 0xf6 }, 3, .none },
-    .{ .neg, .m, &.{ .rm8  }, &.{ 0xf6 }, 3, .rex  },
-    .{ .neg, .m, &.{ .rm16 }, &.{ 0xf7 }, 3, .none },
-    .{ .neg, .m, &.{ .rm32 }, &.{ 0xf7 }, 3, .none },
-    .{ .neg, .m, &.{ .rm64 }, &.{ 0xf7 }, 3, .long },
+    .{ .neg, .m, &.{ .rm8  }, &.{ 0xf6 }, 3, .none, .none },
+    .{ .neg, .m, &.{ .rm8  }, &.{ 0xf6 }, 3, .rex,  .none },
+    .{ .neg, .m, &.{ .rm16 }, &.{ 0xf7 }, 3, .none, .none },
+    .{ .neg, .m, &.{ .rm32 }, &.{ 0xf7 }, 3, .none, .none },
+    .{ .neg, .m, &.{ .rm64 }, &.{ 0xf7 }, 3, .long, .none },
 
-    .{ .nop, .np, &.{}, &.{ 0x90 }, 0, .none },
+    .{ .nop, .np, &.{}, &.{ 0x90 }, 0, .none, .none },
 
-    .{ .not, .m, &.{ .rm8  }, &.{ 0xf6 }, 2, .none },
-    .{ .not, .m, &.{ .rm8  }, &.{ 0xf6 }, 2, .rex  },
-    .{ .not, .m, &.{ .rm16 }, &.{ 0xf7 }, 2, .none },
-    .{ .not, .m, &.{ .rm32 }, &.{ 0xf7 }, 2, .none },
-    .{ .not, .m, &.{ .rm64 }, &.{ 0xf7 }, 2, .long },
+    .{ .not, .m, &.{ .rm8  }, &.{ 0xf6 }, 2, .none, .none },
+    .{ .not, .m, &.{ .rm8  }, &.{ 0xf6 }, 2, .rex,  .none },
+    .{ .not, .m, &.{ .rm16 }, &.{ 0xf7 }, 2, .none, .none },
+    .{ .not, .m, &.{ .rm32 }, &.{ 0xf7 }, 2, .none, .none },
+    .{ .not, .m, &.{ .rm64 }, &.{ 0xf7 }, 2, .long, .none },
 
-    .{ .@"or", .zi, &.{ .al,   .imm8   }, &.{ 0x0c }, 0, .none },
-    .{ .@"or", .zi, &.{ .ax,   .imm16  }, &.{ 0x0d }, 0, .none },
-    .{ .@"or", .zi, &.{ .eax,  .imm32  }, &.{ 0x0d }, 0, .none },
-    .{ .@"or", .zi, &.{ .rax,  .imm32s }, &.{ 0x0d }, 0, .long },
-    .{ .@"or", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 1, .none },
-    .{ .@"or", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 1, .rex  },
-    .{ .@"or", .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 1, .none },
-    .{ .@"or", .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 1, .none },
-    .{ .@"or", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 1, .long },
-    .{ .@"or", .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 1, .none },
-    .{ .@"or", .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 1, .none },
-    .{ .@"or", .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 1, .long },
-    .{ .@"or", .mr, &.{ .rm8,  .r8     }, &.{ 0x08 }, 0, .none },
-    .{ .@"or", .mr, &.{ .rm8,  .r8     }, &.{ 0x08 }, 0, .rex  },
-    .{ .@"or", .mr, &.{ .rm16, .r16    }, &.{ 0x09 }, 0, .none },
-    .{ .@"or", .mr, &.{ .rm32, .r32    }, &.{ 0x09 }, 0, .none },
-    .{ .@"or", .mr, &.{ .rm64, .r64    }, &.{ 0x09 }, 0, .long },
-    .{ .@"or", .rm, &.{ .r8,   .rm8    }, &.{ 0x0a }, 0, .none },
-    .{ .@"or", .rm, &.{ .r8,   .rm8    }, &.{ 0x0a }, 0, .rex  },
-    .{ .@"or", .rm, &.{ .r16,  .rm16   }, &.{ 0x0b }, 0, .none },
-    .{ .@"or", .rm, &.{ .r32,  .rm32   }, &.{ 0x0b }, 0, .none },
-    .{ .@"or", .rm, &.{ .r64,  .rm64   }, &.{ 0x0b }, 0, .long },
+    .{ .@"or", .zi, &.{ .al,   .imm8   }, &.{ 0x0c }, 0, .none, .none },
+    .{ .@"or", .zi, &.{ .ax,   .imm16  }, &.{ 0x0d }, 0, .none, .none },
+    .{ .@"or", .zi, &.{ .eax,  .imm32  }, &.{ 0x0d }, 0, .none, .none },
+    .{ .@"or", .zi, &.{ .rax,  .imm32s }, &.{ 0x0d }, 0, .long, .none },
+    .{ .@"or", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 1, .none, .none },
+    .{ .@"or", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 1, .rex,  .none },
+    .{ .@"or", .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 1, .none, .none },
+    .{ .@"or", .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 1, .none, .none },
+    .{ .@"or", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 1, .long, .none },
+    .{ .@"or", .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 1, .none, .none },
+    .{ .@"or", .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 1, .none, .none },
+    .{ .@"or", .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 1, .long, .none },
+    .{ .@"or", .mr, &.{ .rm8,  .r8     }, &.{ 0x08 }, 0, .none, .none },
+    .{ .@"or", .mr, &.{ .rm8,  .r8     }, &.{ 0x08 }, 0, .rex,  .none },
+    .{ .@"or", .mr, &.{ .rm16, .r16    }, &.{ 0x09 }, 0, .none, .none },
+    .{ .@"or", .mr, &.{ .rm32, .r32    }, &.{ 0x09 }, 0, .none, .none },
+    .{ .@"or", .mr, &.{ .rm64, .r64    }, &.{ 0x09 }, 0, .long, .none },
+    .{ .@"or", .rm, &.{ .r8,   .rm8    }, &.{ 0x0a }, 0, .none, .none },
+    .{ .@"or", .rm, &.{ .r8,   .rm8    }, &.{ 0x0a }, 0, .rex,  .none },
+    .{ .@"or", .rm, &.{ .r16,  .rm16   }, &.{ 0x0b }, 0, .none, .none },
+    .{ .@"or", .rm, &.{ .r32,  .rm32   }, &.{ 0x0b }, 0, .none, .none },
+    .{ .@"or", .rm, &.{ .r64,  .rm64   }, &.{ 0x0b }, 0, .long, .none },
 
-    .{ .pop, .o, &.{ .r16  }, &.{ 0x58 }, 0, .none },
-    .{ .pop, .o, &.{ .r64  }, &.{ 0x58 }, 0, .none },
-    .{ .pop, .m, &.{ .rm16 }, &.{ 0x8f }, 0, .none },
-    .{ .pop, .m, &.{ .rm64 }, &.{ 0x8f }, 0, .none },
+    .{ .pop, .o, &.{ .r16  }, &.{ 0x58 }, 0, .none, .none },
+    .{ .pop, .o, &.{ .r64  }, &.{ 0x58 }, 0, .none, .none },
+    .{ .pop, .m, &.{ .rm16 }, &.{ 0x8f }, 0, .none, .none },
+    .{ .pop, .m, &.{ .rm64 }, &.{ 0x8f }, 0, .none, .none },
 
-    .{ .popcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },
-    .{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },
-    .{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long },
+    .{ .popcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .none },
+    .{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .none },
+    .{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long, .none },
 
-    .{ .push, .o, &.{ .r16   }, &.{ 0x50 }, 0, .none },
-    .{ .push, .o, &.{ .r64   }, &.{ 0x50 }, 0, .none },
-    .{ .push, .m, &.{ .rm16  }, &.{ 0xff }, 6, .none },
-    .{ .push, .m, &.{ .rm64  }, &.{ 0xff }, 6, .none },
-    .{ .push, .i, &.{ .imm8  }, &.{ 0x6a }, 0, .none },
-    .{ .push, .i, &.{ .imm16 }, &.{ 0x68 }, 0, .none },
-    .{ .push, .i, &.{ .imm32 }, &.{ 0x68 }, 0, .none },
+    .{ .push, .o, &.{ .r16   }, &.{ 0x50 }, 0, .none, .none },
+    .{ .push, .o, &.{ .r64   }, &.{ 0x50 }, 0, .none, .none },
+    .{ .push, .m, &.{ .rm16  }, &.{ 0xff }, 6, .none, .none },
+    .{ .push, .m, &.{ .rm64  }, &.{ 0xff }, 6, .none, .none },
+    .{ .push, .i, &.{ .imm8  }, &.{ 0x6a }, 0, .none, .none },
+    .{ .push, .i, &.{ .imm16 }, &.{ 0x68 }, 0, .none, .none },
+    .{ .push, .i, &.{ .imm32 }, &.{ 0x68 }, 0, .none, .none },
 
-    .{ .ret, .np, &.{}, &.{ 0xc3 }, 0, .none },
+    .{ .ret, .np, &.{}, &.{ 0xc3 }, 0, .none, .none },
 
-    .{ .rcl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 2, .none },
-    .{ .rcl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 2, .rex  },
-    .{ .rcl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 2, .none },
-    .{ .rcl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 2, .rex  },
-    .{ .rcl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 2, .none },
-    .{ .rcl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 2, .rex  },
-    .{ .rcl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 2, .none },
-    .{ .rcl, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 2, .none },
-    .{ .rcl, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 2, .none },
-    .{ .rcl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 2, .none },
-    .{ .rcl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 2, .long },
-    .{ .rcl, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 2, .none },
-    .{ .rcl, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 2, .long },
-    .{ .rcl, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 2, .none },
-    .{ .rcl, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 2, .long },
+    .{ .rcl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 2, .none, .none },
+    .{ .rcl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 2, .rex,  .none },
+    .{ .rcl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 2, .none, .none },
+    .{ .rcl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 2, .rex,  .none },
+    .{ .rcl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 2, .none, .none },
+    .{ .rcl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 2, .rex,  .none },
+    .{ .rcl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 2, .none, .none },
+    .{ .rcl, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 2, .none, .none },
+    .{ .rcl, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 2, .none, .none },
+    .{ .rcl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 2, .none, .none },
+    .{ .rcl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 2, .long, .none },
+    .{ .rcl, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 2, .none, .none },
+    .{ .rcl, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 2, .long, .none },
+    .{ .rcl, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 2, .none, .none },
+    .{ .rcl, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 2, .long, .none },
 
-    .{ .rcr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 3, .none },
-    .{ .rcr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 3, .rex  },
-    .{ .rcr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 3, .none },
-    .{ .rcr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 3, .rex  },
-    .{ .rcr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 3, .none },
-    .{ .rcr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 3, .rex  },
-    .{ .rcr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 3, .none },
-    .{ .rcr, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 3, .none },
-    .{ .rcr, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 3, .none },
-    .{ .rcr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 3, .none },
-    .{ .rcr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 3, .long },
-    .{ .rcr, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 3, .none },
-    .{ .rcr, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 3, .long },
-    .{ .rcr, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 3, .none },
-    .{ .rcr, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 3, .long },
+    .{ .rcr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 3, .none, .none },
+    .{ .rcr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 3, .rex,  .none },
+    .{ .rcr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 3, .none, .none },
+    .{ .rcr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 3, .rex,  .none },
+    .{ .rcr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 3, .none, .none },
+    .{ .rcr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 3, .rex,  .none },
+    .{ .rcr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 3, .none, .none },
+    .{ .rcr, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 3, .none, .none },
+    .{ .rcr, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 3, .none, .none },
+    .{ .rcr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 3, .none, .none },
+    .{ .rcr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 3, .long, .none },
+    .{ .rcr, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 3, .none, .none },
+    .{ .rcr, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 3, .long, .none },
+    .{ .rcr, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 3, .none, .none },
+    .{ .rcr, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 3, .long, .none },
 
-    .{ .rol, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 0, .none },
-    .{ .rol, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 0, .rex  },
-    .{ .rol, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 0, .none },
-    .{ .rol, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 0, .rex  },
-    .{ .rol, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 0, .none },
-    .{ .rol, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 0, .rex  },
-    .{ .rol, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 0, .none },
-    .{ .rol, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 0, .none },
-    .{ .rol, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 0, .none },
-    .{ .rol, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 0, .none },
-    .{ .rol, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 0, .long },
-    .{ .rol, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 0, .none },
-    .{ .rol, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 0, .long },
-    .{ .rol, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 0, .none },
-    .{ .rol, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 0, .long },
+    .{ .rol, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 0, .none, .none },
+    .{ .rol, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 0, .rex,  .none },
+    .{ .rol, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 0, .none, .none },
+    .{ .rol, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 0, .rex,  .none },
+    .{ .rol, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 0, .none, .none },
+    .{ .rol, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 0, .rex,  .none },
+    .{ .rol, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 0, .none, .none },
+    .{ .rol, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 0, .none, .none },
+    .{ .rol, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 0, .none, .none },
+    .{ .rol, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 0, .none, .none },
+    .{ .rol, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 0, .long, .none },
+    .{ .rol, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 0, .none, .none },
+    .{ .rol, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 0, .long, .none },
+    .{ .rol, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 0, .none, .none },
+    .{ .rol, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 0, .long, .none },
 
-    .{ .ror, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 1, .none },
-    .{ .ror, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 1, .rex  },
-    .{ .ror, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 1, .none },
-    .{ .ror, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 1, .rex  },
-    .{ .ror, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 1, .none },
-    .{ .ror, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 1, .rex  },
-    .{ .ror, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 1, .none },
-    .{ .ror, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 1, .none },
-    .{ .ror, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 1, .none },
-    .{ .ror, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 1, .none },
-    .{ .ror, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 1, .long },
-    .{ .ror, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 1, .none },
-    .{ .ror, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 1, .long },
-    .{ .ror, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 1, .none },
-    .{ .ror, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 1, .long },
+    .{ .ror, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 1, .none, .none },
+    .{ .ror, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 1, .rex,  .none },
+    .{ .ror, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 1, .none, .none },
+    .{ .ror, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 1, .rex,  .none },
+    .{ .ror, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 1, .none, .none },
+    .{ .ror, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 1, .rex,  .none },
+    .{ .ror, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 1, .none, .none },
+    .{ .ror, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 1, .none, .none },
+    .{ .ror, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 1, .none, .none },
+    .{ .ror, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 1, .none, .none },
+    .{ .ror, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 1, .long, .none },
+    .{ .ror, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 1, .none, .none },
+    .{ .ror, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 1, .long, .none },
+    .{ .ror, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 1, .none, .none },
+    .{ .ror, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 1, .long, .none },
 
-    .{ .sal, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .none },
-    .{ .sal, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .rex  },
-    .{ .sal, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .none },
-    .{ .sal, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none },
-    .{ .sal, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long },
-    .{ .sal, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .none },
-    .{ .sal, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .rex  },
-    .{ .sal, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 4, .none },
-    .{ .sal, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 4, .none },
-    .{ .sal, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 4, .long },
-    .{ .sal, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .none },
-    .{ .sal, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .rex  },
-    .{ .sal, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 4, .none },
-    .{ .sal, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 4, .none },
-    .{ .sal, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 4, .long },
+    .{ .sal, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .none, .none },
+    .{ .sal, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .rex,  .none },
+    .{ .sal, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .none, .none },
+    .{ .sal, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none, .none },
+    .{ .sal, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long, .none },
+    .{ .sal, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .none, .none },
+    .{ .sal, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .rex,  .none },
+    .{ .sal, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 4, .none, .none },
+    .{ .sal, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 4, .none, .none },
+    .{ .sal, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 4, .long, .none },
+    .{ .sal, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .none, .none },
+    .{ .sal, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .rex,  .none },
+    .{ .sal, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 4, .none, .none },
+    .{ .sal, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 4, .none, .none },
+    .{ .sal, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 4, .long, .none },
 
-    .{ .sar, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 7, .none },
-    .{ .sar, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 7, .rex  },
-    .{ .sar, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 7, .none },
-    .{ .sar, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 7, .none },
-    .{ .sar, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 7, .long },
-    .{ .sar, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 7, .none },
-    .{ .sar, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 7, .rex  },
-    .{ .sar, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 7, .none },
-    .{ .sar, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 7, .none },
-    .{ .sar, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 7, .long },
-    .{ .sar, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 7, .none },
-    .{ .sar, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 7, .rex  },
-    .{ .sar, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 7, .none },
-    .{ .sar, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 7, .none },
-    .{ .sar, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 7, .long },
+    .{ .sar, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 7, .none, .none },
+    .{ .sar, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 7, .rex,  .none },
+    .{ .sar, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 7, .none, .none },
+    .{ .sar, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 7, .none, .none },
+    .{ .sar, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 7, .long, .none },
+    .{ .sar, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 7, .none, .none },
+    .{ .sar, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 7, .rex,  .none },
+    .{ .sar, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 7, .none, .none },
+    .{ .sar, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 7, .none, .none },
+    .{ .sar, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 7, .long, .none },
+    .{ .sar, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 7, .none, .none },
+    .{ .sar, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 7, .rex,  .none },
+    .{ .sar, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 7, .none, .none },
+    .{ .sar, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 7, .none, .none },
+    .{ .sar, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 7, .long, .none },
 
-    .{ .sbb, .zi, &.{ .al,   .imm8   }, &.{ 0x1c }, 0, .none },
-    .{ .sbb, .zi, &.{ .ax,   .imm16  }, &.{ 0x1d }, 0, .none },
-    .{ .sbb, .zi, &.{ .eax,  .imm32  }, &.{ 0x1d }, 0, .none },
-    .{ .sbb, .zi, &.{ .rax,  .imm32s }, &.{ 0x1d }, 0, .long },
-    .{ .sbb, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 3, .none },
-    .{ .sbb, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 3, .rex  },
-    .{ .sbb, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 3, .none },
-    .{ .sbb, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 3, .none },
-    .{ .sbb, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 3, .long },
-    .{ .sbb, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 3, .none },
-    .{ .sbb, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 3, .none },
-    .{ .sbb, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 3, .long },
-    .{ .sbb, .mr, &.{ .rm8,  .r8     }, &.{ 0x18 }, 0, .none },
-    .{ .sbb, .mr, &.{ .rm8,  .r8     }, &.{ 0x18 }, 0, .rex  },
-    .{ .sbb, .mr, &.{ .rm16, .r16    }, &.{ 0x19 }, 0, .none },
-    .{ .sbb, .mr, &.{ .rm32, .r32    }, &.{ 0x19 }, 0, .none },
-    .{ .sbb, .mr, &.{ .rm64, .r64    }, &.{ 0x19 }, 0, .long },
-    .{ .sbb, .rm, &.{ .r8,   .rm8    }, &.{ 0x1a }, 0, .none },
-    .{ .sbb, .rm, &.{ .r8,   .rm8    }, &.{ 0x1a }, 0, .rex  },
-    .{ .sbb, .rm, &.{ .r16,  .rm16   }, &.{ 0x1b }, 0, .none },
-    .{ .sbb, .rm, &.{ .r32,  .rm32   }, &.{ 0x1b }, 0, .none },
-    .{ .sbb, .rm, &.{ .r64,  .rm64   }, &.{ 0x1b }, 0, .long },
+    .{ .sbb, .zi, &.{ .al,   .imm8   }, &.{ 0x1c }, 0, .none, .none },
+    .{ .sbb, .zi, &.{ .ax,   .imm16  }, &.{ 0x1d }, 0, .none, .none },
+    .{ .sbb, .zi, &.{ .eax,  .imm32  }, &.{ 0x1d }, 0, .none, .none },
+    .{ .sbb, .zi, &.{ .rax,  .imm32s }, &.{ 0x1d }, 0, .long, .none },
+    .{ .sbb, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 3, .none, .none },
+    .{ .sbb, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 3, .rex,  .none },
+    .{ .sbb, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 3, .none, .none },
+    .{ .sbb, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 3, .none, .none },
+    .{ .sbb, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 3, .long, .none },
+    .{ .sbb, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 3, .none, .none },
+    .{ .sbb, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 3, .none, .none },
+    .{ .sbb, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 3, .long, .none },
+    .{ .sbb, .mr, &.{ .rm8,  .r8     }, &.{ 0x18 }, 0, .none, .none },
+    .{ .sbb, .mr, &.{ .rm8,  .r8     }, &.{ 0x18 }, 0, .rex,  .none },
+    .{ .sbb, .mr, &.{ .rm16, .r16    }, &.{ 0x19 }, 0, .none, .none },
+    .{ .sbb, .mr, &.{ .rm32, .r32    }, &.{ 0x19 }, 0, .none, .none },
+    .{ .sbb, .mr, &.{ .rm64, .r64    }, &.{ 0x19 }, 0, .long, .none },
+    .{ .sbb, .rm, &.{ .r8,   .rm8    }, &.{ 0x1a }, 0, .none, .none },
+    .{ .sbb, .rm, &.{ .r8,   .rm8    }, &.{ 0x1a }, 0, .rex,  .none },
+    .{ .sbb, .rm, &.{ .r16,  .rm16   }, &.{ 0x1b }, 0, .none, .none },
+    .{ .sbb, .rm, &.{ .r32,  .rm32   }, &.{ 0x1b }, 0, .none, .none },
+    .{ .sbb, .rm, &.{ .r64,  .rm64   }, &.{ 0x1b }, 0, .long, .none },
 
-    .{ .scas,  .np, &.{ .m8  }, &.{ 0xae }, 0, .none  },
-    .{ .scas,  .np, &.{ .m16 }, &.{ 0xaf }, 0, .none  },
-    .{ .scas,  .np, &.{ .m32 }, &.{ 0xaf }, 0, .none  },
-    .{ .scas,  .np, &.{ .m64 }, &.{ 0xaf }, 0, .long  },
+    .{ .scas,  .np, &.{ .m8  }, &.{ 0xae }, 0, .none, .none },
+    .{ .scas,  .np, &.{ .m16 }, &.{ 0xaf }, 0, .none, .none },
+    .{ .scas,  .np, &.{ .m32 }, &.{ 0xaf }, 0, .none, .none },
+    .{ .scas,  .np, &.{ .m64 }, &.{ 0xaf }, 0, .long, .none },
 
-    .{ .scasb, .np, &.{}, &.{ 0xae }, 0, .none  },
-    .{ .scasw, .np, &.{}, &.{ 0xaf }, 0, .short },
-    .{ .scasd, .np, &.{}, &.{ 0xaf }, 0, .none  },
-    .{ .scasq, .np, &.{}, &.{ 0xaf }, 0, .long  },
+    .{ .scasb, .np, &.{}, &.{ 0xae }, 0, .none,  .none },
+    .{ .scasw, .np, &.{}, &.{ 0xaf }, 0, .short, .none },
+    .{ .scasd, .np, &.{}, &.{ 0xaf }, 0, .none,  .none },
+    .{ .scasq, .np, &.{}, &.{ 0xaf }, 0, .long,  .none },
 
-    .{ .seta,   .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none },
-    .{ .seta,   .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex  },
-    .{ .setae,  .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none },
-    .{ .setae,  .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex  },
-    .{ .setb,   .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none },
-    .{ .setb,   .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex  },
-    .{ .setbe,  .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .none },
-    .{ .setbe,  .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .rex  },
-    .{ .setc,   .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none },
-    .{ .setc,   .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex  },
-    .{ .sete,   .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none },
-    .{ .sete,   .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex  },
-    .{ .setg,   .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .none },
-    .{ .setg,   .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .rex  },
-    .{ .setge,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .none },
-    .{ .setge,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .rex  },
-    .{ .setl,   .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .none },
-    .{ .setl,   .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .rex  },
-    .{ .setle,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .none },
-    .{ .setle,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .rex  },
-    .{ .setna,  .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .none },
-    .{ .setna,  .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .rex  },
-    .{ .setnae, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none },
-    .{ .setnae, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex  },
-    .{ .setnb,  .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none },
-    .{ .setnb,  .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex  },
-    .{ .setnbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none },
-    .{ .setnbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex  },
-    .{ .setnc,  .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none },
-    .{ .setnc,  .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex  },
-    .{ .setne,  .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .none },
-    .{ .setne,  .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .rex  },
-    .{ .setng,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .none },
-    .{ .setng,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .rex  },
-    .{ .setnge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .none },
-    .{ .setnge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .rex  },
-    .{ .setnl,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .none },
-    .{ .setnl,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .rex  },
-    .{ .setnle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .none },
-    .{ .setnle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .rex  },
-    .{ .setno,  .m, &.{ .rm8 }, &.{ 0x0f, 0x91 }, 0, .none },
-    .{ .setno,  .m, &.{ .rm8 }, &.{ 0x0f, 0x91 }, 0, .rex  },
-    .{ .setnp,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .none },
-    .{ .setnp,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .rex  },
-    .{ .setns,  .m, &.{ .rm8 }, &.{ 0x0f, 0x99 }, 0, .none },
-    .{ .setns,  .m, &.{ .rm8 }, &.{ 0x0f, 0x99 }, 0, .rex  },
-    .{ .setnz,  .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .none },
-    .{ .setnz,  .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .rex  },
-    .{ .seto,   .m, &.{ .rm8 }, &.{ 0x0f, 0x90 }, 0, .none },
-    .{ .seto,   .m, &.{ .rm8 }, &.{ 0x0f, 0x90 }, 0, .rex  },
-    .{ .setp,   .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .none },
-    .{ .setp,   .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .rex  },
-    .{ .setpe,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .none },
-    .{ .setpe,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .rex  },
-    .{ .setpo,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .none },
-    .{ .setpo,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .rex  },
-    .{ .sets,   .m, &.{ .rm8 }, &.{ 0x0f, 0x98 }, 0, .none },
-    .{ .sets,   .m, &.{ .rm8 }, &.{ 0x0f, 0x98 }, 0, .rex  },
-    .{ .setz,   .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none },
-    .{ .setz,   .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex  },
+    .{ .seta,   .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none, .none },
+    .{ .seta,   .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex,  .none },
+    .{ .setae,  .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none, .none },
+    .{ .setae,  .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex,  .none },
+    .{ .setb,   .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none, .none },
+    .{ .setb,   .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex,  .none },
+    .{ .setbe,  .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .none, .none },
+    .{ .setbe,  .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .rex,  .none },
+    .{ .setc,   .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none, .none },
+    .{ .setc,   .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex,  .none },
+    .{ .sete,   .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none, .none },
+    .{ .sete,   .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex,  .none },
+    .{ .setg,   .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .none, .none },
+    .{ .setg,   .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .rex,  .none },
+    .{ .setge,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .none, .none },
+    .{ .setge,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .rex,  .none },
+    .{ .setl,   .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .none, .none },
+    .{ .setl,   .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .rex,  .none },
+    .{ .setle,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .none, .none },
+    .{ .setle,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .rex,  .none },
+    .{ .setna,  .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .none, .none },
+    .{ .setna,  .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .rex,  .none },
+    .{ .setnae, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none, .none },
+    .{ .setnae, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex,  .none },
+    .{ .setnb,  .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none, .none },
+    .{ .setnb,  .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex,  .none },
+    .{ .setnbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none, .none },
+    .{ .setnbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex,  .none },
+    .{ .setnc,  .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none, .none },
+    .{ .setnc,  .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex,  .none },
+    .{ .setne,  .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .none, .none },
+    .{ .setne,  .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .rex,  .none },
+    .{ .setng,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .none, .none },
+    .{ .setng,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .rex,  .none },
+    .{ .setnge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .none, .none },
+    .{ .setnge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .rex,  .none },
+    .{ .setnl,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .none, .none },
+    .{ .setnl,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .rex,  .none },
+    .{ .setnle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .none, .none },
+    .{ .setnle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .rex,  .none },
+    .{ .setno,  .m, &.{ .rm8 }, &.{ 0x0f, 0x91 }, 0, .none, .none },
+    .{ .setno,  .m, &.{ .rm8 }, &.{ 0x0f, 0x91 }, 0, .rex,  .none },
+    .{ .setnp,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .none, .none },
+    .{ .setnp,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .rex,  .none },
+    .{ .setns,  .m, &.{ .rm8 }, &.{ 0x0f, 0x99 }, 0, .none, .none },
+    .{ .setns,  .m, &.{ .rm8 }, &.{ 0x0f, 0x99 }, 0, .rex,  .none },
+    .{ .setnz,  .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .none, .none },
+    .{ .setnz,  .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .rex,  .none },
+    .{ .seto,   .m, &.{ .rm8 }, &.{ 0x0f, 0x90 }, 0, .none, .none },
+    .{ .seto,   .m, &.{ .rm8 }, &.{ 0x0f, 0x90 }, 0, .rex,  .none },
+    .{ .setp,   .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .none, .none },
+    .{ .setp,   .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .rex,  .none },
+    .{ .setpe,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .none, .none },
+    .{ .setpe,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .rex,  .none },
+    .{ .setpo,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .none, .none },
+    .{ .setpo,  .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .rex,  .none },
+    .{ .sets,   .m, &.{ .rm8 }, &.{ 0x0f, 0x98 }, 0, .none, .none },
+    .{ .sets,   .m, &.{ .rm8 }, &.{ 0x0f, 0x98 }, 0, .rex,  .none },
+    .{ .setz,   .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none, .none },
+    .{ .setz,   .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex,  .none },
 
-    .{ .sfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none },
+    .{ .sfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none, .none },
 
-    .{ .shl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .none },
-    .{ .shl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .rex  },
-    .{ .shl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .none },
-    .{ .shl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none },
-    .{ .shl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long },
-    .{ .shl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .none },
-    .{ .shl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .rex  },
-    .{ .shl, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 4, .none },
-    .{ .shl, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 4, .none },
-    .{ .shl, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 4, .long },
-    .{ .shl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .none },
-    .{ .shl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .rex  },
-    .{ .shl, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 4, .none },
-    .{ .shl, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 4, .none },
-    .{ .shl, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 4, .long },
+    .{ .shl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .none, .none },
+    .{ .shl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .rex,  .none },
+    .{ .shl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .none, .none },
+    .{ .shl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none, .none },
+    .{ .shl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long, .none },
+    .{ .shl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .none, .none },
+    .{ .shl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .rex,  .none },
+    .{ .shl, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 4, .none, .none },
+    .{ .shl, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 4, .none, .none },
+    .{ .shl, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 4, .long, .none },
+    .{ .shl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .none, .none },
+    .{ .shl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .rex,  .none },
+    .{ .shl, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 4, .none, .none },
+    .{ .shl, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 4, .none, .none },
+    .{ .shl, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 4, .long, .none },
 
-    .{ .shld, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none },
-    .{ .shld, .mrc, &.{ .rm16, .r16, .cl   }, &.{ 0x0f, 0xa5 }, 0, .none },
-    .{ .shld, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none },
-    .{ .shld, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .long },
-    .{ .shld, .mrc, &.{ .rm32, .r32, .cl   }, &.{ 0x0f, 0xa5 }, 0, .none },
-    .{ .shld, .mrc, &.{ .rm64, .r64, .cl   }, &.{ 0x0f, 0xa5 }, 0, .long },
+    .{ .shld, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none, .none },
+    .{ .shld, .mrc, &.{ .rm16, .r16, .cl   }, &.{ 0x0f, 0xa5 }, 0, .none, .none },
+    .{ .shld, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none, .none },
+    .{ .shld, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .long, .none },
+    .{ .shld, .mrc, &.{ .rm32, .r32, .cl   }, &.{ 0x0f, 0xa5 }, 0, .none, .none },
+    .{ .shld, .mrc, &.{ .rm64, .r64, .cl   }, &.{ 0x0f, 0xa5 }, 0, .long, .none },
 
-    .{ .shr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 5, .none },
-    .{ .shr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 5, .rex  },
-    .{ .shr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 5, .none },
-    .{ .shr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 5, .none },
-    .{ .shr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 5, .long },
-    .{ .shr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 5, .none },
-    .{ .shr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 5, .rex  },
-    .{ .shr, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 5, .none },
-    .{ .shr, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 5, .none },
-    .{ .shr, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 5, .long },
-    .{ .shr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 5, .none },
-    .{ .shr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 5, .rex  },
-    .{ .shr, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 5, .none },
-    .{ .shr, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 5, .none },
-    .{ .shr, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 5, .long },
+    .{ .shr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 5, .none, .none },
+    .{ .shr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 5, .rex,  .none },
+    .{ .shr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 5, .none, .none },
+    .{ .shr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 5, .none, .none },
+    .{ .shr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 5, .long, .none },
+    .{ .shr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 5, .none, .none },
+    .{ .shr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 5, .rex,  .none },
+    .{ .shr, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 5, .none, .none },
+    .{ .shr, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 5, .none, .none },
+    .{ .shr, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 5, .long, .none },
+    .{ .shr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 5, .none, .none },
+    .{ .shr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 5, .rex,  .none },
+    .{ .shr, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 5, .none, .none },
+    .{ .shr, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 5, .none, .none },
+    .{ .shr, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 5, .long, .none },
 
-    .{ .shrd, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xac }, 0, .none },
-    .{ .shrd, .mrc, &.{ .rm16, .r16, .cl   }, &.{ 0x0f, 0xad }, 0, .none },
-    .{ .shrd, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xac }, 0, .none },
-    .{ .shrd, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xac }, 0, .long },
-    .{ .shrd, .mrc, &.{ .rm32, .r32, .cl   }, &.{ 0x0f, 0xad }, 0, .none },
-    .{ .shrd, .mrc, &.{ .rm64, .r64, .cl   }, &.{ 0x0f, 0xad }, 0, .long },
+    .{ .shrd, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xac }, 0, .none, .none },
+    .{ .shrd, .mrc, &.{ .rm16, .r16, .cl   }, &.{ 0x0f, 0xad }, 0, .none, .none },
+    .{ .shrd, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xac }, 0, .none, .none },
+    .{ .shrd, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xac }, 0, .long, .none },
+    .{ .shrd, .mrc, &.{ .rm32, .r32, .cl   }, &.{ 0x0f, 0xad }, 0, .none, .none },
+    .{ .shrd, .mrc, &.{ .rm64, .r64, .cl   }, &.{ 0x0f, 0xad }, 0, .long, .none },
 
-    .{ .stos,  .np, &.{ .m8  }, &.{ 0xaa }, 0, .none  },
-    .{ .stos,  .np, &.{ .m16 }, &.{ 0xab }, 0, .none  },
-    .{ .stos,  .np, &.{ .m32 }, &.{ 0xab }, 0, .none  },
-    .{ .stos,  .np, &.{ .m64 }, &.{ 0xab }, 0, .long  },
+    .{ .stos,  .np, &.{ .m8  }, &.{ 0xaa }, 0, .none, .none },
+    .{ .stos,  .np, &.{ .m16 }, &.{ 0xab }, 0, .none, .none },
+    .{ .stos,  .np, &.{ .m32 }, &.{ 0xab }, 0, .none, .none },
+    .{ .stos,  .np, &.{ .m64 }, &.{ 0xab }, 0, .long, .none },
 
-    .{ .stosb, .np, &.{}, &.{ 0xaa }, 0, .none  },
-    .{ .stosw, .np, &.{}, &.{ 0xab }, 0, .short },
-    .{ .stosd, .np, &.{}, &.{ 0xab }, 0, .none  },
-    .{ .stosq, .np, &.{}, &.{ 0xab }, 0, .long  },
+    .{ .stosb, .np, &.{}, &.{ 0xaa }, 0, .none,  .none },
+    .{ .stosw, .np, &.{}, &.{ 0xab }, 0, .short, .none },
+    .{ .stosd, .np, &.{}, &.{ 0xab }, 0, .none,  .none },
+    .{ .stosq, .np, &.{}, &.{ 0xab }, 0, .long,  .none },
 
-    .{ .sub, .zi, &.{ .al,   .imm8   }, &.{ 0x2c }, 0, .none },
-    .{ .sub, .zi, &.{ .ax,   .imm16  }, &.{ 0x2d }, 0, .none },
-    .{ .sub, .zi, &.{ .eax,  .imm32  }, &.{ 0x2d }, 0, .none },
-    .{ .sub, .zi, &.{ .rax,  .imm32s }, &.{ 0x2d }, 0, .long },
-    .{ .sub, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 5, .none },
-    .{ .sub, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 5, .rex  },
-    .{ .sub, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 5, .none },
-    .{ .sub, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 5, .none },
-    .{ .sub, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 5, .long },
-    .{ .sub, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 5, .none },
-    .{ .sub, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 5, .none },
-    .{ .sub, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 5, .long },
-    .{ .sub, .mr, &.{ .rm8,  .r8     }, &.{ 0x28 }, 0, .none },
-    .{ .sub, .mr, &.{ .rm8,  .r8     }, &.{ 0x28 }, 0, .rex  },
-    .{ .sub, .mr, &.{ .rm16, .r16    }, &.{ 0x29 }, 0, .none },
-    .{ .sub, .mr, &.{ .rm32, .r32    }, &.{ 0x29 }, 0, .none },
-    .{ .sub, .mr, &.{ .rm64, .r64    }, &.{ 0x29 }, 0, .long },
-    .{ .sub, .rm, &.{ .r8,   .rm8    }, &.{ 0x2a }, 0, .none },
-    .{ .sub, .rm, &.{ .r8,   .rm8    }, &.{ 0x2a }, 0, .rex  },
-    .{ .sub, .rm, &.{ .r16,  .rm16   }, &.{ 0x2b }, 0, .none },
-    .{ .sub, .rm, &.{ .r32,  .rm32   }, &.{ 0x2b }, 0, .none },
-    .{ .sub, .rm, &.{ .r64,  .rm64   }, &.{ 0x2b }, 0, .long },
+    .{ .sub, .zi, &.{ .al,   .imm8   }, &.{ 0x2c }, 0, .none, .none },
+    .{ .sub, .zi, &.{ .ax,   .imm16  }, &.{ 0x2d }, 0, .none, .none },
+    .{ .sub, .zi, &.{ .eax,  .imm32  }, &.{ 0x2d }, 0, .none, .none },
+    .{ .sub, .zi, &.{ .rax,  .imm32s }, &.{ 0x2d }, 0, .long, .none },
+    .{ .sub, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 5, .none, .none },
+    .{ .sub, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 5, .rex,  .none },
+    .{ .sub, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 5, .none, .none },
+    .{ .sub, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 5, .none, .none },
+    .{ .sub, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 5, .long, .none },
+    .{ .sub, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 5, .none, .none },
+    .{ .sub, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 5, .none, .none },
+    .{ .sub, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 5, .long, .none },
+    .{ .sub, .mr, &.{ .rm8,  .r8     }, &.{ 0x28 }, 0, .none, .none },
+    .{ .sub, .mr, &.{ .rm8,  .r8     }, &.{ 0x28 }, 0, .rex,  .none },
+    .{ .sub, .mr, &.{ .rm16, .r16    }, &.{ 0x29 }, 0, .none, .none },
+    .{ .sub, .mr, &.{ .rm32, .r32    }, &.{ 0x29 }, 0, .none, .none },
+    .{ .sub, .mr, &.{ .rm64, .r64    }, &.{ 0x29 }, 0, .long, .none },
+    .{ .sub, .rm, &.{ .r8,   .rm8    }, &.{ 0x2a }, 0, .none, .none },
+    .{ .sub, .rm, &.{ .r8,   .rm8    }, &.{ 0x2a }, 0, .rex,  .none },
+    .{ .sub, .rm, &.{ .r16,  .rm16   }, &.{ 0x2b }, 0, .none, .none },
+    .{ .sub, .rm, &.{ .r32,  .rm32   }, &.{ 0x2b }, 0, .none, .none },
+    .{ .sub, .rm, &.{ .r64,  .rm64   }, &.{ 0x2b }, 0, .long, .none },
 
-    .{ .syscall, .np, &.{}, &.{ 0x0f, 0x05 }, 0, .none }
-,
-    .{ .@"test", .zi, &.{ .al,   .imm8   }, &.{ 0xa8 }, 0, .none },
-    .{ .@"test", .zi, &.{ .ax,   .imm16  }, &.{ 0xa9 }, 0, .none },
-    .{ .@"test", .zi, &.{ .eax,  .imm32  }, &.{ 0xa9 }, 0, .none },
-    .{ .@"test", .zi, &.{ .rax,  .imm32s }, &.{ 0xa9 }, 0, .long },
-    .{ .@"test", .mi, &.{ .rm8,  .imm8   }, &.{ 0xf6 }, 0, .none },
-    .{ .@"test", .mi, &.{ .rm8,  .imm8   }, &.{ 0xf6 }, 0, .rex  },
-    .{ .@"test", .mi, &.{ .rm16, .imm16  }, &.{ 0xf7 }, 0, .none },
-    .{ .@"test", .mi, &.{ .rm32, .imm32  }, &.{ 0xf7 }, 0, .none },
-    .{ .@"test", .mi, &.{ .rm64, .imm32s }, &.{ 0xf7 }, 0, .long },
-    .{ .@"test", .mr, &.{ .rm8,  .r8     }, &.{ 0x84 }, 0, .none },
-    .{ .@"test", .mr, &.{ .rm8,  .r8     }, &.{ 0x84 }, 0, .rex  },
-    .{ .@"test", .mr, &.{ .rm16, .r16    }, &.{ 0x85 }, 0, .none },
-    .{ .@"test", .mr, &.{ .rm32, .r32    }, &.{ 0x85 }, 0, .none },
-    .{ .@"test", .mr, &.{ .rm64, .r64    }, &.{ 0x85 }, 0, .long },
+    .{ .syscall, .np, &.{}, &.{ 0x0f, 0x05 }, 0, .none, .none },
 
-    .{ .tzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },
-    .{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },
-    .{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long },
+    .{ .@"test", .zi, &.{ .al,   .imm8   }, &.{ 0xa8 }, 0, .none, .none },
+    .{ .@"test", .zi, &.{ .ax,   .imm16  }, &.{ 0xa9 }, 0, .none, .none },
+    .{ .@"test", .zi, &.{ .eax,  .imm32  }, &.{ 0xa9 }, 0, .none, .none },
+    .{ .@"test", .zi, &.{ .rax,  .imm32s }, &.{ 0xa9 }, 0, .long, .none },
+    .{ .@"test", .mi, &.{ .rm8,  .imm8   }, &.{ 0xf6 }, 0, .none, .none },
+    .{ .@"test", .mi, &.{ .rm8,  .imm8   }, &.{ 0xf6 }, 0, .rex,  .none },
+    .{ .@"test", .mi, &.{ .rm16, .imm16  }, &.{ 0xf7 }, 0, .none, .none },
+    .{ .@"test", .mi, &.{ .rm32, .imm32  }, &.{ 0xf7 }, 0, .none, .none },
+    .{ .@"test", .mi, &.{ .rm64, .imm32s }, &.{ 0xf7 }, 0, .long, .none },
+    .{ .@"test", .mr, &.{ .rm8,  .r8     }, &.{ 0x84 }, 0, .none, .none },
+    .{ .@"test", .mr, &.{ .rm8,  .r8     }, &.{ 0x84 }, 0, .rex,  .none },
+    .{ .@"test", .mr, &.{ .rm16, .r16    }, &.{ 0x85 }, 0, .none, .none },
+    .{ .@"test", .mr, &.{ .rm32, .r32    }, &.{ 0x85 }, 0, .none, .none },
+    .{ .@"test", .mr, &.{ .rm64, .r64    }, &.{ 0x85 }, 0, .long, .none },
 
-    .{ .ud2, .np, &.{}, &.{ 0x0f, 0x0b }, 0, .none },
+    .{ .tzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .none },
+    .{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .none },
+    .{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long, .none },
 
-    .{ .xadd, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xc0 }, 0, .none },
-    .{ .xadd, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xc0 }, 0, .rex  },
-    .{ .xadd, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xc1 }, 0, .none },
-    .{ .xadd, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xc1 }, 0, .none },
-    .{ .xadd, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xc1 }, 0, .long },
+    .{ .ud2, .np, &.{}, &.{ 0x0f, 0x0b }, 0, .none, .none },
 
-    .{ .xchg, .o,  &.{ .ax,   .r16  }, &.{ 0x90 }, 0, .none },
-    .{ .xchg, .o,  &.{ .r16,  .ax   }, &.{ 0x90 }, 0, .none },
-    .{ .xchg, .o,  &.{ .eax,  .r32  }, &.{ 0x90 }, 0, .none },
-    .{ .xchg, .o,  &.{ .rax,  .r64  }, &.{ 0x90 }, 0, .long },
-    .{ .xchg, .o,  &.{ .r32,  .eax  }, &.{ 0x90 }, 0, .none },
-    .{ .xchg, .o,  &.{ .r64,  .rax  }, &.{ 0x90 }, 0, .long },
-    .{ .xchg, .mr, &.{ .rm8,  .r8   }, &.{ 0x86 }, 0, .none },
-    .{ .xchg, .mr, &.{ .rm8,  .r8   }, &.{ 0x86 }, 0, .rex  },
-    .{ .xchg, .rm, &.{ .r8,   .rm8  }, &.{ 0x86 }, 0, .none },
-    .{ .xchg, .rm, &.{ .r8,   .rm8  }, &.{ 0x86 }, 0, .rex  },
-    .{ .xchg, .mr, &.{ .rm16, .r16  }, &.{ 0x87 }, 0, .none },
-    .{ .xchg, .rm, &.{ .r16,  .rm16 }, &.{ 0x87 }, 0, .none },
-    .{ .xchg, .mr, &.{ .rm32, .r32  }, &.{ 0x87 }, 0, .none },
-    .{ .xchg, .mr, &.{ .rm64, .r64  }, &.{ 0x87 }, 0, .long },
-    .{ .xchg, .rm, &.{ .r32,  .rm32 }, &.{ 0x87 }, 0, .none },
-    .{ .xchg, .rm, &.{ .r64,  .rm64 }, &.{ 0x87 }, 0, .long },
+    .{ .xadd, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xc0 }, 0, .none, .none },
+    .{ .xadd, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xc0 }, 0, .rex,  .none },
+    .{ .xadd, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xc1 }, 0, .none, .none },
+    .{ .xadd, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xc1 }, 0, .none, .none },
+    .{ .xadd, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xc1 }, 0, .long, .none },
 
-    .{ .xor, .zi, &.{ .al,   .imm8   }, &.{ 0x34 }, 0, .none },
-    .{ .xor, .zi, &.{ .ax,   .imm16  }, &.{ 0x35 }, 0, .none },
-    .{ .xor, .zi, &.{ .eax,  .imm32  }, &.{ 0x35 }, 0, .none },
-    .{ .xor, .zi, &.{ .rax,  .imm32s }, &.{ 0x35 }, 0, .long },
-    .{ .xor, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 6, .none },
-    .{ .xor, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 6, .rex  },
-    .{ .xor, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 6, .none },
-    .{ .xor, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 6, .none },
-    .{ .xor, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 6, .long },
-    .{ .xor, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 6, .none },
-    .{ .xor, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 6, .none },
-    .{ .xor, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 6, .long },
-    .{ .xor, .mr, &.{ .rm8,  .r8     }, &.{ 0x30 }, 0, .none },
-    .{ .xor, .mr, &.{ .rm8,  .r8     }, &.{ 0x30 }, 0, .rex  },
-    .{ .xor, .mr, &.{ .rm16, .r16    }, &.{ 0x31 }, 0, .none },
-    .{ .xor, .mr, &.{ .rm32, .r32    }, &.{ 0x31 }, 0, .none },
-    .{ .xor, .mr, &.{ .rm64, .r64    }, &.{ 0x31 }, 0, .long },
-    .{ .xor, .rm, &.{ .r8,   .rm8    }, &.{ 0x32 }, 0, .none },
-    .{ .xor, .rm, &.{ .r8,   .rm8    }, &.{ 0x32 }, 0, .rex  },
-    .{ .xor, .rm, &.{ .r16,  .rm16   }, &.{ 0x33 }, 0, .none },
-    .{ .xor, .rm, &.{ .r32,  .rm32   }, &.{ 0x33 }, 0, .none },
-    .{ .xor, .rm, &.{ .r64,  .rm64   }, &.{ 0x33 }, 0, .long },
+    .{ .xchg, .o,  &.{ .ax,   .r16  }, &.{ 0x90 }, 0, .none, .none },
+    .{ .xchg, .o,  &.{ .r16,  .ax   }, &.{ 0x90 }, 0, .none, .none },
+    .{ .xchg, .o,  &.{ .eax,  .r32  }, &.{ 0x90 }, 0, .none, .none },
+    .{ .xchg, .o,  &.{ .rax,  .r64  }, &.{ 0x90 }, 0, .long, .none },
+    .{ .xchg, .o,  &.{ .r32,  .eax  }, &.{ 0x90 }, 0, .none, .none },
+    .{ .xchg, .o,  &.{ .r64,  .rax  }, &.{ 0x90 }, 0, .long, .none },
+    .{ .xchg, .mr, &.{ .rm8,  .r8   }, &.{ 0x86 }, 0, .none, .none },
+    .{ .xchg, .mr, &.{ .rm8,  .r8   }, &.{ 0x86 }, 0, .rex,  .none },
+    .{ .xchg, .rm, &.{ .r8,   .rm8  }, &.{ 0x86 }, 0, .none, .none },
+    .{ .xchg, .rm, &.{ .r8,   .rm8  }, &.{ 0x86 }, 0, .rex,  .none },
+    .{ .xchg, .mr, &.{ .rm16, .r16  }, &.{ 0x87 }, 0, .none, .none },
+    .{ .xchg, .rm, &.{ .r16,  .rm16 }, &.{ 0x87 }, 0, .none, .none },
+    .{ .xchg, .mr, &.{ .rm32, .r32  }, &.{ 0x87 }, 0, .none, .none },
+    .{ .xchg, .mr, &.{ .rm64, .r64  }, &.{ 0x87 }, 0, .long, .none },
+    .{ .xchg, .rm, &.{ .r32,  .rm32 }, &.{ 0x87 }, 0, .none, .none },
+    .{ .xchg, .rm, &.{ .r64,  .rm64 }, &.{ 0x87 }, 0, .long, .none },
+
+    .{ .xor, .zi, &.{ .al,   .imm8   }, &.{ 0x34 }, 0, .none, .none },
+    .{ .xor, .zi, &.{ .ax,   .imm16  }, &.{ 0x35 }, 0, .none, .none },
+    .{ .xor, .zi, &.{ .eax,  .imm32  }, &.{ 0x35 }, 0, .none, .none },
+    .{ .xor, .zi, &.{ .rax,  .imm32s }, &.{ 0x35 }, 0, .long, .none },
+    .{ .xor, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 6, .none, .none },
+    .{ .xor, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 6, .rex,  .none },
+    .{ .xor, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 6, .none, .none },
+    .{ .xor, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 6, .none, .none },
+    .{ .xor, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 6, .long, .none },
+    .{ .xor, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 6, .none, .none },
+    .{ .xor, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 6, .none, .none },
+    .{ .xor, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 6, .long, .none },
+    .{ .xor, .mr, &.{ .rm8,  .r8     }, &.{ 0x30 }, 0, .none, .none },
+    .{ .xor, .mr, &.{ .rm8,  .r8     }, &.{ 0x30 }, 0, .rex,  .none },
+    .{ .xor, .mr, &.{ .rm16, .r16    }, &.{ 0x31 }, 0, .none, .none },
+    .{ .xor, .mr, &.{ .rm32, .r32    }, &.{ 0x31 }, 0, .none, .none },
+    .{ .xor, .mr, &.{ .rm64, .r64    }, &.{ 0x31 }, 0, .long, .none },
+    .{ .xor, .rm, &.{ .r8,   .rm8    }, &.{ 0x32 }, 0, .none, .none },
+    .{ .xor, .rm, &.{ .r8,   .rm8    }, &.{ 0x32 }, 0, .rex,  .none },
+    .{ .xor, .rm, &.{ .r16,  .rm16   }, &.{ 0x33 }, 0, .none, .none },
+    .{ .xor, .rm, &.{ .r32,  .rm32   }, &.{ 0x33 }, 0, .none, .none },
+    .{ .xor, .rm, &.{ .r64,  .rm64   }, &.{ 0x33 }, 0, .long, .none },
 
     // SSE
-    .{ .addss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .sse },
+    .{ .addss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .none, .sse },
 
-    .{ .andnps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x55 }, 0, .sse },
+    .{ .andnps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x55 }, 0, .none, .sse },
 
-    .{ .andps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x54 }, 0, .sse },
+    .{ .andps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x54 }, 0, .none, .sse },
 
-    .{ .cmpss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0xf3, 0x0f, 0xc2 }, 0, .sse },
+    .{ .cmpss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0xf3, 0x0f, 0xc2 }, 0, .none, .sse },
 
-    .{ .cvtsi2ss, .rm, &.{ .xmm, .rm32 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .sse },
-    .{ .cvtsi2ss, .rm, &.{ .xmm, .rm64 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .sse_long },
+    .{ .cvtsi2ss, .rm, &.{ .xmm, .rm32 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .none, .sse },
+    .{ .cvtsi2ss, .rm, &.{ .xmm, .rm64 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .long, .sse },
 
-    .{ .divss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .sse },
+    .{ .divss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .none, .sse },
 
-    .{ .maxss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5f }, 0, .sse },
+    .{ .maxss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5f }, 0, .none, .sse },
 
-    .{ .minss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5d }, 0, .sse },
+    .{ .minss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5d }, 0, .none, .sse },
 
-    .{ .movaps, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .sse },
-    .{ .movaps, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x29 }, 0, .sse },
+    .{ .movaps, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .none, .sse },
+    .{ .movaps, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x29 }, 0, .none, .sse },
 
-    .{ .movss, .rm, &.{ .xmm,     .xmm_m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .sse },
-    .{ .movss, .mr, &.{ .xmm_m32, .xmm     }, &.{ 0xf3, 0x0f, 0x11 }, 0, .sse },
+    .{ .movss, .rm, &.{ .xmm,     .xmm_m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .none, .sse },
+    .{ .movss, .mr, &.{ .xmm_m32, .xmm     }, &.{ 0xf3, 0x0f, 0x11 }, 0, .none, .sse },
 
-    .{ .movups, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .sse },
-    .{ .movups, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x11 }, 0, .sse },
+    .{ .movups, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .none, .sse },
+    .{ .movups, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x11 }, 0, .none, .sse },
 
-    .{ .mulss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .sse },
+    .{ .mulss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .none, .sse },
 
-    .{ .orps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .sse },
+    .{ .orps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .none, .sse },
 
-    .{ .subss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .sse },
+    .{ .subss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .none, .sse },
 
-    .{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{       0x0f, 0x51 }, 0, .sse },
-    .{ .sqrtss, .rm, &.{ .xmm, .xmm_m32  }, &.{ 0xf3, 0x0f, 0x51 }, 0, .sse },
+    .{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{       0x0f, 0x51 }, 0, .none, .sse },
+    .{ .sqrtss, .rm, &.{ .xmm, .xmm_m32  }, &.{ 0xf3, 0x0f, 0x51 }, 0, .none, .sse },
 
-    .{ .ucomiss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x0f, 0x2e }, 0, .sse },
+    .{ .ucomiss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x0f, 0x2e }, 0, .none, .sse },
 
-    .{ .xorps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x57 }, 0, .sse },
+    .{ .xorps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x57 }, 0, .none, .sse },
 
     // SSE2
-    .{ .addsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x58 }, 0, .sse2 },
+    .{ .addsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x58 }, 0, .none, .sse2 },
 
-    .{ .andnpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x55 }, 0, .sse2 },
+    .{ .andnpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x55 }, 0, .none, .sse2 },
 
-    .{ .andpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x54 }, 0, .sse2 },
+    .{ .andpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x54 }, 0, .none, .sse2 },
 
-    .{ .cmpsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0xf2, 0x0f, 0xc2 }, 0, .sse2 },
+    .{ .cmpsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0xf2, 0x0f, 0xc2 }, 0, .none, .sse2 },
 
-    .{ .cvtsd2ss, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .sse2 },
+    .{ .cvtsd2ss, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .none, .sse2 },
 
-    .{ .cvtsi2sd, .rm, &.{ .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .sse2 },
-    .{ .cvtsi2sd, .rm, &.{ .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .sse2_long },
+    .{ .cvtsi2sd, .rm, &.{ .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .none, .sse2 },
+    .{ .cvtsi2sd, .rm, &.{ .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .long, .sse2 },
 
-    .{ .cvtss2sd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5a }, 0, .sse2 },
+    .{ .cvtss2sd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5a }, 0, .none, .sse2 },
 
-    .{ .divsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5e }, 0, .sse2 },
+    .{ .divsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5e }, 0, .none, .sse2 },
 
-    .{ .maxsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5f }, 0, .sse2 },
+    .{ .maxsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5f }, 0, .none, .sse2 },
 
-    .{ .minsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .sse2 },
+    .{ .minsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .none, .sse2 },
 
-    .{ .movapd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .sse2 },
-    .{ .movapd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x29 }, 0, .sse2 },
+    .{ .movapd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .none, .sse2 },
+    .{ .movapd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x29 }, 0, .none, .sse2 },
 
-    .{ .movd, .rm, &.{ .xmm,  .rm32 }, &.{ 0x66, 0x0f, 0x6e }, 0, .sse2 },
-    .{ .movd, .mr, &.{ .rm32, .xmm  }, &.{ 0x66, 0x0f, 0x7e }, 0, .sse2 },
+    .{ .movd, .rm, &.{ .xmm,  .rm32 }, &.{ 0x66, 0x0f, 0x6e }, 0, .none, .sse2 },
+    .{ .movd, .mr, &.{ .rm32, .xmm  }, &.{ 0x66, 0x0f, 0x7e }, 0, .none, .sse2 },
 
-    .{ .movq, .rm, &.{ .xmm,  .rm64 }, &.{ 0x66, 0x0f, 0x6e }, 0, .sse2_long },
-    .{ .movq, .mr, &.{ .rm64, .xmm  }, &.{ 0x66, 0x0f, 0x7e }, 0, .sse2_long },
+    .{ .movq, .rm, &.{ .xmm,  .rm64 }, &.{ 0x66, 0x0f, 0x6e }, 0, .long, .sse2 },
+    .{ .movq, .mr, &.{ .rm64, .xmm  }, &.{ 0x66, 0x0f, 0x7e }, 0, .long, .sse2 },
 
-    .{ .movq, .rm, &.{ .xmm,     .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .sse2 },
-    .{ .movq, .mr, &.{ .xmm_m64, .xmm     }, &.{ 0x66, 0x0f, 0xd6 }, 0, .sse2 },
+    .{ .movq, .rm, &.{ .xmm,     .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .none, .sse2 },
+    .{ .movq, .mr, &.{ .xmm_m64, .xmm     }, &.{ 0x66, 0x0f, 0xd6 }, 0, .none, .sse2 },
 
-    .{ .movupd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .sse2 },
-    .{ .movupd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x11 }, 0, .sse2 },
+    .{ .movupd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .none, .sse2 },
+    .{ .movupd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x11 }, 0, .none, .sse2 },
 
-    .{ .mulsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x59 }, 0, .sse2 },
+    .{ .mulsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x59 }, 0, .none, .sse2 },
 
-    .{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .sse2 },
+    .{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .none, .sse2 },
 
-    .{ .pextrw, .mri, &.{ .r16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .sse2 },
+    .{ .pextrw, .mri, &.{ .r16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .none, .sse2 },
 
-    .{ .pinsrw, .rmi, &.{ .xmm, .rm16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .sse2 },
+    .{ .pinsrw, .rmi, &.{ .xmm, .rm16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .none, .sse2 },
 
-    .{ .sqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .sse2 },
-    .{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0xf2, 0x0f, 0x51 }, 0, .sse2 },
+    .{ .sqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .none, .sse2 },
+    .{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0xf2, 0x0f, 0x51 }, 0, .none, .sse2 },
 
-    .{ .subsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .sse2 },
+    .{ .subsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .none, .sse2 },
 
-    .{ .movsd, .rm, &.{ .xmm,     .xmm_m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .sse2 },
-    .{ .movsd, .mr, &.{ .xmm_m64, .xmm     }, &.{ 0xf2, 0x0f, 0x11 }, 0, .sse2 },
+    .{ .movsd, .rm, &.{ .xmm,     .xmm_m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .none, .sse2 },
+    .{ .movsd, .mr, &.{ .xmm_m64, .xmm     }, &.{ 0xf2, 0x0f, 0x11 }, 0, .none, .sse2 },
 
-    .{ .ucomisd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x2e }, 0, .sse2 },
+    .{ .ucomisd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x2e }, 0, .none, .sse2 },
 
-    .{ .xorpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .sse2 },
+    .{ .xorpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .none, .sse2 },
 
     // SSE4.1
-    .{ .pextrw, .mri, &.{ .rm16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .sse4_1 },
+    .{ .pextrw, .mri, &.{ .rm16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .none, .sse4_1 },
 
-    .{ .roundss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .sse4_1 },
-    .{ .roundsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .sse4_1 },
+    .{ .roundss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .none, .sse4_1 },
+    .{ .roundsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .none, .sse4_1 },
+
+    // F16C
+    .{ .vcvtph2ps, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128, .f16c },
+
+    .{ .vcvtps2ph, .mri, &.{ .xmm_m64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_128, .f16c },
 };
 // zig fmt: on
diff --git a/test/behavior/vector.zig b/test/behavior/vector.zig
index b0e717d131..41b0bfc39b 100644
--- a/test/behavior/vector.zig
+++ b/test/behavior/vector.zig
@@ -168,7 +168,8 @@ test "array to vector" {
 
 test "array to vector with element type coercion" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .f16c)) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From ae588a09f2c2146ada0f914c7d279f69a0d79396 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Fri, 5 May 2023 22:16:13 -0400
Subject: [PATCH 121/725] x86_64: implement f16 cmp

---
 src/arch/x86_64/CodeGen.zig   |   53 +-
 src/arch/x86_64/Encoding.zig  |  163 +++--
 src/arch/x86_64/Lower.zig     |   42 +-
 src/arch/x86_64/Mir.zig       |   80 +-
 src/arch/x86_64/encoder.zig   |   49 +-
 src/arch/x86_64/encodings.zig | 1297 +++++++++++++++++----------------
 6 files changed, 957 insertions(+), 727 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index b7fd81db68..d24428467a 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -6737,26 +6737,43 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
     defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
     const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
-    try self.genBinOpMir(switch (ty.zigTypeTag()) {
-        else => .cmp,
+    switch (ty.zigTypeTag()) {
+        else => try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv),
         .Float => switch (ty.floatBits(self.target.*)) {
-            32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
-                .ucomiss
-            else
-                return self.fail("TODO implement airCmp for {} without sse", .{
-                    ty.fmt(self.bin_file.options.module.?),
-                }),
-            64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
-                .ucomisd
-            else
-                return self.fail("TODO implement airCmp for {} without sse2", .{
-                    ty.fmt(self.bin_file.options.module.?),
-                }),
+            16 => if (self.hasFeature(.f16c)) {
+                const dst_reg = dst_mcv.getReg().?.to128();
+
+                const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
+                const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
+                defer self.register_manager.unlockReg(tmp_lock);
+
+                if (src_mcv.isRegister())
+                    try self.asmRegisterRegisterRegister(
+                        .vpunpcklwd,
+                        dst_reg,
+                        dst_reg,
+                        src_mcv.getReg().?.to128(),
+                    )
+                else
+                    try self.asmRegisterMemoryImmediate(
+                        .vpinsrw,
+                        dst_reg,
+                        src_mcv.mem(.word),
+                        Immediate.u(1),
+                    );
+                try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
+                try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);
+                try self.genBinOpMir(.ucomiss, ty, dst_mcv, .{ .register = tmp_reg });
+            } else return self.fail("TODO implement airCmp for {}", .{
+                ty.fmt(self.bin_file.options.module.?),
+            }),
+            32 => try self.genBinOpMir(.ucomiss, ty, dst_mcv, src_mcv),
+            64 => try self.genBinOpMir(.ucomisd, ty, dst_mcv, src_mcv),
             else => return self.fail("TODO implement airCmp for {}", .{
                 ty.fmt(self.bin_file.options.module.?),
             }),
         },
-    }, ty, dst_mcv, src_mcv);
+    }
 
     const signedness = if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned;
     const result = MCValue{
@@ -7834,8 +7851,8 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             else switch (abi_size) {
                 2 => return try self.asmRegisterRegisterImmediate(
                     if (dst_reg.class() == .floating_point) .pinsrw else .pextrw,
-                    registerAlias(dst_reg, abi_size),
-                    registerAlias(src_reg, abi_size),
+                    registerAlias(dst_reg, 4),
+                    registerAlias(src_reg, 4),
                     Immediate.u(0),
                 ),
                 4 => .movd,
@@ -8045,7 +8062,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
                 try self.asmMemoryRegisterImmediate(
                     .pextrw,
                     dst_mem,
-                    registerAlias(src_reg, abi_size),
+                    src_reg.to128(),
                     Immediate.u(0),
                 )
             else
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index 05c48ecddf..ada1e891fb 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -58,9 +58,9 @@ pub fn findByMnemonic(
     var shortest_len: ?usize = null;
     next: for (mnemonic_to_encodings_map[@enumToInt(mnemonic)]) |data| {
         switch (data.mode) {
-            .rex => if (!rex_required) continue,
-            .long => {},
-            else => if (rex_required) continue,
+            .none, .short => if (rex_required) continue,
+            .rex, .rex_short => if (!rex_required) continue,
+            else => {},
         }
         for (input_ops, data.ops) |input_op, data_op|
             if (!input_op.isSubset(data_op)) continue :next;
@@ -90,24 +90,26 @@ pub fn findByOpcode(opc: []const u8, prefixes: struct {
         if (!std.mem.eql(u8, opc, enc.opcode())) continue;
         if (prefixes.rex.w) {
             switch (data.mode) {
-                .short, .fpu, .sse, .sse2, .sse4_1, .none => continue,
-                .long, .sse_long, .sse2_long, .rex => {},
+                .none, .short, .rex, .rex_short, .vex_128, .vex_256 => continue,
+                .long, .vex_128_long, .vex_256_long => {},
             }
         } else if (prefixes.rex.present and !prefixes.rex.isSet()) {
             switch (data.mode) {
-                .rex => {},
+                .rex, .rex_short => {},
                 else => continue,
             }
         } else if (prefixes.legacy.prefix_66) {
-            switch (enc.operandBitSize()) {
-                16 => {},
-                else => continue,
+            switch (data.mode) {
+                .short, .rex_short => {},
+                .none, .rex, .vex_128, .vex_256 => continue,
+                .long, .vex_128_long, .vex_256_long => continue,
             }
         } else {
             switch (data.mode) {
-                .none => switch (enc.operandBitSize()) {
-                    16 => continue,
-                    else => {},
+                .none => switch (data.mode) {
+                    .short, .rex_short => continue,
+                    .none, .rex, .vex_128, .vex_256 => {},
+                    .long, .vex_128_long, .vex_256_long => {},
                 },
                 else => continue,
             }
@@ -131,28 +133,11 @@ pub fn mandatoryPrefix(encoding: *const Encoding) ?u8 {
 
 pub fn modRmExt(encoding: Encoding) u3 {
     return switch (encoding.data.op_en) {
-        .m, .mi, .m1, .mc => encoding.data.modrm_ext,
+        .m, .mi, .m1, .mc, .vmi => encoding.data.modrm_ext,
         else => unreachable,
     };
 }
 
-pub fn operandBitSize(encoding: Encoding) u64 {
-    return switch (encoding.data.mode) {
-        .short => 16,
-        .long => 64,
-        else => switch (encoding.data.op_en) {
-            .np => switch (encoding.data.ops[0]) {
-                .o16 => 16,
-                .o32 => 32,
-                .o64 => 64,
-                else => 32,
-            },
-            .td => encoding.data.ops[1].bitSize(),
-            else => encoding.data.ops[0].bitSize(),
-        },
-    };
-}
-
 pub fn format(
     encoding: Encoding,
     comptime fmt: []const u8,
@@ -220,17 +205,17 @@ pub fn format(
             };
             try writer.print("+{s} ", .{tag});
         },
-        .m, .mi, .m1, .mc => try writer.print("/{d} ", .{encoding.modRmExt()}),
-        .mr, .rm, .rmi, .mri, .mrc, .rrm, .rrmi => try writer.writeAll("/r "),
+        .m, .mi, .m1, .mc, .vmi => try writer.print("/{d} ", .{encoding.modRmExt()}),
+        .mr, .rm, .rmi, .mri, .mrc, .rvm, .rvmi => try writer.writeAll("/r "),
     }
 
     switch (encoding.data.op_en) {
-        .i, .d, .zi, .oi, .mi, .rmi, .mri, .rrmi => {
+        .i, .d, .zi, .oi, .mi, .rmi, .mri, .vmi, .rvmi => {
             const op = switch (encoding.data.op_en) {
                 .i, .d => encoding.data.ops[0],
                 .zi, .oi, .mi => encoding.data.ops[1],
-                .rmi, .mri => encoding.data.ops[2],
-                .rrmi => encoding.data.ops[3],
+                .rmi, .mri, .vmi => encoding.data.ops[2],
+                .rvmi => encoding.data.ops[3],
                 else => unreachable,
             };
             const tag = switch (op) {
@@ -245,7 +230,7 @@ pub fn format(
             };
             try writer.print("{s} ", .{tag});
         },
-        .np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc, .rrm => {},
+        .np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc, .rvm => {},
     }
 
     try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
@@ -315,8 +300,7 @@ pub const Mnemonic = enum {
     movaps, movss, movups,
     mulss,
     orps,
-    pextrw,
-    pinsrw,
+    pextrw, pinsrw,
     sqrtps,
     sqrtss,
     subss,
@@ -335,14 +319,25 @@ pub const Mnemonic = enum {
     movupd,
     mulsd,
     orpd,
-    sqrtpd,
-    sqrtsd,
+    pshufhw, pshuflw,
+    psrld, psrlq, psrlw,
+    punpckhbw, punpckhdq, punpckhqdq, punpckhwd,
+    punpcklbw, punpckldq, punpcklqdq, punpcklwd,
+    sqrtpd, sqrtsd,
     subsd,
     ucomisd,
     xorpd,
+    // SSE3
+    movddup, movshdup, movsldup,
     // SSE4.1
-    roundss,
-    roundsd,
+    roundsd, roundss,
+    // AVX
+    vmovddup, vmovshdup, vmovsldup,
+    vpextrw, vpinsrw,
+    vpshufhw, vpshuflw,
+    vpsrld, vpsrlq, vpsrlw,
+    vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd,
+    vpunpcklbw, vpunpckldq, vpunpcklqdq, vpunpcklwd,
     // F16C
     vcvtph2ps, vcvtps2ph,
     // zig fmt: on
@@ -357,7 +352,7 @@ pub const OpEn = enum {
     fd, td,
     m1, mc, mi, mr, rm,
     rmi, mri, mrc,
-    rrm, rrmi,
+    vmi, rvm, rvmi,
     // zig fmt: on
 };
 
@@ -372,6 +367,7 @@ pub const Op = enum {
     cl,
     r8, r16, r32, r64,
     rm8, rm16, rm32, rm64,
+    r32_m16, r64_m16,
     m8, m16, m32, m64, m80, m128,
     rel8, rel16, rel32,
     m,
@@ -450,16 +446,49 @@ pub const Op = enum {
         }
     }
 
-    pub fn bitSize(op: Op) u64 {
+    pub fn immBitSize(op: Op) u64 {
         return switch (op) {
             .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
+            .al, .cl, .r8, .rm8 => unreachable,
+            .ax, .r16, .rm16 => unreachable,
+            .eax, .r32, .rm32, .r32_m16 => unreachable,
+            .rax, .r64, .rm64, .r64_m16 => unreachable,
+            .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,
+            .m8, .m16, .m32, .m64, .m80, .m128 => unreachable,
             .unity => 1,
-            .imm8, .imm8s, .al, .cl, .r8, .m8, .rm8, .rel8 => 8,
-            .imm16, .imm16s, .ax, .r16, .m16, .rm16, .rel16 => 16,
-            .imm32, .imm32s, .eax, .r32, .m32, .rm32, .rel32, .xmm_m32 => 32,
-            .imm64, .rax, .r64, .m64, .rm64, .xmm_m64 => 64,
+            .imm8, .imm8s, .rel8 => 8,
+            .imm16, .imm16s, .rel16 => 16,
+            .imm32, .imm32s, .rel32 => 32,
+            .imm64 => 64,
+        };
+    }
+
+    pub fn regBitSize(op: Op) u64 {
+        return switch (op) {
+            .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
+            .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
+            .rel8, .rel16, .rel32 => unreachable,
+            .m8, .m16, .m32, .m64, .m80, .m128 => unreachable,
+            .al, .cl, .r8, .rm8 => 8,
+            .ax, .r16, .rm16 => 16,
+            .eax, .r32, .rm32, .r32_m16 => 32,
+            .rax, .r64, .rm64, .r64_m16 => 64,
+            .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
+        };
+    }
+
+    pub fn memBitSize(op: Op) u64 {
+        return switch (op) {
+            .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
+            .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
+            .rel8, .rel16, .rel32 => unreachable,
+            .al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64, .xmm => unreachable,
+            .m8, .rm8 => 8,
+            .m16, .rm16, .r32_m16, .r64_m16 => 16,
+            .m32, .rm32, .xmm_m32 => 32,
+            .m64, .rm64, .xmm_m64 => 64,
             .m80 => 80,
-            .m128, .xmm, .xmm_m128 => 128,
+            .m128, .xmm_m128 => 128,
         };
     }
 
@@ -482,6 +511,7 @@ pub const Op = enum {
             .al, .ax, .eax, .rax,
             .r8, .r16, .r32, .r64,
             .rm8, .rm16, .rm32, .rm64,
+            .r32_m16, .r64_m16,
             .xmm, .xmm_m32, .xmm_m64, .xmm_m128,
             => true,
             else => false,
@@ -506,6 +536,7 @@ pub const Op = enum {
         // zig fmt: off
         return switch (op) {
             .rm8, .rm16, .rm32, .rm64,
+            .r32_m16, .r64_m16,
             .m8, .m16, .m32, .m64, .m80, .m128,
             .m,
             .xmm_m32, .xmm_m64, .xmm_m128,
@@ -528,18 +559,12 @@ pub const Op = enum {
             .al, .ax, .eax, .rax, .cl => .general_purpose,
             .r8, .r16, .r32, .r64 => .general_purpose,
             .rm8, .rm16, .rm32, .rm64 => .general_purpose,
+            .r32_m16, .r64_m16 => .general_purpose,
             .sreg => .segment,
             .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .floating_point,
         };
     }
 
-    pub fn isFloatingPointRegister(op: Op) bool {
-        return switch (op) {
-            .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => true,
-            else => false,
-        };
-    }
-
     /// Given an operand `op` checks if `target` is a subset for the purposes of the encoding.
     pub fn isSubset(op: Op, target: Op) bool {
         switch (op) {
@@ -553,30 +578,27 @@ pub const Op = enum {
                 if (op.isRegister() and target.isRegister()) {
                     return switch (target) {
                         .cl, .al, .ax, .eax, .rax => op == target,
-                        else => op.class() == target.class() and switch (target.class()) {
-                            .floating_point => true,
-                            else => op.bitSize() == target.bitSize(),
-                        },
+                        else => op.class() == target.class() and op.regBitSize() == target.regBitSize(),
                     };
                 }
                 if (op.isMemory() and target.isMemory()) {
                     switch (target) {
                         .m => return true,
-                        else => return op.bitSize() == target.bitSize(),
+                        else => return op.memBitSize() == target.memBitSize(),
                     }
                 }
                 if (op.isImmediate() and target.isImmediate()) {
                     switch (target) {
-                        .imm64 => if (op.bitSize() <= 64) return true,
-                        .imm32s, .rel32 => if (op.bitSize() < 32 or (op.bitSize() == 32 and op.isSigned()))
+                        .imm64 => if (op.immBitSize() <= 64) return true,
+                        .imm32s, .rel32 => if (op.immBitSize() < 32 or (op.immBitSize() == 32 and op.isSigned()))
                             return true,
-                        .imm32 => if (op.bitSize() <= 32) return true,
-                        .imm16s, .rel16 => if (op.bitSize() < 16 or (op.bitSize() == 16 and op.isSigned()))
+                        .imm32 => if (op.immBitSize() <= 32) return true,
+                        .imm16s, .rel16 => if (op.immBitSize() < 16 or (op.immBitSize() == 16 and op.isSigned()))
                             return true,
-                        .imm16 => if (op.bitSize() <= 16) return true,
-                        .imm8s, .rel8 => if (op.bitSize() < 8 or (op.bitSize() == 8 and op.isSigned()))
+                        .imm16 => if (op.immBitSize() <= 16) return true,
+                        .imm8s, .rel8 => if (op.immBitSize() < 8 or (op.immBitSize() == 8 and op.isSigned()))
                             return true,
-                        .imm8 => if (op.bitSize() <= 8) return true,
+                        .imm8 => if (op.immBitSize() <= 8) return true,
                         else => {},
                     }
                     return op == target;
@@ -590,8 +612,9 @@ pub const Op = enum {
 pub const Mode = enum {
     none,
     short,
-    rex,
     long,
+    rex,
+    rex_short,
     vex_128,
     vex_128_long,
     vex_256,
@@ -600,9 +623,11 @@ pub const Mode = enum {
 
 pub const Feature = enum {
     none,
+    avx,
     f16c,
     sse,
     sse2,
+    sse3,
     sse4_1,
     x87,
 };
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index 9571f50e7c..d9482d4b39 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -108,12 +108,12 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
         .orps,
         .pextrw,
         .pinsrw,
-        .roundss,
         .sqrtps,
         .sqrtss,
         .subss,
         .ucomiss,
         .xorps,
+
         .addsd,
         .andnpd,
         .andpd,
@@ -127,13 +127,51 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
         .movsd,
         .mulsd,
         .orpd,
-        .roundsd,
+        .pshufhw,
+        .pshuflw,
+        .psrld,
+        .psrlq,
+        .psrlw,
+        .punpckhbw,
+        .punpckhdq,
+        .punpckhqdq,
+        .punpckhwd,
+        .punpcklbw,
+        .punpckldq,
+        .punpcklqdq,
+        .punpcklwd,
         .sqrtpd,
         .sqrtsd,
         .subsd,
         .ucomisd,
         .xorpd,
 
+        .movddup,
+        .movshdup,
+        .movsldup,
+
+        .roundsd,
+        .roundss,
+
+        .vmovddup,
+        .vmovshdup,
+        .vmovsldup,
+        .vpextrw,
+        .vpinsrw,
+        .vpshufhw,
+        .vpshuflw,
+        .vpsrld,
+        .vpsrlq,
+        .vpsrlw,
+        .vpunpckhbw,
+        .vpunpckhdq,
+        .vpunpckhqdq,
+        .vpunpckhwd,
+        .vpunpcklbw,
+        .vpunpckldq,
+        .vpunpcklqdq,
+        .vpunpcklwd,
+
         .vcvtph2ps,
         .vcvtps2ph,
         => try lower.mirGeneric(inst),
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index c4e19fdc0e..9e39d23bd4 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -196,8 +196,6 @@ pub const Inst = struct {
         pextrw,
         /// Insert word
         pinsrw,
-        /// Round scalar single-precision floating-point values
-        roundss,
         /// Square root of scalar single precision floating-point value
         sqrtps,
         /// Subtract scalar single-precision floating-point values
@@ -208,6 +206,7 @@ pub const Inst = struct {
         ucomiss,
         /// Bitwise logical xor of packed single precision floating-point values
         xorps,
+
         /// Add double precision floating point values
         addsd,
         /// Bitwise logical and not of packed double precision floating-point values
@@ -234,8 +233,32 @@ pub const Inst = struct {
         mulsd,
         /// Bitwise logical or of packed double precision floating-point values
         orpd,
-        /// Round scalar double-precision floating-point values
-        roundsd,
+        /// Shuffle packed high words
+        pshufhw,
+        /// Shuffle packed low words
+        pshuflw,
+        /// Shift packed data right logical
+        psrld,
+        /// Shift packed data right logical
+        psrlq,
+        /// Shift packed data right logical
+        psrlw,
+        /// Unpack high data
+        punpckhbw,
+        /// Unpack high data
+        punpckhdq,
+        /// Unpack high data
+        punpckhqdq,
+        /// Unpack high data
+        punpckhwd,
+        /// Unpack low data
+        punpcklbw,
+        /// Unpack low data
+        punpckldq,
+        /// Unpack low data
+        punpcklqdq,
+        /// Unpack low data
+        punpcklwd,
         /// Square root of double precision floating-point values
         sqrtpd,
         /// Square root of scalar double precision floating-point value
@@ -247,6 +270,55 @@ pub const Inst = struct {
         /// Bitwise logical xor of packed double precision floating-point values
         xorpd,
 
+        /// Replicate double floating-point values
+        movddup,
+        /// Replicate single floating-point values
+        movshdup,
+        /// Replicate single floating-point values
+        movsldup,
+
+        /// Round scalar double-precision floating-point values
+        roundsd,
+        /// Round scalar single-precision floating-point values
+        roundss,
+
+        /// Replicate double floating-point values
+        vmovddup,
+        /// Replicate single floating-point values
+        vmovshdup,
+        /// Replicate single floating-point values
+        vmovsldup,
+        /// Extract word
+        vpextrw,
+        /// Insert word
+        vpinsrw,
+        /// Shuffle packed high words
+        vpshufhw,
+        /// Shuffle packed low words
+        vpshuflw,
+        /// Shift packed data right logical
+        vpsrld,
+        /// Shift packed data right logical
+        vpsrlq,
+        /// Shift packed data right logical
+        vpsrlw,
+        /// Unpack high data
+        vpunpckhbw,
+        /// Unpack high data
+        vpunpckhdq,
+        /// Unpack high data
+        vpunpckhqdq,
+        /// Unpack high data
+        vpunpckhwd,
+        /// Unpack low data
+        vpunpcklbw,
+        /// Unpack low data
+        vpunpckldq,
+        /// Unpack low data
+        vpunpcklqdq,
+        /// Unpack low data
+        vpunpcklwd,
+
         /// Convert 16-bit floating-point values to single-precision floating-point values
         vcvtph2ps,
         /// Convert single-precision floating-point values to 16-bit floating-point values
diff --git a/src/arch/x86_64/encoder.zig b/src/arch/x86_64/encoder.zig
index 94f4eb56d5..495edb5f2a 100644
--- a/src/arch/x86_64/encoder.zig
+++ b/src/arch/x86_64/encoder.zig
@@ -151,15 +151,12 @@ pub const Instruction = struct {
                         moffs.offset,
                     }),
                 },
-                .imm => |imm| try writer.print("0x{x}", .{imm.asUnsigned(enc_op.bitSize())}),
+                .imm => |imm| try writer.print("0x{x}", .{imm.asUnsigned(enc_op.immBitSize())}),
             }
         }
 
         pub fn fmtPrint(op: Operand, enc_op: Encoding.Op) std.fmt.Formatter(fmt) {
-            return .{ .data = .{
-                .op = op,
-                .enc_op = enc_op,
-            } };
+            return .{ .data = .{ .op = op, .enc_op = enc_op } };
         }
     };
 
@@ -210,7 +207,7 @@ pub const Instruction = struct {
         const data = enc.data;
 
         switch (data.mode) {
-            .none, .short, .rex, .long => {
+            .none, .short, .long, .rex, .rex_short => {
                 try inst.encodeLegacyPrefixes(encoder);
                 try inst.encodeMandatoryPrefix(encoder);
                 try inst.encodeRexPrefix(encoder);
@@ -232,15 +229,16 @@ pub const Instruction = struct {
             else => {
                 const mem_op = switch (data.op_en) {
                     .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],
-                    .rm, .rmi => inst.ops[1],
+                    .rm, .rmi, .vmi => inst.ops[1],
+                    .rvm, .rvmi => inst.ops[2],
                     else => unreachable,
                 };
                 switch (mem_op) {
                     .reg => |reg| {
                         const rm = switch (data.op_en) {
-                            .m, .mi, .m1, .mc => enc.modRmExt(),
+                            .m, .mi, .m1, .mc, .vmi => enc.modRmExt(),
                             .mr, .mri, .mrc => inst.ops[1].reg.lowEnc(),
-                            .rm, .rmi => inst.ops[0].reg.lowEnc(),
+                            .rm, .rmi, .rvm, .rvmi => inst.ops[0].reg.lowEnc(),
                             else => unreachable,
                         };
                         try encoder.modRm_direct(rm, reg.lowEnc());
@@ -259,7 +257,8 @@ pub const Instruction = struct {
 
                 switch (data.op_en) {
                     .mi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder),
-                    .rmi, .mri => try encodeImm(inst.ops[2].imm, data.ops[2], encoder),
+                    .rmi, .mri, .vmi => try encodeImm(inst.ops[2].imm, data.ops[2], encoder),
+                    .rvmi => try encodeImm(inst.ops[3].imm, data.ops[3], encoder),
                     else => {},
                 }
             },
@@ -291,11 +290,9 @@ pub const Instruction = struct {
             .rep, .repe, .repz => legacy.prefix_f3 = true,
         }
 
-        if (data.mode == .none) {
-            const bit_size = enc.operandBitSize();
-            if (bit_size == 16) {
-                legacy.set16BitOverride();
-            }
+        switch (data.mode) {
+            .short, .rex_short => legacy.set16BitOverride(),
+            else => {},
         }
 
         const segment_override: ?Register = switch (op_en) {
@@ -318,7 +315,7 @@ pub const Instruction = struct {
                 }
             else
                 null,
-            .rrm, .rrmi => unreachable,
+            .vmi, .rvm, .rvmi => unreachable,
         };
         if (segment_override) |seg| {
             legacy.setSegmentOverride(seg);
@@ -353,7 +350,7 @@ pub const Instruction = struct {
                 rex.b = b_x_op.isBaseExtended();
                 rex.x = b_x_op.isIndexExtended();
             },
-            .rrm, .rrmi => unreachable,
+            .vmi, .rvm, .rvmi => unreachable,
         }
 
         try encoder.rex(rex);
@@ -375,18 +372,19 @@ pub const Instruction = struct {
         switch (op_en) {
             .np, .i, .zi, .fd, .td, .d => {},
             .o, .oi => vex.b = inst.ops[0].reg.isExtended(),
-            .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rrm, .rrmi => {
+            .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .vmi, .rvm, .rvmi => {
                 const r_op = switch (op_en) {
-                    .rm, .rmi, .rrm, .rrmi => inst.ops[0],
+                    .rm, .rmi, .rvm, .rvmi => inst.ops[0],
                     .mr, .mri, .mrc => inst.ops[1],
-                    else => .none,
+                    .m, .mi, .m1, .mc, .vmi => .none,
+                    else => unreachable,
                 };
                 vex.r = r_op.isBaseExtended();
 
                 const b_x_op = switch (op_en) {
-                    .rm, .rmi => inst.ops[1],
+                    .rm, .rmi, .vmi => inst.ops[1],
                     .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],
-                    .rrm, .rrmi => inst.ops[2],
+                    .rvm, .rvmi => inst.ops[2],
                     else => unreachable,
                 };
                 vex.b = b_x_op.isBaseExtended();
@@ -417,7 +415,8 @@ pub const Instruction = struct {
 
         switch (op_en) {
             else => {},
-            .rrm, .rrmi => vex.v = inst.ops[1].reg,
+            .vmi => vex.v = inst.ops[0].reg,
+            .rvm, .rvmi => vex.v = inst.ops[1].reg,
         }
 
         try encoder.vex(vex);
@@ -515,8 +514,8 @@ pub const Instruction = struct {
     }
 
     fn encodeImm(imm: Immediate, kind: Encoding.Op, encoder: anytype) !void {
-        const raw = imm.asUnsigned(kind.bitSize());
-        switch (kind.bitSize()) {
+        const raw = imm.asUnsigned(kind.immBitSize());
+        switch (kind.immBitSize()) {
             8 => try encoder.imm8(@intCast(u8, raw)),
             16 => try encoder.imm16(@intCast(u16, raw)),
             32 => try encoder.imm32(@intCast(u32, raw)),
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index 52b8cc29d6..5d2630e9a8 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -13,264 +13,264 @@ pub const Entry = struct { Mnemonic, OpEn, []const Op, []const u8, modrm_ext, Mo
 // zig fmt: off
 pub const table = [_]Entry{
     // General-purpose
-    .{ .adc, .zi, &.{ .al,   .imm8   }, &.{ 0x14 }, 0, .none, .none },
-    .{ .adc, .zi, &.{ .ax,   .imm16  }, &.{ 0x15 }, 0, .none, .none },
-    .{ .adc, .zi, &.{ .eax,  .imm32  }, &.{ 0x15 }, 0, .none, .none },
-    .{ .adc, .zi, &.{ .rax,  .imm32s }, &.{ 0x15 }, 0, .long, .none },
-    .{ .adc, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 2, .none, .none },
-    .{ .adc, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 2, .rex,  .none },
-    .{ .adc, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 2, .none, .none },
-    .{ .adc, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 2, .none, .none },
-    .{ .adc, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 2, .long, .none },
-    .{ .adc, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 2, .none, .none },
-    .{ .adc, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 2, .none, .none },
-    .{ .adc, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 2, .long, .none },
-    .{ .adc, .mr, &.{ .rm8,  .r8     }, &.{ 0x10 }, 0, .none, .none },
-    .{ .adc, .mr, &.{ .rm8,  .r8     }, &.{ 0x10 }, 0, .rex,  .none },
-    .{ .adc, .mr, &.{ .rm16, .r16    }, &.{ 0x11 }, 0, .none, .none },
-    .{ .adc, .mr, &.{ .rm32, .r32    }, &.{ 0x11 }, 0, .none, .none },
-    .{ .adc, .mr, &.{ .rm64, .r64    }, &.{ 0x11 }, 0, .long, .none },
-    .{ .adc, .rm, &.{ .r8,   .rm8    }, &.{ 0x12 }, 0, .none, .none },
-    .{ .adc, .rm, &.{ .r8,   .rm8    }, &.{ 0x12 }, 0, .rex,  .none },
-    .{ .adc, .rm, &.{ .r16,  .rm16   }, &.{ 0x13 }, 0, .none, .none },
-    .{ .adc, .rm, &.{ .r32,  .rm32   }, &.{ 0x13 }, 0, .none, .none },
-    .{ .adc, .rm, &.{ .r64,  .rm64   }, &.{ 0x13 }, 0, .long, .none },
+    .{ .adc, .zi, &.{ .al,   .imm8   }, &.{ 0x14 }, 0, .none,  .none },
+    .{ .adc, .zi, &.{ .ax,   .imm16  }, &.{ 0x15 }, 0, .short, .none },
+    .{ .adc, .zi, &.{ .eax,  .imm32  }, &.{ 0x15 }, 0, .none,  .none },
+    .{ .adc, .zi, &.{ .rax,  .imm32s }, &.{ 0x15 }, 0, .long,  .none },
+    .{ .adc, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 2, .none,  .none },
+    .{ .adc, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 2, .rex,   .none },
+    .{ .adc, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 2, .short, .none },
+    .{ .adc, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 2, .none,  .none },
+    .{ .adc, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 2, .long,  .none },
+    .{ .adc, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 2, .short, .none },
+    .{ .adc, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 2, .none,  .none },
+    .{ .adc, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 2, .long,  .none },
+    .{ .adc, .mr, &.{ .rm8,  .r8     }, &.{ 0x10 }, 0, .none,  .none },
+    .{ .adc, .mr, &.{ .rm8,  .r8     }, &.{ 0x10 }, 0, .rex,   .none },
+    .{ .adc, .mr, &.{ .rm16, .r16    }, &.{ 0x11 }, 0, .short, .none },
+    .{ .adc, .mr, &.{ .rm32, .r32    }, &.{ 0x11 }, 0, .none,  .none },
+    .{ .adc, .mr, &.{ .rm64, .r64    }, &.{ 0x11 }, 0, .long,  .none },
+    .{ .adc, .rm, &.{ .r8,   .rm8    }, &.{ 0x12 }, 0, .none,  .none },
+    .{ .adc, .rm, &.{ .r8,   .rm8    }, &.{ 0x12 }, 0, .rex,   .none },
+    .{ .adc, .rm, &.{ .r16,  .rm16   }, &.{ 0x13 }, 0, .short, .none },
+    .{ .adc, .rm, &.{ .r32,  .rm32   }, &.{ 0x13 }, 0, .none,  .none },
+    .{ .adc, .rm, &.{ .r64,  .rm64   }, &.{ 0x13 }, 0, .long,  .none },
 
-    .{ .add, .zi, &.{ .al,   .imm8   }, &.{ 0x04 }, 0, .none, .none },
-    .{ .add, .zi, &.{ .ax,   .imm16  }, &.{ 0x05 }, 0, .none, .none },
-    .{ .add, .zi, &.{ .eax,  .imm32  }, &.{ 0x05 }, 0, .none, .none },
-    .{ .add, .zi, &.{ .rax,  .imm32s }, &.{ 0x05 }, 0, .long, .none },
-    .{ .add, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 0, .none, .none },
-    .{ .add, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 0, .rex,  .none },
-    .{ .add, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 0, .none, .none },
-    .{ .add, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 0, .none, .none },
-    .{ .add, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 0, .long, .none },
-    .{ .add, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 0, .none, .none },
-    .{ .add, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 0, .none, .none },
-    .{ .add, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 0, .long, .none },
-    .{ .add, .mr, &.{ .rm8,  .r8     }, &.{ 0x00 }, 0, .none, .none },
-    .{ .add, .mr, &.{ .rm8,  .r8     }, &.{ 0x00 }, 0, .rex,  .none },
-    .{ .add, .mr, &.{ .rm16, .r16    }, &.{ 0x01 }, 0, .none, .none },
-    .{ .add, .mr, &.{ .rm32, .r32    }, &.{ 0x01 }, 0, .none, .none },
-    .{ .add, .mr, &.{ .rm64, .r64    }, &.{ 0x01 }, 0, .long, .none },
-    .{ .add, .rm, &.{ .r8,   .rm8    }, &.{ 0x02 }, 0, .none, .none },
-    .{ .add, .rm, &.{ .r8,   .rm8    }, &.{ 0x02 }, 0, .rex,  .none },
-    .{ .add, .rm, &.{ .r16,  .rm16   }, &.{ 0x03 }, 0, .none, .none },
-    .{ .add, .rm, &.{ .r32,  .rm32   }, &.{ 0x03 }, 0, .none, .none },
-    .{ .add, .rm, &.{ .r64,  .rm64   }, &.{ 0x03 }, 0, .long, .none },
+    .{ .add, .zi, &.{ .al,   .imm8   }, &.{ 0x04 }, 0, .none,  .none },
+    .{ .add, .zi, &.{ .ax,   .imm16  }, &.{ 0x05 }, 0, .short, .none },
+    .{ .add, .zi, &.{ .eax,  .imm32  }, &.{ 0x05 }, 0, .none,  .none },
+    .{ .add, .zi, &.{ .rax,  .imm32s }, &.{ 0x05 }, 0, .long,  .none },
+    .{ .add, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 0, .none,  .none },
+    .{ .add, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 0, .rex,   .none },
+    .{ .add, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 0, .short, .none },
+    .{ .add, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 0, .none,  .none },
+    .{ .add, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 0, .long,  .none },
+    .{ .add, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 0, .short, .none },
+    .{ .add, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 0, .none,  .none },
+    .{ .add, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 0, .long,  .none },
+    .{ .add, .mr, &.{ .rm8,  .r8     }, &.{ 0x00 }, 0, .none,  .none },
+    .{ .add, .mr, &.{ .rm8,  .r8     }, &.{ 0x00 }, 0, .rex,   .none },
+    .{ .add, .mr, &.{ .rm16, .r16    }, &.{ 0x01 }, 0, .short, .none },
+    .{ .add, .mr, &.{ .rm32, .r32    }, &.{ 0x01 }, 0, .none,  .none },
+    .{ .add, .mr, &.{ .rm64, .r64    }, &.{ 0x01 }, 0, .long,  .none },
+    .{ .add, .rm, &.{ .r8,   .rm8    }, &.{ 0x02 }, 0, .none,  .none },
+    .{ .add, .rm, &.{ .r8,   .rm8    }, &.{ 0x02 }, 0, .rex,   .none },
+    .{ .add, .rm, &.{ .r16,  .rm16   }, &.{ 0x03 }, 0, .short, .none },
+    .{ .add, .rm, &.{ .r32,  .rm32   }, &.{ 0x03 }, 0, .none,  .none },
+    .{ .add, .rm, &.{ .r64,  .rm64   }, &.{ 0x03 }, 0, .long,  .none },
 
-    .{ .@"and", .zi, &.{ .al,   .imm8   }, &.{ 0x24 }, 0, .none, .none },
-    .{ .@"and", .zi, &.{ .ax,   .imm16  }, &.{ 0x25 }, 0, .none, .none },
-    .{ .@"and", .zi, &.{ .eax,  .imm32  }, &.{ 0x25 }, 0, .none, .none },
-    .{ .@"and", .zi, &.{ .rax,  .imm32s }, &.{ 0x25 }, 0, .long, .none },
-    .{ .@"and", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 4, .none, .none },
-    .{ .@"and", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 4, .rex,  .none },
-    .{ .@"and", .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 4, .none, .none },
-    .{ .@"and", .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 4, .none, .none },
-    .{ .@"and", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 4, .long, .none },
-    .{ .@"and", .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 4, .none, .none },
-    .{ .@"and", .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 4, .none, .none },
-    .{ .@"and", .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 4, .long, .none },
-    .{ .@"and", .mr, &.{ .rm8,  .r8     }, &.{ 0x20 }, 0, .none, .none },
-    .{ .@"and", .mr, &.{ .rm8,  .r8     }, &.{ 0x20 }, 0, .rex,  .none },
-    .{ .@"and", .mr, &.{ .rm16, .r16    }, &.{ 0x21 }, 0, .none, .none },
-    .{ .@"and", .mr, &.{ .rm32, .r32    }, &.{ 0x21 }, 0, .none, .none },
-    .{ .@"and", .mr, &.{ .rm64, .r64    }, &.{ 0x21 }, 0, .long, .none },
-    .{ .@"and", .rm, &.{ .r8,   .rm8    }, &.{ 0x22 }, 0, .none, .none },
-    .{ .@"and", .rm, &.{ .r8,   .rm8    }, &.{ 0x22 }, 0, .rex,  .none },
-    .{ .@"and", .rm, &.{ .r16,  .rm16   }, &.{ 0x23 }, 0, .none, .none },
-    .{ .@"and", .rm, &.{ .r32,  .rm32   }, &.{ 0x23 }, 0, .none, .none },
-    .{ .@"and", .rm, &.{ .r64,  .rm64   }, &.{ 0x23 }, 0, .long, .none },
+    .{ .@"and", .zi, &.{ .al,   .imm8   }, &.{ 0x24 }, 0, .none,  .none },
+    .{ .@"and", .zi, &.{ .ax,   .imm16  }, &.{ 0x25 }, 0, .short, .none },
+    .{ .@"and", .zi, &.{ .eax,  .imm32  }, &.{ 0x25 }, 0, .none,  .none },
+    .{ .@"and", .zi, &.{ .rax,  .imm32s }, &.{ 0x25 }, 0, .long,  .none },
+    .{ .@"and", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 4, .none,  .none },
+    .{ .@"and", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 4, .rex,   .none },
+    .{ .@"and", .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 4, .short, .none },
+    .{ .@"and", .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 4, .none,  .none },
+    .{ .@"and", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 4, .long,  .none },
+    .{ .@"and", .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 4, .short, .none },
+    .{ .@"and", .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 4, .none,  .none },
+    .{ .@"and", .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 4, .long,  .none },
+    .{ .@"and", .mr, &.{ .rm8,  .r8     }, &.{ 0x20 }, 0, .none,  .none },
+    .{ .@"and", .mr, &.{ .rm8,  .r8     }, &.{ 0x20 }, 0, .rex,   .none },
+    .{ .@"and", .mr, &.{ .rm16, .r16    }, &.{ 0x21 }, 0, .short, .none },
+    .{ .@"and", .mr, &.{ .rm32, .r32    }, &.{ 0x21 }, 0, .none,  .none },
+    .{ .@"and", .mr, &.{ .rm64, .r64    }, &.{ 0x21 }, 0, .long,  .none },
+    .{ .@"and", .rm, &.{ .r8,   .rm8    }, &.{ 0x22 }, 0, .none,  .none },
+    .{ .@"and", .rm, &.{ .r8,   .rm8    }, &.{ 0x22 }, 0, .rex,   .none },
+    .{ .@"and", .rm, &.{ .r16,  .rm16   }, &.{ 0x23 }, 0, .short, .none },
+    .{ .@"and", .rm, &.{ .r32,  .rm32   }, &.{ 0x23 }, 0, .none,  .none },
+    .{ .@"and", .rm, &.{ .r64,  .rm64   }, &.{ 0x23 }, 0, .long,  .none },
 
-    .{ .bsf, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbc }, 0, .none, .none },
-    .{ .bsf, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbc }, 0, .none, .none },
-    .{ .bsf, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbc }, 0, .long, .none },
+    .{ .bsf, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbc }, 0, .short, .none },
+    .{ .bsf, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbc }, 0, .none,  .none },
+    .{ .bsf, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbc }, 0, .long,  .none },
 
-    .{ .bsr, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbd }, 0, .none, .none },
-    .{ .bsr, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbd }, 0, .none, .none },
-    .{ .bsr, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbd }, 0, .long, .none },
+    .{ .bsr, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbd }, 0, .short, .none },
+    .{ .bsr, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbd }, 0, .none,  .none },
+    .{ .bsr, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbd }, 0, .long,  .none },
 
     .{ .bswap, .o, &.{ .r32 }, &.{ 0x0f, 0xc8 }, 0, .none, .none },
     .{ .bswap, .o, &.{ .r64 }, &.{ 0x0f, 0xc8 }, 0, .long, .none },
 
-    .{ .bt, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xa3 }, 0, .none, .none },
-    .{ .bt, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xa3 }, 0, .none, .none },
-    .{ .bt, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xa3 }, 0, .long, .none },
-    .{ .bt, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 4, .none, .none },
-    .{ .bt, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 4, .none, .none },
-    .{ .bt, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 4, .long, .none },
+    .{ .bt, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xa3 }, 0, .short, .none },
+    .{ .bt, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xa3 }, 0, .none,  .none },
+    .{ .bt, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xa3 }, 0, .long,  .none },
+    .{ .bt, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 4, .short, .none },
+    .{ .bt, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 4, .none,  .none },
+    .{ .bt, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 4, .long,  .none },
 
-    .{ .btc, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xbb }, 0, .none, .none },
-    .{ .btc, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xbb }, 0, .none, .none },
-    .{ .btc, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xbb }, 0, .long, .none },
-    .{ .btc, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 7, .none, .none },
-    .{ .btc, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 7, .none, .none },
-    .{ .btc, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 7, .long, .none },
+    .{ .btc, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xbb }, 0, .short, .none },
+    .{ .btc, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xbb }, 0, .none,  .none },
+    .{ .btc, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xbb }, 0, .long,  .none },
+    .{ .btc, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 7, .short, .none },
+    .{ .btc, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 7, .none,  .none },
+    .{ .btc, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 7, .long,  .none },
 
-    .{ .btr, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xb3 }, 0, .none, .none },
-    .{ .btr, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xb3 }, 0, .none, .none },
-    .{ .btr, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xb3 }, 0, .long, .none },
-    .{ .btr, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 6, .none, .none },
-    .{ .btr, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 6, .none, .none },
-    .{ .btr, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 6, .long, .none },
+    .{ .btr, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xb3 }, 0, .short, .none },
+    .{ .btr, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xb3 }, 0, .none,  .none },
+    .{ .btr, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xb3 }, 0, .long,  .none },
+    .{ .btr, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 6, .short, .none },
+    .{ .btr, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 6, .none,  .none },
+    .{ .btr, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 6, .long,  .none },
 
-    .{ .bts, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xab }, 0, .none, .none },
-    .{ .bts, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xab }, 0, .none, .none },
-    .{ .bts, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xab }, 0, .long, .none },
-    .{ .bts, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 5, .none, .none },
-    .{ .bts, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 5, .none, .none },
-    .{ .bts, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 5, .long, .none },
+    .{ .bts, .mr, &.{ .rm16, .r16  }, &.{ 0x0f, 0xab }, 0, .short, .none },
+    .{ .bts, .mr, &.{ .rm32, .r32  }, &.{ 0x0f, 0xab }, 0, .none,  .none },
+    .{ .bts, .mr, &.{ .rm64, .r64  }, &.{ 0x0f, 0xab }, 0, .long,  .none },
+    .{ .bts, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 5, .short, .none },
+    .{ .bts, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 5, .none,  .none },
+    .{ .bts, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 5, .long,  .none },
 
     // This is M encoding according to Intel, but D makes more sense here.
     .{ .call, .d, &.{ .rel32 }, &.{ 0xe8 }, 0, .none, .none },
     .{ .call, .m, &.{ .rm64  }, &.{ 0xff }, 2, .none, .none },
 
-    .{ .cbw,  .np, &.{ .o16 }, &.{ 0x98 }, 0, .none, .none },
-    .{ .cwde, .np, &.{ .o32 }, &.{ 0x98 }, 0, .none, .none },
-    .{ .cdqe, .np, &.{ .o64 }, &.{ 0x98 }, 0, .long, .none },
+    .{ .cbw,  .np, &.{ .o16 }, &.{ 0x98 }, 0, .short, .none },
+    .{ .cwde, .np, &.{ .o32 }, &.{ 0x98 }, 0, .none,  .none },
+    .{ .cdqe, .np, &.{ .o64 }, &.{ 0x98 }, 0, .long,  .none },
 
-    .{ .cwd, .np, &.{ .o16 }, &.{ 0x99 }, 0, .none, .none },
-    .{ .cdq, .np, &.{ .o32 }, &.{ 0x99 }, 0, .none, .none },
-    .{ .cqo, .np, &.{ .o64 }, &.{ 0x99 }, 0, .long, .none },
+    .{ .cwd, .np, &.{ .o16 }, &.{ 0x99 }, 0, .short, .none },
+    .{ .cdq, .np, &.{ .o32 }, &.{ 0x99 }, 0, .none,  .none },
+    .{ .cqo, .np, &.{ .o64 }, &.{ 0x99 }, 0, .long,  .none },
 
-    .{ .cmova,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
-    .{ .cmova,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
-    .{ .cmova,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },
-    .{ .cmovae,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
-    .{ .cmovae,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
-    .{ .cmovae,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
-    .{ .cmovb,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
-    .{ .cmovb,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
-    .{ .cmovb,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
-    .{ .cmovbe,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
-    .{ .cmovbe,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
-    .{ .cmovbe,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },
-    .{ .cmovc,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
-    .{ .cmovc,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
-    .{ .cmovc,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
-    .{ .cmove,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
-    .{ .cmove,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
-    .{ .cmove,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },
-    .{ .cmovg,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
-    .{ .cmovg,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
-    .{ .cmovg,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },
-    .{ .cmovge,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
-    .{ .cmovge,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
-    .{ .cmovge,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },
-    .{ .cmovl,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
-    .{ .cmovl,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
-    .{ .cmovl,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },
-    .{ .cmovle,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
-    .{ .cmovle,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
-    .{ .cmovle,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },
-    .{ .cmovna,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
-    .{ .cmovna,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
-    .{ .cmovna,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },
-    .{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
-    .{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
-    .{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
-    .{ .cmovnb,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
-    .{ .cmovnb,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
-    .{ .cmovnb,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
-    .{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
-    .{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
-    .{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },
-    .{ .cmovnc,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
-    .{ .cmovnc,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
-    .{ .cmovnc,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
-    .{ .cmovne,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
-    .{ .cmovne,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
-    .{ .cmovne,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },
-    .{ .cmovng,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
-    .{ .cmovng,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
-    .{ .cmovng,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },
-    .{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
-    .{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
-    .{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },
-    .{ .cmovnl,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
-    .{ .cmovnl,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
-    .{ .cmovnl,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },
-    .{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
-    .{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
-    .{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },
-    .{ .cmovno,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .none, .none },
-    .{ .cmovno,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none, .none },
-    .{ .cmovno,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long, .none },
-    .{ .cmovnp,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
-    .{ .cmovnp,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
-    .{ .cmovnp,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },
-    .{ .cmovns,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .none, .none },
-    .{ .cmovns,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none, .none },
-    .{ .cmovns,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long, .none },
-    .{ .cmovnz,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
-    .{ .cmovnz,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
-    .{ .cmovnz,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },
-    .{ .cmovo,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .none, .none },
-    .{ .cmovo,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none, .none },
-    .{ .cmovo,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long, .none },
-    .{ .cmovp,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
-    .{ .cmovp,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
-    .{ .cmovp,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },
-    .{ .cmovpe,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
-    .{ .cmovpe,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
-    .{ .cmovpe,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },
-    .{ .cmovpo,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
-    .{ .cmovpo,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
-    .{ .cmovpo,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },
-    .{ .cmovs,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .none, .none },
-    .{ .cmovs,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none, .none },
-    .{ .cmovs,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long, .none },
-    .{ .cmovz,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
-    .{ .cmovz,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
-    .{ .cmovz,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },
+    .{ .cmova,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .none },
+    .{ .cmova,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none,  .none },
+    .{ .cmova,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long,  .none },
+    .{ .cmovae,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
+    .{ .cmovae,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none,  .none },
+    .{ .cmovae,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long,  .none },
+    .{ .cmovb,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
+    .{ .cmovb,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none,  .none },
+    .{ .cmovb,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long,  .none },
+    .{ .cmovbe,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .none },
+    .{ .cmovbe,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none,  .none },
+    .{ .cmovbe,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long,  .none },
+    .{ .cmovc,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
+    .{ .cmovc,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none,  .none },
+    .{ .cmovc,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long,  .none },
+    .{ .cmove,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .none },
+    .{ .cmove,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none,  .none },
+    .{ .cmove,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long,  .none },
+    .{ .cmovg,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .none },
+    .{ .cmovg,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none,  .none },
+    .{ .cmovg,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long,  .none },
+    .{ .cmovge,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .none },
+    .{ .cmovge,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none,  .none },
+    .{ .cmovge,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long,  .none },
+    .{ .cmovl,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .none },
+    .{ .cmovl,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none,  .none },
+    .{ .cmovl,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long,  .none },
+    .{ .cmovle,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .none },
+    .{ .cmovle,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none,  .none },
+    .{ .cmovle,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long,  .none },
+    .{ .cmovna,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .none },
+    .{ .cmovna,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none,  .none },
+    .{ .cmovna,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long,  .none },
+    .{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
+    .{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none,  .none },
+    .{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long,  .none },
+    .{ .cmovnb,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
+    .{ .cmovnb,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none,  .none },
+    .{ .cmovnb,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long,  .none },
+    .{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .none },
+    .{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none,  .none },
+    .{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long,  .none },
+    .{ .cmovnc,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
+    .{ .cmovnc,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none,  .none },
+    .{ .cmovnc,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long,  .none },
+    .{ .cmovne,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .none },
+    .{ .cmovne,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none,  .none },
+    .{ .cmovne,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long,  .none },
+    .{ .cmovng,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .none },
+    .{ .cmovng,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none,  .none },
+    .{ .cmovng,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long,  .none },
+    .{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .none },
+    .{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none,  .none },
+    .{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long,  .none },
+    .{ .cmovnl,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .none },
+    .{ .cmovnl,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none,  .none },
+    .{ .cmovnl,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long,  .none },
+    .{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .none },
+    .{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none,  .none },
+    .{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long,  .none },
+    .{ .cmovno,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .short, .none },
+    .{ .cmovno,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none,  .none },
+    .{ .cmovno,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long,  .none },
+    .{ .cmovnp,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .none },
+    .{ .cmovnp,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none,  .none },
+    .{ .cmovnp,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long,  .none },
+    .{ .cmovns,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .short, .none },
+    .{ .cmovns,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none,  .none },
+    .{ .cmovns,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long,  .none },
+    .{ .cmovnz,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .none },
+    .{ .cmovnz,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none,  .none },
+    .{ .cmovnz,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long,  .none },
+    .{ .cmovo,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .short, .none },
+    .{ .cmovo,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none,  .none },
+    .{ .cmovo,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long,  .none },
+    .{ .cmovp,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .none },
+    .{ .cmovp,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none,  .none },
+    .{ .cmovp,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long,  .none },
+    .{ .cmovpe,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .none },
+    .{ .cmovpe,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none,  .none },
+    .{ .cmovpe,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long,  .none },
+    .{ .cmovpo,  .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .none },
+    .{ .cmovpo,  .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none,  .none },
+    .{ .cmovpo,  .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long,  .none },
+    .{ .cmovs,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .short, .none },
+    .{ .cmovs,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none,  .none },
+    .{ .cmovs,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long,  .none },
+    .{ .cmovz,   .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .none },
+    .{ .cmovz,   .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none,  .none },
+    .{ .cmovz,   .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long,  .none },
 
-    .{ .cmp, .zi, &.{ .al,   .imm8   }, &.{ 0x3c }, 0, .none, .none },
-    .{ .cmp, .zi, &.{ .ax,   .imm16  }, &.{ 0x3d }, 0, .none, .none },
-    .{ .cmp, .zi, &.{ .eax,  .imm32  }, &.{ 0x3d }, 0, .none, .none },
-    .{ .cmp, .zi, &.{ .rax,  .imm32s }, &.{ 0x3d }, 0, .long, .none },
-    .{ .cmp, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 7, .none, .none },
-    .{ .cmp, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 7, .rex,  .none },
-    .{ .cmp, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 7, .none, .none },
-    .{ .cmp, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 7, .none, .none },
-    .{ .cmp, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 7, .long, .none },
-    .{ .cmp, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 7, .none, .none },
-    .{ .cmp, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 7, .none, .none },
-    .{ .cmp, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 7, .long, .none },
-    .{ .cmp, .mr, &.{ .rm8,  .r8     }, &.{ 0x38 }, 0, .none, .none },
-    .{ .cmp, .mr, &.{ .rm8,  .r8     }, &.{ 0x38 }, 0, .rex,  .none },
-    .{ .cmp, .mr, &.{ .rm16, .r16    }, &.{ 0x39 }, 0, .none, .none },
-    .{ .cmp, .mr, &.{ .rm32, .r32    }, &.{ 0x39 }, 0, .none, .none },
-    .{ .cmp, .mr, &.{ .rm64, .r64    }, &.{ 0x39 }, 0, .long, .none },
-    .{ .cmp, .rm, &.{ .r8,   .rm8    }, &.{ 0x3a }, 0, .none, .none },
-    .{ .cmp, .rm, &.{ .r8,   .rm8    }, &.{ 0x3a }, 0, .rex,  .none },
-    .{ .cmp, .rm, &.{ .r16,  .rm16   }, &.{ 0x3b }, 0, .none, .none },
-    .{ .cmp, .rm, &.{ .r32,  .rm32   }, &.{ 0x3b }, 0, .none, .none },
-    .{ .cmp, .rm, &.{ .r64,  .rm64   }, &.{ 0x3b }, 0, .long, .none },
+    .{ .cmp, .zi, &.{ .al,   .imm8   }, &.{ 0x3c }, 0, .none,  .none },
+    .{ .cmp, .zi, &.{ .ax,   .imm16  }, &.{ 0x3d }, 0, .short, .none },
+    .{ .cmp, .zi, &.{ .eax,  .imm32  }, &.{ 0x3d }, 0, .none,  .none },
+    .{ .cmp, .zi, &.{ .rax,  .imm32s }, &.{ 0x3d }, 0, .long,  .none },
+    .{ .cmp, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 7, .none,  .none },
+    .{ .cmp, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 7, .rex,   .none },
+    .{ .cmp, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 7, .short, .none },
+    .{ .cmp, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 7, .none,  .none },
+    .{ .cmp, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 7, .long,  .none },
+    .{ .cmp, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 7, .short, .none },
+    .{ .cmp, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 7, .none,  .none },
+    .{ .cmp, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 7, .long,  .none },
+    .{ .cmp, .mr, &.{ .rm8,  .r8     }, &.{ 0x38 }, 0, .none,  .none },
+    .{ .cmp, .mr, &.{ .rm8,  .r8     }, &.{ 0x38 }, 0, .rex,   .none },
+    .{ .cmp, .mr, &.{ .rm16, .r16    }, &.{ 0x39 }, 0, .short, .none },
+    .{ .cmp, .mr, &.{ .rm32, .r32    }, &.{ 0x39 }, 0, .none,  .none },
+    .{ .cmp, .mr, &.{ .rm64, .r64    }, &.{ 0x39 }, 0, .long,  .none },
+    .{ .cmp, .rm, &.{ .r8,   .rm8    }, &.{ 0x3a }, 0, .none,  .none },
+    .{ .cmp, .rm, &.{ .r8,   .rm8    }, &.{ 0x3a }, 0, .rex,   .none },
+    .{ .cmp, .rm, &.{ .r16,  .rm16   }, &.{ 0x3b }, 0, .short, .none },
+    .{ .cmp, .rm, &.{ .r32,  .rm32   }, &.{ 0x3b }, 0, .none,  .none },
+    .{ .cmp, .rm, &.{ .r64,  .rm64   }, &.{ 0x3b }, 0, .long,  .none },
 
-    .{ .cmps,  .np, &.{ .m8,   .m8   }, &.{ 0xa6 }, 0, .none, .none },
-    .{ .cmps,  .np, &.{ .m16,  .m16  }, &.{ 0xa7 }, 0, .none, .none },
-    .{ .cmps,  .np, &.{ .m32,  .m32  }, &.{ 0xa7 }, 0, .none, .none },
-    .{ .cmps,  .np, &.{ .m64,  .m64  }, &.{ 0xa7 }, 0, .long, .none },
+    .{ .cmps,  .np, &.{ .m8,   .m8   }, &.{ 0xa6 }, 0, .none,  .none },
+    .{ .cmps,  .np, &.{ .m16,  .m16  }, &.{ 0xa7 }, 0, .short, .none },
+    .{ .cmps,  .np, &.{ .m32,  .m32  }, &.{ 0xa7 }, 0, .none,  .none },
+    .{ .cmps,  .np, &.{ .m64,  .m64  }, &.{ 0xa7 }, 0, .long,  .none },
 
     .{ .cmpsb, .np, &.{}, &.{ 0xa6 }, 0, .none,  .none },
     .{ .cmpsw, .np, &.{}, &.{ 0xa7 }, 0, .short, .none },
     .{ .cmpsd, .np, &.{}, &.{ 0xa7 }, 0, .none,  .none },
     .{ .cmpsq, .np, &.{}, &.{ 0xa7 }, 0, .long,  .none },
 
-    .{ .cmpxchg, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xb0 }, 0, .none, .none },
-    .{ .cmpxchg, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xb0 }, 0, .rex,  .none },
-    .{ .cmpxchg, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xb1 }, 0, .none, .none },
-    .{ .cmpxchg, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xb1 }, 0, .none, .none },
-    .{ .cmpxchg, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xb1 }, 0, .long, .none },
+    .{ .cmpxchg, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xb0 }, 0, .none,  .none },
+    .{ .cmpxchg, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xb0 }, 0, .rex,   .none },
+    .{ .cmpxchg, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xb1 }, 0, .short, .none },
+    .{ .cmpxchg, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xb1 }, 0, .none,  .none },
+    .{ .cmpxchg, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xb1 }, 0, .long,  .none },
 
     .{ .cmpxchg8b,  .m, &.{ .m64  }, &.{ 0x0f, 0xc7 }, 1, .none, .none },
     .{ .cmpxchg16b, .m, &.{ .m128 }, &.{ 0x0f, 0xc7 }, 1, .long, .none },
 
-    .{ .div, .m, &.{ .rm8  }, &.{ 0xf6 }, 6, .none, .none },
-    .{ .div, .m, &.{ .rm8  }, &.{ 0xf6 }, 6, .rex,  .none },
-    .{ .div, .m, &.{ .rm16 }, &.{ 0xf7 }, 6, .none, .none },
-    .{ .div, .m, &.{ .rm32 }, &.{ 0xf7 }, 6, .none, .none },
-    .{ .div, .m, &.{ .rm64 }, &.{ 0xf7 }, 6, .long, .none },
+    .{ .div, .m, &.{ .rm8  }, &.{ 0xf6 }, 6, .none,  .none },
+    .{ .div, .m, &.{ .rm8  }, &.{ 0xf6 }, 6, .rex,   .none },
+    .{ .div, .m, &.{ .rm16 }, &.{ 0xf7 }, 6, .short, .none },
+    .{ .div, .m, &.{ .rm32 }, &.{ 0xf7 }, 6, .none,  .none },
+    .{ .div, .m, &.{ .rm64 }, &.{ 0xf7 }, 6, .long,  .none },
 
     .{ .fisttp, .m, &.{ .m16 }, &.{ 0xdf }, 1, .none, .x87 },
     .{ .fisttp, .m, &.{ .m32 }, &.{ 0xdb }, 1, .none, .x87 },
@@ -280,26 +280,26 @@ pub const table = [_]Entry{
     .{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .none, .x87 },
     .{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .none, .x87 },
 
-    .{ .idiv, .m, &.{ .rm8  }, &.{ 0xf6 }, 7, .none, .none },
-    .{ .idiv, .m, &.{ .rm8  }, &.{ 0xf6 }, 7, .rex,  .none },
-    .{ .idiv, .m, &.{ .rm16 }, &.{ 0xf7 }, 7, .none, .none },
-    .{ .idiv, .m, &.{ .rm32 }, &.{ 0xf7 }, 7, .none, .none },
-    .{ .idiv, .m, &.{ .rm64 }, &.{ 0xf7 }, 7, .long, .none },
+    .{ .idiv, .m, &.{ .rm8  }, &.{ 0xf6 }, 7, .none,  .none },
+    .{ .idiv, .m, &.{ .rm8  }, &.{ 0xf6 }, 7, .rex,   .none },
+    .{ .idiv, .m, &.{ .rm16 }, &.{ 0xf7 }, 7, .short, .none },
+    .{ .idiv, .m, &.{ .rm32 }, &.{ 0xf7 }, 7, .none,  .none },
+    .{ .idiv, .m, &.{ .rm64 }, &.{ 0xf7 }, 7, .long,  .none },
 
-    .{ .imul, .m,   &.{ .rm8                 }, &.{ 0xf6       }, 5, .none, .none },
-    .{ .imul, .m,   &.{ .rm8                 }, &.{ 0xf6       }, 5, .rex,  .none },
-    .{ .imul, .m,   &.{ .rm16,               }, &.{ 0xf7       }, 5, .none, .none },
-    .{ .imul, .m,   &.{ .rm32,               }, &.{ 0xf7       }, 5, .none, .none },
-    .{ .imul, .m,   &.{ .rm64,               }, &.{ 0xf7       }, 5, .long, .none },
-    .{ .imul, .rm,  &.{ .r16,  .rm16,        }, &.{ 0x0f, 0xaf }, 0, .none, .none },
-    .{ .imul, .rm,  &.{ .r32,  .rm32,        }, &.{ 0x0f, 0xaf }, 0, .none, .none },
-    .{ .imul, .rm,  &.{ .r64,  .rm64,        }, &.{ 0x0f, 0xaf }, 0, .long, .none },
-    .{ .imul, .rmi, &.{ .r16,  .rm16, .imm8s }, &.{ 0x6b       }, 0, .none, .none },
-    .{ .imul, .rmi, &.{ .r32,  .rm32, .imm8s }, &.{ 0x6b       }, 0, .none, .none },
-    .{ .imul, .rmi, &.{ .r64,  .rm64, .imm8s }, &.{ 0x6b       }, 0, .long, .none },
-    .{ .imul, .rmi, &.{ .r16,  .rm16, .imm16 }, &.{ 0x69       }, 0, .none, .none },
-    .{ .imul, .rmi, &.{ .r32,  .rm32, .imm32 }, &.{ 0x69       }, 0, .none, .none },
-    .{ .imul, .rmi, &.{ .r64,  .rm64, .imm32 }, &.{ 0x69       }, 0, .long, .none },
+    .{ .imul, .m,   &.{ .rm8                 }, &.{ 0xf6       }, 5, .none,  .none },
+    .{ .imul, .m,   &.{ .rm8                 }, &.{ 0xf6       }, 5, .rex,   .none },
+    .{ .imul, .m,   &.{ .rm16,               }, &.{ 0xf7       }, 5, .short, .none },
+    .{ .imul, .m,   &.{ .rm32,               }, &.{ 0xf7       }, 5, .none,  .none },
+    .{ .imul, .m,   &.{ .rm64,               }, &.{ 0xf7       }, 5, .long,  .none },
+    .{ .imul, .rm,  &.{ .r16,  .rm16,        }, &.{ 0x0f, 0xaf }, 0, .short, .none },
+    .{ .imul, .rm,  &.{ .r32,  .rm32,        }, &.{ 0x0f, 0xaf }, 0, .none,  .none },
+    .{ .imul, .rm,  &.{ .r64,  .rm64,        }, &.{ 0x0f, 0xaf }, 0, .long,  .none },
+    .{ .imul, .rmi, &.{ .r16,  .rm16, .imm8s }, &.{ 0x6b       }, 0, .short, .none },
+    .{ .imul, .rmi, &.{ .r32,  .rm32, .imm8s }, &.{ 0x6b       }, 0, .none,  .none },
+    .{ .imul, .rmi, &.{ .r64,  .rm64, .imm8s }, &.{ 0x6b       }, 0, .long,  .none },
+    .{ .imul, .rmi, &.{ .r16,  .rm16, .imm16 }, &.{ 0x69       }, 0, .short, .none },
+    .{ .imul, .rmi, &.{ .r32,  .rm32, .imm32 }, &.{ 0x69       }, 0, .none,  .none },
+    .{ .imul, .rmi, &.{ .r64,  .rm64, .imm32 }, &.{ 0x69       }, 0, .long,  .none },
 
     .{ .int3, .np, &.{}, &.{ 0xcc }, 0, .none, .none },
 
@@ -338,281 +338,283 @@ pub const table = [_]Entry{
     .{ .jmp, .d, &.{ .rel32 }, &.{ 0xe9 }, 0, .none, .none },
     .{ .jmp, .m, &.{ .rm64  }, &.{ 0xff }, 4, .none, .none },
 
-    .{ .lea, .rm, &.{ .r16, .m }, &.{ 0x8d }, 0, .none, .none },
-    .{ .lea, .rm, &.{ .r32, .m }, &.{ 0x8d }, 0, .none, .none },
-    .{ .lea, .rm, &.{ .r64, .m }, &.{ 0x8d }, 0, .long, .none },
+    .{ .lea, .rm, &.{ .r16, .m }, &.{ 0x8d }, 0, .short, .none },
+    .{ .lea, .rm, &.{ .r32, .m }, &.{ 0x8d }, 0, .none,  .none },
+    .{ .lea, .rm, &.{ .r64, .m }, &.{ 0x8d }, 0, .long,  .none },
 
     .{ .lfence, .np, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none, .none },
 
-    .{ .lods,  .np, &.{ .m8  }, &.{ 0xac }, 0, .none, .none },
-    .{ .lods,  .np, &.{ .m16 }, &.{ 0xad }, 0, .none, .none },
-    .{ .lods,  .np, &.{ .m32 }, &.{ 0xad }, 0, .none, .none },
-    .{ .lods,  .np, &.{ .m64 }, &.{ 0xad }, 0, .long, .none },
+    .{ .lods,  .np, &.{ .m8  }, &.{ 0xac }, 0, .none,  .none },
+    .{ .lods,  .np, &.{ .m16 }, &.{ 0xad }, 0, .short, .none },
+    .{ .lods,  .np, &.{ .m32 }, &.{ 0xad }, 0, .none,  .none },
+    .{ .lods,  .np, &.{ .m64 }, &.{ 0xad }, 0, .long,  .none },
 
     .{ .lodsb, .np, &.{}, &.{ 0xac }, 0, .none,  .none },
     .{ .lodsw, .np, &.{}, &.{ 0xad }, 0, .short, .none },
     .{ .lodsd, .np, &.{}, &.{ 0xad }, 0, .none,  .none },
     .{ .lodsq, .np, &.{}, &.{ 0xad }, 0, .long,  .none },
 
-    .{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .none },
-    .{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .none },
-    .{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long, .none },
+    .{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .short, .none },
+    .{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none,  .none },
+    .{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long,  .none },
 
     .{ .mfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none, .none },
 
-    .{ .mov, .mr, &.{ .rm8,   .r8     }, &.{ 0x88 }, 0, .none, .none },
-    .{ .mov, .mr, &.{ .rm8,   .r8     }, &.{ 0x88 }, 0, .rex,  .none },
-    .{ .mov, .mr, &.{ .rm16,  .r16    }, &.{ 0x89 }, 0, .none, .none },
-    .{ .mov, .mr, &.{ .rm32,  .r32    }, &.{ 0x89 }, 0, .none, .none },
-    .{ .mov, .mr, &.{ .rm64,  .r64    }, &.{ 0x89 }, 0, .long, .none },
-    .{ .mov, .rm, &.{ .r8,    .rm8    }, &.{ 0x8a }, 0, .none, .none },
-    .{ .mov, .rm, &.{ .r8,    .rm8    }, &.{ 0x8a }, 0, .rex,  .none },
-    .{ .mov, .rm, &.{ .r16,   .rm16   }, &.{ 0x8b }, 0, .none, .none },
-    .{ .mov, .rm, &.{ .r32,   .rm32   }, &.{ 0x8b }, 0, .none, .none },
-    .{ .mov, .rm, &.{ .r64,   .rm64   }, &.{ 0x8b }, 0, .long, .none },
-    .{ .mov, .mr, &.{ .rm16,  .sreg   }, &.{ 0x8c }, 0, .none, .none },
-    .{ .mov, .mr, &.{ .rm64,  .sreg   }, &.{ 0x8c }, 0, .long, .none },
-    .{ .mov, .rm, &.{ .sreg,  .rm16   }, &.{ 0x8e }, 0, .none, .none },
-    .{ .mov, .rm, &.{ .sreg,  .rm64   }, &.{ 0x8e }, 0, .long, .none },
-    .{ .mov, .fd, &.{ .al,    .moffs  }, &.{ 0xa0 }, 0, .none, .none },
-    .{ .mov, .fd, &.{ .ax,    .moffs  }, &.{ 0xa1 }, 0, .none, .none },
-    .{ .mov, .fd, &.{ .eax,   .moffs  }, &.{ 0xa1 }, 0, .none, .none },
-    .{ .mov, .fd, &.{ .rax,   .moffs  }, &.{ 0xa1 }, 0, .long, .none },
-    .{ .mov, .td, &.{ .moffs, .al     }, &.{ 0xa2 }, 0, .none, .none },
-    .{ .mov, .td, &.{ .moffs, .ax     }, &.{ 0xa3 }, 0, .none, .none },
-    .{ .mov, .td, &.{ .moffs, .eax    }, &.{ 0xa3 }, 0, .none, .none },
-    .{ .mov, .td, &.{ .moffs, .rax    }, &.{ 0xa3 }, 0, .long, .none },
-    .{ .mov, .oi, &.{ .r8,    .imm8   }, &.{ 0xb0 }, 0, .none, .none },
-    .{ .mov, .oi, &.{ .r8,    .imm8   }, &.{ 0xb0 }, 0, .rex,  .none },
-    .{ .mov, .oi, &.{ .r16,   .imm16  }, &.{ 0xb8 }, 0, .none, .none },
-    .{ .mov, .oi, &.{ .r32,   .imm32  }, &.{ 0xb8 }, 0, .none, .none },
-    .{ .mov, .oi, &.{ .r64,   .imm64  }, &.{ 0xb8 }, 0, .long, .none },
-    .{ .mov, .mi, &.{ .rm8,   .imm8   }, &.{ 0xc6 }, 0, .none, .none },
-    .{ .mov, .mi, &.{ .rm8,   .imm8   }, &.{ 0xc6 }, 0, .rex,  .none },
-    .{ .mov, .mi, &.{ .rm16,  .imm16  }, &.{ 0xc7 }, 0, .none, .none },
-    .{ .mov, .mi, &.{ .rm32,  .imm32  }, &.{ 0xc7 }, 0, .none, .none },
-    .{ .mov, .mi, &.{ .rm64,  .imm32s }, &.{ 0xc7 }, 0, .long, .none },
+    .{ .mov, .mr, &.{ .rm8,     .r8      }, &.{ 0x88 }, 0, .none,  .none },
+    .{ .mov, .mr, &.{ .rm8,     .r8      }, &.{ 0x88 }, 0, .rex,   .none },
+    .{ .mov, .mr, &.{ .rm16,    .r16     }, &.{ 0x89 }, 0, .short, .none },
+    .{ .mov, .mr, &.{ .rm32,    .r32     }, &.{ 0x89 }, 0, .none,  .none },
+    .{ .mov, .mr, &.{ .rm64,    .r64     }, &.{ 0x89 }, 0, .long,  .none },
+    .{ .mov, .rm, &.{ .r8,      .rm8     }, &.{ 0x8a }, 0, .none,  .none },
+    .{ .mov, .rm, &.{ .r8,      .rm8     }, &.{ 0x8a }, 0, .rex,   .none },
+    .{ .mov, .rm, &.{ .r16,     .rm16    }, &.{ 0x8b }, 0, .short, .none },
+    .{ .mov, .rm, &.{ .r32,     .rm32    }, &.{ 0x8b }, 0, .none,  .none },
+    .{ .mov, .rm, &.{ .r64,     .rm64    }, &.{ 0x8b }, 0, .long,  .none },
+    .{ .mov, .mr, &.{ .rm16,    .sreg    }, &.{ 0x8c }, 0, .short, .none },
+    .{ .mov, .mr, &.{ .r32_m16, .sreg    }, &.{ 0x8c }, 0, .none,  .none },
+    .{ .mov, .mr, &.{ .r64_m16, .sreg    }, &.{ 0x8c }, 0, .long,  .none },
+    .{ .mov, .rm, &.{ .sreg,    .rm16    }, &.{ 0x8e }, 0, .short, .none },
+    .{ .mov, .rm, &.{ .sreg,    .r32_m16 }, &.{ 0x8e }, 0, .none,  .none },
+    .{ .mov, .rm, &.{ .sreg,    .r64_m16 }, &.{ 0x8e }, 0, .long,  .none },
+    .{ .mov, .fd, &.{ .al,      .moffs   }, &.{ 0xa0 }, 0, .none,  .none },
+    .{ .mov, .fd, &.{ .ax,      .moffs   }, &.{ 0xa1 }, 0, .none,  .none },
+    .{ .mov, .fd, &.{ .eax,     .moffs   }, &.{ 0xa1 }, 0, .none,  .none },
+    .{ .mov, .fd, &.{ .rax,     .moffs   }, &.{ 0xa1 }, 0, .long,  .none },
+    .{ .mov, .td, &.{ .moffs,   .al      }, &.{ 0xa2 }, 0, .none,  .none },
+    .{ .mov, .td, &.{ .moffs,   .ax      }, &.{ 0xa3 }, 0, .none,  .none },
+    .{ .mov, .td, &.{ .moffs,   .eax     }, &.{ 0xa3 }, 0, .none,  .none },
+    .{ .mov, .td, &.{ .moffs,   .rax     }, &.{ 0xa3 }, 0, .long,  .none },
+    .{ .mov, .oi, &.{ .r8,      .imm8    }, &.{ 0xb0 }, 0, .none,  .none },
+    .{ .mov, .oi, &.{ .r8,      .imm8    }, &.{ 0xb0 }, 0, .rex,   .none },
+    .{ .mov, .oi, &.{ .r16,     .imm16   }, &.{ 0xb8 }, 0, .short, .none },
+    .{ .mov, .oi, &.{ .r32,     .imm32   }, &.{ 0xb8 }, 0, .none,  .none },
+    .{ .mov, .oi, &.{ .r64,     .imm64   }, &.{ 0xb8 }, 0, .long,  .none },
+    .{ .mov, .mi, &.{ .rm8,     .imm8    }, &.{ 0xc6 }, 0, .none,  .none },
+    .{ .mov, .mi, &.{ .rm8,     .imm8    }, &.{ 0xc6 }, 0, .rex,   .none },
+    .{ .mov, .mi, &.{ .rm16,    .imm16   }, &.{ 0xc7 }, 0, .short, .none },
+    .{ .mov, .mi, &.{ .rm32,    .imm32   }, &.{ 0xc7 }, 0, .none,  .none },
+    .{ .mov, .mi, &.{ .rm64,    .imm32s  }, &.{ 0xc7 }, 0, .long,  .none },
 
-    .{ .movbe, .rm, &.{ .r16, .m16 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none, .none },
-    .{ .movbe, .rm, &.{ .r32, .m32 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none, .none },
-    .{ .movbe, .rm, &.{ .r64, .m64 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .long, .none },
-    .{ .movbe, .mr, &.{ .m16, .r16 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none, .none },
-    .{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none, .none },
-    .{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long, .none },
+    .{ .movbe, .rm, &.{ .r16, .m16 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .short, .none },
+    .{ .movbe, .rm, &.{ .r32, .m32 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none,  .none },
+    .{ .movbe, .rm, &.{ .r64, .m64 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .long,  .none },
+    .{ .movbe, .mr, &.{ .m16, .r16 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .short, .none },
+    .{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none,  .none },
+    .{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long,  .none },
 
-    .{ .movs,  .np, &.{ .m8,  .m8  }, &.{ 0xa4 }, 0, .none, .none },
-    .{ .movs,  .np, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .none, .none },
-    .{ .movs,  .np, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none, .none },
-    .{ .movs,  .np, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long, .none },
+    .{ .movs,  .np, &.{ .m8,  .m8  }, &.{ 0xa4 }, 0, .none,  .none },
+    .{ .movs,  .np, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .short, .none },
+    .{ .movs,  .np, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none,  .none },
+    .{ .movs,  .np, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long,  .none },
 
     .{ .movsb, .np, &.{}, &.{ 0xa4 }, 0, .none,  .none },
     .{ .movsw, .np, &.{}, &.{ 0xa5 }, 0, .short, .none },
     .{ .movsd, .np, &.{}, &.{ 0xa5 }, 0, .none,  .none },
     .{ .movsq, .np, &.{}, &.{ 0xa5 }, 0, .long,  .none },
 
-    .{ .movsx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xbe }, 0, .none, .none },
-    .{ .movsx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xbe }, 0, .rex,  .none },
-    .{ .movsx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xbe }, 0, .none, .none },
-    .{ .movsx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xbe }, 0, .rex,  .none },
-    .{ .movsx, .rm, &.{ .r64, .rm8  }, &.{ 0x0f, 0xbe }, 0, .long, .none },
-    .{ .movsx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xbf }, 0, .none, .none },
-    .{ .movsx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xbf }, 0, .long, .none },
+    .{ .movsx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xbe }, 0, .short,     .none },
+    .{ .movsx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xbe }, 0, .rex_short, .none },
+    .{ .movsx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xbe }, 0, .none,      .none },
+    .{ .movsx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xbe }, 0, .rex,       .none },
+    .{ .movsx, .rm, &.{ .r64, .rm8  }, &.{ 0x0f, 0xbe }, 0, .long,      .none },
+    .{ .movsx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xbf }, 0, .none,      .none },
+    .{ .movsx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xbf }, 0, .long,      .none },
 
     // This instruction is discouraged.
     .{ .movsxd, .rm, &.{ .r32, .rm32 }, &.{ 0x63 }, 0, .none, .none },
     .{ .movsxd, .rm, &.{ .r64, .rm32 }, &.{ 0x63 }, 0, .long, .none },
 
-    .{ .movzx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .none, .none },
-    .{ .movzx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .none, .none },
-    .{ .movzx, .rm, &.{ .r64, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .long, .none },
-    .{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .none, .none },
-    .{ .movzx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .long, .none },
+    .{ .movzx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .short, .none },
+    .{ .movzx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .none,  .none },
+    .{ .movzx, .rm, &.{ .r64, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .long,  .none },
+    .{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .none,  .none },
+    .{ .movzx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .long,  .none },
 
-    .{ .mul, .m, &.{ .rm8  }, &.{ 0xf6 }, 4, .none, .none },
-    .{ .mul, .m, &.{ .rm8  }, &.{ 0xf6 }, 4, .rex,  .none },
-    .{ .mul, .m, &.{ .rm16 }, &.{ 0xf7 }, 4, .none, .none },
-    .{ .mul, .m, &.{ .rm32 }, &.{ 0xf7 }, 4, .none, .none },
-    .{ .mul, .m, &.{ .rm64 }, &.{ 0xf7 }, 4, .long, .none },
+    .{ .mul, .m, &.{ .rm8  }, &.{ 0xf6 }, 4, .none,  .none },
+    .{ .mul, .m, &.{ .rm8  }, &.{ 0xf6 }, 4, .rex,   .none },
+    .{ .mul, .m, &.{ .rm16 }, &.{ 0xf7 }, 4, .short, .none },
+    .{ .mul, .m, &.{ .rm32 }, &.{ 0xf7 }, 4, .none,  .none },
+    .{ .mul, .m, &.{ .rm64 }, &.{ 0xf7 }, 4, .long,  .none },
 
-    .{ .neg, .m, &.{ .rm8  }, &.{ 0xf6 }, 3, .none, .none },
-    .{ .neg, .m, &.{ .rm8  }, &.{ 0xf6 }, 3, .rex,  .none },
-    .{ .neg, .m, &.{ .rm16 }, &.{ 0xf7 }, 3, .none, .none },
-    .{ .neg, .m, &.{ .rm32 }, &.{ 0xf7 }, 3, .none, .none },
-    .{ .neg, .m, &.{ .rm64 }, &.{ 0xf7 }, 3, .long, .none },
+    .{ .neg, .m, &.{ .rm8  }, &.{ 0xf6 }, 3, .none,  .none },
+    .{ .neg, .m, &.{ .rm8  }, &.{ 0xf6 }, 3, .rex,   .none },
+    .{ .neg, .m, &.{ .rm16 }, &.{ 0xf7 }, 3, .short, .none },
+    .{ .neg, .m, &.{ .rm32 }, &.{ 0xf7 }, 3, .none,  .none },
+    .{ .neg, .m, &.{ .rm64 }, &.{ 0xf7 }, 3, .long,  .none },
 
     .{ .nop, .np, &.{}, &.{ 0x90 }, 0, .none, .none },
 
-    .{ .not, .m, &.{ .rm8  }, &.{ 0xf6 }, 2, .none, .none },
-    .{ .not, .m, &.{ .rm8  }, &.{ 0xf6 }, 2, .rex,  .none },
-    .{ .not, .m, &.{ .rm16 }, &.{ 0xf7 }, 2, .none, .none },
-    .{ .not, .m, &.{ .rm32 }, &.{ 0xf7 }, 2, .none, .none },
-    .{ .not, .m, &.{ .rm64 }, &.{ 0xf7 }, 2, .long, .none },
+    .{ .not, .m, &.{ .rm8  }, &.{ 0xf6 }, 2, .none,  .none },
+    .{ .not, .m, &.{ .rm8  }, &.{ 0xf6 }, 2, .rex,   .none },
+    .{ .not, .m, &.{ .rm16 }, &.{ 0xf7 }, 2, .short, .none },
+    .{ .not, .m, &.{ .rm32 }, &.{ 0xf7 }, 2, .none,  .none },
+    .{ .not, .m, &.{ .rm64 }, &.{ 0xf7 }, 2, .long,  .none },
 
-    .{ .@"or", .zi, &.{ .al,   .imm8   }, &.{ 0x0c }, 0, .none, .none },
-    .{ .@"or", .zi, &.{ .ax,   .imm16  }, &.{ 0x0d }, 0, .none, .none },
-    .{ .@"or", .zi, &.{ .eax,  .imm32  }, &.{ 0x0d }, 0, .none, .none },
-    .{ .@"or", .zi, &.{ .rax,  .imm32s }, &.{ 0x0d }, 0, .long, .none },
-    .{ .@"or", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 1, .none, .none },
-    .{ .@"or", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 1, .rex,  .none },
-    .{ .@"or", .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 1, .none, .none },
-    .{ .@"or", .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 1, .none, .none },
-    .{ .@"or", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 1, .long, .none },
-    .{ .@"or", .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 1, .none, .none },
-    .{ .@"or", .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 1, .none, .none },
-    .{ .@"or", .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 1, .long, .none },
-    .{ .@"or", .mr, &.{ .rm8,  .r8     }, &.{ 0x08 }, 0, .none, .none },
-    .{ .@"or", .mr, &.{ .rm8,  .r8     }, &.{ 0x08 }, 0, .rex,  .none },
-    .{ .@"or", .mr, &.{ .rm16, .r16    }, &.{ 0x09 }, 0, .none, .none },
-    .{ .@"or", .mr, &.{ .rm32, .r32    }, &.{ 0x09 }, 0, .none, .none },
-    .{ .@"or", .mr, &.{ .rm64, .r64    }, &.{ 0x09 }, 0, .long, .none },
-    .{ .@"or", .rm, &.{ .r8,   .rm8    }, &.{ 0x0a }, 0, .none, .none },
-    .{ .@"or", .rm, &.{ .r8,   .rm8    }, &.{ 0x0a }, 0, .rex,  .none },
-    .{ .@"or", .rm, &.{ .r16,  .rm16   }, &.{ 0x0b }, 0, .none, .none },
-    .{ .@"or", .rm, &.{ .r32,  .rm32   }, &.{ 0x0b }, 0, .none, .none },
-    .{ .@"or", .rm, &.{ .r64,  .rm64   }, &.{ 0x0b }, 0, .long, .none },
+    .{ .@"or", .zi, &.{ .al,   .imm8   }, &.{ 0x0c }, 0, .none,  .none },
+    .{ .@"or", .zi, &.{ .ax,   .imm16  }, &.{ 0x0d }, 0, .short, .none },
+    .{ .@"or", .zi, &.{ .eax,  .imm32  }, &.{ 0x0d }, 0, .none,  .none },
+    .{ .@"or", .zi, &.{ .rax,  .imm32s }, &.{ 0x0d }, 0, .long,  .none },
+    .{ .@"or", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 1, .none,  .none },
+    .{ .@"or", .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 1, .rex,   .none },
+    .{ .@"or", .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 1, .short, .none },
+    .{ .@"or", .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 1, .none,  .none },
+    .{ .@"or", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 1, .long,  .none },
+    .{ .@"or", .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 1, .short, .none },
+    .{ .@"or", .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 1, .none,  .none },
+    .{ .@"or", .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 1, .long,  .none },
+    .{ .@"or", .mr, &.{ .rm8,  .r8     }, &.{ 0x08 }, 0, .none,  .none },
+    .{ .@"or", .mr, &.{ .rm8,  .r8     }, &.{ 0x08 }, 0, .rex,   .none },
+    .{ .@"or", .mr, &.{ .rm16, .r16    }, &.{ 0x09 }, 0, .short, .none },
+    .{ .@"or", .mr, &.{ .rm32, .r32    }, &.{ 0x09 }, 0, .none,  .none },
+    .{ .@"or", .mr, &.{ .rm64, .r64    }, &.{ 0x09 }, 0, .long,  .none },
+    .{ .@"or", .rm, &.{ .r8,   .rm8    }, &.{ 0x0a }, 0, .none,  .none },
+    .{ .@"or", .rm, &.{ .r8,   .rm8    }, &.{ 0x0a }, 0, .rex,   .none },
+    .{ .@"or", .rm, &.{ .r16,  .rm16   }, &.{ 0x0b }, 0, .short, .none },
+    .{ .@"or", .rm, &.{ .r32,  .rm32   }, &.{ 0x0b }, 0, .none,  .none },
+    .{ .@"or", .rm, &.{ .r64,  .rm64   }, &.{ 0x0b }, 0, .long,  .none },
 
-    .{ .pop, .o, &.{ .r16  }, &.{ 0x58 }, 0, .none, .none },
-    .{ .pop, .o, &.{ .r64  }, &.{ 0x58 }, 0, .none, .none },
-    .{ .pop, .m, &.{ .rm16 }, &.{ 0x8f }, 0, .none, .none },
-    .{ .pop, .m, &.{ .rm64 }, &.{ 0x8f }, 0, .none, .none },
+    .{ .pop, .o, &.{ .r16  }, &.{ 0x58 }, 0, .short, .none },
+    .{ .pop, .o, &.{ .r64  }, &.{ 0x58 }, 0, .none,  .none },
+    .{ .pop, .m, &.{ .rm16 }, &.{ 0x8f }, 0, .short, .none },
+    .{ .pop, .m, &.{ .rm64 }, &.{ 0x8f }, 0, .none,  .none },
 
-    .{ .popcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .none },
-    .{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .none },
-    .{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long, .none },
+    .{ .popcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .short, .none },
+    .{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none,  .none },
+    .{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long,  .none },
 
-    .{ .push, .o, &.{ .r16   }, &.{ 0x50 }, 0, .none, .none },
-    .{ .push, .o, &.{ .r64   }, &.{ 0x50 }, 0, .none, .none },
-    .{ .push, .m, &.{ .rm16  }, &.{ 0xff }, 6, .none, .none },
-    .{ .push, .m, &.{ .rm64  }, &.{ 0xff }, 6, .none, .none },
-    .{ .push, .i, &.{ .imm8  }, &.{ 0x6a }, 0, .none, .none },
-    .{ .push, .i, &.{ .imm16 }, &.{ 0x68 }, 0, .none, .none },
-    .{ .push, .i, &.{ .imm32 }, &.{ 0x68 }, 0, .none, .none },
+    .{ .push, .o, &.{ .r16   }, &.{ 0x50 }, 0, .short, .none },
+    .{ .push, .o, &.{ .r64   }, &.{ 0x50 }, 0, .none,  .none },
+    .{ .push, .m, &.{ .rm16  }, &.{ 0xff }, 6, .short, .none },
+    .{ .push, .m, &.{ .rm64  }, &.{ 0xff }, 6, .none,  .none },
+    .{ .push, .i, &.{ .imm8  }, &.{ 0x6a }, 0, .none,  .none },
+    .{ .push, .i, &.{ .imm16 }, &.{ 0x68 }, 0, .short, .none },
+    .{ .push, .i, &.{ .imm32 }, &.{ 0x68 }, 0, .none,  .none },
 
     .{ .ret, .np, &.{}, &.{ 0xc3 }, 0, .none, .none },
 
-    .{ .rcl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 2, .none, .none },
-    .{ .rcl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 2, .rex,  .none },
-    .{ .rcl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 2, .none, .none },
-    .{ .rcl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 2, .rex,  .none },
-    .{ .rcl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 2, .none, .none },
-    .{ .rcl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 2, .rex,  .none },
-    .{ .rcl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 2, .none, .none },
-    .{ .rcl, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 2, .none, .none },
-    .{ .rcl, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 2, .none, .none },
-    .{ .rcl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 2, .none, .none },
-    .{ .rcl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 2, .long, .none },
-    .{ .rcl, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 2, .none, .none },
-    .{ .rcl, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 2, .long, .none },
-    .{ .rcl, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 2, .none, .none },
-    .{ .rcl, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 2, .long, .none },
+    .{ .rcl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 2, .none,  .none },
+    .{ .rcl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 2, .rex,   .none },
+    .{ .rcl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 2, .none,  .none },
+    .{ .rcl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 2, .rex,   .none },
+    .{ .rcl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 2, .none,  .none },
+    .{ .rcl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 2, .rex,   .none },
+    .{ .rcl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 2, .short, .none },
+    .{ .rcl, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 2, .short, .none },
+    .{ .rcl, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 2, .short, .none },
+    .{ .rcl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 2, .none,  .none },
+    .{ .rcl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 2, .long,  .none },
+    .{ .rcl, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 2, .none,  .none },
+    .{ .rcl, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 2, .long,  .none },
+    .{ .rcl, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 2, .none,  .none },
+    .{ .rcl, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 2, .long,  .none },
 
-    .{ .rcr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 3, .none, .none },
-    .{ .rcr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 3, .rex,  .none },
-    .{ .rcr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 3, .none, .none },
-    .{ .rcr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 3, .rex,  .none },
-    .{ .rcr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 3, .none, .none },
-    .{ .rcr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 3, .rex,  .none },
-    .{ .rcr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 3, .none, .none },
-    .{ .rcr, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 3, .none, .none },
-    .{ .rcr, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 3, .none, .none },
-    .{ .rcr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 3, .none, .none },
-    .{ .rcr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 3, .long, .none },
-    .{ .rcr, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 3, .none, .none },
-    .{ .rcr, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 3, .long, .none },
-    .{ .rcr, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 3, .none, .none },
-    .{ .rcr, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 3, .long, .none },
+    .{ .rcr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 3, .none,  .none },
+    .{ .rcr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 3, .rex,   .none },
+    .{ .rcr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 3, .none,  .none },
+    .{ .rcr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 3, .rex,   .none },
+    .{ .rcr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 3, .none,  .none },
+    .{ .rcr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 3, .rex,   .none },
+    .{ .rcr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 3, .short, .none },
+    .{ .rcr, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 3, .short, .none },
+    .{ .rcr, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 3, .short, .none },
+    .{ .rcr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 3, .none,  .none },
+    .{ .rcr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 3, .long,  .none },
+    .{ .rcr, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 3, .none,  .none },
+    .{ .rcr, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 3, .long,  .none },
+    .{ .rcr, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 3, .none,  .none },
+    .{ .rcr, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 3, .long,  .none },
 
-    .{ .rol, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 0, .none, .none },
-    .{ .rol, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 0, .rex,  .none },
-    .{ .rol, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 0, .none, .none },
-    .{ .rol, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 0, .rex,  .none },
-    .{ .rol, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 0, .none, .none },
-    .{ .rol, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 0, .rex,  .none },
-    .{ .rol, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 0, .none, .none },
-    .{ .rol, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 0, .none, .none },
-    .{ .rol, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 0, .none, .none },
-    .{ .rol, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 0, .none, .none },
-    .{ .rol, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 0, .long, .none },
-    .{ .rol, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 0, .none, .none },
-    .{ .rol, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 0, .long, .none },
-    .{ .rol, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 0, .none, .none },
-    .{ .rol, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 0, .long, .none },
+    .{ .rol, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 0, .none,  .none },
+    .{ .rol, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 0, .rex,   .none },
+    .{ .rol, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 0, .none,  .none },
+    .{ .rol, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 0, .rex,   .none },
+    .{ .rol, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 0, .none,  .none },
+    .{ .rol, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 0, .rex,   .none },
+    .{ .rol, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 0, .short, .none },
+    .{ .rol, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 0, .short, .none },
+    .{ .rol, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 0, .short, .none },
+    .{ .rol, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 0, .none,  .none },
+    .{ .rol, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 0, .long,  .none },
+    .{ .rol, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 0, .none,  .none },
+    .{ .rol, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 0, .long,  .none },
+    .{ .rol, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 0, .none,  .none },
+    .{ .rol, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 0, .long,  .none },
 
-    .{ .ror, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 1, .none, .none },
-    .{ .ror, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 1, .rex,  .none },
-    .{ .ror, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 1, .none, .none },
-    .{ .ror, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 1, .rex,  .none },
-    .{ .ror, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 1, .none, .none },
-    .{ .ror, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 1, .rex,  .none },
-    .{ .ror, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 1, .none, .none },
-    .{ .ror, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 1, .none, .none },
-    .{ .ror, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 1, .none, .none },
-    .{ .ror, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 1, .none, .none },
-    .{ .ror, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 1, .long, .none },
-    .{ .ror, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 1, .none, .none },
-    .{ .ror, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 1, .long, .none },
-    .{ .ror, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 1, .none, .none },
-    .{ .ror, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 1, .long, .none },
+    .{ .ror, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 1, .none,  .none },
+    .{ .ror, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 1, .rex,   .none },
+    .{ .ror, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 1, .none,  .none },
+    .{ .ror, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 1, .rex,   .none },
+    .{ .ror, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 1, .none,  .none },
+    .{ .ror, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 1, .rex,   .none },
+    .{ .ror, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 1, .short, .none },
+    .{ .ror, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 1, .short, .none },
+    .{ .ror, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 1, .short, .none },
+    .{ .ror, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 1, .none,  .none },
+    .{ .ror, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 1, .long,  .none },
+    .{ .ror, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 1, .none,  .none },
+    .{ .ror, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 1, .long,  .none },
+    .{ .ror, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 1, .none,  .none },
+    .{ .ror, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 1, .long,  .none },
 
-    .{ .sal, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .none, .none },
-    .{ .sal, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .rex,  .none },
-    .{ .sal, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .none, .none },
-    .{ .sal, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none, .none },
-    .{ .sal, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long, .none },
-    .{ .sal, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .none, .none },
-    .{ .sal, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .rex,  .none },
-    .{ .sal, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 4, .none, .none },
-    .{ .sal, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 4, .none, .none },
-    .{ .sal, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 4, .long, .none },
-    .{ .sal, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .none, .none },
-    .{ .sal, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .rex,  .none },
-    .{ .sal, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 4, .none, .none },
-    .{ .sal, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 4, .none, .none },
-    .{ .sal, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 4, .long, .none },
+    .{ .sal, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .none,  .none },
+    .{ .sal, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .rex,   .none },
+    .{ .sal, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .short, .none },
+    .{ .sal, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none,  .none },
+    .{ .sal, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long,  .none },
+    .{ .sal, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .none,  .none },
+    .{ .sal, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .rex,   .none },
+    .{ .sal, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 4, .short, .none },
+    .{ .sal, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 4, .none,  .none },
+    .{ .sal, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 4, .long,  .none },
+    .{ .sal, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .none,  .none },
+    .{ .sal, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .rex,   .none },
+    .{ .sal, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 4, .short, .none },
+    .{ .sal, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 4, .none,  .none },
+    .{ .sal, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 4, .long,  .none },
 
-    .{ .sar, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 7, .none, .none },
-    .{ .sar, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 7, .rex,  .none },
-    .{ .sar, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 7, .none, .none },
-    .{ .sar, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 7, .none, .none },
-    .{ .sar, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 7, .long, .none },
-    .{ .sar, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 7, .none, .none },
-    .{ .sar, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 7, .rex,  .none },
-    .{ .sar, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 7, .none, .none },
-    .{ .sar, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 7, .none, .none },
-    .{ .sar, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 7, .long, .none },
-    .{ .sar, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 7, .none, .none },
-    .{ .sar, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 7, .rex,  .none },
-    .{ .sar, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 7, .none, .none },
-    .{ .sar, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 7, .none, .none },
-    .{ .sar, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 7, .long, .none },
+    .{ .sar, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 7, .none,  .none },
+    .{ .sar, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 7, .rex,   .none },
+    .{ .sar, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 7, .short, .none },
+    .{ .sar, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 7, .none,  .none },
+    .{ .sar, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 7, .long,  .none },
+    .{ .sar, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 7, .none,  .none },
+    .{ .sar, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 7, .rex,   .none },
+    .{ .sar, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 7, .short, .none },
+    .{ .sar, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 7, .none,  .none },
+    .{ .sar, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 7, .long,  .none },
+    .{ .sar, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 7, .none,  .none },
+    .{ .sar, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 7, .rex,   .none },
+    .{ .sar, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 7, .short, .none },
+    .{ .sar, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 7, .none,  .none },
+    .{ .sar, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 7, .long,  .none },
 
-    .{ .sbb, .zi, &.{ .al,   .imm8   }, &.{ 0x1c }, 0, .none, .none },
-    .{ .sbb, .zi, &.{ .ax,   .imm16  }, &.{ 0x1d }, 0, .none, .none },
-    .{ .sbb, .zi, &.{ .eax,  .imm32  }, &.{ 0x1d }, 0, .none, .none },
-    .{ .sbb, .zi, &.{ .rax,  .imm32s }, &.{ 0x1d }, 0, .long, .none },
-    .{ .sbb, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 3, .none, .none },
-    .{ .sbb, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 3, .rex,  .none },
-    .{ .sbb, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 3, .none, .none },
-    .{ .sbb, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 3, .none, .none },
-    .{ .sbb, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 3, .long, .none },
-    .{ .sbb, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 3, .none, .none },
-    .{ .sbb, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 3, .none, .none },
-    .{ .sbb, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 3, .long, .none },
-    .{ .sbb, .mr, &.{ .rm8,  .r8     }, &.{ 0x18 }, 0, .none, .none },
-    .{ .sbb, .mr, &.{ .rm8,  .r8     }, &.{ 0x18 }, 0, .rex,  .none },
-    .{ .sbb, .mr, &.{ .rm16, .r16    }, &.{ 0x19 }, 0, .none, .none },
-    .{ .sbb, .mr, &.{ .rm32, .r32    }, &.{ 0x19 }, 0, .none, .none },
-    .{ .sbb, .mr, &.{ .rm64, .r64    }, &.{ 0x19 }, 0, .long, .none },
-    .{ .sbb, .rm, &.{ .r8,   .rm8    }, &.{ 0x1a }, 0, .none, .none },
-    .{ .sbb, .rm, &.{ .r8,   .rm8    }, &.{ 0x1a }, 0, .rex,  .none },
-    .{ .sbb, .rm, &.{ .r16,  .rm16   }, &.{ 0x1b }, 0, .none, .none },
-    .{ .sbb, .rm, &.{ .r32,  .rm32   }, &.{ 0x1b }, 0, .none, .none },
-    .{ .sbb, .rm, &.{ .r64,  .rm64   }, &.{ 0x1b }, 0, .long, .none },
+    .{ .sbb, .zi, &.{ .al,   .imm8   }, &.{ 0x1c }, 0, .none,  .none },
+    .{ .sbb, .zi, &.{ .ax,   .imm16  }, &.{ 0x1d }, 0, .short, .none },
+    .{ .sbb, .zi, &.{ .eax,  .imm32  }, &.{ 0x1d }, 0, .none,  .none },
+    .{ .sbb, .zi, &.{ .rax,  .imm32s }, &.{ 0x1d }, 0, .long,  .none },
+    .{ .sbb, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 3, .none,  .none },
+    .{ .sbb, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 3, .rex,   .none },
+    .{ .sbb, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 3, .short, .none },
+    .{ .sbb, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 3, .none,  .none },
+    .{ .sbb, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 3, .long,  .none },
+    .{ .sbb, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 3, .short, .none },
+    .{ .sbb, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 3, .none,  .none },
+    .{ .sbb, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 3, .long,  .none },
+    .{ .sbb, .mr, &.{ .rm8,  .r8     }, &.{ 0x18 }, 0, .none,  .none },
+    .{ .sbb, .mr, &.{ .rm8,  .r8     }, &.{ 0x18 }, 0, .rex,   .none },
+    .{ .sbb, .mr, &.{ .rm16, .r16    }, &.{ 0x19 }, 0, .short, .none },
+    .{ .sbb, .mr, &.{ .rm32, .r32    }, &.{ 0x19 }, 0, .none,  .none },
+    .{ .sbb, .mr, &.{ .rm64, .r64    }, &.{ 0x19 }, 0, .long,  .none },
+    .{ .sbb, .rm, &.{ .r8,   .rm8    }, &.{ 0x1a }, 0, .none,  .none },
+    .{ .sbb, .rm, &.{ .r8,   .rm8    }, &.{ 0x1a }, 0, .rex,   .none },
+    .{ .sbb, .rm, &.{ .r16,  .rm16   }, &.{ 0x1b }, 0, .short, .none },
+    .{ .sbb, .rm, &.{ .r32,  .rm32   }, &.{ 0x1b }, 0, .none,  .none },
+    .{ .sbb, .rm, &.{ .r64,  .rm64   }, &.{ 0x1b }, 0, .long,  .none },
 
-    .{ .scas,  .np, &.{ .m8  }, &.{ 0xae }, 0, .none, .none },
-    .{ .scas,  .np, &.{ .m16 }, &.{ 0xaf }, 0, .none, .none },
-    .{ .scas,  .np, &.{ .m32 }, &.{ 0xaf }, 0, .none, .none },
-    .{ .scas,  .np, &.{ .m64 }, &.{ 0xaf }, 0, .long, .none },
+    .{ .scas,  .np, &.{ .m8  }, &.{ 0xae }, 0, .none,  .none },
+    .{ .scas,  .np, &.{ .m16 }, &.{ 0xaf }, 0, .short, .none },
+    .{ .scas,  .np, &.{ .m32 }, &.{ 0xaf }, 0, .none,  .none },
+    .{ .scas,  .np, &.{ .m64 }, &.{ 0xaf }, 0, .long,  .none },
 
     .{ .scasb, .np, &.{}, &.{ 0xae }, 0, .none,  .none },
     .{ .scasw, .np, &.{}, &.{ 0xaf }, 0, .short, .none },
@@ -682,153 +684,153 @@ pub const table = [_]Entry{
 
     .{ .sfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none, .none },
 
-    .{ .shl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .none, .none },
-    .{ .shl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .rex,  .none },
-    .{ .shl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .none, .none },
-    .{ .shl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none, .none },
-    .{ .shl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long, .none },
-    .{ .shl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .none, .none },
-    .{ .shl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .rex,  .none },
-    .{ .shl, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 4, .none, .none },
-    .{ .shl, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 4, .none, .none },
-    .{ .shl, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 4, .long, .none },
-    .{ .shl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .none, .none },
-    .{ .shl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .rex,  .none },
-    .{ .shl, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 4, .none, .none },
-    .{ .shl, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 4, .none, .none },
-    .{ .shl, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 4, .long, .none },
+    .{ .shl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .none,  .none },
+    .{ .shl, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 4, .rex,   .none },
+    .{ .shl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .short, .none },
+    .{ .shl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none,  .none },
+    .{ .shl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long,  .none },
+    .{ .shl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .none,  .none },
+    .{ .shl, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 4, .rex,   .none },
+    .{ .shl, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 4, .short, .none },
+    .{ .shl, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 4, .none,  .none },
+    .{ .shl, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 4, .long,  .none },
+    .{ .shl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .none,  .none },
+    .{ .shl, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 4, .rex,   .none },
+    .{ .shl, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 4, .short, .none },
+    .{ .shl, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 4, .none,  .none },
+    .{ .shl, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 4, .long,  .none },
 
-    .{ .shld, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none, .none },
-    .{ .shld, .mrc, &.{ .rm16, .r16, .cl   }, &.{ 0x0f, 0xa5 }, 0, .none, .none },
-    .{ .shld, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none, .none },
-    .{ .shld, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .long, .none },
-    .{ .shld, .mrc, &.{ .rm32, .r32, .cl   }, &.{ 0x0f, 0xa5 }, 0, .none, .none },
-    .{ .shld, .mrc, &.{ .rm64, .r64, .cl   }, &.{ 0x0f, 0xa5 }, 0, .long, .none },
+    .{ .shld, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .short, .none },
+    .{ .shld, .mrc, &.{ .rm16, .r16, .cl   }, &.{ 0x0f, 0xa5 }, 0, .short, .none },
+    .{ .shld, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none,  .none },
+    .{ .shld, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .long,  .none },
+    .{ .shld, .mrc, &.{ .rm32, .r32, .cl   }, &.{ 0x0f, 0xa5 }, 0, .none,  .none },
+    .{ .shld, .mrc, &.{ .rm64, .r64, .cl   }, &.{ 0x0f, 0xa5 }, 0, .long,  .none },
 
-    .{ .shr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 5, .none, .none },
-    .{ .shr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 5, .rex,  .none },
-    .{ .shr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 5, .none, .none },
-    .{ .shr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 5, .none, .none },
-    .{ .shr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 5, .long, .none },
-    .{ .shr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 5, .none, .none },
-    .{ .shr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 5, .rex,  .none },
-    .{ .shr, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 5, .none, .none },
-    .{ .shr, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 5, .none, .none },
-    .{ .shr, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 5, .long, .none },
-    .{ .shr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 5, .none, .none },
-    .{ .shr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 5, .rex,  .none },
-    .{ .shr, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 5, .none, .none },
-    .{ .shr, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 5, .none, .none },
-    .{ .shr, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 5, .long, .none },
+    .{ .shr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 5, .none,  .none },
+    .{ .shr, .m1, &.{ .rm8,  .unity }, &.{ 0xd0 }, 5, .rex,   .none },
+    .{ .shr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 5, .short, .none },
+    .{ .shr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 5, .none,  .none },
+    .{ .shr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 5, .long,  .none },
+    .{ .shr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 5, .none,  .none },
+    .{ .shr, .mc, &.{ .rm8,  .cl    }, &.{ 0xd2 }, 5, .rex,   .none },
+    .{ .shr, .mc, &.{ .rm16, .cl    }, &.{ 0xd3 }, 5, .short, .none },
+    .{ .shr, .mc, &.{ .rm32, .cl    }, &.{ 0xd3 }, 5, .none,  .none },
+    .{ .shr, .mc, &.{ .rm64, .cl    }, &.{ 0xd3 }, 5, .long,  .none },
+    .{ .shr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 5, .none,  .none },
+    .{ .shr, .mi, &.{ .rm8,  .imm8  }, &.{ 0xc0 }, 5, .rex,   .none },
+    .{ .shr, .mi, &.{ .rm16, .imm8  }, &.{ 0xc1 }, 5, .short, .none },
+    .{ .shr, .mi, &.{ .rm32, .imm8  }, &.{ 0xc1 }, 5, .none,  .none },
+    .{ .shr, .mi, &.{ .rm64, .imm8  }, &.{ 0xc1 }, 5, .long,  .none },
 
-    .{ .shrd, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xac }, 0, .none, .none },
-    .{ .shrd, .mrc, &.{ .rm16, .r16, .cl   }, &.{ 0x0f, 0xad }, 0, .none, .none },
-    .{ .shrd, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xac }, 0, .none, .none },
-    .{ .shrd, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xac }, 0, .long, .none },
-    .{ .shrd, .mrc, &.{ .rm32, .r32, .cl   }, &.{ 0x0f, 0xad }, 0, .none, .none },
-    .{ .shrd, .mrc, &.{ .rm64, .r64, .cl   }, &.{ 0x0f, 0xad }, 0, .long, .none },
+    .{ .shrd, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xac }, 0, .short, .none },
+    .{ .shrd, .mrc, &.{ .rm16, .r16, .cl   }, &.{ 0x0f, 0xad }, 0, .short, .none },
+    .{ .shrd, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xac }, 0, .none,  .none },
+    .{ .shrd, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xac }, 0, .long,  .none },
+    .{ .shrd, .mrc, &.{ .rm32, .r32, .cl   }, &.{ 0x0f, 0xad }, 0, .none,  .none },
+    .{ .shrd, .mrc, &.{ .rm64, .r64, .cl   }, &.{ 0x0f, 0xad }, 0, .long,  .none },
 
-    .{ .stos,  .np, &.{ .m8  }, &.{ 0xaa }, 0, .none, .none },
-    .{ .stos,  .np, &.{ .m16 }, &.{ 0xab }, 0, .none, .none },
-    .{ .stos,  .np, &.{ .m32 }, &.{ 0xab }, 0, .none, .none },
-    .{ .stos,  .np, &.{ .m64 }, &.{ 0xab }, 0, .long, .none },
+    .{ .stos,  .np, &.{ .m8  }, &.{ 0xaa }, 0, .none,  .none },
+    .{ .stos,  .np, &.{ .m16 }, &.{ 0xab }, 0, .short, .none },
+    .{ .stos,  .np, &.{ .m32 }, &.{ 0xab }, 0, .none,  .none },
+    .{ .stos,  .np, &.{ .m64 }, &.{ 0xab }, 0, .long,  .none },
 
     .{ .stosb, .np, &.{}, &.{ 0xaa }, 0, .none,  .none },
     .{ .stosw, .np, &.{}, &.{ 0xab }, 0, .short, .none },
     .{ .stosd, .np, &.{}, &.{ 0xab }, 0, .none,  .none },
     .{ .stosq, .np, &.{}, &.{ 0xab }, 0, .long,  .none },
 
-    .{ .sub, .zi, &.{ .al,   .imm8   }, &.{ 0x2c }, 0, .none, .none },
-    .{ .sub, .zi, &.{ .ax,   .imm16  }, &.{ 0x2d }, 0, .none, .none },
-    .{ .sub, .zi, &.{ .eax,  .imm32  }, &.{ 0x2d }, 0, .none, .none },
-    .{ .sub, .zi, &.{ .rax,  .imm32s }, &.{ 0x2d }, 0, .long, .none },
-    .{ .sub, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 5, .none, .none },
-    .{ .sub, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 5, .rex,  .none },
-    .{ .sub, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 5, .none, .none },
-    .{ .sub, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 5, .none, .none },
-    .{ .sub, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 5, .long, .none },
-    .{ .sub, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 5, .none, .none },
-    .{ .sub, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 5, .none, .none },
-    .{ .sub, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 5, .long, .none },
-    .{ .sub, .mr, &.{ .rm8,  .r8     }, &.{ 0x28 }, 0, .none, .none },
-    .{ .sub, .mr, &.{ .rm8,  .r8     }, &.{ 0x28 }, 0, .rex,  .none },
-    .{ .sub, .mr, &.{ .rm16, .r16    }, &.{ 0x29 }, 0, .none, .none },
-    .{ .sub, .mr, &.{ .rm32, .r32    }, &.{ 0x29 }, 0, .none, .none },
-    .{ .sub, .mr, &.{ .rm64, .r64    }, &.{ 0x29 }, 0, .long, .none },
-    .{ .sub, .rm, &.{ .r8,   .rm8    }, &.{ 0x2a }, 0, .none, .none },
-    .{ .sub, .rm, &.{ .r8,   .rm8    }, &.{ 0x2a }, 0, .rex,  .none },
-    .{ .sub, .rm, &.{ .r16,  .rm16   }, &.{ 0x2b }, 0, .none, .none },
-    .{ .sub, .rm, &.{ .r32,  .rm32   }, &.{ 0x2b }, 0, .none, .none },
-    .{ .sub, .rm, &.{ .r64,  .rm64   }, &.{ 0x2b }, 0, .long, .none },
+    .{ .sub, .zi, &.{ .al,   .imm8   }, &.{ 0x2c }, 0, .none,  .none },
+    .{ .sub, .zi, &.{ .ax,   .imm16  }, &.{ 0x2d }, 0, .short, .none },
+    .{ .sub, .zi, &.{ .eax,  .imm32  }, &.{ 0x2d }, 0, .none,  .none },
+    .{ .sub, .zi, &.{ .rax,  .imm32s }, &.{ 0x2d }, 0, .long,  .none },
+    .{ .sub, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 5, .none,  .none },
+    .{ .sub, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 5, .rex,   .none },
+    .{ .sub, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 5, .short, .none },
+    .{ .sub, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 5, .none,  .none },
+    .{ .sub, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 5, .long,  .none },
+    .{ .sub, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 5, .short, .none },
+    .{ .sub, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 5, .none,  .none },
+    .{ .sub, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 5, .long,  .none },
+    .{ .sub, .mr, &.{ .rm8,  .r8     }, &.{ 0x28 }, 0, .none,  .none },
+    .{ .sub, .mr, &.{ .rm8,  .r8     }, &.{ 0x28 }, 0, .rex,   .none },
+    .{ .sub, .mr, &.{ .rm16, .r16    }, &.{ 0x29 }, 0, .short, .none },
+    .{ .sub, .mr, &.{ .rm32, .r32    }, &.{ 0x29 }, 0, .none,  .none },
+    .{ .sub, .mr, &.{ .rm64, .r64    }, &.{ 0x29 }, 0, .long,  .none },
+    .{ .sub, .rm, &.{ .r8,   .rm8    }, &.{ 0x2a }, 0, .none,  .none },
+    .{ .sub, .rm, &.{ .r8,   .rm8    }, &.{ 0x2a }, 0, .rex,   .none },
+    .{ .sub, .rm, &.{ .r16,  .rm16   }, &.{ 0x2b }, 0, .short, .none },
+    .{ .sub, .rm, &.{ .r32,  .rm32   }, &.{ 0x2b }, 0, .none,  .none },
+    .{ .sub, .rm, &.{ .r64,  .rm64   }, &.{ 0x2b }, 0, .long,  .none },
 
     .{ .syscall, .np, &.{}, &.{ 0x0f, 0x05 }, 0, .none, .none },
 
-    .{ .@"test", .zi, &.{ .al,   .imm8   }, &.{ 0xa8 }, 0, .none, .none },
-    .{ .@"test", .zi, &.{ .ax,   .imm16  }, &.{ 0xa9 }, 0, .none, .none },
-    .{ .@"test", .zi, &.{ .eax,  .imm32  }, &.{ 0xa9 }, 0, .none, .none },
-    .{ .@"test", .zi, &.{ .rax,  .imm32s }, &.{ 0xa9 }, 0, .long, .none },
-    .{ .@"test", .mi, &.{ .rm8,  .imm8   }, &.{ 0xf6 }, 0, .none, .none },
-    .{ .@"test", .mi, &.{ .rm8,  .imm8   }, &.{ 0xf6 }, 0, .rex,  .none },
-    .{ .@"test", .mi, &.{ .rm16, .imm16  }, &.{ 0xf7 }, 0, .none, .none },
-    .{ .@"test", .mi, &.{ .rm32, .imm32  }, &.{ 0xf7 }, 0, .none, .none },
-    .{ .@"test", .mi, &.{ .rm64, .imm32s }, &.{ 0xf7 }, 0, .long, .none },
-    .{ .@"test", .mr, &.{ .rm8,  .r8     }, &.{ 0x84 }, 0, .none, .none },
-    .{ .@"test", .mr, &.{ .rm8,  .r8     }, &.{ 0x84 }, 0, .rex,  .none },
-    .{ .@"test", .mr, &.{ .rm16, .r16    }, &.{ 0x85 }, 0, .none, .none },
-    .{ .@"test", .mr, &.{ .rm32, .r32    }, &.{ 0x85 }, 0, .none, .none },
-    .{ .@"test", .mr, &.{ .rm64, .r64    }, &.{ 0x85 }, 0, .long, .none },
+    .{ .@"test", .zi, &.{ .al,   .imm8   }, &.{ 0xa8 }, 0, .none,  .none },
+    .{ .@"test", .zi, &.{ .ax,   .imm16  }, &.{ 0xa9 }, 0, .short, .none },
+    .{ .@"test", .zi, &.{ .eax,  .imm32  }, &.{ 0xa9 }, 0, .none,  .none },
+    .{ .@"test", .zi, &.{ .rax,  .imm32s }, &.{ 0xa9 }, 0, .long,  .none },
+    .{ .@"test", .mi, &.{ .rm8,  .imm8   }, &.{ 0xf6 }, 0, .none,  .none },
+    .{ .@"test", .mi, &.{ .rm8,  .imm8   }, &.{ 0xf6 }, 0, .rex,   .none },
+    .{ .@"test", .mi, &.{ .rm16, .imm16  }, &.{ 0xf7 }, 0, .short, .none },
+    .{ .@"test", .mi, &.{ .rm32, .imm32  }, &.{ 0xf7 }, 0, .none,  .none },
+    .{ .@"test", .mi, &.{ .rm64, .imm32s }, &.{ 0xf7 }, 0, .long,  .none },
+    .{ .@"test", .mr, &.{ .rm8,  .r8     }, &.{ 0x84 }, 0, .none,  .none },
+    .{ .@"test", .mr, &.{ .rm8,  .r8     }, &.{ 0x84 }, 0, .rex,   .none },
+    .{ .@"test", .mr, &.{ .rm16, .r16    }, &.{ 0x85 }, 0, .short, .none },
+    .{ .@"test", .mr, &.{ .rm32, .r32    }, &.{ 0x85 }, 0, .none,  .none },
+    .{ .@"test", .mr, &.{ .rm64, .r64    }, &.{ 0x85 }, 0, .long,  .none },
 
-    .{ .tzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .none },
-    .{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .none },
-    .{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long, .none },
+    .{ .tzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .short, .none },
+    .{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none,  .none },
+    .{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long,  .none },
 
     .{ .ud2, .np, &.{}, &.{ 0x0f, 0x0b }, 0, .none, .none },
 
-    .{ .xadd, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xc0 }, 0, .none, .none },
-    .{ .xadd, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xc0 }, 0, .rex,  .none },
-    .{ .xadd, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xc1 }, 0, .none, .none },
-    .{ .xadd, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xc1 }, 0, .none, .none },
-    .{ .xadd, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xc1 }, 0, .long, .none },
+    .{ .xadd, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xc0 }, 0, .none,  .none },
+    .{ .xadd, .mr, &.{ .rm8,  .r8  }, &.{ 0x0f, 0xc0 }, 0, .rex,   .none },
+    .{ .xadd, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xc1 }, 0, .short, .none },
+    .{ .xadd, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xc1 }, 0, .none,  .none },
+    .{ .xadd, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xc1 }, 0, .long,  .none },
 
-    .{ .xchg, .o,  &.{ .ax,   .r16  }, &.{ 0x90 }, 0, .none, .none },
-    .{ .xchg, .o,  &.{ .r16,  .ax   }, &.{ 0x90 }, 0, .none, .none },
-    .{ .xchg, .o,  &.{ .eax,  .r32  }, &.{ 0x90 }, 0, .none, .none },
-    .{ .xchg, .o,  &.{ .rax,  .r64  }, &.{ 0x90 }, 0, .long, .none },
-    .{ .xchg, .o,  &.{ .r32,  .eax  }, &.{ 0x90 }, 0, .none, .none },
-    .{ .xchg, .o,  &.{ .r64,  .rax  }, &.{ 0x90 }, 0, .long, .none },
-    .{ .xchg, .mr, &.{ .rm8,  .r8   }, &.{ 0x86 }, 0, .none, .none },
-    .{ .xchg, .mr, &.{ .rm8,  .r8   }, &.{ 0x86 }, 0, .rex,  .none },
-    .{ .xchg, .rm, &.{ .r8,   .rm8  }, &.{ 0x86 }, 0, .none, .none },
-    .{ .xchg, .rm, &.{ .r8,   .rm8  }, &.{ 0x86 }, 0, .rex,  .none },
-    .{ .xchg, .mr, &.{ .rm16, .r16  }, &.{ 0x87 }, 0, .none, .none },
-    .{ .xchg, .rm, &.{ .r16,  .rm16 }, &.{ 0x87 }, 0, .none, .none },
-    .{ .xchg, .mr, &.{ .rm32, .r32  }, &.{ 0x87 }, 0, .none, .none },
-    .{ .xchg, .mr, &.{ .rm64, .r64  }, &.{ 0x87 }, 0, .long, .none },
-    .{ .xchg, .rm, &.{ .r32,  .rm32 }, &.{ 0x87 }, 0, .none, .none },
-    .{ .xchg, .rm, &.{ .r64,  .rm64 }, &.{ 0x87 }, 0, .long, .none },
+    .{ .xchg, .o,  &.{ .ax,   .r16  }, &.{ 0x90 }, 0, .short, .none },
+    .{ .xchg, .o,  &.{ .r16,  .ax   }, &.{ 0x90 }, 0, .short, .none },
+    .{ .xchg, .o,  &.{ .eax,  .r32  }, &.{ 0x90 }, 0, .none,  .none },
+    .{ .xchg, .o,  &.{ .rax,  .r64  }, &.{ 0x90 }, 0, .long,  .none },
+    .{ .xchg, .o,  &.{ .r32,  .eax  }, &.{ 0x90 }, 0, .none,  .none },
+    .{ .xchg, .o,  &.{ .r64,  .rax  }, &.{ 0x90 }, 0, .long,  .none },
+    .{ .xchg, .mr, &.{ .rm8,  .r8   }, &.{ 0x86 }, 0, .none,  .none },
+    .{ .xchg, .mr, &.{ .rm8,  .r8   }, &.{ 0x86 }, 0, .rex,   .none },
+    .{ .xchg, .rm, &.{ .r8,   .rm8  }, &.{ 0x86 }, 0, .none,  .none },
+    .{ .xchg, .rm, &.{ .r8,   .rm8  }, &.{ 0x86 }, 0, .rex,   .none },
+    .{ .xchg, .mr, &.{ .rm16, .r16  }, &.{ 0x87 }, 0, .short, .none },
+    .{ .xchg, .rm, &.{ .r16,  .rm16 }, &.{ 0x87 }, 0, .short, .none },
+    .{ .xchg, .mr, &.{ .rm32, .r32  }, &.{ 0x87 }, 0, .none,  .none },
+    .{ .xchg, .mr, &.{ .rm64, .r64  }, &.{ 0x87 }, 0, .long,  .none },
+    .{ .xchg, .rm, &.{ .r32,  .rm32 }, &.{ 0x87 }, 0, .none,  .none },
+    .{ .xchg, .rm, &.{ .r64,  .rm64 }, &.{ 0x87 }, 0, .long,  .none },
 
-    .{ .xor, .zi, &.{ .al,   .imm8   }, &.{ 0x34 }, 0, .none, .none },
-    .{ .xor, .zi, &.{ .ax,   .imm16  }, &.{ 0x35 }, 0, .none, .none },
-    .{ .xor, .zi, &.{ .eax,  .imm32  }, &.{ 0x35 }, 0, .none, .none },
-    .{ .xor, .zi, &.{ .rax,  .imm32s }, &.{ 0x35 }, 0, .long, .none },
-    .{ .xor, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 6, .none, .none },
-    .{ .xor, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 6, .rex,  .none },
-    .{ .xor, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 6, .none, .none },
-    .{ .xor, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 6, .none, .none },
-    .{ .xor, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 6, .long, .none },
-    .{ .xor, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 6, .none, .none },
-    .{ .xor, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 6, .none, .none },
-    .{ .xor, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 6, .long, .none },
-    .{ .xor, .mr, &.{ .rm8,  .r8     }, &.{ 0x30 }, 0, .none, .none },
-    .{ .xor, .mr, &.{ .rm8,  .r8     }, &.{ 0x30 }, 0, .rex,  .none },
-    .{ .xor, .mr, &.{ .rm16, .r16    }, &.{ 0x31 }, 0, .none, .none },
-    .{ .xor, .mr, &.{ .rm32, .r32    }, &.{ 0x31 }, 0, .none, .none },
-    .{ .xor, .mr, &.{ .rm64, .r64    }, &.{ 0x31 }, 0, .long, .none },
-    .{ .xor, .rm, &.{ .r8,   .rm8    }, &.{ 0x32 }, 0, .none, .none },
-    .{ .xor, .rm, &.{ .r8,   .rm8    }, &.{ 0x32 }, 0, .rex,  .none },
-    .{ .xor, .rm, &.{ .r16,  .rm16   }, &.{ 0x33 }, 0, .none, .none },
-    .{ .xor, .rm, &.{ .r32,  .rm32   }, &.{ 0x33 }, 0, .none, .none },
-    .{ .xor, .rm, &.{ .r64,  .rm64   }, &.{ 0x33 }, 0, .long, .none },
+    .{ .xor, .zi, &.{ .al,   .imm8   }, &.{ 0x34 }, 0, .none,  .none },
+    .{ .xor, .zi, &.{ .ax,   .imm16  }, &.{ 0x35 }, 0, .short, .none },
+    .{ .xor, .zi, &.{ .eax,  .imm32  }, &.{ 0x35 }, 0, .none,  .none },
+    .{ .xor, .zi, &.{ .rax,  .imm32s }, &.{ 0x35 }, 0, .long,  .none },
+    .{ .xor, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 6, .none,  .none },
+    .{ .xor, .mi, &.{ .rm8,  .imm8   }, &.{ 0x80 }, 6, .rex,   .none },
+    .{ .xor, .mi, &.{ .rm16, .imm16  }, &.{ 0x81 }, 6, .short, .none },
+    .{ .xor, .mi, &.{ .rm32, .imm32  }, &.{ 0x81 }, 6, .none,  .none },
+    .{ .xor, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 6, .long,  .none },
+    .{ .xor, .mi, &.{ .rm16, .imm8s  }, &.{ 0x83 }, 6, .short, .none },
+    .{ .xor, .mi, &.{ .rm32, .imm8s  }, &.{ 0x83 }, 6, .none,  .none },
+    .{ .xor, .mi, &.{ .rm64, .imm8s  }, &.{ 0x83 }, 6, .long,  .none },
+    .{ .xor, .mr, &.{ .rm8,  .r8     }, &.{ 0x30 }, 0, .none,  .none },
+    .{ .xor, .mr, &.{ .rm8,  .r8     }, &.{ 0x30 }, 0, .rex,   .none },
+    .{ .xor, .mr, &.{ .rm16, .r16    }, &.{ 0x31 }, 0, .short, .none },
+    .{ .xor, .mr, &.{ .rm32, .r32    }, &.{ 0x31 }, 0, .none,  .none },
+    .{ .xor, .mr, &.{ .rm64, .r64    }, &.{ 0x31 }, 0, .long,  .none },
+    .{ .xor, .rm, &.{ .r8,   .rm8    }, &.{ 0x32 }, 0, .none,  .none },
+    .{ .xor, .rm, &.{ .r8,   .rm8    }, &.{ 0x32 }, 0, .rex,   .none },
+    .{ .xor, .rm, &.{ .r16,  .rm16   }, &.{ 0x33 }, 0, .short, .none },
+    .{ .xor, .rm, &.{ .r32,  .rm32   }, &.{ 0x33 }, 0, .none,  .none },
+    .{ .xor, .rm, &.{ .r64,  .rm64   }, &.{ 0x33 }, 0, .long,  .none },
 
     // SSE
     .{ .addss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .none, .sse },
@@ -911,9 +913,39 @@ pub const table = [_]Entry{
 
     .{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .none, .sse2 },
 
-    .{ .pextrw, .mri, &.{ .r16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .none, .sse2 },
+    .{ .pextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .none, .sse2 },
+    .{ .pextrw, .rmi, &.{ .r64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .long, .sse2 },
 
-    .{ .pinsrw, .rmi, &.{ .xmm, .rm16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .none, .sse2 },
+    .{ .pinsrw, .rmi, &.{ .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .none, .sse2 },
+
+    .{ .pshufhw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .none, .sse2 },
+
+    .{ .pshuflw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf2, 0x0f, 0x70 }, 0, .none, .sse2 },
+
+    .{ .psrld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .none, .sse2 },
+    .{ .psrld, .mi, &.{ .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x72 }, 2, .none, .sse2 },
+
+    .{ .psrlq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .none, .sse2 },
+    .{ .psrlq, .mi, &.{ .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x73 }, 2, .none, .sse2 },
+
+    .{ .psrlw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .none, .sse2 },
+    .{ .psrlw, .mi, &.{ .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x71 }, 2, .none, .sse2 },
+
+    .{ .punpckhbw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .none, .sse2 },
+
+    .{ .punpckhdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .none, .sse2 },
+
+    .{ .punpckhqdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6d }, 0, .none, .sse2 },
+
+    .{ .punpckhwd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .none, .sse2 },
+
+    .{ .punpcklbw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x60 }, 0, .none, .sse2 },
+
+    .{ .punpckldq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .none, .sse2 },
+
+    .{ .punpcklqdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .none, .sse2 },
+
+    .{ .punpcklwd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x61 }, 0, .none, .sse2 },
 
     .{ .sqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .none, .sse2 },
     .{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0xf2, 0x0f, 0x51 }, 0, .none, .sse2 },
@@ -927,12 +959,59 @@ pub const table = [_]Entry{
 
     .{ .xorpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .none, .sse2 },
 
+    // SSE3
+    .{ .movddup, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .none, .sse3 },
+
+    .{ .movshdup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x16 }, 0, .none, .sse3 },
+
+    .{ .movsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .none, .sse3 },
+
     // SSE4.1
-    .{ .pextrw, .mri, &.{ .rm16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .none, .sse4_1 },
+    .{ .pextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .none, .sse4_1 },
+    .{ .pextrw, .mri, &.{ .r64_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .long, .sse4_1 },
 
     .{ .roundss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .none, .sse4_1 },
     .{ .roundsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .none, .sse4_1 },
 
+    // AVX
+    .{ .vmovddup, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_128, .avx },
+
+    .{ .vmovshdup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x16 }, 0, .vex_128, .avx },
+
+    .{ .vmovsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .vex_128, .avx },
+
+    .{ .vpextrw, .mri, &.{ .r32,     .xmm, .imm8 }, &.{ 0x66, 0x0f,       0x15 }, 0, .vex_128,      .avx },
+    .{ .vpextrw, .mri, &.{ .r64,     .xmm, .imm8 }, &.{ 0x66, 0x0f,       0x15 }, 0, .vex_128_long, .avx },
+    .{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128,      .avx },
+    .{ .vpextrw, .mri, &.{ .r64_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_long, .avx },
+
+    .{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128, .avx },
+
+    .{ .vpsrld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .vex_128, .avx },
+    .{ .vpsrld, .vmi, &.{ .xmm, .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x72 }, 2, .vex_128, .avx },
+
+    .{ .vpsrlq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .vex_128, .avx },
+    .{ .vpsrlq, .vmi, &.{ .xmm, .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x73 }, 2, .vex_128, .avx },
+
+    .{ .vpsrlw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .vex_128, .avx },
+    .{ .vpsrlw, .vmi, &.{ .xmm, .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x71 }, 2, .vex_128, .avx },
+
+    .{ .vpunpckhbw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_128, .avx },
+
+    .{ .vpunpckhdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_128, .avx },
+
+    .{ .vpunpckhqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6d }, 0, .vex_128, .avx },
+
+    .{ .vpunpckhwd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_128, .avx },
+
+    .{ .vpunpcklbw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x60 }, 0, .vex_128, .avx },
+
+    .{ .vpunpckldq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .vex_128, .avx },
+
+    .{ .vpunpcklqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .vex_128, .avx },
+
+    .{ .vpunpcklwd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x61 }, 0, .vex_128, .avx },
+
     // F16C
     .{ .vcvtph2ps, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128, .f16c },
 

From 5d4288c5f6c69bdd4cbd9b3580016828e38f087d Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sat, 6 May 2023 06:00:22 -0400
Subject: [PATCH 122/725] x86_64: fix unordered float equality

---
 src/arch/x86_64/CodeGen.zig        | 421 ++++++++++++++++++-----------
 src/arch/x86_64/Emit.zig           | 273 ++++++++++---------
 src/arch/x86_64/Lower.zig          | 220 +++++++++++----
 src/arch/x86_64/Mir.zig            |  34 ++-
 src/arch/x86_64/bits.zig           |   9 +
 src/arch/x86_64/encoder.zig        |   4 +-
 test/behavior/bugs/12891.zig       |   6 -
 test/behavior/field_parent_ptr.zig |   1 -
 8 files changed, 602 insertions(+), 366 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index d24428467a..87eceec347 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -205,16 +205,7 @@ pub const MCValue = union(enum) {
 
     fn isMemory(mcv: MCValue) bool {
         return switch (mcv) {
-            .memory,
-            .load_direct,
-            .lea_direct,
-            .load_got,
-            .lea_got,
-            .load_tlv,
-            .lea_tlv,
-            .load_frame,
-            .lea_frame,
-            => true,
+            .memory, .indirect, .load_frame => true,
             else => false,
         };
     }
@@ -937,7 +928,7 @@ fn formatWipMir(
         .target = data.self.target,
         .src_loc = data.self.src_loc,
     };
-    for (lower.lowerMir(data.self.mir_instructions.get(data.inst)) catch |err| switch (err) {
+    for ((lower.lowerMir(data.inst) catch |err| switch (err) {
         error.LowerFail => {
             defer {
                 lower.err_msg.?.deinit(data.self.gpa);
@@ -955,7 +946,7 @@ fn formatWipMir(
             return;
         },
         else => |e| return e,
-    }) |lower_inst| try writer.print("  | {}", .{lower_inst});
+    }).insts) |lowered_inst| try writer.print("  | {}", .{lowered_inst});
 }
 fn fmtWipMir(self: *Self, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {
     return .{ .data = .{ .self = self, .inst = inst } };
@@ -1016,7 +1007,14 @@ fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
     _ = try self.addInst(.{
         .tag = .setcc,
         .ops = .r_cc,
-        .data = .{ .r_cc = .{ .r = reg, .cc = cc } },
+        .data = .{ .r_cc = .{
+            .r = reg,
+            .scratch = if (cc == .z_and_np or cc == .nz_or_p)
+                (try self.register_manager.allocReg(null, gp)).to8()
+            else
+                .none,
+            .cc = cc,
+        } },
     });
 }
 
@@ -1028,23 +1026,36 @@ fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
             .rip => .m_rip_cc,
             else => unreachable,
         },
-        .data = .{ .x_cc = .{ .cc = cc, .payload = switch (m) {
-            .sib => try self.addExtra(Mir.MemorySib.encode(m)),
-            .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
-            else => unreachable,
-        } } },
+        .data = .{ .x_cc = .{
+            .scratch = if (cc == .z_and_np or cc == .nz_or_p)
+                (try self.register_manager.allocReg(null, gp)).to8()
+            else
+                .none,
+            .cc = cc,
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
+        } },
     });
 }
 
+/// A `cc` of `.z_and_np` clobbers `reg2`!
 fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
     _ = try self.addInst(.{
         .tag = .cmovcc,
         .ops = .rr_cc,
-        .data = .{ .rr_cc = .{ .r1 = reg1, .r2 = reg2, .cc = cc } },
+        .data = .{ .rr_cc = .{
+            .r1 = reg1,
+            .r2 = reg2,
+            .cc = cc,
+        } },
     });
 }
 
 fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
+    assert(cc != .z_and_np); // not supported
     _ = try self.addInst(.{
         .tag = .cmovcc,
         .ops = switch (m) {
@@ -1052,11 +1063,15 @@ fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condi
             .rip => .rm_rip_cc,
             else => unreachable,
         },
-        .data = .{ .rx_cc = .{ .r = reg, .cc = cc, .payload = switch (m) {
-            .sib => try self.addExtra(Mir.MemorySib.encode(m)),
-            .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
-            else => unreachable,
-        } } },
+        .data = .{ .rx_cc = .{
+            .r = reg,
+            .cc = cc,
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
+        } },
     });
 }
 
@@ -1131,10 +1146,13 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
         .tag = tag,
         .ops = ops,
         .data = switch (ops) {
-            .ri_s, .ri_u => .{ .ri = .{ .r = reg, .i = switch (imm) {
-                .signed => |s| @bitCast(u32, s),
-                .unsigned => |u| @intCast(u32, u),
-            } } },
+            .ri_s, .ri_u => .{ .ri = .{
+                .r = reg,
+                .i = switch (imm) {
+                    .signed => |s| @bitCast(u32, s),
+                    .unsigned => |u| @intCast(u32, u),
+                },
+            } },
             .ri64 => .{ .rx = .{
                 .r = reg,
                 .payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
@@ -1171,10 +1189,14 @@ fn asmRegisterRegisterImmediate(
             .signed => .rri_s,
             .unsigned => .rri_u,
         },
-        .data = .{ .rri = .{ .r1 = reg1, .r2 = reg2, .i = switch (imm) {
-            .signed => |s| @bitCast(u32, s),
-            .unsigned => |u| @intCast(u32, u),
-        } } },
+        .data = .{ .rri = .{
+            .r1 = reg1,
+            .r2 = reg2,
+            .i = switch (imm) {
+                .signed => |s| @bitCast(u32, s),
+                .unsigned => |u| @intCast(u32, u),
+            },
+        } },
     });
 }
 
@@ -1202,11 +1224,14 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !
             .rip => .rm_rip,
             else => unreachable,
         },
-        .data = .{ .rx = .{ .r = reg, .payload = switch (m) {
-            .sib => try self.addExtra(Mir.MemorySib.encode(m)),
-            .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
-            else => unreachable,
-        } } },
+        .data = .{ .rx = .{
+            .r = reg,
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
+        } },
     });
 }
 
@@ -1224,11 +1249,43 @@ fn asmRegisterMemoryImmediate(
             .rip => .rmi_rip,
             else => unreachable,
         },
-        .data = .{ .rix = .{ .r = reg, .i = @intCast(u8, imm.unsigned), .payload = switch (m) {
-            .sib => try self.addExtra(Mir.MemorySib.encode(m)),
-            .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+        .data = .{ .rix = .{
+            .r = reg,
+            .i = @intCast(u8, imm.unsigned),
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
+        } },
+    });
+}
+
+fn asmRegisterRegisterMemoryImmediate(
+    self: *Self,
+    tag: Mir.Inst.Tag,
+    reg1: Register,
+    reg2: Register,
+    m: Memory,
+    imm: Immediate,
+) !void {
+    _ = try self.addInst(.{
+        .tag = tag,
+        .ops = switch (m) {
+            .sib => .rrmi_sib,
+            .rip => .rrmi_rip,
             else => unreachable,
-        } } },
+        },
+        .data = .{ .rrix = .{
+            .r1 = reg1,
+            .r2 = reg2,
+            .i = @intCast(u8, imm.unsigned),
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
+        } },
     });
 }
 
@@ -1240,11 +1297,14 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !
             .rip => .mr_rip,
             else => unreachable,
         },
-        .data = .{ .rx = .{ .r = reg, .payload = switch (m) {
-            .sib => try self.addExtra(Mir.MemorySib.encode(m)),
-            .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
-            else => unreachable,
-        } } },
+        .data = .{ .rx = .{
+            .r = reg,
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
+        } },
     });
 }
 
@@ -1262,14 +1322,17 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate)
             },
             else => unreachable,
         },
-        .data = .{ .ix = .{ .i = switch (imm) {
-            .signed => |s| @bitCast(u32, s),
-            .unsigned => |u| @intCast(u32, u),
-        }, .payload = switch (m) {
-            .sib => try self.addExtra(Mir.MemorySib.encode(m)),
-            .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
-            else => unreachable,
-        } } },
+        .data = .{ .ix = .{
+            .i = switch (imm) {
+                .signed => |s| @bitCast(u32, s),
+                .unsigned => |u| @intCast(u32, u),
+            },
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
+        } },
     });
 }
 
@@ -6612,11 +6675,13 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
                 _ = try self.addInst(.{
                     .tag = .mov_linker,
                     .ops = .import_reloc,
-                    .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
-                        .reg = @enumToInt(Register.rax),
-                        .atom_index = atom_index,
-                        .sym_index = sym_index,
-                    }) },
+                    .data = .{ .rx = .{
+                        .r = .rax,
+                        .payload = try self.addExtra(Mir.Reloc{
+                            .atom_index = atom_index,
+                            .sym_index = sym_index,
+                        }),
+                    } },
                 });
                 try self.asmRegister(.call, .rax);
             } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
@@ -6695,8 +6760,6 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
 fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
     const bin_op = self.air.instructions.items(.data)[inst].bin_op;
     const ty = self.air.typeOf(bin_op.lhs);
-    const ty_abi_size = ty.abiSize(self.target.*);
-    const can_reuse = ty_abi_size <= 8;
 
     try self.spillEflagsIfOccupied();
     self.eflags_inst = inst;
@@ -6715,69 +6778,93 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
     };
     defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
 
-    const dst_mem_ok = !ty.isRuntimeFloat();
-    var flipped = false;
-    const dst_mcv: MCValue = if (can_reuse and !lhs_mcv.isImmediate() and
-        (dst_mem_ok or lhs_mcv.isRegister()) and self.liveness.operandDies(inst, 0))
-        lhs_mcv
-    else if (can_reuse and !rhs_mcv.isImmediate() and
-        (dst_mem_ok or rhs_mcv.isRegister()) and self.liveness.operandDies(inst, 1))
-    dst: {
-        flipped = true;
-        break :dst rhs_mcv;
-    } else if (dst_mem_ok) dst: {
-        const dst_mcv = try self.allocTempRegOrMem(ty, true);
-        try self.genCopy(ty, dst_mcv, lhs_mcv);
-        break :dst dst_mcv;
-    } else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };
-    const dst_lock = switch (dst_mcv) {
-        .register => |reg| self.register_manager.lockReg(reg),
-        else => null,
-    };
-    defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
-
-    const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
-    switch (ty.zigTypeTag()) {
-        else => try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv),
-        .Float => switch (ty.floatBits(self.target.*)) {
-            16 => if (self.hasFeature(.f16c)) {
-                const dst_reg = dst_mcv.getReg().?.to128();
-
-                const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
-                const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
-                defer self.register_manager.unlockReg(tmp_lock);
-
-                if (src_mcv.isRegister())
-                    try self.asmRegisterRegisterRegister(
-                        .vpunpcklwd,
-                        dst_reg,
-                        dst_reg,
-                        src_mcv.getReg().?.to128(),
-                    )
-                else
-                    try self.asmRegisterMemoryImmediate(
-                        .vpinsrw,
-                        dst_reg,
-                        src_mcv.mem(.word),
-                        Immediate.u(1),
-                    );
-                try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
-                try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);
-                try self.genBinOpMir(.ucomiss, ty, dst_mcv, .{ .register = tmp_reg });
-            } else return self.fail("TODO implement airCmp for {}", .{
-                ty.fmt(self.bin_file.options.module.?),
-            }),
-            32 => try self.genBinOpMir(.ucomiss, ty, dst_mcv, src_mcv),
-            64 => try self.genBinOpMir(.ucomisd, ty, dst_mcv, src_mcv),
-            else => return self.fail("TODO implement airCmp for {}", .{
-                ty.fmt(self.bin_file.options.module.?),
-            }),
-        },
-    }
-
-    const signedness = if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned;
     const result = MCValue{
-        .eflags = Condition.fromCompareOperator(signedness, if (flipped) op.reverse() else op),
+        .eflags = switch (ty.zigTypeTag()) {
+            else => result: {
+                var flipped = false;
+                const dst_mcv: MCValue = if (lhs_mcv.isRegister() or lhs_mcv.isMemory())
+                    lhs_mcv
+                else if (rhs_mcv.isRegister() or rhs_mcv.isMemory()) dst: {
+                    flipped = true;
+                    break :dst rhs_mcv;
+                } else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };
+                const dst_lock = switch (dst_mcv) {
+                    .register => |reg| self.register_manager.lockReg(reg),
+                    else => null,
+                };
+                defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+                const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
+
+                try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv);
+                break :result Condition.fromCompareOperator(
+                    if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned,
+                    if (flipped) op.reverse() else op,
+                );
+            },
+            .Float => result: {
+                const flipped = switch (op) {
+                    .lt, .lte => true,
+                    .eq, .gte, .gt, .neq => false,
+                };
+
+                const dst_mcv = if (flipped) rhs_mcv else lhs_mcv;
+                const dst_reg = if (dst_mcv.isRegister())
+                    dst_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(ty, dst_mcv);
+                const dst_lock = self.register_manager.lockReg(dst_reg);
+                defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+                const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
+
+                switch (ty.floatBits(self.target.*)) {
+                    16 => if (self.hasFeature(.f16c)) {
+                        const tmp1_reg = (try self.register_manager.allocReg(null, sse)).to128();
+                        const tmp1_mcv = MCValue{ .register = tmp1_reg };
+                        const tmp1_lock = self.register_manager.lockRegAssumeUnused(tmp1_reg);
+                        defer self.register_manager.unlockReg(tmp1_lock);
+
+                        const tmp2_reg = (try self.register_manager.allocReg(null, sse)).to128();
+                        const tmp2_mcv = MCValue{ .register = tmp2_reg };
+                        const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg);
+                        defer self.register_manager.unlockReg(tmp2_lock);
+
+                        if (src_mcv.isRegister())
+                            try self.asmRegisterRegisterRegister(
+                                .vpunpcklwd,
+                                tmp1_reg,
+                                dst_reg.to128(),
+                                src_mcv.getReg().?.to128(),
+                            )
+                        else
+                            try self.asmRegisterRegisterMemoryImmediate(
+                                .vpinsrw,
+                                tmp1_reg,
+                                dst_reg.to128(),
+                                src_mcv.mem(.word),
+                                Immediate.u(1),
+                            );
+                        try self.asmRegisterRegister(.vcvtph2ps, tmp1_reg, tmp1_reg);
+                        try self.asmRegisterRegister(.vmovshdup, tmp2_reg, tmp1_reg);
+                        try self.genBinOpMir(.ucomiss, ty, tmp1_mcv, tmp2_mcv);
+                    } else return self.fail("TODO implement airCmp for {}", .{
+                        ty.fmt(self.bin_file.options.module.?),
+                    }),
+                    32 => try self.genBinOpMir(.ucomiss, ty, .{ .register = dst_reg }, src_mcv),
+                    64 => try self.genBinOpMir(.ucomisd, ty, .{ .register = dst_reg }, src_mcv),
+                    else => return self.fail("TODO implement airCmp for {}", .{
+                        ty.fmt(self.bin_file.options.module.?),
+                    }),
+                }
+
+                break :result switch (if (flipped) op.reverse() else op) {
+                    .lt, .lte => unreachable, // required to have been canonicalized to gt(e)
+                    .gt => .a,
+                    .gte => .ae,
+                    .eq => .z_and_np,
+                    .neq => .nz_or_p,
+                };
+            },
+        },
     };
     return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
 }
@@ -7929,11 +8016,13 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                     _ = try self.addInst(.{
                         .tag = .mov_linker,
                         .ops = .direct_reloc,
-                        .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
-                            .reg = @enumToInt(dst_reg.to64()),
-                            .atom_index = atom_index,
-                            .sym_index = sym_index,
-                        }) },
+                        .data = .{ .rx = .{
+                            .r = dst_reg.to64(),
+                            .payload = try self.addExtra(Mir.Reloc{
+                                .atom_index = atom_index,
+                                .sym_index = sym_index,
+                            }),
+                        } },
                     });
                     return;
                 },
@@ -7975,11 +8064,13 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                     .lea_got => .got_reloc,
                     else => unreachable,
                 },
-                .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
-                    .reg = @enumToInt(dst_reg.to64()),
-                    .atom_index = atom_index,
-                    .sym_index = sym_index,
-                }) },
+                .data = .{ .rx = .{
+                    .r = dst_reg.to64(),
+                    .payload = try self.addExtra(Mir.Reloc{
+                        .atom_index = atom_index,
+                        .sym_index = sym_index,
+                    }),
+                } },
             });
         },
         .lea_tlv => |sym_index| {
@@ -7988,11 +8079,13 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                 _ = try self.addInst(.{
                     .tag = .lea_linker,
                     .ops = .tlv_reloc,
-                    .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
-                        .reg = @enumToInt(Register.rdi),
-                        .atom_index = atom_index,
-                        .sym_index = sym_index,
-                    }) },
+                    .data = .{ .rx = .{
+                        .r = .rdi,
+                        .payload = try self.addExtra(Mir.Reloc{
+                            .atom_index = atom_index,
+                            .sym_index = sym_index,
+                        }),
+                    } },
                 });
                 // TODO: spill registers before calling
                 try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
@@ -8463,14 +8556,20 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
 
     try self.spillEflagsIfOccupied();
     if (val_abi_size <= 8) {
-        _ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
-            .r = registerAlias(new_reg.?, val_abi_size),
-            .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
-        } } });
+        _ = try self.addInst(.{
+            .tag = .cmpxchg,
+            .ops = .lock_mr_sib,
+            .data = .{ .rx = .{
+                .r = registerAlias(new_reg.?, val_abi_size),
+                .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
+            } },
+        });
     } else {
-        _ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{
-            .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
-        } });
+        _ = try self.addInst(.{
+            .tag = .cmpxchgb,
+            .ops = .lock_m_sib,
+            .data = .{ .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)) },
+        });
     }
 
     const result: MCValue = result: {
@@ -8571,14 +8670,18 @@ fn atomicOp(
             if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
                 try self.genUnOpMir(.neg, val_ty, dst_mcv);
             }
-            _ = try self.addInst(.{ .tag = tag, .ops = switch (tag) {
-                .mov, .xchg => .mr_sib,
-                .xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
-                else => unreachable,
-            }, .data = .{ .rx = .{
-                .r = registerAlias(dst_reg, val_abi_size),
-                .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
-            } } });
+            _ = try self.addInst(.{
+                .tag = tag,
+                .ops = switch (tag) {
+                    .mov, .xchg => .mr_sib,
+                    .xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
+                    else => unreachable,
+                },
+                .data = .{ .rx = .{
+                    .r = registerAlias(dst_reg, val_abi_size),
+                    .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
+                } },
+            });
 
             return if (unused) .unreach else dst_mcv;
         },
@@ -8645,10 +8748,14 @@ fn atomicOp(
                     }
                 },
             };
-            _ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
-                .r = registerAlias(tmp_reg, val_abi_size),
-                .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
-            } } });
+            _ = try self.addInst(.{
+                .tag = .cmpxchg,
+                .ops = .lock_mr_sib,
+                .data = .{ .rx = .{
+                    .r = registerAlias(tmp_reg, val_abi_size),
+                    .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
+                } },
+            });
             _ = try self.asmJccReloc(loop, .ne);
             return if (unused) .unreach else .{ .register = .rax };
         } else {
diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig
index c6c8f7995c..3574d52878 100644
--- a/src/arch/x86_64/Emit.zig
+++ b/src/arch/x86_64/Emit.zig
@@ -18,142 +18,149 @@ pub const Error = Lower.Error || error{
 };
 
 pub fn emitMir(emit: *Emit) Error!void {
-    for (0..emit.lower.mir.instructions.len) |i| {
-        const index = @intCast(Mir.Inst.Index, i);
-        const inst = emit.lower.mir.instructions.get(index);
-
-        const start_offset = @intCast(u32, emit.code.items.len);
-        try emit.code_offset_mapping.putNoClobber(emit.lower.allocator, index, start_offset);
-        for (try emit.lower.lowerMir(inst)) |lower_inst| try lower_inst.encode(emit.code.writer(), .{});
-        const end_offset = @intCast(u32, emit.code.items.len);
-
-        switch (inst.tag) {
-            else => {},
-
-            .jmp_reloc => try emit.relocs.append(emit.lower.allocator, .{
-                .source = start_offset,
-                .target = inst.data.inst,
-                .offset = end_offset - 4,
-                .length = 5,
-            }),
-
-            .call_extern => if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
-                // Add relocation to the decl.
-                const atom_index = macho_file.getAtomIndexForSymbol(
-                    .{ .sym_index = inst.data.relocation.atom_index, .file = null },
-                ).?;
-                const target = macho_file.getGlobalByIndex(inst.data.relocation.sym_index);
-                try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
-                    .type = .branch,
+    for (0..emit.lower.mir.instructions.len) |mir_i| {
+        const mir_index = @intCast(Mir.Inst.Index, mir_i);
+        try emit.code_offset_mapping.putNoClobber(
+            emit.lower.allocator,
+            mir_index,
+            @intCast(u32, emit.code.items.len),
+        );
+        const lowered = try emit.lower.lowerMir(mir_index);
+        var lowered_relocs = lowered.relocs;
+        for (lowered.insts, 0..) |lowered_inst, lowered_index| {
+            const start_offset = @intCast(u32, emit.code.items.len);
+            try lowered_inst.encode(emit.code.writer(), .{});
+            const end_offset = @intCast(u32, emit.code.items.len);
+            while (lowered_relocs.len > 0 and
+                lowered_relocs[0].lowered_inst_index == lowered_index) : ({
+                lowered_relocs = lowered_relocs[1..];
+            }) switch (lowered_relocs[0].target) {
+                .inst => |target| try emit.relocs.append(emit.lower.allocator, .{
+                    .source = start_offset,
                     .target = target,
                     .offset = end_offset - 4,
-                    .addend = 0,
-                    .pcrel = true,
-                    .length = 2,
-                });
-            } else if (emit.bin_file.cast(link.File.Coff)) |coff_file| {
-                // Add relocation to the decl.
-                const atom_index = coff_file.getAtomIndexForSymbol(
-                    .{ .sym_index = inst.data.relocation.atom_index, .file = null },
-                ).?;
-                const target = coff_file.getGlobalByIndex(inst.data.relocation.sym_index);
-                try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
-                    .type = .direct,
-                    .target = target,
-                    .offset = end_offset - 4,
-                    .addend = 0,
-                    .pcrel = true,
-                    .length = 2,
-                });
-            } else return emit.fail("TODO implement {} for {}", .{ inst.tag, emit.bin_file.tag }),
+                    .length = @intCast(u5, end_offset - start_offset),
+                }),
+                .@"extern" => |symbol| if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
+                    // Add relocation to the decl.
+                    const atom_index = macho_file.getAtomIndexForSymbol(
+                        .{ .sym_index = symbol.atom_index, .file = null },
+                    ).?;
+                    const target = macho_file.getGlobalByIndex(symbol.sym_index);
+                    try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
+                        .type = .branch,
+                        .target = target,
+                        .offset = end_offset - 4,
+                        .addend = 0,
+                        .pcrel = true,
+                        .length = 2,
+                    });
+                } else if (emit.bin_file.cast(link.File.Coff)) |coff_file| {
+                    // Add relocation to the decl.
+                    const atom_index = coff_file.getAtomIndexForSymbol(
+                        .{ .sym_index = symbol.atom_index, .file = null },
+                    ).?;
+                    const target = coff_file.getGlobalByIndex(symbol.sym_index);
+                    try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
+                        .type = .direct,
+                        .target = target,
+                        .offset = end_offset - 4,
+                        .addend = 0,
+                        .pcrel = true,
+                        .length = 2,
+                    });
+                } else return emit.fail("TODO implement extern reloc for {s}", .{
+                    @tagName(emit.bin_file.tag),
+                }),
+                .linker_got,
+                .linker_direct,
+                .linker_import,
+                .linker_tlv,
+                => |symbol| if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
+                    const atom_index = macho_file.getAtomIndexForSymbol(.{
+                        .sym_index = symbol.atom_index,
+                        .file = null,
+                    }).?;
+                    try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
+                        .type = switch (lowered_relocs[0].target) {
+                            .linker_got => .got,
+                            .linker_direct => .signed,
+                            .linker_tlv => .tlv,
+                            else => unreachable,
+                        },
+                        .target = .{ .sym_index = symbol.sym_index, .file = null },
+                        .offset = @intCast(u32, end_offset - 4),
+                        .addend = 0,
+                        .pcrel = true,
+                        .length = 2,
+                    });
+                } else if (emit.bin_file.cast(link.File.Coff)) |coff_file| {
+                    const atom_index = coff_file.getAtomIndexForSymbol(.{
+                        .sym_index = symbol.atom_index,
+                        .file = null,
+                    }).?;
+                    try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
+                        .type = switch (lowered_relocs[0].target) {
+                            .linker_got => .got,
+                            .linker_direct => .direct,
+                            .linker_import => .import,
+                            else => unreachable,
+                        },
+                        .target = switch (lowered_relocs[0].target) {
+                            .linker_got,
+                            .linker_direct,
+                            => .{ .sym_index = symbol.sym_index, .file = null },
+                            .linker_import => coff_file.getGlobalByIndex(symbol.sym_index),
+                            else => unreachable,
+                        },
+                        .offset = @intCast(u32, end_offset - 4),
+                        .addend = 0,
+                        .pcrel = true,
+                        .length = 2,
+                    });
+                } else return emit.fail("TODO implement linker reloc for {s}", .{
+                    @tagName(emit.bin_file.tag),
+                }),
+            };
+        }
+        std.debug.assert(lowered_relocs.len == 0);
 
-            .mov_linker, .lea_linker => if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
-                const metadata =
-                    emit.lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
-                const atom_index = macho_file.getAtomIndexForSymbol(.{
-                    .sym_index = metadata.atom_index,
-                    .file = null,
-                }).?;
-                try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
-                    .type = switch (inst.ops) {
-                        .got_reloc => .got,
-                        .direct_reloc => .signed,
-                        .tlv_reloc => .tlv,
-                        else => unreachable,
-                    },
-                    .target = .{ .sym_index = metadata.sym_index, .file = null },
-                    .offset = @intCast(u32, end_offset - 4),
-                    .addend = 0,
-                    .pcrel = true,
-                    .length = 2,
-                });
-            } else if (emit.bin_file.cast(link.File.Coff)) |coff_file| {
-                const metadata =
-                    emit.lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
-                const atom_index = coff_file.getAtomIndexForSymbol(.{
-                    .sym_index = metadata.atom_index,
-                    .file = null,
-                }).?;
-                try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
-                    .type = switch (inst.ops) {
-                        .got_reloc => .got,
-                        .direct_reloc => .direct,
-                        .import_reloc => .import,
-                        else => unreachable,
-                    },
-                    .target = switch (inst.ops) {
-                        .got_reloc,
-                        .direct_reloc,
-                        => .{ .sym_index = metadata.sym_index, .file = null },
-                        .import_reloc => coff_file.getGlobalByIndex(metadata.sym_index),
-                        else => unreachable,
-                    },
-                    .offset = @intCast(u32, end_offset - 4),
-                    .addend = 0,
-                    .pcrel = true,
-                    .length = 2,
-                });
-            } else return emit.fail("TODO implement {} for {}", .{ inst.tag, emit.bin_file.tag }),
-
-            .jcc => try emit.relocs.append(emit.lower.allocator, .{
-                .source = start_offset,
-                .target = inst.data.inst_cc.inst,
-                .offset = end_offset - 4,
-                .length = 6,
-            }),
-
-            .dbg_line => try emit.dbgAdvancePCAndLine(
-                inst.data.line_column.line,
-                inst.data.line_column.column,
-            ),
-
-            .dbg_prologue_end => {
-                switch (emit.debug_output) {
-                    .dwarf => |dw| {
-                        try dw.setPrologueEnd();
-                        log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
-                            emit.prev_di_line, emit.prev_di_column,
-                        });
-                        try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
-                    },
-                    .plan9 => {},
-                    .none => {},
-                }
-            },
-
-            .dbg_epilogue_begin => {
-                switch (emit.debug_output) {
-                    .dwarf => |dw| {
-                        try dw.setEpilogueBegin();
-                        log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
-                            emit.prev_di_line, emit.prev_di_column,
-                        });
-                        try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
-                    },
-                    .plan9 => {},
-                    .none => {},
-                }
-            },
+        if (lowered.insts.len == 0) {
+            const mir_inst = emit.lower.mir.instructions.get(mir_index);
+            switch (mir_inst.tag) {
+                else => unreachable,
+                .dead => {},
+                .dbg_line => try emit.dbgAdvancePCAndLine(
+                    mir_inst.data.line_column.line,
+                    mir_inst.data.line_column.column,
+                ),
+                .dbg_prologue_end => {
+                    switch (emit.debug_output) {
+                        .dwarf => |dw| {
+                            try dw.setPrologueEnd();
+                            log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
+                                emit.prev_di_line, emit.prev_di_column,
+                            });
+                            try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
+                        },
+                        .plan9 => {},
+                        .none => {},
+                    }
+                },
+                .dbg_epilogue_begin => {
+                    switch (emit.debug_output) {
+                        .dwarf => |dw| {
+                            try dw.setEpilogueBegin();
+                            log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
+                                emit.prev_di_line, emit.prev_di_column,
+                            });
+                            try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
+                        },
+                        .plan9 => {},
+                        .none => {},
+                    }
+                },
+            }
         }
     }
     try emit.fixupRelocs();
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index d9482d4b39..d82d5ec300 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -5,13 +5,22 @@ mir: Mir,
 target: *const std.Target,
 err_msg: ?*ErrorMsg = null,
 src_loc: Module.SrcLoc,
-result: [
+result_insts_len: u8 = undefined,
+result_relocs_len: u8 = undefined,
+result_insts: [
     std.mem.max(usize, &.{
-        abi.Win64.callee_preserved_regs.len,
-        abi.SysV.callee_preserved_regs.len,
+        2, // cmovcc: cmovcc \ cmovcc
+        3, // setcc: setcc \ setcc \ logicop
+        2, // jcc: jcc \ jcc
+        abi.Win64.callee_preserved_regs.len, // push_regs/pop_regs
+        abi.SysV.callee_preserved_regs.len, // push_regs/pop_regs
     })
 ]Instruction = undefined,
-result_len: usize = undefined,
+result_relocs: [
+    std.mem.max(usize, &.{
+        2, // jcc: jcc \ jcc
+    })
+]Reloc = undefined,
 
 pub const Error = error{
     OutOfMemory,
@@ -20,13 +29,35 @@ pub const Error = error{
     CannotEncode,
 };
 
-/// The returned slice is overwritten by the next call to lowerMir.
-pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
-    lower.result = undefined;
-    errdefer lower.result = undefined;
-    lower.result_len = 0;
-    defer lower.result_len = undefined;
+pub const Reloc = struct {
+    lowered_inst_index: u8,
+    target: Target,
 
+    const Target = union(enum) {
+        inst: Mir.Inst.Index,
+        @"extern": Mir.Reloc,
+        linker_got: Mir.Reloc,
+        linker_direct: Mir.Reloc,
+        linker_import: Mir.Reloc,
+        linker_tlv: Mir.Reloc,
+    };
+};
+
+/// The returned slice is overwritten by the next call to lowerMir.
+pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
+    insts: []const Instruction,
+    relocs: []const Reloc,
+} {
+    lower.result_insts = undefined;
+    lower.result_relocs = undefined;
+    errdefer lower.result_insts = undefined;
+    errdefer lower.result_relocs = undefined;
+    lower.result_insts_len = 0;
+    lower.result_relocs_len = 0;
+    defer lower.result_insts_len = undefined;
+    defer lower.result_relocs_len = undefined;
+
+    const inst = lower.mir.instructions.get(index);
     switch (inst.tag) {
         .adc,
         .add,
@@ -185,22 +216,26 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
 
         .cmpxchgb => try lower.mirCmpxchgBytes(inst),
 
-        .jmp_reloc => try lower.emit(.none, .jmp, &.{.{ .imm = Immediate.s(0) }}),
+        .jmp_reloc => try lower.emitInstWithReloc(.none, .jmp, &.{
+            .{ .imm = Immediate.s(0) },
+        }, .{ .inst = inst.data.inst }),
 
-        .call_extern => try lower.emit(.none, .call, &.{.{ .imm = Immediate.s(0) }}),
+        .call_extern => try lower.emitInstWithReloc(.none, .call, &.{
+            .{ .imm = Immediate.s(0) },
+        }, .{ .@"extern" = inst.data.relocation }),
 
-        .lea_linker => try lower.mirLeaLinker(inst),
-        .mov_linker => try lower.mirMovLinker(inst),
+        .lea_linker => try lower.mirLinker(.lea, inst),
+        .mov_linker => try lower.mirLinker(.mov, inst),
 
         .mov_moffs => try lower.mirMovMoffs(inst),
 
         .movsx => try lower.mirMovsx(inst),
         .cmovcc => try lower.mirCmovcc(inst),
         .setcc => try lower.mirSetcc(inst),
-        .jcc => try lower.emit(.none, mnem_cc(.j, inst.data.inst_cc.cc), &.{.{ .imm = Immediate.s(0) }}),
+        .jcc => try lower.mirJcc(index, inst),
 
-        .push_regs => try lower.mirPushPopRegisterList(inst, .push),
-        .pop_regs => try lower.mirPushPopRegisterList(inst, .pop),
+        .push_regs => try lower.mirRegisterList(.push, inst),
+        .pop_regs => try lower.mirRegisterList(.pop, inst),
 
         .dbg_line,
         .dbg_prologue_end,
@@ -209,7 +244,10 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
         => {},
     }
 
-    return lower.result[0..lower.result_len];
+    return .{
+        .insts = lower.result_insts[0..lower.result_insts_len],
+        .relocs = lower.result_relocs[0..lower.result_relocs_len],
+    };
 }
 
 pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
@@ -221,7 +259,10 @@ pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
 
 fn mnem_cc(comptime base: @Type(.EnumLiteral), cc: bits.Condition) Mnemonic {
     return switch (cc) {
-        inline else => |c| @field(Mnemonic, @tagName(base) ++ @tagName(c)),
+        inline else => |c| if (@hasField(Mnemonic, @tagName(base) ++ @tagName(c)))
+            @field(Mnemonic, @tagName(base) ++ @tagName(c))
+        else
+            unreachable,
     };
 }
 
@@ -247,6 +288,8 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
         .rmi_rip,
         .mri_sib,
         .mri_rip,
+        .rrmi_sib,
+        .rrmi_rip,
         => Immediate.u(i),
 
         .ri64 => Immediate.u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
@@ -267,6 +310,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
         .mr_sib,
         .mrr_sib,
         .mri_sib,
+        .rrmi_sib,
         .lock_m_sib,
         .lock_mi_sib_u,
         .lock_mi_sib_s,
@@ -283,6 +327,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
         .mr_rip,
         .mrr_rip,
         .mri_rip,
+        .rrmi_rip,
         .lock_m_rip,
         .lock_mi_rip_u,
         .lock_mi_rip_s,
@@ -298,13 +343,28 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
     });
 }
 
-fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
-    lower.result[lower.result_len] = try Instruction.new(prefix, mnemonic, ops);
-    lower.result_len += 1;
+fn emitInst(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
+    lower.result_insts[lower.result_insts_len] = try Instruction.new(prefix, mnemonic, ops);
+    lower.result_insts_len += 1;
+}
+
+fn emitInstWithReloc(
+    lower: *Lower,
+    prefix: Prefix,
+    mnemonic: Mnemonic,
+    ops: []const Operand,
+    target: Reloc.Target,
+) Error!void {
+    lower.result_relocs[lower.result_relocs_len] = .{
+        .lowered_inst_index = lower.result_insts_len,
+        .target = target,
+    };
+    lower.result_relocs_len += 1;
+    try lower.emitInst(prefix, mnemonic, ops);
 }
 
 fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
-    try lower.emit(switch (inst.ops) {
+    try lower.emitInst(switch (inst.ops) {
         else => .none,
         .lock_m_sib,
         .lock_m_rip,
@@ -389,13 +449,19 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .reg = inst.data.rix.r },
             .{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
         },
+        .rrmi_sib, .rrmi_rip => &.{
+            .{ .reg = inst.data.rrix.r1 },
+            .{ .reg = inst.data.rrix.r2 },
+            .{ .mem = lower.mem(inst.ops, inst.data.rrix.payload) },
+            .{ .imm = lower.imm(inst.ops, inst.data.rrix.i) },
+        },
         else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
     });
 }
 
 fn mirString(lower: *Lower, inst: Mir.Inst) Error!void {
     switch (inst.ops) {
-        .string => try lower.emit(switch (inst.data.string.repeat) {
+        .string => try lower.emitInst(switch (inst.data.string.repeat) {
             inline else => |repeat| @field(Prefix, @tagName(repeat)),
         }, switch (inst.tag) {
             inline .cmps, .lods, .movs, .scas, .stos => |tag| switch (inst.data.string.width) {
@@ -414,7 +480,7 @@ fn mirCmpxchgBytes(lower: *Lower, inst: Mir.Inst) Error!void {
         },
         else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
     };
-    try lower.emit(switch (inst.ops) {
+    try lower.emitInst(switch (inst.ops) {
         .m_sib, .m_rip => .none,
         .lock_m_sib, .lock_m_rip => .lock,
         else => unreachable,
@@ -426,7 +492,7 @@ fn mirCmpxchgBytes(lower: *Lower, inst: Mir.Inst) Error!void {
 }
 
 fn mirMovMoffs(lower: *Lower, inst: Mir.Inst) Error!void {
-    try lower.emit(switch (inst.ops) {
+    try lower.emitInst(switch (inst.ops) {
         .rax_moffs, .moffs_rax => .none,
         .lock_moffs_rax => .lock,
         else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
@@ -455,7 +521,7 @@ fn mirMovsx(lower: *Lower, inst: Mir.Inst) Error!void {
         },
         else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
     };
-    try lower.emit(.none, switch (ops[0].bitSize()) {
+    try lower.emitInst(.none, switch (ops[0].bitSize()) {
         32, 64 => switch (ops[1].bitSize()) {
             32 => .movsxd,
             else => .movsx,
@@ -465,32 +531,82 @@ fn mirMovsx(lower: *Lower, inst: Mir.Inst) Error!void {
 }
 
 fn mirCmovcc(lower: *Lower, inst: Mir.Inst) Error!void {
-    switch (inst.ops) {
-        .rr_cc => try lower.emit(.none, mnem_cc(.cmov, inst.data.rr_cc.cc), &.{
+    const data: struct { cc: bits.Condition, ops: [2]Operand } = switch (inst.ops) {
+        .rr_cc => .{ .cc = inst.data.rr_cc.cc, .ops = .{
             .{ .reg = inst.data.rr_cc.r1 },
             .{ .reg = inst.data.rr_cc.r2 },
-        }),
-        .rm_sib_cc, .rm_rip_cc => try lower.emit(.none, mnem_cc(.cmov, inst.data.rx_cc.cc), &.{
+        } },
+        .rm_sib_cc, .rm_rip_cc => .{ .cc = inst.data.rx_cc.cc, .ops = .{
             .{ .reg = inst.data.rx_cc.r },
             .{ .mem = lower.mem(inst.ops, inst.data.rx_cc.payload) },
-        }),
+        } },
         else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
+    };
+    switch (data.cc) {
+        else => |cc| try lower.emitInst(.none, mnem_cc(.cmov, cc), &data.ops),
+        .z_and_np => {
+            try lower.emitInst(.none, mnem_cc(.cmov, .nz), &.{ data.ops[1], data.ops[0] });
+            try lower.emitInst(.none, mnem_cc(.cmov, .np), &data.ops);
+        },
+        .nz_or_p => {
+            try lower.emitInst(.none, mnem_cc(.cmov, .nz), &data.ops);
+            try lower.emitInst(.none, mnem_cc(.cmov, .p), &data.ops);
+        },
     }
 }
 
 fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {
-    switch (inst.ops) {
-        .r_cc => try lower.emit(.none, mnem_cc(.set, inst.data.r_cc.cc), &.{
+    const data: struct { cc: bits.Condition, ops: [2]Operand } = switch (inst.ops) {
+        .r_cc => .{ .cc = inst.data.r_cc.cc, .ops = .{
             .{ .reg = inst.data.r_cc.r },
-        }),
-        .m_sib_cc, .m_rip_cc => try lower.emit(.none, mnem_cc(.set, inst.data.x_cc.cc), &.{
+            .{ .reg = inst.data.r_cc.scratch },
+        } },
+        .m_sib_cc, .m_rip_cc => .{ .cc = inst.data.x_cc.cc, .ops = .{
             .{ .mem = lower.mem(inst.ops, inst.data.x_cc.payload) },
-        }),
+            .{ .reg = inst.data.x_cc.scratch },
+        } },
         else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
+    };
+    switch (data.cc) {
+        else => |cc| try lower.emitInst(.none, mnem_cc(.set, cc), data.ops[0..1]),
+        .z_and_np => {
+            try lower.emitInst(.none, mnem_cc(.set, .z), data.ops[0..1]);
+            try lower.emitInst(.none, mnem_cc(.set, .np), data.ops[1..2]);
+            try lower.emitInst(.none, .@"and", data.ops[0..2]);
+        },
+        .nz_or_p => {
+            try lower.emitInst(.none, mnem_cc(.set, .nz), data.ops[0..1]);
+            try lower.emitInst(.none, mnem_cc(.set, .p), data.ops[1..2]);
+            try lower.emitInst(.none, .@"or", data.ops[0..2]);
+        },
     }
 }
 
-fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst, comptime mnemonic: Mnemonic) Error!void {
+fn mirJcc(lower: *Lower, index: Mir.Inst.Index, inst: Mir.Inst) Error!void {
+    switch (inst.data.inst_cc.cc) {
+        else => |cc| try lower.emitInstWithReloc(.none, mnem_cc(.j, cc), &.{
+            .{ .imm = Immediate.s(0) },
+        }, .{ .inst = inst.data.inst_cc.inst }),
+        .z_and_np => {
+            try lower.emitInstWithReloc(.none, mnem_cc(.j, .nz), &.{
+                .{ .imm = Immediate.s(0) },
+            }, .{ .inst = index + 1 });
+            try lower.emitInstWithReloc(.none, mnem_cc(.j, .np), &.{
+                .{ .imm = Immediate.s(0) },
+            }, .{ .inst = inst.data.inst_cc.inst });
+        },
+        .nz_or_p => {
+            try lower.emitInstWithReloc(.none, mnem_cc(.j, .nz), &.{
+                .{ .imm = Immediate.s(0) },
+            }, .{ .inst = inst.data.inst_cc.inst });
+            try lower.emitInstWithReloc(.none, mnem_cc(.j, .p), &.{
+                .{ .imm = Immediate.s(0) },
+            }, .{ .inst = inst.data.inst_cc.inst });
+        },
+    }
+}
+
+fn mirRegisterList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
     const reg_list = Mir.RegisterList.fromInt(inst.data.payload);
     const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);
     var it = reg_list.iterator(.{ .direction = switch (mnemonic) {
@@ -498,24 +614,20 @@ fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst, comptime mnemonic: Mnem
         .pop => .forward,
         else => unreachable,
     } });
-    while (it.next()) |i| try lower.emit(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});
+    while (it.next()) |i| try lower.emitInst(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});
 }
 
-fn mirLeaLinker(lower: *Lower, inst: Mir.Inst) Error!void {
-    const metadata = lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
-    const reg = @intToEnum(Register, metadata.reg);
-    try lower.emit(.none, .lea, &.{
-        .{ .reg = reg },
-        .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
-    });
-}
-
-fn mirMovLinker(lower: *Lower, inst: Mir.Inst) Error!void {
-    const metadata = lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
-    const reg = @intToEnum(Register, metadata.reg);
-    try lower.emit(.none, .mov, &.{
-        .{ .reg = reg },
-        .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
+fn mirLinker(lower: *Lower, mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
+    const reloc = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
+    try lower.emitInstWithReloc(.none, mnemonic, &.{
+        .{ .reg = inst.data.rx.r },
+        .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(inst.data.rx.r.bitSize()), 0) },
+    }, switch (inst.ops) {
+        .got_reloc => .{ .linker_got = reloc },
+        .direct_reloc => .{ .linker_direct = reloc },
+        .import_reloc => .{ .linker_import = reloc },
+        .tlv_reloc => .{ .linker_tlv = reloc },
+        else => unreachable,
     });
 }
 
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index 9e39d23bd4..e261f6dc38 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -434,6 +434,12 @@ pub const Inst = struct {
         /// Register, memory (SIB), immediate (byte) operands.
         /// Uses `rix` payload with extra data of type `MemorySib`.
         rmi_sib,
+        /// Register, register, memory (RIP), immediate (byte) operands.
+        /// Uses `rrix` payload with extra data of type `MemoryRip`.
+        rrmi_rip,
+        /// Register, register, memory (SIB), immediate (byte) operands.
+        /// Uses `rrix` payload with extra data of type `MemorySib`.
+        rrmi_sib,
         /// Register, memory (RIP), immediate (byte) operands.
         /// Uses `rix` payload with extra data of type `MemoryRip`.
         rmi_rip,
@@ -524,16 +530,16 @@ pub const Inst = struct {
         /// Uses `reloc` payload.
         reloc,
         /// Linker relocation - GOT indirection.
-        /// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
+        /// Uses `rx` payload with extra data of type `Reloc`.
         got_reloc,
         /// Linker relocation - direct reference.
-        /// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
+        /// Uses `rx` payload with extra data of type `Reloc`.
         direct_reloc,
         /// Linker relocation - imports table indirection (binding).
-        /// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
+        /// Uses `rx` payload with extra data of type `Reloc`.
         import_reloc,
         /// Linker relocation - threadlocal variable via GOT indirection.
-        /// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
+        /// Uses `rx` payload with extra data of type `Reloc`.
         tlv_reloc,
     };
 
@@ -567,12 +573,14 @@ pub const Inst = struct {
         },
         /// Condition code (CC), followed by custom payload found in extra.
         x_cc: struct {
+            scratch: Register,
             cc: bits.Condition,
             payload: u32,
         },
         /// Register with condition code (CC).
         r_cc: struct {
             r: Register,
+            scratch: Register,
             cc: bits.Condition,
         },
         /// Register, register with condition code (CC).
@@ -614,6 +622,13 @@ pub const Inst = struct {
             i: u8,
             payload: u32,
         },
+        /// Register, register, byte immediate, followed by Custom payload found in extra.
+        rrix: struct {
+            r1: Register,
+            r2: Register,
+            i: u8,
+            payload: u32,
+        },
         /// String instruction prefix and width.
         string: struct {
             repeat: bits.StringRepeat,
@@ -622,12 +637,7 @@ pub const Inst = struct {
         /// Relocation for the linker where:
         /// * `atom_index` is the index of the source
         /// * `sym_index` is the index of the target
-        relocation: struct {
-            /// Index of the containing atom.
-            atom_index: u32,
-            /// Index into the linker's symbol table.
-            sym_index: u32,
-        },
+        relocation: Reloc,
         /// Debug line and column position
         line_column: struct {
             line: u32,
@@ -646,9 +656,7 @@ pub const Inst = struct {
     }
 };
 
-pub const LeaRegisterReloc = struct {
-    /// Destination register.
-    reg: u32,
+pub const Reloc = struct {
     /// Index of the containing atom.
     atom_index: u32,
     /// Index into the linker's symbol table.
diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig
index 5d06865566..77dc0cfb7c 100644
--- a/src/arch/x86_64/bits.zig
+++ b/src/arch/x86_64/bits.zig
@@ -72,6 +72,12 @@ pub const Condition = enum(u5) {
     /// zero
     z,
 
+    // Pseudo conditions
+    /// zero and not parity
+    z_and_np,
+    /// not zero or parity
+    nz_or_p,
+
     /// Converts a std.math.CompareOperator into a condition flag,
     /// i.e. returns the condition that is true iff the result of the
     /// comparison is true. Assumes signed comparison
@@ -143,6 +149,9 @@ pub const Condition = enum(u5) {
             .po => .pe,
             .s => .ns,
             .z => .nz,
+
+            .z_and_np => .nz_or_p,
+            .nz_or_p => .z_and_np,
         };
     }
 };
diff --git a/src/arch/x86_64/encoder.zig b/src/arch/x86_64/encoder.zig
index 495edb5f2a..47211591ec 100644
--- a/src/arch/x86_64/encoder.zig
+++ b/src/arch/x86_64/encoder.zig
@@ -245,9 +245,9 @@ pub const Instruction = struct {
                     },
                     .mem => |mem| {
                         const op = switch (data.op_en) {
-                            .m, .mi, .m1, .mc => .none,
+                            .m, .mi, .m1, .mc, .vmi => .none,
                             .mr, .mri, .mrc => inst.ops[1],
-                            .rm, .rmi => inst.ops[0],
+                            .rm, .rmi, .rvm, .rvmi => inst.ops[0],
                             else => unreachable,
                         };
                         try encodeMemory(enc, mem, op, encoder);
diff --git a/test/behavior/bugs/12891.zig b/test/behavior/bugs/12891.zig
index e558783705..354d9e856e 100644
--- a/test/behavior/bugs/12891.zig
+++ b/test/behavior/bugs/12891.zig
@@ -29,7 +29,6 @@ test "inf >= 1" {
 test "isNan(nan * 1)" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const nan_times_one = comptime std.math.nan(f64) * 1;
     try std.testing.expect(std.math.isNan(nan_times_one));
@@ -37,7 +36,6 @@ test "isNan(nan * 1)" {
 test "runtime isNan(nan * 1)" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const nan_times_one = std.math.nan(f64) * 1;
     try std.testing.expect(std.math.isNan(nan_times_one));
@@ -45,7 +43,6 @@ test "runtime isNan(nan * 1)" {
 test "isNan(nan * 0)" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const nan_times_zero = comptime std.math.nan(f64) * 0;
     try std.testing.expect(std.math.isNan(nan_times_zero));
@@ -55,7 +52,6 @@ test "isNan(nan * 0)" {
 test "isNan(inf * 0)" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const inf_times_zero = comptime std.math.inf(f64) * 0;
     try std.testing.expect(std.math.isNan(inf_times_zero));
@@ -65,7 +61,6 @@ test "isNan(inf * 0)" {
 test "runtime isNan(nan * 0)" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const nan_times_zero = std.math.nan(f64) * 0;
     try std.testing.expect(std.math.isNan(nan_times_zero));
@@ -75,7 +70,6 @@ test "runtime isNan(nan * 0)" {
 test "runtime isNan(inf * 0)" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
 
     const inf_times_zero = std.math.inf(f64) * 0;
     try std.testing.expect(std.math.isNan(inf_times_zero));
diff --git a/test/behavior/field_parent_ptr.zig b/test/behavior/field_parent_ptr.zig
index bf99fd1795..c56bcad0d2 100644
--- a/test/behavior/field_parent_ptr.zig
+++ b/test/behavior/field_parent_ptr.zig
@@ -2,7 +2,6 @@ const expect = @import("std").testing.expect;
 const builtin = @import("builtin");
 
 test "@fieldParentPtr non-first field" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     try testParentFieldPtr(&foo.c);

From 4b75352c78731f76cfeac0b5c78c03f232022096 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sat, 6 May 2023 16:05:52 -0400
Subject: [PATCH 123/725] x86_64: implement packed floating point fields

---
 src/arch/x86_64/CodeGen.zig | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 87eceec347..befd5be0fd 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -4922,7 +4922,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
                 }
 
                 if (field_extra_bits > 0) try self.truncateRegister(field_ty, dst_reg);
-                break :result .{ .register = dst_reg };
+
+                const dst_mcv = MCValue{ .register = dst_reg };
+                const dst_rc = regClassForType(field_ty);
+                if (dst_rc.eql(gp)) break :result dst_mcv;
+
+                const result_reg = try self.register_manager.allocReg(inst, dst_rc);
+                try self.genSetReg(result_reg, field_ty, dst_mcv);
+                break :result .{ .register = result_reg };
             },
             .register => |reg| {
                 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
@@ -7896,7 +7903,8 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
 
 fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerError!void {
     const abi_size = @intCast(u32, ty.abiSize(self.target.*));
-    if (abi_size > 8) return self.fail("genSetReg called with a value larger than one register", .{});
+    if (abi_size * 8 > dst_reg.bitSize())
+        return self.fail("genSetReg called with a value larger than dst_reg", .{});
     switch (src_mcv) {
         .none,
         .unreach,

From 0bd92da0e20058942497d3f2d572799f9710959c Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sat, 6 May 2023 16:24:41 -0400
Subject: [PATCH 124/725] target: fix typos in x86 feature descriptions

---
 lib/std/target/x86.zig | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/std/target/x86.zig b/lib/std/target/x86.zig
index c46367e755..bf3b8cb953 100644
--- a/lib/std/target/x86.zig
+++ b/lib/std/target/x86.zig
@@ -326,7 +326,7 @@ pub const all_features = blk: {
     };
     result[@enumToInt(Feature.avx512ifma)] = .{
         .llvm_name = "avx512ifma",
-        .description = "Enable AVX-512 Integer Fused Multiple-Add",
+        .description = "Enable AVX-512 Integer Fused Multiply-Add",
         .dependencies = featureSet(&[_]Feature{
             .avx512f,
         }),
@@ -599,14 +599,14 @@ pub const all_features = blk: {
     };
     result[@enumToInt(Feature.fma)] = .{
         .llvm_name = "fma",
-        .description = "Enable three-operand fused multiple-add",
+        .description = "Enable three-operand fused multiply-add",
         .dependencies = featureSet(&[_]Feature{
             .avx,
         }),
     };
     result[@enumToInt(Feature.fma4)] = .{
         .llvm_name = "fma4",
-        .description = "Enable four-operand fused multiple-add",
+        .description = "Enable four-operand fused multiply-add",
         .dependencies = featureSet(&[_]Feature{
             .avx,
             .sse4a,

From 3a5e3c52e0f09112989a2a40345305bfe9508431 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sat, 6 May 2023 20:31:48 -0400
Subject: [PATCH 125/725] x86_64: implement `@mulAdd`

---
 src/arch/x86_64/CodeGen.zig   | 169 +++++++++++++++++++++++++++++++++-
 src/arch/x86_64/Encoding.zig  |  24 ++++-
 src/arch/x86_64/Lower.zig     |  22 +++++
 src/arch/x86_64/Mir.zig       |  31 +++++++
 src/arch/x86_64/bits.zig      |  16 +++-
 src/arch/x86_64/encodings.zig |  23 +++++
 test/behavior/muladd.zig      |   6 +-
 7 files changed, 277 insertions(+), 14 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index befd5be0fd..fffb814d7f 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -1200,6 +1200,32 @@ fn asmRegisterRegisterImmediate(
     });
 }
 
+fn asmRegisterRegisterMemory(
+    self: *Self,
+    tag: Mir.Inst.Tag,
+    reg1: Register,
+    reg2: Register,
+    m: Memory,
+) !void {
+    _ = try self.addInst(.{
+        .tag = tag,
+        .ops = switch (m) {
+            .sib => .rrm_sib,
+            .rip => .rrm_rip,
+            else => unreachable,
+        },
+        .data = .{ .rrx = .{
+            .r1 = reg1,
+            .r2 = reg2,
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
+        } },
+    });
+}
+
 fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
     _ = try self.addInst(.{
         .tag = tag,
@@ -9369,9 +9395,146 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
 fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
     const pl_op = self.air.instructions.items(.data)[inst].pl_op;
     const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
-    _ = extra;
-    return self.fail("TODO implement airMulAdd for x86_64", .{});
-    //return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });
+    const ty = self.air.typeOfIndex(inst);
+
+    if (!self.hasFeature(.fma)) return self.fail("TODO implement airMulAdd for {}", .{
+        ty.fmt(self.bin_file.options.module.?),
+    });
+
+    const ops = [3]Air.Inst.Ref{ extra.lhs, extra.rhs, pl_op.operand };
+    var mcvs: [3]MCValue = undefined;
+    var locks = [1]?RegisterManager.RegisterLock{null} ** 3;
+    defer for (locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
+    var order = [1]u2{0} ** 3;
+    var unused = std.StaticBitSet(3).initFull();
+    for (ops, &mcvs, &locks, 0..) |op, *mcv, *lock, op_i| {
+        const op_index = @intCast(u2, op_i);
+        mcv.* = try self.resolveInst(op);
+        if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) {
+            order[op_index] = 1;
+            unused.unset(0);
+        } else if (unused.isSet(2) and mcv.isMemory()) {
+            order[op_index] = 3;
+            unused.unset(2);
+        }
+        switch (mcv.*) {
+            .register => |reg| lock.* = self.register_manager.lockReg(reg),
+            else => {},
+        }
+    }
+    for (&order, &mcvs, &locks) |*mop_index, *mcv, *lock| {
+        if (mop_index.* != 0) continue;
+        mop_index.* = 1 + @intCast(u2, unused.toggleFirstSet().?);
+        if (mop_index.* > 1 and mcv.isRegister()) continue;
+        const reg = try self.copyToTmpRegister(ty, mcv.*);
+        mcv.* = .{ .register = reg };
+        if (lock.*) |old_lock| self.register_manager.unlockReg(old_lock);
+        lock.* = self.register_manager.lockRegAssumeUnused(reg);
+    }
+
+    const tag: ?Mir.Inst.Tag =
+        if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or mem.eql(u2, &order, &.{ 3, 1, 2 }))
+        switch (ty.zigTypeTag()) {
+            .Float => switch (ty.floatBits(self.target.*)) {
+                32 => .vfmadd132ss,
+                64 => .vfmadd132sd,
+                else => null,
+            },
+            .Vector => switch (ty.childType().zigTypeTag()) {
+                .Float => switch (ty.childType().floatBits(self.target.*)) {
+                    32 => switch (ty.vectorLen()) {
+                        1 => .vfmadd132ss,
+                        2...8 => .vfmadd132ps,
+                        else => null,
+                    },
+                    64 => switch (ty.vectorLen()) {
+                        1 => .vfmadd132sd,
+                        2...4 => .vfmadd132pd,
+                        else => null,
+                    },
+                    else => null,
+                },
+                else => null,
+            },
+            else => unreachable,
+        }
+    else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
+        switch (ty.zigTypeTag()) {
+            .Float => switch (ty.floatBits(self.target.*)) {
+                32 => .vfmadd213ss,
+                64 => .vfmadd213sd,
+                else => null,
+            },
+            .Vector => switch (ty.childType().zigTypeTag()) {
+                .Float => switch (ty.childType().floatBits(self.target.*)) {
+                    32 => switch (ty.vectorLen()) {
+                        1 => .vfmadd213ss,
+                        2...8 => .vfmadd213ps,
+                        else => null,
+                    },
+                    64 => switch (ty.vectorLen()) {
+                        1 => .vfmadd213sd,
+                        2...4 => .vfmadd213pd,
+                        else => null,
+                    },
+                    else => null,
+                },
+                else => null,
+            },
+            else => unreachable,
+        }
+    else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
+        switch (ty.zigTypeTag()) {
+            .Float => switch (ty.floatBits(self.target.*)) {
+                32 => .vfmadd231ss,
+                64 => .vfmadd231sd,
+                else => null,
+            },
+            .Vector => switch (ty.childType().zigTypeTag()) {
+                .Float => switch (ty.childType().floatBits(self.target.*)) {
+                    32 => switch (ty.vectorLen()) {
+                        1 => .vfmadd231ss,
+                        2...8 => .vfmadd231ps,
+                        else => null,
+                    },
+                    64 => switch (ty.vectorLen()) {
+                        1 => .vfmadd231sd,
+                        2...4 => .vfmadd231pd,
+                        else => null,
+                    },
+                    else => null,
+                },
+                else => null,
+            },
+            else => null,
+        }
+    else
+        unreachable;
+    if (tag == null) return self.fail("TODO implement airMulAdd for {}", .{
+        ty.fmt(self.bin_file.options.module.?),
+    });
+
+    var mops: [3]MCValue = undefined;
+    for (order, mcvs) |mop_index, mcv| mops[mop_index - 1] = mcv;
+
+    const abi_size = @intCast(u32, ty.abiSize(self.target.*));
+    const mop1_reg = registerAlias(mops[0].getReg().?, abi_size);
+    const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);
+    if (mops[2].isRegister())
+        try self.asmRegisterRegisterRegister(
+            tag.?,
+            mop1_reg,
+            mop2_reg,
+            registerAlias(mops[2].getReg().?, abi_size),
+        )
+    else
+        try self.asmRegisterRegisterMemory(
+            tag.?,
+            mop1_reg,
+            mop2_reg,
+            mops[2].mem(Memory.PtrSize.fromSize(abi_size)),
+        );
+    return self.finishAir(inst, mops[0], ops);
 }
 
 fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index ada1e891fb..94bfa63999 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -340,6 +340,11 @@ pub const Mnemonic = enum {
     vpunpcklbw, vpunpckldq, vpunpcklqdq, vpunpcklwd,
     // F16C
     vcvtph2ps, vcvtps2ph,
+    // FMA
+    vfmadd132pd, vfmadd213pd, vfmadd231pd,
+    vfmadd132ps, vfmadd213ps, vfmadd231ps,
+    vfmadd132sd, vfmadd213sd, vfmadd231sd,
+    vfmadd132ss, vfmadd213ss, vfmadd231ss,
     // zig fmt: on
 };
 
@@ -368,12 +373,13 @@ pub const Op = enum {
     r8, r16, r32, r64,
     rm8, rm16, rm32, rm64,
     r32_m16, r64_m16,
-    m8, m16, m32, m64, m80, m128,
+    m8, m16, m32, m64, m80, m128, m256,
     rel8, rel16, rel32,
     m,
     moffs,
     sreg,
     xmm, xmm_m32, xmm_m64, xmm_m128,
+    ymm, ymm_m256,
     // zig fmt: on
 
     pub fn fromOperand(operand: Instruction.Operand) Op {
@@ -385,6 +391,7 @@ pub const Op = enum {
                     .segment => return .sreg,
                     .floating_point => return switch (reg.bitSize()) {
                         128 => .xmm,
+                        256 => .ymm,
                         else => unreachable,
                     },
                     .general_purpose => {
@@ -418,6 +425,7 @@ pub const Op = enum {
                         64 => .m64,
                         80 => .m80,
                         128 => .m128,
+                        256 => .m256,
                         else => unreachable,
                     };
                 },
@@ -454,7 +462,8 @@ pub const Op = enum {
             .eax, .r32, .rm32, .r32_m16 => unreachable,
             .rax, .r64, .rm64, .r64_m16 => unreachable,
             .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,
-            .m8, .m16, .m32, .m64, .m80, .m128 => unreachable,
+            .ymm, .ymm_m256 => unreachable,
+            .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
             .unity => 1,
             .imm8, .imm8s, .rel8 => 8,
             .imm16, .imm16s, .rel16 => 16,
@@ -468,12 +477,13 @@ pub const Op = enum {
             .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
             .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
             .rel8, .rel16, .rel32 => unreachable,
-            .m8, .m16, .m32, .m64, .m80, .m128 => unreachable,
+            .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
             .al, .cl, .r8, .rm8 => 8,
             .ax, .r16, .rm16 => 16,
             .eax, .r32, .rm32, .r32_m16 => 32,
             .rax, .r64, .rm64, .r64_m16 => 64,
             .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
+            .ymm, .ymm_m256 => 256,
         };
     }
 
@@ -482,13 +492,14 @@ pub const Op = enum {
             .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
             .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
             .rel8, .rel16, .rel32 => unreachable,
-            .al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64, .xmm => unreachable,
+            .al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64, .xmm, .ymm => unreachable,
             .m8, .rm8 => 8,
             .m16, .rm16, .r32_m16, .r64_m16 => 16,
             .m32, .rm32, .xmm_m32 => 32,
             .m64, .rm64, .xmm_m64 => 64,
             .m80 => 80,
             .m128, .xmm_m128 => 128,
+            .m256, .ymm_m256 => 256,
         };
     }
 
@@ -513,6 +524,7 @@ pub const Op = enum {
             .rm8, .rm16, .rm32, .rm64,
             .r32_m16, .r64_m16,
             .xmm, .xmm_m32, .xmm_m64, .xmm_m128,
+            .ymm, .ymm_m256,
             => true,
             else => false,
         };
@@ -539,7 +551,7 @@ pub const Op = enum {
             .r32_m16, .r64_m16,
             .m8, .m16, .m32, .m64, .m80, .m128,
             .m,
-            .xmm_m32, .xmm_m64, .xmm_m128,
+            .xmm_m32, .xmm_m64, .xmm_m128, .ymm_m256,
             =>  true,
             else => false,
         };
@@ -562,6 +574,7 @@ pub const Op = enum {
             .r32_m16, .r64_m16 => .general_purpose,
             .sreg => .segment,
             .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .floating_point,
+            .ymm, .ymm_m256 => .floating_point,
         };
     }
 
@@ -625,6 +638,7 @@ pub const Feature = enum {
     none,
     avx,
     f16c,
+    fma,
     sse,
     sse2,
     sse3,
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index d82d5ec300..a37f28c0c3 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -205,6 +205,19 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
 
         .vcvtph2ps,
         .vcvtps2ph,
+
+        .vfmadd132pd,
+        .vfmadd213pd,
+        .vfmadd231pd,
+        .vfmadd132ps,
+        .vfmadd213ps,
+        .vfmadd231ps,
+        .vfmadd132sd,
+        .vfmadd213sd,
+        .vfmadd231sd,
+        .vfmadd132ss,
+        .vfmadd213ss,
+        .vfmadd231ss,
         => try lower.mirGeneric(inst),
 
         .cmps,
@@ -288,6 +301,8 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
         .rmi_rip,
         .mri_sib,
         .mri_rip,
+        .rrm_sib,
+        .rrm_rip,
         .rrmi_sib,
         .rrmi_rip,
         => Immediate.u(i),
@@ -310,6 +325,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
         .mr_sib,
         .mrr_sib,
         .mri_sib,
+        .rrm_sib,
         .rrmi_sib,
         .lock_m_sib,
         .lock_mi_sib_u,
@@ -327,6 +343,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
         .mr_rip,
         .mrr_rip,
         .mri_rip,
+        .rrm_rip,
         .rrmi_rip,
         .lock_m_rip,
         .lock_mi_rip_u,
@@ -449,6 +466,11 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .reg = inst.data.rix.r },
             .{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
         },
+        .rrm_sib, .rrm_rip => &.{
+            .{ .reg = inst.data.rrx.r1 },
+            .{ .reg = inst.data.rrx.r2 },
+            .{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },
+        },
         .rrmi_sib, .rrmi_rip => &.{
             .{ .reg = inst.data.rrix.r1 },
             .{ .reg = inst.data.rrix.r2 },
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index e261f6dc38..92a9a74fbb 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -324,6 +324,31 @@ pub const Inst = struct {
         /// Convert single-precision floating-point values to 16-bit floating-point values
         vcvtps2ph,
 
+        /// Fused multiply-add of packed double-precision floating-point values
+        vfmadd132pd,
+        /// Fused multiply-add of packed double-precision floating-point values
+        vfmadd213pd,
+        /// Fused multiply-add of packed double-precision floating-point values
+        vfmadd231pd,
+        /// Fused multiply-add of packed single-precision floating-point values
+        vfmadd132ps,
+        /// Fused multiply-add of packed single-precision floating-point values
+        vfmadd213ps,
+        /// Fused multiply-add of packed single-precision floating-point values
+        vfmadd231ps,
+        /// Fused multiply-add of scalar double-precision floating-point values
+        vfmadd132sd,
+        /// Fused multiply-add of scalar double-precision floating-point values
+        vfmadd213sd,
+        /// Fused multiply-add of scalar double-precision floating-point values
+        vfmadd231sd,
+        /// Fused multiply-add of scalar single-precision floating-point values
+        vfmadd132ss,
+        /// Fused multiply-add of scalar single-precision floating-point values
+        vfmadd213ss,
+        /// Fused multiply-add of scalar single-precision floating-point values
+        vfmadd231ss,
+
         /// Compare string operands
         cmps,
         /// Load string
@@ -434,6 +459,12 @@ pub const Inst = struct {
         /// Register, memory (SIB), immediate (byte) operands.
         /// Uses `rix` payload with extra data of type `MemorySib`.
         rmi_sib,
+        /// Register, register, memory (RIP).
+        /// Uses `rrix` payload with extra data of type `MemoryRip`.
+        rrm_rip,
+        /// Register, register, memory (SIB).
+        /// Uses `rrix` payload with extra data of type `MemorySib`.
+        rrm_sib,
         /// Register, register, memory (RIP), immediate (byte) operands.
         /// Uses `rrix` payload with extra data of type `MemoryRip`.
         rrmi_rip,
diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig
index 77dc0cfb7c..b73a37d6cb 100644
--- a/src/arch/x86_64/bits.zig
+++ b/src/arch/x86_64/bits.zig
@@ -485,7 +485,9 @@ pub const Memory = union(enum) {
         dword,
         qword,
         tbyte,
-        dqword,
+        xword,
+        yword,
+        zword,
 
         pub fn fromSize(size: u32) PtrSize {
             return switch (size) {
@@ -493,7 +495,9 @@ pub const Memory = union(enum) {
                 2...2 => .word,
                 3...4 => .dword,
                 5...8 => .qword,
-                9...16 => .dqword,
+                9...16 => .xword,
+                17...32 => .yword,
+                33...64 => .zword,
                 else => unreachable,
             };
         }
@@ -505,7 +509,9 @@ pub const Memory = union(enum) {
                 32 => .dword,
                 64 => .qword,
                 80 => .tbyte,
-                128 => .dqword,
+                128 => .xword,
+                256 => .yword,
+                512 => .zword,
                 else => unreachable,
             };
         }
@@ -517,7 +523,9 @@ pub const Memory = union(enum) {
                 .dword => 32,
                 .qword => 64,
                 .tbyte => 80,
-                .dqword => 128,
+                .xword => 128,
+                .yword => 256,
+                .zword => 512,
             };
         }
     };
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index 5d2630e9a8..dd05728e24 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -1016,5 +1016,28 @@ pub const table = [_]Entry{
     .{ .vcvtph2ps, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128, .f16c },
 
     .{ .vcvtps2ph, .mri, &.{ .xmm_m64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_128, .f16c },
+
+    // FMA
+    .{ .vfmadd132pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128_long, .fma },
+    .{ .vfmadd132pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256_long, .fma },
+    .{ .vfmadd213pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128_long, .fma },
+    .{ .vfmadd213pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256_long, .fma },
+    .{ .vfmadd231pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128_long, .fma },
+    .{ .vfmadd231pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256_long, .fma },
+
+    .{ .vfmadd132ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128, .fma },
+    .{ .vfmadd132ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256, .fma },
+    .{ .vfmadd213ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128, .fma },
+    .{ .vfmadd213ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256, .fma },
+    .{ .vfmadd231ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128, .fma },
+    .{ .vfmadd231ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256, .fma },
+
+    .{ .vfmadd132sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_128_long, .fma },
+    .{ .vfmadd213sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_128_long, .fma },
+    .{ .vfmadd231sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_128_long, .fma },
+
+    .{ .vfmadd132ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_128, .fma },
+    .{ .vfmadd213ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_128, .fma },
+    .{ .vfmadd231ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_128, .fma },
 };
 // zig fmt: on
diff --git a/test/behavior/muladd.zig b/test/behavior/muladd.zig
index aa36c99784..8656dc4f45 100644
--- a/test/behavior/muladd.zig
+++ b/test/behavior/muladd.zig
@@ -1,8 +1,10 @@
+const std = @import("std");
 const builtin = @import("builtin");
-const expect = @import("std").testing.expect;
+const expect = std.testing.expect;
 
 test "@mulAdd" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .fma)) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From cba195c1170fff77c5210f023e019d72f13b9614 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sat, 6 May 2023 22:27:39 -0400
Subject: [PATCH 126/725] x86_64: implement some float and float vector
 movement

This allows actually storing value of these supported types in
registers, and not restricting them to stack slots.
---
 src/arch/x86_64/CodeGen.zig   | 127 ++++++++++++++++++++++++++--------
 src/arch/x86_64/Encoding.zig  |  18 +++--
 src/arch/x86_64/Lower.zig     |   6 ++
 src/arch/x86_64/Mir.zig       |  12 ++++
 src/arch/x86_64/encoder.zig   |  13 ++--
 src/arch/x86_64/encodings.zig |  30 ++++++++
 test/behavior/math.zig        |   3 +-
 test/behavior/muladd.zig      |  10 +--
 8 files changed, 176 insertions(+), 43 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index fffb814d7f..3e47ef63f6 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -2008,6 +2008,11 @@ fn computeFrameLayout(self: *Self) !FrameLayout {
     };
 }
 
+fn getFrameAddrAlignment(self: *Self, frame_addr: FrameAddr) u32 {
+    const alloc_align = @as(u32, 1) << self.frame_allocs.get(@enumToInt(frame_addr.index)).abi_align;
+    return @min(alloc_align, @bitCast(u32, frame_addr.off) & (alloc_align - 1));
+}
+
 fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
     const frame_allocs_slice = self.frame_allocs.slice();
     const frame_size = frame_allocs_slice.items(.abi_size);
@@ -2051,24 +2056,36 @@ fn allocTempRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool) !MCValue {
     return self.allocRegOrMemAdvanced(elem_ty, null, reg_ok);
 }
 
-fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
-    const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) orelse {
+fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
+    const abi_size = math.cast(u32, ty.abiSize(self.target.*)) orelse {
         const mod = self.bin_file.options.module.?;
-        return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(mod)});
+        return self.fail("type '{}' too big to fit into stack frame", .{ty.fmt(mod)});
     };
 
-    if (reg_ok) {
-        // Make sure the type can fit in a register before we try to allocate one.
-        const ptr_bits = self.target.cpu.arch.ptrBitWidth();
-        const ptr_bytes: u64 = @divExact(ptr_bits, 8);
-        if (abi_size <= ptr_bytes) {
-            if (self.register_manager.tryAllocReg(inst, regClassForType(elem_ty))) |reg| {
+    if (reg_ok) need_mem: {
+        if (abi_size <= @as(u32, switch (ty.zigTypeTag()) {
+            .Float => switch (ty.floatBits(self.target.*)) {
+                16, 32, 64, 128 => 16,
+                80 => break :need_mem,
+                else => unreachable,
+            },
+            .Vector => switch (ty.childType().zigTypeTag()) {
+                .Float => switch (ty.childType().floatBits(self.target.*)) {
+                    16, 32, 64 => if (self.hasFeature(.avx)) 32 else 16,
+                    80, 128 => break :need_mem,
+                    else => unreachable,
+                },
+                else => break :need_mem,
+            },
+            else => 8,
+        })) {
+            if (self.register_manager.tryAllocReg(inst, regClassForType(ty))) |reg| {
                 return MCValue{ .register = registerAlias(reg, abi_size) };
             }
         }
     }
 
-    const frame_index = try self.allocFrameIndex(FrameAlloc.initType(elem_ty, self.target.*));
+    const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ty, self.target.*));
     return .{ .load_frame = .{ .index = frame_index } };
 }
 
@@ -4442,12 +4459,19 @@ fn airRound(self: *Self, inst: Air.Inst.Index, mode: Immediate) !void {
         }),
     };
     assert(dst_mcv.isRegister());
+    const abi_size = @intCast(u32, ty.abiSize(self.target.*));
+    const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
     if (src_mcv.isRegister())
-        try self.asmRegisterRegisterImmediate(mir_tag, dst_mcv.getReg().?, src_mcv.getReg().?, mode)
+        try self.asmRegisterRegisterImmediate(
+            mir_tag,
+            dst_reg,
+            registerAlias(src_mcv.getReg().?, abi_size),
+            mode,
+        )
     else
         try self.asmRegisterMemoryImmediate(
             mir_tag,
-            dst_mcv.getReg().?,
+            dst_reg,
             src_mcv.mem(Memory.PtrSize.fromSize(@intCast(u32, ty.abiSize(self.target.*)))),
             mode,
         );
@@ -7847,19 +7871,43 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAirResult(inst, result);
 }
 
-fn movMirTag(self: *Self, ty: Type) !Mir.Inst.Tag {
-    return switch (ty.zigTypeTag()) {
-        else => .mov,
+fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.Tag {
+    switch (ty.zigTypeTag()) {
+        else => return .mov,
         .Float => switch (ty.floatBits(self.target.*)) {
             16 => unreachable, // needs special handling
-            32 => .movss,
-            64 => .movsd,
-            128 => .movaps,
-            else => return self.fail("TODO movMirTag from {}", .{
-                ty.fmt(self.bin_file.options.module.?),
-            }),
+            32 => return if (self.hasFeature(.avx)) .vmovss else .movss,
+            64 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,
+            128 => return if (self.hasFeature(.avx))
+                if (aligned) .vmovaps else .vmovups
+            else if (aligned) .movaps else .movups,
+            else => {},
         },
-    };
+        .Vector => switch (ty.childType().zigTypeTag()) {
+            .Float => switch (ty.childType().floatBits(self.target.*)) {
+                16 => unreachable, // needs special handling
+                32 => switch (ty.vectorLen()) {
+                    1 => return if (self.hasFeature(.avx)) .vmovss else .movss,
+                    2...4 => return if (self.hasFeature(.avx))
+                        if (aligned) .vmovaps else .vmovups
+                    else if (aligned) .movaps else .movups,
+                    5...8 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,
+                    else => {},
+                },
+                64 => switch (ty.vectorLen()) {
+                    1 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,
+                    2 => return if (self.hasFeature(.avx))
+                        if (aligned) .vmovaps else .vmovups
+                    else if (aligned) .movaps else .movups,
+                    3...4 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,
+                    else => {},
+                },
+                else => {},
+            },
+            else => {},
+        },
+    }
+    return self.fail("TODO movMirTag for {}", .{ty.fmt(self.bin_file.options.module.?)});
 }
 
 fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
@@ -8016,7 +8064,11 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                             0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
                             else => .lea,
                         },
-                        .indirect, .load_frame => try self.movMirTag(ty),
+                        .indirect => try self.movMirTag(ty, false),
+                        .load_frame => |frame_addr| try self.movMirTag(
+                            ty,
+                            self.getFrameAddrAlignment(frame_addr) >= ty.abiAlignment(self.target.*),
+                        ),
                         .lea_frame => .lea,
                         else => unreachable,
                     },
@@ -8040,7 +8092,11 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                         )
                     else
                         self.asmRegisterMemory(
-                            try self.movMirTag(ty),
+                            try self.movMirTag(ty, mem.isAlignedGeneric(
+                                u32,
+                                @bitCast(u32, small_addr),
+                                ty.abiAlignment(self.target.*),
+                            )),
                             registerAlias(dst_reg, abi_size),
                             src_mem,
                         );
@@ -8080,7 +8136,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                 )
             else
                 try self.asmRegisterMemory(
-                    try self.movMirTag(ty),
+                    try self.movMirTag(ty, false),
                     registerAlias(dst_reg, abi_size),
                     src_mem,
                 );
@@ -8194,7 +8250,24 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
                 )
             else
                 try self.asmMemoryRegister(
-                    try self.movMirTag(ty),
+                    try self.movMirTag(ty, switch (base) {
+                        .none => mem.isAlignedGeneric(
+                            u32,
+                            @bitCast(u32, disp),
+                            ty.abiAlignment(self.target.*),
+                        ),
+                        .reg => |reg| switch (reg) {
+                            .es, .cs, .ss, .ds => mem.isAlignedGeneric(
+                                u32,
+                                @bitCast(u32, disp),
+                                ty.abiAlignment(self.target.*),
+                            ),
+                            else => false,
+                        },
+                        .frame => |frame_index| self.getFrameAddrAlignment(
+                            .{ .index = frame_index, .off = disp },
+                        ) >= ty.abiAlignment(self.target.*),
+                    }),
                     dst_mem,
                     registerAlias(src_reg, abi_size),
                 );
@@ -8415,7 +8488,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
         defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
 
         const dest = try self.allocRegOrMem(inst, true);
-        try self.genCopy(self.air.typeOfIndex(inst), dest, operand);
+        try self.genCopy(if (!dest.isMemory() or operand.isMemory()) dst_ty else src_ty, dest, operand);
         break :result dest;
     };
     return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index 94bfa63999..1fd1112aaf 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -206,7 +206,7 @@ pub fn format(
             try writer.print("+{s} ", .{tag});
         },
         .m, .mi, .m1, .mc, .vmi => try writer.print("/{d} ", .{encoding.modRmExt()}),
-        .mr, .rm, .rmi, .mri, .mrc, .rvm, .rvmi => try writer.writeAll("/r "),
+        .mr, .rm, .rmi, .mri, .mrc, .rvm, .rvmi, .mvr => try writer.writeAll("/r "),
     }
 
     switch (encoding.data.op_en) {
@@ -230,7 +230,7 @@ pub fn format(
             };
             try writer.print("{s} ", .{tag});
         },
-        .np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc, .rvm => {},
+        .np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc, .rvm, .mvr => {},
     }
 
     try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
@@ -332,7 +332,12 @@ pub const Mnemonic = enum {
     // SSE4.1
     roundsd, roundss,
     // AVX
-    vmovddup, vmovshdup, vmovsldup,
+    vmovapd, vmovaps,
+    vmovddup,
+    vmovsd,
+    vmovshdup, vmovsldup,
+    vmovss,
+    vmovupd, vmovups,
     vpextrw, vpinsrw,
     vpshufhw, vpshuflw,
     vpsrld, vpsrlq, vpsrlw,
@@ -357,7 +362,7 @@ pub const OpEn = enum {
     fd, td,
     m1, mc, mi, mr, rm,
     rmi, mri, mrc,
-    vmi, rvm, rvmi,
+    vmi, rvm, rvmi, mvr,
     // zig fmt: on
 };
 
@@ -549,9 +554,10 @@ pub const Op = enum {
         return switch (op) {
             .rm8, .rm16, .rm32, .rm64,
             .r32_m16, .r64_m16,
-            .m8, .m16, .m32, .m64, .m80, .m128,
+            .m8, .m16, .m32, .m64, .m80, .m128, .m256,
             .m,
-            .xmm_m32, .xmm_m64, .xmm_m128, .ymm_m256,
+            .xmm_m32, .xmm_m64, .xmm_m128,
+            .ymm_m256,
             =>  true,
             else => false,
         };
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index a37f28c0c3..a246a97d4b 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -184,9 +184,15 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .roundsd,
         .roundss,
 
+        .vmovapd,
+        .vmovaps,
         .vmovddup,
+        .vmovsd,
         .vmovshdup,
         .vmovsldup,
+        .vmovss,
+        .vmovupd,
+        .vmovups,
         .vpextrw,
         .vpinsrw,
         .vpshufhw,
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index 92a9a74fbb..de7f2cff53 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -282,12 +282,24 @@ pub const Inst = struct {
         /// Round scalar single-precision floating-point values
         roundss,
 
+        /// Move aligned packed double-precision floating-point values
+        vmovapd,
+        /// Move aligned packed single-precision floating-point values
+        vmovaps,
         /// Replicate double floating-point values
         vmovddup,
+        /// Move or merge scalar double-precision floating-point value
+        vmovsd,
         /// Replicate single floating-point values
         vmovshdup,
         /// Replicate single floating-point values
         vmovsldup,
+        /// Move or merge scalar single-precision floating-point value
+        vmovss,
+        /// Move unaligned packed double-precision floating-point values
+        vmovupd,
+        /// Move unaligned packed single-precision floating-point values
+        vmovups,
         /// Extract word
         vpextrw,
         /// Insert word
diff --git a/src/arch/x86_64/encoder.zig b/src/arch/x86_64/encoder.zig
index 47211591ec..fa6ce676cb 100644
--- a/src/arch/x86_64/encoder.zig
+++ b/src/arch/x86_64/encoder.zig
@@ -228,7 +228,7 @@ pub const Instruction = struct {
             .td => try encoder.imm64(inst.ops[0].mem.moffs.offset),
             else => {
                 const mem_op = switch (data.op_en) {
-                    .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],
+                    .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],
                     .rm, .rmi, .vmi => inst.ops[1],
                     .rvm, .rvmi => inst.ops[2],
                     else => unreachable,
@@ -239,6 +239,7 @@ pub const Instruction = struct {
                             .m, .mi, .m1, .mc, .vmi => enc.modRmExt(),
                             .mr, .mri, .mrc => inst.ops[1].reg.lowEnc(),
                             .rm, .rmi, .rvm, .rvmi => inst.ops[0].reg.lowEnc(),
+                            .mvr => inst.ops[2].reg.lowEnc(),
                             else => unreachable,
                         };
                         try encoder.modRm_direct(rm, reg.lowEnc());
@@ -248,6 +249,7 @@ pub const Instruction = struct {
                             .m, .mi, .m1, .mc, .vmi => .none,
                             .mr, .mri, .mrc => inst.ops[1],
                             .rm, .rmi, .rvm, .rvmi => inst.ops[0],
+                            .mvr => inst.ops[2],
                             else => unreachable,
                         };
                         try encodeMemory(enc, mem, op, encoder);
@@ -315,7 +317,7 @@ pub const Instruction = struct {
                 }
             else
                 null,
-            .vmi, .rvm, .rvmi => unreachable,
+            .vmi, .rvm, .rvmi, .mvr => unreachable,
         };
         if (segment_override) |seg| {
             legacy.setSegmentOverride(seg);
@@ -350,7 +352,7 @@ pub const Instruction = struct {
                 rex.b = b_x_op.isBaseExtended();
                 rex.x = b_x_op.isIndexExtended();
             },
-            .vmi, .rvm, .rvmi => unreachable,
+            .vmi, .rvm, .rvmi, .mvr => unreachable,
         }
 
         try encoder.rex(rex);
@@ -372,10 +374,11 @@ pub const Instruction = struct {
         switch (op_en) {
             .np, .i, .zi, .fd, .td, .d => {},
             .o, .oi => vex.b = inst.ops[0].reg.isExtended(),
-            .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .vmi, .rvm, .rvmi => {
+            .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .vmi, .rvm, .rvmi, .mvr => {
                 const r_op = switch (op_en) {
                     .rm, .rmi, .rvm, .rvmi => inst.ops[0],
                     .mr, .mri, .mrc => inst.ops[1],
+                    .mvr => inst.ops[2],
                     .m, .mi, .m1, .mc, .vmi => .none,
                     else => unreachable,
                 };
@@ -383,7 +386,7 @@ pub const Instruction = struct {
 
                 const b_x_op = switch (op_en) {
                     .rm, .rmi, .vmi => inst.ops[1],
-                    .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],
+                    .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],
                     .rvm, .rvmi => inst.ops[2],
                     else => unreachable,
                 };
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index dd05728e24..607a87b8d9 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -974,12 +974,42 @@ pub const table = [_]Entry{
     .{ .roundsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .none, .sse4_1 },
 
     // AVX
+    .{ .vmovapd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .vex_128, .avx },
+    .{ .vmovapd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x29 }, 0, .vex_128, .avx },
+    .{ .vmovapd, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x66, 0x0f, 0x28 }, 0, .vex_256, .avx },
+    .{ .vmovapd, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x66, 0x0f, 0x29 }, 0, .vex_256, .avx },
+
+    .{ .vmovaps, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .vex_128, .avx },
+    .{ .vmovaps, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x29 }, 0, .vex_128, .avx },
+    .{ .vmovaps, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x0f, 0x28 }, 0, .vex_256, .avx },
+    .{ .vmovaps, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x0f, 0x29 }, 0, .vex_256, .avx },
+
     .{ .vmovddup, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_128, .avx },
 
+    .{ .vmovsd, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_128, .avx },
+    .{ .vmovsd, .rm,  &.{       .xmm, .m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_128, .avx },
+    .{ .vmovsd, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_128, .avx },
+    .{ .vmovsd, .mr,  &.{       .m64, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_128, .avx },
+
     .{ .vmovshdup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x16 }, 0, .vex_128, .avx },
 
     .{ .vmovsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .vex_128, .avx },
 
+    .{ .vmovss, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf3, 0x0f, 0x10 }, 0, .vex_128, .avx },
+    .{ .vmovss, .rm,  &.{       .xmm, .m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .vex_128, .avx },
+    .{ .vmovss, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .vex_128, .avx },
+    .{ .vmovss, .mr,  &.{       .m32, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .vex_128, .avx },
+
+    .{ .vmovupd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .vex_128, .avx },
+    .{ .vmovupd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x11 }, 0, .vex_128, .avx },
+    .{ .vmovupd, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x66, 0x0f, 0x10 }, 0, .vex_256, .avx },
+    .{ .vmovupd, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x66, 0x0f, 0x11 }, 0, .vex_256, .avx },
+
+    .{ .vmovups, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .vex_128, .avx },
+    .{ .vmovups, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x11 }, 0, .vex_128, .avx },
+    .{ .vmovups, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x0f, 0x10 }, 0, .vex_256, .avx },
+    .{ .vmovups, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x0f, 0x11 }, 0, .vex_256, .avx },
+
     .{ .vpextrw, .mri, &.{ .r32,     .xmm, .imm8 }, &.{ 0x66, 0x0f,       0x15 }, 0, .vex_128,      .avx },
     .{ .vpextrw, .mri, &.{ .r64,     .xmm, .imm8 }, &.{ 0x66, 0x0f,       0x15 }, 0, .vex_128_long, .avx },
     .{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128,      .avx },
diff --git a/test/behavior/math.zig b/test/behavior/math.zig
index 0362bd3a2b..7e16111059 100644
--- a/test/behavior/math.zig
+++ b/test/behavior/math.zig
@@ -399,7 +399,8 @@ fn testBinaryNot128(comptime Type: type, x: Type) !void {
 
 test "division" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/muladd.zig b/test/behavior/muladd.zig
index 8656dc4f45..bfb94de270 100644
--- a/test/behavior/muladd.zig
+++ b/test/behavior/muladd.zig
@@ -2,9 +2,11 @@ const std = @import("std");
 const builtin = @import("builtin");
 const expect = std.testing.expect;
 
+const stage2_x86_64_without_hardware_fma_support = builtin.zig_backend == .stage2_x86_64 and
+    !std.Target.x86.featureSetHas(builtin.cpu.features, .fma);
+
 test "@mulAdd" {
-    if (builtin.zig_backend == .stage2_x86_64 and
-        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .fma)) return error.SkipZigTest; // TODO
+    if (stage2_x86_64_without_hardware_fma_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -118,7 +120,7 @@ fn vector32() !void {
 
 test "vector f32" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (stage2_x86_64_without_hardware_fma_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -141,7 +143,7 @@ fn vector64() !void {
 
 test "vector f64" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (stage2_x86_64_without_hardware_fma_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From 406c4035435657aaefe6f8e96642d0db326c7989 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sat, 6 May 2023 23:45:36 -0400
Subject: [PATCH 127/725] x86_64: add missing `movsx` and `movzx` encodings

---
 src/arch/x86_64/encodings.zig | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index 607a87b8d9..5096ca5627 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -418,17 +418,21 @@ pub const table = [_]Entry{
     .{ .movsx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xbe }, 0, .rex,       .none },
     .{ .movsx, .rm, &.{ .r64, .rm8  }, &.{ 0x0f, 0xbe }, 0, .long,      .none },
     .{ .movsx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xbf }, 0, .none,      .none },
+    .{ .movsx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xbf }, 0, .rex,       .none },
     .{ .movsx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xbf }, 0, .long,      .none },
 
     // This instruction is discouraged.
     .{ .movsxd, .rm, &.{ .r32, .rm32 }, &.{ 0x63 }, 0, .none, .none },
     .{ .movsxd, .rm, &.{ .r64, .rm32 }, &.{ 0x63 }, 0, .long, .none },
 
-    .{ .movzx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .short, .none },
-    .{ .movzx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .none,  .none },
-    .{ .movzx, .rm, &.{ .r64, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .long,  .none },
-    .{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .none,  .none },
-    .{ .movzx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .long,  .none },
+    .{ .movzx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .short,     .none },
+    .{ .movzx, .rm, &.{ .r16, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .rex_short, .none },
+    .{ .movzx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .none,      .none },
+    .{ .movzx, .rm, &.{ .r32, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .rex,       .none },
+    .{ .movzx, .rm, &.{ .r64, .rm8  }, &.{ 0x0f, 0xb6 }, 0, .long,      .none },
+    .{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .none,      .none },
+    .{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .rex,       .none },
+    .{ .movzx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .long,      .none },
 
     .{ .mul, .m, &.{ .rm8  }, &.{ 0xf6 }, 4, .none,  .none },
     .{ .mul, .m, &.{ .rm8  }, &.{ 0xf6 }, 4, .rex,   .none },

From 1c53f0a6b0b1866663346c473da310203a317e90 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sun, 7 May 2023 00:27:30 -0400
Subject: [PATCH 128/725] Dwarf: workaround crash

---
 src/link/Dwarf.zig      | 54 ++++++++++++++++++++++-------------------
 test/behavior/error.zig |  3 ---
 2 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/src/link/Dwarf.zig b/src/link/Dwarf.zig
index 7a008ca732..2ec0dedc6f 100644
--- a/src/link/Dwarf.zig
+++ b/src/link/Dwarf.zig
@@ -526,7 +526,7 @@ pub const DeclState = struct {
             .ErrorUnion => {
                 const error_ty = ty.errorUnionSet();
                 const payload_ty = ty.errorUnionPayload();
-                const payload_align = payload_ty.abiAlignment(target);
+                const payload_align = if (payload_ty.isNoReturn()) 0 else payload_ty.abiAlignment(target);
                 const error_align = Type.anyerror.abiAlignment(target);
                 const abi_size = ty.abiSize(target);
                 const payload_off = if (error_align >= payload_align) Type.anyerror.abiSize(target) else 0;
@@ -540,31 +540,35 @@ pub const DeclState = struct {
                 const name = try ty.nameAllocArena(arena, module);
                 try dbg_info_buffer.writer().print("{s}\x00", .{name});
 
-                // DW.AT.member
-                try dbg_info_buffer.ensureUnusedCapacity(7);
-                dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_member));
-                // DW.AT.name, DW.FORM.string
-                dbg_info_buffer.appendSliceAssumeCapacity("value");
-                dbg_info_buffer.appendAssumeCapacity(0);
-                // DW.AT.type, DW.FORM.ref4
-                var index = dbg_info_buffer.items.len;
-                try dbg_info_buffer.resize(index + 4);
-                try self.addTypeRelocGlobal(atom_index, payload_ty, @intCast(u32, index));
-                // DW.AT.data_member_location, DW.FORM.sdata
-                try leb128.writeULEB128(dbg_info_buffer.writer(), payload_off);
+                if (!payload_ty.isNoReturn()) {
+                    // DW.AT.member
+                    try dbg_info_buffer.ensureUnusedCapacity(7);
+                    dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_member));
+                    // DW.AT.name, DW.FORM.string
+                    dbg_info_buffer.appendSliceAssumeCapacity("value");
+                    dbg_info_buffer.appendAssumeCapacity(0);
+                    // DW.AT.type, DW.FORM.ref4
+                    const index = dbg_info_buffer.items.len;
+                    try dbg_info_buffer.resize(index + 4);
+                    try self.addTypeRelocGlobal(atom_index, payload_ty, @intCast(u32, index));
+                    // DW.AT.data_member_location, DW.FORM.sdata
+                    try leb128.writeULEB128(dbg_info_buffer.writer(), payload_off);
+                }
 
-                // DW.AT.member
-                try dbg_info_buffer.ensureUnusedCapacity(5);
-                dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_member));
-                // DW.AT.name, DW.FORM.string
-                dbg_info_buffer.appendSliceAssumeCapacity("err");
-                dbg_info_buffer.appendAssumeCapacity(0);
-                // DW.AT.type, DW.FORM.ref4
-                index = dbg_info_buffer.items.len;
-                try dbg_info_buffer.resize(index + 4);
-                try self.addTypeRelocGlobal(atom_index, error_ty, @intCast(u32, index));
-                // DW.AT.data_member_location, DW.FORM.sdata
-                try leb128.writeULEB128(dbg_info_buffer.writer(), error_off);
+                {
+                    // DW.AT.member
+                    try dbg_info_buffer.ensureUnusedCapacity(5);
+                    dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_member));
+                    // DW.AT.name, DW.FORM.string
+                    dbg_info_buffer.appendSliceAssumeCapacity("err");
+                    dbg_info_buffer.appendAssumeCapacity(0);
+                    // DW.AT.type, DW.FORM.ref4
+                    const index = dbg_info_buffer.items.len;
+                    try dbg_info_buffer.resize(index + 4);
+                    try self.addTypeRelocGlobal(atom_index, error_ty, @intCast(u32, index));
+                    // DW.AT.data_member_location, DW.FORM.sdata
+                    try leb128.writeULEB128(dbg_info_buffer.writer(), error_off);
+                }
 
                 // DW.AT.structure_type delimit children
                 try dbg_info_buffer.append(0);
diff --git a/test/behavior/error.zig b/test/behavior/error.zig
index 0cd9be05ca..91b5561d62 100644
--- a/test/behavior/error.zig
+++ b/test/behavior/error.zig
@@ -757,7 +757,6 @@ test "error union of noreturn used with if" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     NoReturn.a = 64;
@@ -772,7 +771,6 @@ test "error union of noreturn used with try" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     NoReturn.a = 64;
@@ -784,7 +782,6 @@ test "error union of noreturn used with catch" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 
     NoReturn.a = 64;

From 05580b9453e4ae2d9b62fe4178651937d8b73989 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sun, 7 May 2023 03:14:31 -0400
Subject: [PATCH 129/725] x86_64: implement float cast from `f16` to `f64`

---
 src/arch/x86_64/CodeGen.zig   |  95 ++++++++++++-----
 src/arch/x86_64/Encoding.zig  | 163 +++++++++++++++++------------
 src/arch/x86_64/Lower.zig     |   4 +
 src/arch/x86_64/Mir.zig       |   8 ++
 src/arch/x86_64/encoder.zig   |  33 ++----
 src/arch/x86_64/encodings.zig | 189 ++++++++++++++++++----------------
 test/behavior/floatop.zig     |   3 +-
 7 files changed, 288 insertions(+), 207 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 3e47ef63f6..38497400f2 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -2287,26 +2287,46 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
         src_mcv
     else
         try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
-    const dst_lock = self.register_manager.lockReg(dst_mcv.register);
+    const dst_reg = dst_mcv.getReg().?.to128();
+    const dst_lock = self.register_manager.lockReg(dst_reg);
     defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
-    if (src_bits == 32 and dst_bits == 16 and self.hasFeature(.f16c))
-        try self.asmRegisterRegisterImmediate(
-            .vcvtps2ph,
-            dst_mcv.register,
-            if (src_mcv.isRegister()) src_mcv.getReg().? else src_reg: {
-                const src_reg = dst_mcv.register;
-                try self.genSetReg(src_reg, src_ty, src_mcv);
-                break :src_reg src_reg;
+    if (dst_bits == 16 and self.hasFeature(.f16c)) {
+        switch (src_bits) {
+            32 => {
+                const mat_src_reg = if (src_mcv.isRegister())
+                    src_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(src_ty, src_mcv);
+                try self.asmRegisterRegisterImmediate(
+                    .vcvtps2ph,
+                    dst_reg,
+                    mat_src_reg.to128(),
+                    Immediate.u(0b1_00),
+                );
             },
-            Immediate.u(0b1_00),
-        )
-    else if (src_bits == 64 and dst_bits == 32)
-        try self.genBinOpMir(.cvtsd2ss, src_ty, dst_mcv, src_mcv)
-    else
-        return self.fail("TODO implement airFptrunc from {} to {}", .{
-            src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
-        });
+            else => return self.fail("TODO implement airFptrunc from {} to {}", .{
+                src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
+            }),
+        }
+    } else if (src_bits == 64 and dst_bits == 32) {
+        if (self.hasFeature(.avx)) if (src_mcv.isRegister()) try self.asmRegisterRegisterRegister(
+            .vcvtsd2ss,
+            dst_reg,
+            dst_reg,
+            src_mcv.getReg().?.to128(),
+        ) else try self.asmRegisterRegisterMemory(
+            .vcvtsd2ss,
+            dst_reg,
+            dst_reg,
+            src_mcv.mem(.qword),
+        ) else if (src_mcv.isRegister())
+            try self.asmRegisterRegister(.cvtsd2ss, dst_reg, src_mcv.getReg().?.to128())
+        else
+            try self.asmRegisterMemory(.cvtsd2ss, dst_reg, src_mcv.mem(.qword));
+    } else return self.fail("TODO implement airFptrunc from {} to {}", .{
+        src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
+    });
     return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
 }
 
@@ -2322,22 +2342,41 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
         src_mcv
     else
         try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
-    const dst_lock = self.register_manager.lockReg(dst_mcv.register);
+    const dst_reg = dst_mcv.getReg().?.to128();
+    const dst_lock = self.register_manager.lockReg(dst_reg);
     defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
-    try self.genBinOpMir(
-        if (src_bits == 16 and dst_bits == 32 and self.hasFeature(.f16c))
-            .vcvtph2ps
-        else if (src_bits == 32 and dst_bits == 64)
-            .cvtss2sd
+    if (src_bits == 16 and self.hasFeature(.f16c)) {
+        const mat_src_reg = if (src_mcv.isRegister())
+            src_mcv.getReg().?
         else
-            return self.fail("TODO implement airFpext from {} to {}", .{
+            try self.copyToTmpRegister(src_ty, src_mcv);
+        try self.asmRegisterRegister(.vcvtph2ps, dst_reg, mat_src_reg.to128());
+        switch (dst_bits) {
+            32 => {},
+            64 => try self.asmRegisterRegisterRegister(.vcvtss2sd, dst_reg, dst_reg, dst_reg),
+            else => return self.fail("TODO implement airFpext from {} to {}", .{
                 src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
             }),
-        src_ty,
-        dst_mcv,
-        src_mcv,
-    );
+        }
+    } else if (src_bits == 32 and dst_bits == 64) {
+        if (self.hasFeature(.avx)) if (src_mcv.isRegister()) try self.asmRegisterRegisterRegister(
+            .vcvtss2sd,
+            dst_reg,
+            dst_reg,
+            src_mcv.getReg().?.to128(),
+        ) else try self.asmRegisterRegisterMemory(
+            .vcvtss2sd,
+            dst_reg,
+            dst_reg,
+            src_mcv.mem(.dword),
+        ) else if (src_mcv.isRegister())
+            try self.asmRegisterRegister(.cvtss2sd, dst_reg, src_mcv.getReg().?.to128())
+        else
+            try self.asmRegisterMemory(.cvtss2sd, dst_reg, src_mcv.mem(.dword));
+    } else return self.fail("TODO implement airFpext from {} to {}", .{
+        src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
+    });
     return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
 }
 
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index 1fd1112aaf..bd6e70c975 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -89,30 +89,13 @@ pub fn findByOpcode(opc: []const u8, prefixes: struct {
         if (modrm_ext) |ext| if (ext != data.modrm_ext) continue;
         if (!std.mem.eql(u8, opc, enc.opcode())) continue;
         if (prefixes.rex.w) {
-            switch (data.mode) {
-                .none, .short, .rex, .rex_short, .vex_128, .vex_256 => continue,
-                .long, .vex_128_long, .vex_256_long => {},
-            }
+            if (!data.mode.isLong()) continue;
         } else if (prefixes.rex.present and !prefixes.rex.isSet()) {
-            switch (data.mode) {
-                .rex, .rex_short => {},
-                else => continue,
-            }
+            if (!data.mode.isRex()) continue;
         } else if (prefixes.legacy.prefix_66) {
-            switch (data.mode) {
-                .short, .rex_short => {},
-                .none, .rex, .vex_128, .vex_256 => continue,
-                .long, .vex_128_long, .vex_256_long => continue,
-            }
+            if (!data.mode.isShort()) continue;
         } else {
-            switch (data.mode) {
-                .none => switch (data.mode) {
-                    .short, .rex_short => continue,
-                    .none, .rex, .vex_128, .vex_256 => {},
-                    .long, .vex_128_long, .vex_256_long => {},
-                },
-                else => continue,
-            }
+            if (data.mode.isShort()) continue;
         }
         return enc;
     };
@@ -148,50 +131,39 @@ pub fn format(
     _ = fmt;
 
     var opc = encoding.opcode();
-    switch (encoding.data.mode) {
-        else => {},
-        .long => try writer.writeAll("REX.W + "),
-        .vex_128, .vex_128_long, .vex_256, .vex_256_long => {
-            try writer.writeAll("VEX.");
+    if (encoding.data.mode.isVex()) {
+        try writer.writeAll("VEX.");
 
-            switch (encoding.data.mode) {
-                .vex_128, .vex_128_long => try writer.writeAll("128"),
-                .vex_256, .vex_256_long => try writer.writeAll("256"),
-                else => unreachable,
-            }
+        try writer.writeAll(switch (encoding.data.mode) {
+            .vex_128_w0, .vex_128_w1, .vex_128_wig => "128",
+            .vex_256_w0, .vex_256_w1, .vex_256_wig => "256",
+            .vex_lig_w0, .vex_lig_w1, .vex_lig_wig => "LIG",
+            .vex_lz_w0, .vex_lz_w1, .vex_lz_wig => "LZ",
+            else => unreachable,
+        });
 
-            switch (opc[0]) {
-                else => {},
-                0x66, 0xf3, 0xf2 => {
-                    try writer.print(".{X:0>2}", .{opc[0]});
-                    opc = opc[1..];
-                },
-            }
+        switch (opc[0]) {
+            else => {},
+            0x66, 0xf3, 0xf2 => {
+                try writer.print(".{X:0>2}", .{opc[0]});
+                opc = opc[1..];
+            },
+        }
 
-            try writer.print(".{X:0>2}", .{opc[0]});
-            opc = opc[1..];
+        try writer.print(".{}", .{std.fmt.fmtSliceHexUpper(opc[0 .. opc.len - 1])});
+        opc = opc[opc.len - 1 ..];
 
-            switch (opc[0]) {
-                else => {},
-                0x38, 0x3A => {
-                    try writer.print("{X:0>2}", .{opc[0]});
-                    opc = opc[1..];
-                },
-            }
+        try writer.writeAll(".W");
+        try writer.writeAll(switch (encoding.data.mode) {
+            .vex_128_w0, .vex_256_w0, .vex_lig_w0, .vex_lz_w0 => "0",
+            .vex_128_w1, .vex_256_w1, .vex_lig_w1, .vex_lz_w1 => "1",
+            .vex_128_wig, .vex_256_wig, .vex_lig_wig, .vex_lz_wig => "IG",
+            else => unreachable,
+        });
 
-            try writer.writeByte('.');
-            try writer.writeAll(switch (encoding.data.mode) {
-                .vex_128, .vex_256 => "W0",
-                .vex_128_long, .vex_256_long => "W1",
-                else => unreachable,
-            });
-            try writer.writeByte(' ');
-        },
-    }
-
-    for (opc) |byte| {
-        try writer.print("{x:0>2} ", .{byte});
-    }
+        try writer.writeByte(' ');
+    } else if (encoding.data.mode.isLong()) try writer.writeAll("REX.W + ");
+    for (opc) |byte| try writer.print("{x:0>2} ", .{byte});
 
     switch (encoding.data.op_en) {
         .np, .fd, .td, .i, .zi, .d => {},
@@ -332,6 +304,7 @@ pub const Mnemonic = enum {
     // SSE4.1
     roundsd, roundss,
     // AVX
+    vcvtsd2ss, vcvtsi2sd, vcvtsi2ss, vcvtss2sd,
     vmovapd, vmovaps,
     vmovddup,
     vmovsd,
@@ -629,20 +602,74 @@ pub const Op = enum {
 };
 
 pub const Mode = enum {
+    // zig fmt: off
     none,
-    short,
-    long,
-    rex,
-    rex_short,
-    vex_128,
-    vex_128_long,
-    vex_256,
-    vex_256_long,
+    short, long,
+    rex, rex_short,
+    vex_128_w0, vex_128_w1, vex_128_wig,
+    vex_256_w0, vex_256_w1, vex_256_wig,
+    vex_lig_w0, vex_lig_w1, vex_lig_wig,
+    vex_lz_w0,  vex_lz_w1,  vex_lz_wig,
+    // zig fmt: on
+
+    pub fn isShort(mode: Mode) bool {
+        return switch (mode) {
+            .short, .rex_short => true,
+            else => false,
+        };
+    }
+
+    pub fn isLong(mode: Mode) bool {
+        return switch (mode) {
+            .long,
+            .vex_128_w1,
+            .vex_256_w1,
+            .vex_lig_w1,
+            .vex_lz_w1,
+            => true,
+            else => false,
+        };
+    }
+
+    pub fn isRex(mode: Mode) bool {
+        return switch (mode) {
+            else => false,
+            .rex, .rex_short => true,
+        };
+    }
+
+    pub fn isVex(mode: Mode) bool {
+        return switch (mode) {
+            // zig fmt: off
+            else => false,
+            .vex_128_w0, .vex_128_w1, .vex_128_wig,
+            .vex_256_w0, .vex_256_w1, .vex_256_wig,
+            .vex_lig_w0, .vex_lig_w1, .vex_lig_wig,
+            .vex_lz_w0,  .vex_lz_w1,  .vex_lz_wig,
+            => true,
+            // zig fmt: on
+        };
+    }
+
+    pub fn isVecLong(mode: Mode) bool {
+        return switch (mode) {
+            // zig fmt: off
+            else => unreachable,
+            .vex_128_w0, .vex_128_w1, .vex_128_wig,
+            .vex_lig_w0, .vex_lig_w1, .vex_lig_wig,
+            .vex_lz_w0,  .vex_lz_w1,  .vex_lz_wig,
+            => false,
+            .vex_256_w0, .vex_256_w1, .vex_256_wig,
+            => true,
+            // zig fmt: on
+        };
+    }
 };
 
 pub const Feature = enum {
     none,
     avx,
+    avx2,
     f16c,
     fma,
     sse,
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index a246a97d4b..40a5ccdb10 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -184,6 +184,10 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .roundsd,
         .roundss,
 
+        .vcvtsd2ss,
+        .vcvtsi2sd,
+        .vcvtsi2ss,
+        .vcvtss2sd,
         .vmovapd,
         .vmovaps,
         .vmovddup,
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index de7f2cff53..cb1a578bb6 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -282,6 +282,14 @@ pub const Inst = struct {
         /// Round scalar single-precision floating-point values
         roundss,
 
+        /// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
+        vcvtsd2ss,
+        /// Convert doubleword integer to scalar double-precision floating-point value
+        vcvtsi2sd,
+        /// Convert doubleword integer to scalar single-precision floating-point value
+        vcvtsi2ss,
+        /// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
+        vcvtss2sd,
         /// Move aligned packed double-precision floating-point values
         vmovapd,
         /// Move aligned packed single-precision floating-point values
diff --git a/src/arch/x86_64/encoder.zig b/src/arch/x86_64/encoder.zig
index fa6ce676cb..0ce875240d 100644
--- a/src/arch/x86_64/encoder.zig
+++ b/src/arch/x86_64/encoder.zig
@@ -206,18 +206,15 @@ pub const Instruction = struct {
         const enc = inst.encoding;
         const data = enc.data;
 
-        switch (data.mode) {
-            .none, .short, .long, .rex, .rex_short => {
-                try inst.encodeLegacyPrefixes(encoder);
-                try inst.encodeMandatoryPrefix(encoder);
-                try inst.encodeRexPrefix(encoder);
-                try inst.encodeOpcode(encoder);
-            },
-            .vex_128, .vex_128_long, .vex_256, .vex_256_long => {
-                try inst.encodeVexPrefix(encoder);
-                const opc = inst.encoding.opcode();
-                try encoder.opcode_1byte(opc[opc.len - 1]);
-            },
+        if (data.mode.isVex()) {
+            try inst.encodeVexPrefix(encoder);
+            const opc = inst.encoding.opcode();
+            try encoder.opcode_1byte(opc[opc.len - 1]);
+        } else {
+            try inst.encodeLegacyPrefixes(encoder);
+            try inst.encodeMandatoryPrefix(encoder);
+            try inst.encodeRexPrefix(encoder);
+            try inst.encodeOpcode(encoder);
         }
 
         switch (data.op_en) {
@@ -365,11 +362,7 @@ pub const Instruction = struct {
 
         var vex = Vex{};
 
-        vex.w = switch (inst.encoding.data.mode) {
-            .vex_128, .vex_256 => false,
-            .vex_128_long, .vex_256_long => true,
-            else => unreachable,
-        };
+        vex.w = inst.encoding.data.mode.isLong();
 
         switch (op_en) {
             .np, .i, .zi, .fd, .td, .d => {},
@@ -395,11 +388,7 @@ pub const Instruction = struct {
             },
         }
 
-        vex.l = switch (inst.encoding.data.mode) {
-            .vex_128, .vex_128_long => false,
-            .vex_256, .vex_256_long => true,
-            else => unreachable,
-        };
+        vex.l = inst.encoding.data.mode.isVecLong();
 
         vex.p = if (mand_pre) |mand| switch (mand) {
             0x66 => .@"66",
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index 5096ca5627..5e4dc2f04b 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -918,7 +918,6 @@ pub const table = [_]Entry{
     .{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .none, .sse2 },
 
     .{ .pextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .none, .sse2 },
-    .{ .pextrw, .rmi, &.{ .r64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .long, .sse2 },
 
     .{ .pinsrw, .rmi, &.{ .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .none, .sse2 },
 
@@ -926,31 +925,23 @@ pub const table = [_]Entry{
 
     .{ .pshuflw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf2, 0x0f, 0x70 }, 0, .none, .sse2 },
 
+    .{ .psrlw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .none, .sse2 },
+    .{ .psrlw, .mi, &.{ .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x71 }, 2, .none, .sse2 },
     .{ .psrld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .none, .sse2 },
     .{ .psrld, .mi, &.{ .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x72 }, 2, .none, .sse2 },
-
     .{ .psrlq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .none, .sse2 },
     .{ .psrlq, .mi, &.{ .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x73 }, 2, .none, .sse2 },
 
-    .{ .psrlw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .none, .sse2 },
-    .{ .psrlw, .mi, &.{ .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x71 }, 2, .none, .sse2 },
-
-    .{ .punpckhbw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .none, .sse2 },
-
-    .{ .punpckhdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .none, .sse2 },
-
+    .{ .punpckhbw,  .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .none, .sse2 },
+    .{ .punpckhwd,  .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .none, .sse2 },
+    .{ .punpckhdq,  .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .none, .sse2 },
     .{ .punpckhqdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6d }, 0, .none, .sse2 },
 
-    .{ .punpckhwd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .none, .sse2 },
-
-    .{ .punpcklbw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x60 }, 0, .none, .sse2 },
-
-    .{ .punpckldq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .none, .sse2 },
-
+    .{ .punpcklbw,  .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x60 }, 0, .none, .sse2 },
+    .{ .punpcklwd,  .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x61 }, 0, .none, .sse2 },
+    .{ .punpckldq,  .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .none, .sse2 },
     .{ .punpcklqdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .none, .sse2 },
 
-    .{ .punpcklwd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x61 }, 0, .none, .sse2 },
-
     .{ .sqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .none, .sse2 },
     .{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0xf2, 0x0f, 0x51 }, 0, .none, .sse2 },
 
@@ -972,106 +963,128 @@ pub const table = [_]Entry{
 
     // SSE4.1
     .{ .pextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .none, .sse4_1 },
-    .{ .pextrw, .mri, &.{ .r64_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .long, .sse4_1 },
 
     .{ .roundss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .none, .sse4_1 },
     .{ .roundsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .none, .sse4_1 },
 
     // AVX
-    .{ .vmovapd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .vex_128, .avx },
-    .{ .vmovapd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x29 }, 0, .vex_128, .avx },
-    .{ .vmovapd, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x66, 0x0f, 0x28 }, 0, .vex_256, .avx },
-    .{ .vmovapd, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x66, 0x0f, 0x29 }, 0, .vex_256, .avx },
+    .{ .vcvtsd2ss, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx },
 
-    .{ .vmovaps, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .vex_128, .avx },
-    .{ .vmovaps, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x29 }, 0, .vex_128, .avx },
-    .{ .vmovaps, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x0f, 0x28 }, 0, .vex_256, .avx },
-    .{ .vmovaps, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x0f, 0x29 }, 0, .vex_256, .avx },
+    .{ .vcvtsi2sd, .rvm, &.{ .xmm, .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx },
+    .{ .vcvtsi2sd, .rvm, &.{ .xmm, .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx },
 
-    .{ .vmovddup, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_128, .avx },
+    .{ .vcvtsi2ss, .rvm, &.{ .xmm, .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx },
+    .{ .vcvtsi2ss, .rvm, &.{ .xmm, .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx },
 
-    .{ .vmovsd, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_128, .avx },
-    .{ .vmovsd, .rm,  &.{       .xmm, .m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_128, .avx },
-    .{ .vmovsd, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_128, .avx },
-    .{ .vmovsd, .mr,  &.{       .m64, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_128, .avx },
+    .{ .vcvtss2sd, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx },
 
-    .{ .vmovshdup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x16 }, 0, .vex_128, .avx },
+    .{ .vmovapd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .vex_128_wig, .avx },
+    .{ .vmovapd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x29 }, 0, .vex_128_wig, .avx },
+    .{ .vmovapd, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x66, 0x0f, 0x28 }, 0, .vex_256_wig, .avx },
+    .{ .vmovapd, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x66, 0x0f, 0x29 }, 0, .vex_256_wig, .avx },
 
-    .{ .vmovsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .vex_128, .avx },
+    .{ .vmovaps, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .vex_128_wig, .avx },
+    .{ .vmovaps, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x29 }, 0, .vex_128_wig, .avx },
+    .{ .vmovaps, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x0f, 0x28 }, 0, .vex_256_wig, .avx },
+    .{ .vmovaps, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x0f, 0x29 }, 0, .vex_256_wig, .avx },
 
-    .{ .vmovss, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf3, 0x0f, 0x10 }, 0, .vex_128, .avx },
-    .{ .vmovss, .rm,  &.{       .xmm, .m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .vex_128, .avx },
-    .{ .vmovss, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .vex_128, .avx },
-    .{ .vmovss, .mr,  &.{       .m32, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .vex_128, .avx },
+    .{ .vmovddup, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
+    .{ .vmovddup, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_256_wig, .avx },
 
-    .{ .vmovupd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .vex_128, .avx },
-    .{ .vmovupd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x11 }, 0, .vex_128, .avx },
-    .{ .vmovupd, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x66, 0x0f, 0x10 }, 0, .vex_256, .avx },
-    .{ .vmovupd, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x66, 0x0f, 0x11 }, 0, .vex_256, .avx },
+    .{ .vmovsd, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
+    .{ .vmovsd, .rm,  &.{       .xmm, .m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
+    .{ .vmovsd, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
+    .{ .vmovsd, .mr,  &.{       .m64, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
 
-    .{ .vmovups, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .vex_128, .avx },
-    .{ .vmovups, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x11 }, 0, .vex_128, .avx },
-    .{ .vmovups, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x0f, 0x10 }, 0, .vex_256, .avx },
-    .{ .vmovups, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x0f, 0x11 }, 0, .vex_256, .avx },
+    .{ .vmovshdup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
+    .{ .vmovshdup, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf3, 0x0f, 0x16 }, 0, .vex_256_wig, .avx },
 
-    .{ .vpextrw, .mri, &.{ .r32,     .xmm, .imm8 }, &.{ 0x66, 0x0f,       0x15 }, 0, .vex_128,      .avx },
-    .{ .vpextrw, .mri, &.{ .r64,     .xmm, .imm8 }, &.{ 0x66, 0x0f,       0x15 }, 0, .vex_128_long, .avx },
-    .{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128,      .avx },
-    .{ .vpextrw, .mri, &.{ .r64_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_long, .avx },
+    .{ .vmovsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
+    .{ .vmovsldup, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .vex_256_wig, .avx },
 
-    .{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128, .avx },
+    .{ .vmovss, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf3, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
+    .{ .vmovss, .rm,  &.{       .xmm, .m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
+    .{ .vmovss, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
+    .{ .vmovss, .mr,  &.{       .m32, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
 
-    .{ .vpsrld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .vex_128, .avx },
-    .{ .vpsrld, .vmi, &.{ .xmm, .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x72 }, 2, .vex_128, .avx },
+    .{ .vmovupd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .vex_128_wig, .avx },
+    .{ .vmovupd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x11 }, 0, .vex_128_wig, .avx },
+    .{ .vmovupd, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x66, 0x0f, 0x10 }, 0, .vex_256_wig, .avx },
+    .{ .vmovupd, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x66, 0x0f, 0x11 }, 0, .vex_256_wig, .avx },
 
-    .{ .vpsrlq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .vex_128, .avx },
-    .{ .vpsrlq, .vmi, &.{ .xmm, .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x73 }, 2, .vex_128, .avx },
+    .{ .vmovups, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .vex_128_wig, .avx },
+    .{ .vmovups, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x11 }, 0, .vex_128_wig, .avx },
+    .{ .vmovups, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x0f, 0x10 }, 0, .vex_256_wig, .avx },
+    .{ .vmovups, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x0f, 0x11 }, 0, .vex_256_wig, .avx },
 
-    .{ .vpsrlw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .vex_128, .avx },
-    .{ .vpsrlw, .vmi, &.{ .xmm, .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x71 }, 2, .vex_128, .avx },
+    .{ .vpextrw, .rmi, &.{ .r32,     .xmm, .imm8 }, &.{ 0x66, 0x0f,       0x15 }, 0, .vex_128_wig, .avx },
+    .{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_wig, .avx },
 
-    .{ .vpunpckhbw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_128, .avx },
+    .{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_wig, .avx },
 
-    .{ .vpunpckhdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_128, .avx },
+    .{ .vpsrlw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .vex_128_wig, .avx },
+    .{ .vpsrlw, .vmi, &.{ .xmm, .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x71 }, 2, .vex_128_wig, .avx },
+    .{ .vpsrld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .vex_128_wig, .avx },
+    .{ .vpsrld, .vmi, &.{ .xmm, .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x72 }, 2, .vex_128_wig, .avx },
+    .{ .vpsrlq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .vex_128_wig, .avx },
+    .{ .vpsrlq, .vmi, &.{ .xmm, .xmm, .imm8     }, &.{ 0x66, 0x0f, 0x73 }, 2, .vex_128_wig, .avx },
 
-    .{ .vpunpckhqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6d }, 0, .vex_128, .avx },
+    .{ .vpunpckhbw,  .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_128_wig, .avx },
+    .{ .vpunpckhwd,  .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_128_wig, .avx },
+    .{ .vpunpckhdq,  .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_128_wig, .avx },
+    .{ .vpunpckhqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6d }, 0, .vex_128_wig, .avx },
 
-    .{ .vpunpckhwd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_128, .avx },
-
-    .{ .vpunpcklbw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x60 }, 0, .vex_128, .avx },
-
-    .{ .vpunpckldq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .vex_128, .avx },
-
-    .{ .vpunpcklqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .vex_128, .avx },
-
-    .{ .vpunpcklwd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x61 }, 0, .vex_128, .avx },
+    .{ .vpunpcklbw,  .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x60 }, 0, .vex_128_wig, .avx },
+    .{ .vpunpcklwd,  .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x61 }, 0, .vex_128_wig, .avx },
+    .{ .vpunpckldq,  .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .vex_128_wig, .avx },
+    .{ .vpunpcklqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .vex_128_wig, .avx },
 
     // F16C
-    .{ .vcvtph2ps, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128, .f16c },
+    .{ .vcvtph2ps, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128_w0, .f16c },
+    .{ .vcvtph2ps, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_256_w0, .f16c },
 
-    .{ .vcvtps2ph, .mri, &.{ .xmm_m64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_128, .f16c },
+    .{ .vcvtps2ph, .mri, &.{ .xmm_m64,  .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_128_w0, .f16c },
+    .{ .vcvtps2ph, .mri, &.{ .xmm_m128, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_256_w0, .f16c },
 
     // FMA
-    .{ .vfmadd132pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128_long, .fma },
-    .{ .vfmadd132pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256_long, .fma },
-    .{ .vfmadd213pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128_long, .fma },
-    .{ .vfmadd213pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256_long, .fma },
-    .{ .vfmadd231pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128_long, .fma },
-    .{ .vfmadd231pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256_long, .fma },
+    .{ .vfmadd132pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128_w1, .fma },
+    .{ .vfmadd213pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128_w1, .fma },
+    .{ .vfmadd231pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128_w1, .fma },
+    .{ .vfmadd132pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256_w1, .fma },
+    .{ .vfmadd213pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256_w1, .fma },
+    .{ .vfmadd231pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256_w1, .fma },
 
-    .{ .vfmadd132ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128, .fma },
-    .{ .vfmadd132ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256, .fma },
-    .{ .vfmadd213ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128, .fma },
-    .{ .vfmadd213ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256, .fma },
-    .{ .vfmadd231ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128, .fma },
-    .{ .vfmadd231ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256, .fma },
+    .{ .vfmadd132ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128_w0, .fma },
+    .{ .vfmadd213ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128_w0, .fma },
+    .{ .vfmadd231ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128_w0, .fma },
+    .{ .vfmadd132ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256_w0, .fma },
+    .{ .vfmadd213ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256_w0, .fma },
+    .{ .vfmadd231ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256_w0, .fma },
 
-    .{ .vfmadd132sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_128_long, .fma },
-    .{ .vfmadd213sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_128_long, .fma },
-    .{ .vfmadd231sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_128_long, .fma },
+    .{ .vfmadd132sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_lig_w1, .fma },
+    .{ .vfmadd213sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_lig_w1, .fma },
+    .{ .vfmadd231sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_lig_w1, .fma },
 
-    .{ .vfmadd132ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_128, .fma },
-    .{ .vfmadd213ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_128, .fma },
-    .{ .vfmadd231ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_128, .fma },
+    .{ .vfmadd132ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_lig_w0, .fma },
+    .{ .vfmadd213ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_lig_w0, .fma },
+    .{ .vfmadd231ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_lig_w0, .fma },
+
+    // AVX2
+    .{ .vpsrlw, .rvm, &.{ .ymm, .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpsrlw, .vmi, &.{ .ymm, .ymm, .imm8     }, &.{ 0x66, 0x0f, 0x71 }, 2, .vex_256_wig, .avx2 },
+    .{ .vpsrld, .rvm, &.{ .ymm, .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpsrld, .vmi, &.{ .ymm, .ymm, .imm8     }, &.{ 0x66, 0x0f, 0x72 }, 2, .vex_256_wig, .avx2 },
+    .{ .vpsrlq, .rvm, &.{ .ymm, .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpsrlq, .vmi, &.{ .ymm, .ymm, .imm8     }, &.{ 0x66, 0x0f, 0x73 }, 2, .vex_256_wig, .avx2 },
+
+    .{ .vpunpckhbw,  .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpunpckhwd,  .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpunpckhdq,  .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_256_wig, .avx2 },
+    .{ .vpunpckhqdq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6d }, 0, .vex_256_wig, .avx2 },
+
+    .{ .vpunpcklbw,  .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x60 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpunpcklwd,  .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x61 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpunpckldq,  .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x62 }, 0, .vex_256_wig, .avx2 },
+    .{ .vpunpcklqdq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6c }, 0, .vex_256_wig, .avx2 },
 };
 // zig fmt: on
diff --git a/test/behavior/floatop.zig b/test/behavior/floatop.zig
index b98d782da1..ec24407d9f 100644
--- a/test/behavior/floatop.zig
+++ b/test/behavior/floatop.zig
@@ -52,7 +52,8 @@ fn testFloatComparisons() !void {
 }
 
 test "different sized float comparisons" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .f16c)) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From 5c5da179fb930c9d8be9366a851eb4a36f4044f1 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sun, 7 May 2023 03:47:56 -0400
Subject: [PATCH 130/725] x86_64: implement `@sqrt` for vectors

---
 src/arch/x86_64/CodeGen.zig   | 225 +++++++++++++++++++++-------------
 src/arch/x86_64/Encoding.zig  |   1 +
 src/arch/x86_64/Lower.zig     |   4 +
 src/arch/x86_64/Mir.zig       |   8 ++
 src/arch/x86_64/encodings.zig |  18 ++-
 5 files changed, 166 insertions(+), 90 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 38497400f2..19878bae17 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -4520,25 +4520,69 @@ fn airRound(self: *Self, inst: Air.Inst.Index, mode: Immediate) !void {
 fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     const ty = self.air.typeOf(un_op);
+    const abi_size = @intCast(u32, ty.abiSize(self.target.*));
 
     const src_mcv = try self.resolveInst(un_op);
     const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
         src_mcv
     else
         try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
+    const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
+    const dst_lock = self.register_manager.lockReg(dst_reg);
+    defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
-    try self.genBinOpMir(switch (ty.zigTypeTag()) {
-        .Float => switch (ty.floatBits(self.target.*)) {
-            32 => .sqrtss,
-            64 => .sqrtsd,
-            else => return self.fail("TODO implement airSqrt for {}", .{
-                ty.fmt(self.bin_file.options.module.?),
-            }),
+    const tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {
+        .Float => switch (ty.childType().floatBits(self.target.*)) {
+            32 => if (self.hasFeature(.avx)) .vsqrtss else .sqrtss,
+            64 => if (self.hasFeature(.avx)) .vsqrtsd else .sqrtsd,
+            16, 80, 128 => null,
+            else => unreachable,
         },
-        else => return self.fail("TODO implement airSqrt for {}", .{
-            ty.fmt(self.bin_file.options.module.?),
-        }),
-    }, ty, dst_mcv, src_mcv);
+        .Vector => switch (ty.childType().zigTypeTag()) {
+            .Float => switch (ty.childType().floatBits(self.target.*)) {
+                32 => switch (ty.vectorLen()) {
+                    1 => if (self.hasFeature(.avx)) .vsqrtss else .sqrtss,
+                    2...4 => if (self.hasFeature(.avx)) .vsqrtps else .sqrtps,
+                    5...8 => if (self.hasFeature(.avx)) .vsqrtps else null,
+                    else => null,
+                },
+                64 => switch (ty.vectorLen()) {
+                    1 => if (self.hasFeature(.avx)) .vsqrtsd else .sqrtsd,
+                    2 => if (self.hasFeature(.avx)) .vsqrtpd else .sqrtpd,
+                    3...4 => if (self.hasFeature(.avx)) .vsqrtpd else null,
+                    else => null,
+                },
+                16, 80, 128 => null,
+                else => unreachable,
+            },
+            else => unreachable,
+        },
+        else => unreachable,
+    })) |tag| tag else return self.fail("TODO implement airSqrt for {}", .{
+        ty.fmt(self.bin_file.options.module.?),
+    });
+    switch (tag) {
+        .vsqrtss, .vsqrtsd => if (src_mcv.isRegister()) try self.asmRegisterRegisterRegister(
+            tag,
+            dst_reg,
+            dst_reg,
+            registerAlias(src_mcv.getReg().?, abi_size),
+        ) else try self.asmRegisterRegisterMemory(
+            tag,
+            dst_reg,
+            dst_reg,
+            src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
+        ),
+        else => if (src_mcv.isRegister()) try self.asmRegisterRegister(
+            tag,
+            dst_reg,
+            registerAlias(src_mcv.getReg().?, abi_size),
+        ) else try self.asmRegisterMemory(
+            tag,
+            dst_reg,
+            src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
+        ),
+    }
     return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
 }
 
@@ -9544,85 +9588,92 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
         lock.* = self.register_manager.lockRegAssumeUnused(reg);
     }
 
-    const tag: ?Mir.Inst.Tag =
+    const tag = if (@as(
+        ?Mir.Inst.Tag,
         if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or mem.eql(u2, &order, &.{ 3, 1, 2 }))
-        switch (ty.zigTypeTag()) {
-            .Float => switch (ty.floatBits(self.target.*)) {
-                32 => .vfmadd132ss,
-                64 => .vfmadd132sd,
-                else => null,
-            },
-            .Vector => switch (ty.childType().zigTypeTag()) {
-                .Float => switch (ty.childType().floatBits(self.target.*)) {
-                    32 => switch (ty.vectorLen()) {
-                        1 => .vfmadd132ss,
-                        2...8 => .vfmadd132ps,
-                        else => null,
-                    },
-                    64 => switch (ty.vectorLen()) {
-                        1 => .vfmadd132sd,
-                        2...4 => .vfmadd132pd,
-                        else => null,
-                    },
-                    else => null,
+            switch (ty.zigTypeTag()) {
+                .Float => switch (ty.floatBits(self.target.*)) {
+                    32 => .vfmadd132ss,
+                    64 => .vfmadd132sd,
+                    16, 80, 128 => null,
+                    else => unreachable,
                 },
-                else => null,
-            },
-            else => unreachable,
-        }
-    else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
-        switch (ty.zigTypeTag()) {
-            .Float => switch (ty.floatBits(self.target.*)) {
-                32 => .vfmadd213ss,
-                64 => .vfmadd213sd,
-                else => null,
-            },
-            .Vector => switch (ty.childType().zigTypeTag()) {
-                .Float => switch (ty.childType().floatBits(self.target.*)) {
-                    32 => switch (ty.vectorLen()) {
-                        1 => .vfmadd213ss,
-                        2...8 => .vfmadd213ps,
-                        else => null,
+                .Vector => switch (ty.childType().zigTypeTag()) {
+                    .Float => switch (ty.childType().floatBits(self.target.*)) {
+                        32 => switch (ty.vectorLen()) {
+                            1 => .vfmadd132ss,
+                            2...8 => .vfmadd132ps,
+                            else => null,
+                        },
+                        64 => switch (ty.vectorLen()) {
+                            1 => .vfmadd132sd,
+                            2...4 => .vfmadd132pd,
+                            else => null,
+                        },
+                        16, 80, 128 => null,
+                        else => unreachable,
                     },
-                    64 => switch (ty.vectorLen()) {
-                        1 => .vfmadd213sd,
-                        2...4 => .vfmadd213pd,
-                        else => null,
-                    },
-                    else => null,
+                    else => unreachable,
                 },
-                else => null,
-            },
-            else => unreachable,
-        }
-    else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
-        switch (ty.zigTypeTag()) {
-            .Float => switch (ty.floatBits(self.target.*)) {
-                32 => .vfmadd231ss,
-                64 => .vfmadd231sd,
-                else => null,
-            },
-            .Vector => switch (ty.childType().zigTypeTag()) {
-                .Float => switch (ty.childType().floatBits(self.target.*)) {
-                    32 => switch (ty.vectorLen()) {
-                        1 => .vfmadd231ss,
-                        2...8 => .vfmadd231ps,
-                        else => null,
-                    },
-                    64 => switch (ty.vectorLen()) {
-                        1 => .vfmadd231sd,
-                        2...4 => .vfmadd231pd,
-                        else => null,
-                    },
-                    else => null,
+                else => unreachable,
+            }
+        else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
+            switch (ty.zigTypeTag()) {
+                .Float => switch (ty.floatBits(self.target.*)) {
+                    32 => .vfmadd213ss,
+                    64 => .vfmadd213sd,
+                    16, 80, 128 => null,
+                    else => unreachable,
                 },
-                else => null,
-            },
-            else => null,
-        }
-    else
-        unreachable;
-    if (tag == null) return self.fail("TODO implement airMulAdd for {}", .{
+                .Vector => switch (ty.childType().zigTypeTag()) {
+                    .Float => switch (ty.childType().floatBits(self.target.*)) {
+                        32 => switch (ty.vectorLen()) {
+                            1 => .vfmadd213ss,
+                            2...8 => .vfmadd213ps,
+                            else => null,
+                        },
+                        64 => switch (ty.vectorLen()) {
+                            1 => .vfmadd213sd,
+                            2...4 => .vfmadd213pd,
+                            else => null,
+                        },
+                        16, 80, 128 => null,
+                        else => unreachable,
+                    },
+                    else => unreachable,
+                },
+                else => unreachable,
+            }
+        else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
+            switch (ty.zigTypeTag()) {
+                .Float => switch (ty.floatBits(self.target.*)) {
+                    32 => .vfmadd231ss,
+                    64 => .vfmadd231sd,
+                    16, 80, 128 => null,
+                    else => unreachable,
+                },
+                .Vector => switch (ty.childType().zigTypeTag()) {
+                    .Float => switch (ty.childType().floatBits(self.target.*)) {
+                        32 => switch (ty.vectorLen()) {
+                            1 => .vfmadd231ss,
+                            2...8 => .vfmadd231ps,
+                            else => null,
+                        },
+                        64 => switch (ty.vectorLen()) {
+                            1 => .vfmadd231sd,
+                            2...4 => .vfmadd231pd,
+                            else => null,
+                        },
+                        16, 80, 128 => null,
+                        else => unreachable,
+                    },
+                    else => unreachable,
+                },
+                else => unreachable,
+            }
+        else
+            unreachable,
+    )) |tag| tag else return self.fail("TODO implement airMulAdd for {}", .{
         ty.fmt(self.bin_file.options.module.?),
     });
 
@@ -9634,14 +9685,14 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
     const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);
     if (mops[2].isRegister())
         try self.asmRegisterRegisterRegister(
-            tag.?,
+            tag,
             mop1_reg,
             mop2_reg,
             registerAlias(mops[2].getReg().?, abi_size),
         )
     else
         try self.asmRegisterRegisterMemory(
-            tag.?,
+            tag,
             mop1_reg,
             mop2_reg,
             mops[2].mem(Memory.PtrSize.fromSize(abi_size)),
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index bd6e70c975..b242c98bdc 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -316,6 +316,7 @@ pub const Mnemonic = enum {
     vpsrld, vpsrlq, vpsrlw,
     vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd,
     vpunpcklbw, vpunpckldq, vpunpcklqdq, vpunpcklwd,
+    vsqrtpd, vsqrtps, vsqrtsd, vsqrtss,
     // F16C
     vcvtph2ps, vcvtps2ph,
     // FMA
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index 40a5ccdb10..39ad2313e7 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -212,6 +212,10 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .vpunpckldq,
         .vpunpcklqdq,
         .vpunpcklwd,
+        .vsqrtpd,
+        .vsqrtps,
+        .vsqrtsd,
+        .vsqrtss,
 
         .vcvtph2ps,
         .vcvtps2ph,
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index cb1a578bb6..b6df0fff09 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -338,6 +338,14 @@ pub const Inst = struct {
         vpunpcklqdq,
         /// Unpack low data
         vpunpcklwd,
+        /// Square root of packed double-precision floating-point value
+        vsqrtpd,
+        /// Square root of packed single-precision floating-point value
+        vsqrtps,
+        /// Square root of scalar double-precision floating-point value
+        vsqrtsd,
+        /// Square root of scalar single-precision floating-point value
+        vsqrtss,
 
         /// Convert 16-bit floating-point values to single-precision floating-point values
         vcvtph2ps,
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index 5e4dc2f04b..49ebc344fd 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -869,8 +869,9 @@ pub const table = [_]Entry{
 
     .{ .subss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .none, .sse },
 
-    .{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{       0x0f, 0x51 }, 0, .none, .sse },
-    .{ .sqrtss, .rm, &.{ .xmm, .xmm_m32  }, &.{ 0xf3, 0x0f, 0x51 }, 0, .none, .sse },
+    .{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .none, .sse },
+
+    .{ .sqrtss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x51 }, 0, .none, .sse },
 
     .{ .ucomiss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x0f, 0x2e }, 0, .none, .sse },
 
@@ -943,7 +944,8 @@ pub const table = [_]Entry{
     .{ .punpcklqdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .none, .sse2 },
 
     .{ .sqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .none, .sse2 },
-    .{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0xf2, 0x0f, 0x51 }, 0, .none, .sse2 },
+
+    .{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x51 }, 0, .none, .sse2 },
 
     .{ .subsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .none, .sse2 },
 
@@ -1039,6 +1041,16 @@ pub const table = [_]Entry{
     .{ .vpunpckldq,  .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .vex_128_wig, .avx },
     .{ .vpunpcklqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .vex_128_wig, .avx },
 
+    .{ .vsqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .vex_128_wig, .avx },
+    .{ .vsqrtpd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x51 }, 0, .vex_256_wig, .avx },
+
+    .{ .vsqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .vex_128_wig, .avx },
+    .{ .vsqrtps, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x0f, 0x51 }, 0, .vex_256_wig, .avx },
+
+    .{ .vsqrtsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f }, 0, .vex_lig_wig, .avx },
+
+    .{ .vsqrtss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f }, 0, .vex_lig_wig, .avx },
+
     // F16C
     .{ .vcvtph2ps, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128_w0, .f16c },
     .{ .vcvtph2ps, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_256_w0, .f16c },

From ea957c4cff77f045108863cb5552b3511cb455c1 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sun, 7 May 2023 05:01:37 -0400
Subject: [PATCH 131/725] x86_64: implement `@sqrt` for `f16` scalars and
 vectors

---
 src/arch/x86_64/CodeGen.zig   | 156 +++++++++++++++++++++++-----------
 src/arch/x86_64/encodings.zig |   4 +-
 test/behavior/floatop.zig     |   1 -
 3 files changed, 109 insertions(+), 52 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 19878bae17..6337ad23f5 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -4531,59 +4531,117 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
     const dst_lock = self.register_manager.lockReg(dst_reg);
     defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
-    const tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {
-        .Float => switch (ty.childType().floatBits(self.target.*)) {
-            32 => if (self.hasFeature(.avx)) .vsqrtss else .sqrtss,
-            64 => if (self.hasFeature(.avx)) .vsqrtsd else .sqrtsd,
-            16, 80, 128 => null,
-            else => unreachable,
-        },
-        .Vector => switch (ty.childType().zigTypeTag()) {
-            .Float => switch (ty.childType().floatBits(self.target.*)) {
-                32 => switch (ty.vectorLen()) {
-                    1 => if (self.hasFeature(.avx)) .vsqrtss else .sqrtss,
-                    2...4 => if (self.hasFeature(.avx)) .vsqrtps else .sqrtps,
-                    5...8 => if (self.hasFeature(.avx)) .vsqrtps else null,
-                    else => null,
+    const result: MCValue = result: {
+        const tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {
+            .Float => switch (ty.floatBits(self.target.*)) {
+                16 => if (self.hasFeature(.f16c)) {
+                    const mat_src_reg = if (src_mcv.isRegister())
+                        src_mcv.getReg().?
+                    else
+                        try self.copyToTmpRegister(ty, src_mcv);
+                    try self.asmRegisterRegister(.vcvtph2ps, dst_reg, mat_src_reg.to128());
+                    try self.asmRegisterRegisterRegister(.vsqrtss, dst_reg, dst_reg, dst_reg);
+                    try self.asmRegisterRegisterImmediate(
+                        .vcvtps2ph,
+                        dst_reg,
+                        dst_reg,
+                        Immediate.u(0b1_00),
+                    );
+                    break :result dst_mcv;
+                } else null,
+                32 => if (self.hasFeature(.avx)) .vsqrtss else .sqrtss,
+                64 => if (self.hasFeature(.avx)) .vsqrtsd else .sqrtsd,
+                80, 128 => null,
+                else => unreachable,
+            },
+            .Vector => switch (ty.childType().zigTypeTag()) {
+                .Float => switch (ty.childType().floatBits(self.target.*)) {
+                    16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen()) {
+                        1 => {
+                            const mat_src_reg = if (src_mcv.isRegister())
+                                src_mcv.getReg().?
+                            else
+                                try self.copyToTmpRegister(ty, src_mcv);
+                            try self.asmRegisterRegister(.vcvtph2ps, dst_reg, mat_src_reg.to128());
+                            try self.asmRegisterRegisterRegister(.vsqrtss, dst_reg, dst_reg, dst_reg);
+                            try self.asmRegisterRegisterImmediate(
+                                .vcvtps2ph,
+                                dst_reg,
+                                dst_reg,
+                                Immediate.u(0b1_00),
+                            );
+                            break :result dst_mcv;
+                        },
+                        2...8 => {
+                            const wide_reg = registerAlias(dst_reg, abi_size * 2);
+                            if (src_mcv.isRegister()) try self.asmRegisterRegister(
+                                .vcvtph2ps,
+                                wide_reg,
+                                src_mcv.getReg().?.to128(),
+                            ) else try self.asmRegisterMemory(
+                                .vcvtph2ps,
+                                wide_reg,
+                                src_mcv.mem(Memory.PtrSize.fromSize(
+                                    @intCast(u32, @divExact(wide_reg.bitSize(), 16)),
+                                )),
+                            );
+                            try self.asmRegisterRegister(.vsqrtps, wide_reg, wide_reg);
+                            try self.asmRegisterRegisterImmediate(
+                                .vcvtps2ph,
+                                dst_reg,
+                                wide_reg,
+                                Immediate.u(0b1_00),
+                            );
+                            break :result dst_mcv;
+                        },
+                        else => null,
+                    } else null,
+                    32 => switch (ty.vectorLen()) {
+                        1 => if (self.hasFeature(.avx)) .vsqrtss else .sqrtss,
+                        2...4 => if (self.hasFeature(.avx)) .vsqrtps else .sqrtps,
+                        5...8 => if (self.hasFeature(.avx)) .vsqrtps else null,
+                        else => null,
+                    },
+                    64 => switch (ty.vectorLen()) {
+                        1 => if (self.hasFeature(.avx)) .vsqrtsd else .sqrtsd,
+                        2 => if (self.hasFeature(.avx)) .vsqrtpd else .sqrtpd,
+                        3...4 => if (self.hasFeature(.avx)) .vsqrtpd else null,
+                        else => null,
+                    },
+                    80, 128 => null,
+                    else => unreachable,
                 },
-                64 => switch (ty.vectorLen()) {
-                    1 => if (self.hasFeature(.avx)) .vsqrtsd else .sqrtsd,
-                    2 => if (self.hasFeature(.avx)) .vsqrtpd else .sqrtpd,
-                    3...4 => if (self.hasFeature(.avx)) .vsqrtpd else null,
-                    else => null,
-                },
-                16, 80, 128 => null,
                 else => unreachable,
             },
             else => unreachable,
-        },
-        else => unreachable,
-    })) |tag| tag else return self.fail("TODO implement airSqrt for {}", .{
-        ty.fmt(self.bin_file.options.module.?),
-    });
-    switch (tag) {
-        .vsqrtss, .vsqrtsd => if (src_mcv.isRegister()) try self.asmRegisterRegisterRegister(
-            tag,
-            dst_reg,
-            dst_reg,
-            registerAlias(src_mcv.getReg().?, abi_size),
-        ) else try self.asmRegisterRegisterMemory(
-            tag,
-            dst_reg,
-            dst_reg,
-            src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
-        ),
-        else => if (src_mcv.isRegister()) try self.asmRegisterRegister(
-            tag,
-            dst_reg,
-            registerAlias(src_mcv.getReg().?, abi_size),
-        ) else try self.asmRegisterMemory(
-            tag,
-            dst_reg,
-            src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
-        ),
-    }
-    return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
+        })) |tag| tag else return self.fail("TODO implement airSqrt for {}", .{
+            ty.fmt(self.bin_file.options.module.?),
+        });
+        switch (tag) {
+            .vsqrtss, .vsqrtsd => if (src_mcv.isRegister()) try self.asmRegisterRegisterRegister(
+                tag,
+                dst_reg,
+                dst_reg,
+                registerAlias(src_mcv.getReg().?, abi_size),
+            ) else try self.asmRegisterRegisterMemory(
+                tag,
+                dst_reg,
+                dst_reg,
+                src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
+            ),
+            else => if (src_mcv.isRegister()) try self.asmRegisterRegister(
+                tag,
+                dst_reg,
+                registerAlias(src_mcv.getReg().?, abi_size),
+            ) else try self.asmRegisterMemory(
+                tag,
+                dst_reg,
+                src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
+            ),
+        }
+        break :result dst_mcv;
+    };
+    return self.finishAir(inst, result, .{ un_op, .none, .none });
 }
 
 fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index 49ebc344fd..78bda4fc76 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -1047,9 +1047,9 @@ pub const table = [_]Entry{
     .{ .vsqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .vex_128_wig, .avx },
     .{ .vsqrtps, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x0f, 0x51 }, 0, .vex_256_wig, .avx },
 
-    .{ .vsqrtsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f }, 0, .vex_lig_wig, .avx },
+    .{ .vsqrtsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x51 }, 0, .vex_lig_wig, .avx },
 
-    .{ .vsqrtss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f }, 0, .vex_lig_wig, .avx },
+    .{ .vsqrtss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x51 }, 0, .vex_lig_wig, .avx },
 
     // F16C
     .{ .vcvtph2ps, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128_w0, .f16c },
diff --git a/test/behavior/floatop.zig b/test/behavior/floatop.zig
index ec24407d9f..3f407061f4 100644
--- a/test/behavior/floatop.zig
+++ b/test/behavior/floatop.zig
@@ -135,7 +135,6 @@ fn testSqrt() !void {
 
 test "@sqrt with vectors" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
 

From 057139fda575e0e6038b821256a45669cd70a073 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sun, 7 May 2023 09:06:12 -0400
Subject: [PATCH 132/725] x86_64: implement binary operations for float vectors

---
 src/arch/x86_64/CodeGen.zig   | 664 ++++++++++++++++++++--------------
 src/arch/x86_64/Encoding.zig  |  34 +-
 src/arch/x86_64/Lower.zig     |  49 +++
 src/arch/x86_64/Mir.zig       | 115 +++++-
 src/arch/x86_64/encodings.zig | 101 +++++-
 5 files changed, 662 insertions(+), 301 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 6337ad23f5..8c6f14ec3a 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -1176,6 +1176,21 @@ fn asmRegisterRegisterRegister(
     });
 }
 
+fn asmRegisterRegisterRegisterImmediate(
+    self: *Self,
+    tag: Mir.Inst.Tag,
+    reg1: Register,
+    reg2: Register,
+    reg3: Register,
+    imm: Immediate,
+) !void {
+    _ = try self.addInst(.{
+        .tag = tag,
+        .ops = .rrri,
+        .data = .{ .rrri = .{ .r1 = reg1, .r2 = reg2, .r3 = reg3, .i = @intCast(u8, imm.unsigned) } },
+    });
+}
+
 fn asmRegisterRegisterImmediate(
     self: *Self,
     tag: Mir.Inst.Tag,
@@ -2310,20 +2325,31 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
             }),
         }
     } else if (src_bits == 64 and dst_bits == 32) {
-        if (self.hasFeature(.avx)) if (src_mcv.isRegister()) try self.asmRegisterRegisterRegister(
-            .vcvtsd2ss,
-            dst_reg,
-            dst_reg,
-            src_mcv.getReg().?.to128(),
-        ) else try self.asmRegisterRegisterMemory(
+        if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
             .vcvtsd2ss,
             dst_reg,
             dst_reg,
             src_mcv.mem(.qword),
-        ) else if (src_mcv.isRegister())
-            try self.asmRegisterRegister(.cvtsd2ss, dst_reg, src_mcv.getReg().?.to128())
-        else
-            try self.asmRegisterMemory(.cvtsd2ss, dst_reg, src_mcv.mem(.qword));
+        ) else try self.asmRegisterRegisterRegister(
+            .vcvtsd2ss,
+            dst_reg,
+            dst_reg,
+            (if (src_mcv.isRegister())
+                src_mcv.getReg().?
+            else
+                try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
+        ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
+            .cvtsd2ss,
+            dst_reg,
+            src_mcv.mem(.qword),
+        ) else try self.asmRegisterRegister(
+            .cvtsd2ss,
+            dst_reg,
+            (if (src_mcv.isRegister())
+                src_mcv.getReg().?
+            else
+                try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
+        );
     } else return self.fail("TODO implement airFptrunc from {} to {}", .{
         src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
     });
@@ -2360,20 +2386,31 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
             }),
         }
     } else if (src_bits == 32 and dst_bits == 64) {
-        if (self.hasFeature(.avx)) if (src_mcv.isRegister()) try self.asmRegisterRegisterRegister(
-            .vcvtss2sd,
-            dst_reg,
-            dst_reg,
-            src_mcv.getReg().?.to128(),
-        ) else try self.asmRegisterRegisterMemory(
+        if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
             .vcvtss2sd,
             dst_reg,
             dst_reg,
             src_mcv.mem(.dword),
-        ) else if (src_mcv.isRegister())
-            try self.asmRegisterRegister(.cvtss2sd, dst_reg, src_mcv.getReg().?.to128())
-        else
-            try self.asmRegisterMemory(.cvtss2sd, dst_reg, src_mcv.mem(.dword));
+        ) else try self.asmRegisterRegisterRegister(
+            .vcvtss2sd,
+            dst_reg,
+            dst_reg,
+            (if (src_mcv.isRegister())
+                src_mcv.getReg().?
+            else
+                try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
+        ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
+            .cvtss2sd,
+            dst_reg,
+            src_mcv.mem(.dword),
+        ) else try self.asmRegisterRegister(
+            .cvtss2sd,
+            dst_reg,
+            (if (src_mcv.isRegister())
+                src_mcv.getReg().?
+            else
+                try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
+        );
     } else return self.fail("TODO implement airFpext from {} to {}", .{
         src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
     });
@@ -4532,7 +4569,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
     defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
     const result: MCValue = result: {
-        const tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {
+        const mir_tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {
             .Float => switch (ty.floatBits(self.target.*)) {
                 16 => if (self.hasFeature(.f16c)) {
                     const mat_src_reg = if (src_mcv.isRegister())
@@ -4558,11 +4595,14 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
                 .Float => switch (ty.childType().floatBits(self.target.*)) {
                     16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen()) {
                         1 => {
-                            const mat_src_reg = if (src_mcv.isRegister())
-                                src_mcv.getReg().?
-                            else
-                                try self.copyToTmpRegister(ty, src_mcv);
-                            try self.asmRegisterRegister(.vcvtph2ps, dst_reg, mat_src_reg.to128());
+                            try self.asmRegisterRegister(
+                                .vcvtph2ps,
+                                dst_reg,
+                                (if (src_mcv.isRegister())
+                                    src_mcv.getReg().?
+                                else
+                                    try self.copyToTmpRegister(ty, src_mcv)).to128(),
+                            );
                             try self.asmRegisterRegisterRegister(.vsqrtss, dst_reg, dst_reg, dst_reg);
                             try self.asmRegisterRegisterImmediate(
                                 .vcvtps2ph,
@@ -4574,16 +4614,19 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
                         },
                         2...8 => {
                             const wide_reg = registerAlias(dst_reg, abi_size * 2);
-                            if (src_mcv.isRegister()) try self.asmRegisterRegister(
-                                .vcvtph2ps,
-                                wide_reg,
-                                src_mcv.getReg().?.to128(),
-                            ) else try self.asmRegisterMemory(
+                            if (src_mcv.isMemory()) try self.asmRegisterMemory(
                                 .vcvtph2ps,
                                 wide_reg,
                                 src_mcv.mem(Memory.PtrSize.fromSize(
                                     @intCast(u32, @divExact(wide_reg.bitSize(), 16)),
                                 )),
+                            ) else try self.asmRegisterRegister(
+                                .vcvtph2ps,
+                                wide_reg,
+                                (if (src_mcv.isRegister())
+                                    src_mcv.getReg().?
+                                else
+                                    try self.copyToTmpRegister(ty, src_mcv)).to128(),
                             );
                             try self.asmRegisterRegister(.vsqrtps, wide_reg, wide_reg);
                             try self.asmRegisterRegisterImmediate(
@@ -4617,26 +4660,32 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
         })) |tag| tag else return self.fail("TODO implement airSqrt for {}", .{
             ty.fmt(self.bin_file.options.module.?),
         });
-        switch (tag) {
-            .vsqrtss, .vsqrtsd => if (src_mcv.isRegister()) try self.asmRegisterRegisterRegister(
-                tag,
-                dst_reg,
-                dst_reg,
-                registerAlias(src_mcv.getReg().?, abi_size),
-            ) else try self.asmRegisterRegisterMemory(
-                tag,
+        switch (mir_tag) {
+            .vsqrtss, .vsqrtsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
+                mir_tag,
                 dst_reg,
                 dst_reg,
                 src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
+            ) else try self.asmRegisterRegisterRegister(
+                mir_tag,
+                dst_reg,
+                dst_reg,
+                registerAlias(if (src_mcv.isRegister())
+                    src_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(ty, src_mcv), abi_size),
             ),
-            else => if (src_mcv.isRegister()) try self.asmRegisterRegister(
-                tag,
-                dst_reg,
-                registerAlias(src_mcv.getReg().?, abi_size),
-            ) else try self.asmRegisterMemory(
-                tag,
+            else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
+                mir_tag,
                 dst_reg,
                 src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
+            ) else try self.asmRegisterRegister(
+                mir_tag,
+                dst_reg,
+                registerAlias(if (src_mcv.isRegister())
+                    src_mcv.getReg().?
+                else
+                    try self.copyToTmpRegister(ty, src_mcv), abi_size),
             ),
         }
         break :result dst_mcv;
@@ -5800,25 +5849,22 @@ fn genMulDivBinOp(
     }
 }
 
-/// Result is always a register.
 fn genBinOp(
     self: *Self,
     maybe_inst: ?Air.Inst.Index,
-    tag: Air.Inst.Tag,
+    air_tag: Air.Inst.Tag,
     lhs_air: Air.Inst.Ref,
     rhs_air: Air.Inst.Ref,
 ) !MCValue {
-    const lhs = try self.resolveInst(lhs_air);
-    const rhs = try self.resolveInst(rhs_air);
+    const lhs_mcv = try self.resolveInst(lhs_air);
+    const rhs_mcv = try self.resolveInst(rhs_air);
     const lhs_ty = self.air.typeOf(lhs_air);
     const rhs_ty = self.air.typeOf(rhs_air);
-    if (lhs_ty.zigTypeTag() == .Vector) {
-        return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmt(self.bin_file.options.module.?)});
-    }
+    const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
 
-    switch (lhs) {
+    switch (lhs_mcv) {
         .immediate => |imm| switch (imm) {
-            0 => switch (tag) {
+            0 => switch (air_tag) {
                 .sub, .subwrap => return self.genUnOp(maybe_inst, .neg, rhs_air),
                 else => {},
             },
@@ -5827,9 +5873,10 @@ fn genBinOp(
         else => {},
     }
 
-    const is_commutative = switch (tag) {
+    const is_commutative = switch (air_tag) {
         .add,
         .addwrap,
+        .mul,
         .bool_or,
         .bit_or,
         .bool_and,
@@ -5841,48 +5888,42 @@ fn genBinOp(
 
         else => false,
     };
-    const dst_mem_ok = switch (tag) {
-        .add,
-        .addwrap,
-        .sub,
-        .subwrap,
-        .mul,
-        .div_float,
-        .div_exact,
-        .div_trunc,
-        .div_floor,
-        => !lhs_ty.isRuntimeFloat(),
-
-        else => true,
+    const vec_op = switch (lhs_ty.zigTypeTag()) {
+        else => false,
+        .Float, .Vector => true,
     };
 
-    const lhs_lock: ?RegisterLock = switch (lhs) {
+    const lhs_lock: ?RegisterLock = switch (lhs_mcv) {
         .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
         else => null,
     };
     defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
 
-    const rhs_lock: ?RegisterLock = switch (rhs) {
+    const rhs_lock: ?RegisterLock = switch (rhs_mcv) {
         .register => |reg| self.register_manager.lockReg(reg),
         else => null,
     };
     defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
 
-    var flipped: bool = false;
+    var flipped = false;
+    var copied_to_dst = true;
     const dst_mcv: MCValue = dst: {
         if (maybe_inst) |inst| {
-            if ((dst_mem_ok or lhs.isRegister()) and self.reuseOperand(inst, lhs_air, 0, lhs)) {
-                break :dst lhs;
+            if ((!vec_op or lhs_mcv.isRegister()) and self.reuseOperand(inst, lhs_air, 0, lhs_mcv)) {
+                break :dst lhs_mcv;
             }
-            if (is_commutative and (dst_mem_ok or rhs.isRegister()) and
-                self.reuseOperand(inst, rhs_air, 1, rhs))
+            if (is_commutative and (!vec_op or rhs_mcv.isRegister()) and
+                self.reuseOperand(inst, rhs_air, 1, rhs_mcv))
             {
                 flipped = true;
-                break :dst rhs;
+                break :dst rhs_mcv;
             }
         }
         const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
-        try self.genCopy(lhs_ty, dst_mcv, lhs);
+        if (vec_op and lhs_mcv.isRegister() and self.hasFeature(.avx))
+            copied_to_dst = false
+        else
+            try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
         break :dst dst_mcv;
     };
     const dst_lock: ?RegisterLock = switch (dst_mcv) {
@@ -5891,160 +5932,47 @@ fn genBinOp(
     };
     defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
-    const src_mcv = if (flipped) lhs else rhs;
-    switch (tag) {
-        .add,
-        .addwrap,
-        => try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
-            else => .add,
-            .Float => switch (lhs_ty.floatBits(self.target.*)) {
-                32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
-                    .addss
-                else
-                    return self.fail("TODO implement genBinOp for {s} {} without sse", .{
-                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                    }),
-                64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
-                    .addsd
-                else
-                    return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
-                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                    }),
-                else => return self.fail("TODO implement genBinOp for {s} {}", .{
-                    @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                }),
+    const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
+    if (!vec_op) {
+        switch (air_tag) {
+            .add,
+            .addwrap,
+            => try self.genBinOpMir(.add, lhs_ty, dst_mcv, src_mcv),
+
+            .sub,
+            .subwrap,
+            => try self.genBinOpMir(.sub, lhs_ty, dst_mcv, src_mcv),
+
+            .ptr_add,
+            .ptr_sub,
+            => {
+                const tmp_reg = try self.copyToTmpRegister(rhs_ty, src_mcv);
+                const tmp_mcv = MCValue{ .register = tmp_reg };
+                const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
+                defer self.register_manager.unlockReg(tmp_lock);
+
+                const elem_size = lhs_ty.elemType2().abiSize(self.target.*);
+                try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size });
+                try self.genBinOpMir(switch (air_tag) {
+                    .ptr_add => .add,
+                    .ptr_sub => .sub,
+                    else => unreachable,
+                }, lhs_ty, dst_mcv, tmp_mcv);
             },
-        }, lhs_ty, dst_mcv, src_mcv),
 
-        .sub,
-        .subwrap,
-        => try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
-            else => .sub,
-            .Float => switch (lhs_ty.floatBits(self.target.*)) {
-                32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
-                    .subss
-                else
-                    return self.fail("TODO implement genBinOp for {s} {} without sse", .{
-                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                    }),
-                64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
-                    .subsd
-                else
-                    return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
-                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                    }),
-                else => return self.fail("TODO implement genBinOp for {s} {}", .{
-                    @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                }),
-            },
-        }, lhs_ty, dst_mcv, src_mcv),
+            .bool_or,
+            .bit_or,
+            => try self.genBinOpMir(.@"or", lhs_ty, dst_mcv, src_mcv),
 
-        .mul => try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
-            else => return self.fail("TODO implement genBinOp for {s} {}", .{
-                @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-            }),
-            .Float => switch (lhs_ty.floatBits(self.target.*)) {
-                32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
-                    .mulss
-                else
-                    return self.fail("TODO implement genBinOp for {s} {} without sse", .{
-                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                    }),
-                64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
-                    .mulsd
-                else
-                    return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
-                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                    }),
-                else => return self.fail("TODO implement genBinOp for {s} {}", .{
-                    @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                }),
-            },
-        }, lhs_ty, dst_mcv, src_mcv),
+            .bool_and,
+            .bit_and,
+            => try self.genBinOpMir(.@"and", lhs_ty, dst_mcv, src_mcv),
 
-        .div_float,
-        .div_exact,
-        .div_trunc,
-        .div_floor,
-        => {
-            try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
-                else => return self.fail("TODO implement genBinOp for {s} {}", .{
-                    @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                }),
-                .Float => switch (lhs_ty.floatBits(self.target.*)) {
-                    32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
-                        .divss
-                    else
-                        return self.fail("TODO implement genBinOp for {s} {} without sse", .{
-                            @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                        }),
-                    64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
-                        .divsd
-                    else
-                        return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
-                            @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                        }),
-                    else => return self.fail("TODO implement genBinOp for {s} {}", .{
-                        @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                    }),
-                },
-            }, lhs_ty, dst_mcv, src_mcv);
-            switch (tag) {
-                .div_float,
-                .div_exact,
-                => {},
-                .div_trunc,
-                .div_floor,
-                => if (self.hasFeature(.sse4_1)) {
-                    const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
-                    const dst_alias = registerAlias(dst_mcv.register, abi_size);
-                    try self.asmRegisterRegisterImmediate(switch (lhs_ty.floatBits(self.target.*)) {
-                        32 => .roundss,
-                        64 => .roundsd,
-                        else => unreachable,
-                    }, dst_alias, dst_alias, Immediate.u(switch (tag) {
-                        .div_trunc => 0b1_0_11,
-                        .div_floor => 0b1_0_01,
-                        else => unreachable,
-                    }));
-                } else return self.fail("TODO implement genBinOp for {s} {} without sse4_1", .{
-                    @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                }),
-                else => unreachable,
-            }
-        },
+            .xor => try self.genBinOpMir(.xor, lhs_ty, dst_mcv, src_mcv),
 
-        .ptr_add,
-        .ptr_sub,
-        => {
-            const tmp_reg = try self.copyToTmpRegister(rhs_ty, src_mcv);
-            const tmp_mcv = MCValue{ .register = tmp_reg };
-            const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
-            defer self.register_manager.unlockReg(tmp_lock);
-
-            const elem_size = lhs_ty.elemType2().abiSize(self.target.*);
-            try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size });
-            try self.genBinOpMir(switch (tag) {
-                .ptr_add => .add,
-                .ptr_sub => .sub,
-                else => unreachable,
-            }, lhs_ty, dst_mcv, tmp_mcv);
-        },
-
-        .bool_or,
-        .bit_or,
-        => try self.genBinOpMir(.@"or", lhs_ty, dst_mcv, src_mcv),
-
-        .bool_and,
-        .bit_and,
-        => try self.genBinOpMir(.@"and", lhs_ty, dst_mcv, src_mcv),
-
-        .xor => try self.genBinOpMir(.xor, lhs_ty, dst_mcv, src_mcv),
-
-        .min,
-        .max,
-        => switch (lhs_ty.zigTypeTag()) {
-            .Int => {
+            .min,
+            .max,
+            => {
                 const mat_src_mcv: MCValue = if (switch (src_mcv) {
                     .immediate,
                     .eflags,
@@ -6070,12 +5998,12 @@ fn genBinOp(
 
                 const int_info = lhs_ty.intInfo(self.target.*);
                 const cc: Condition = switch (int_info.signedness) {
-                    .unsigned => switch (tag) {
+                    .unsigned => switch (air_tag) {
                         .min => .a,
                         .max => .b,
                         else => unreachable,
                     },
-                    .signed => switch (tag) {
+                    .signed => switch (air_tag) {
                         .min => .g,
                         .max => .l,
                         else => unreachable,
@@ -6134,26 +6062,222 @@ fn genBinOp(
                 }
                 try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });
             },
-            .Float => try self.genBinOpMir(switch (lhs_ty.floatBits(self.target.*)) {
-                32 => switch (tag) {
-                    .min => .minss,
-                    .max => .maxss,
-                    else => unreachable,
-                },
-                64 => switch (tag) {
-                    .min => .minsd,
-                    .max => .maxsd,
-                    else => unreachable,
-                },
-                else => return self.fail("TODO implement genBinOp for {s} {}", .{
-                    @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-                }),
-            }, lhs_ty, dst_mcv, src_mcv),
-            else => return self.fail("TODO implement genBinOp for {s} {}", .{
-                @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
-            }),
-        },
 
+            else => return self.fail("TODO implement genBinOp for {s} {}", .{
+                @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
+            }),
+        }
+        return dst_mcv;
+    }
+
+    const mir_tag = if (@as(?Mir.Inst.Tag, switch (lhs_ty.zigTypeTag()) {
+        else => unreachable,
+        .Float => switch (lhs_ty.floatBits(self.target.*)) {
+            32 => switch (air_tag) {
+                .add => if (self.hasFeature(.avx)) .vaddss else .addss,
+                .sub => if (self.hasFeature(.avx)) .vsubss else .subss,
+                .mul => if (self.hasFeature(.avx)) .vmulss else .mulss,
+                .div_float,
+                .div_trunc,
+                .div_floor,
+                .div_exact,
+                => if (self.hasFeature(.avx)) .vdivss else .divss,
+                .max => if (self.hasFeature(.avx)) .vmaxss else .maxss,
+                .min => if (self.hasFeature(.avx)) .vminss else .minss,
+                else => unreachable,
+            },
+            64 => switch (air_tag) {
+                .add => if (self.hasFeature(.avx)) .vaddsd else .addsd,
+                .sub => if (self.hasFeature(.avx)) .vsubsd else .subsd,
+                .mul => if (self.hasFeature(.avx)) .vmulsd else .mulsd,
+                .div_float,
+                .div_trunc,
+                .div_floor,
+                .div_exact,
+                => if (self.hasFeature(.avx)) .vdivsd else .divsd,
+                .max => if (self.hasFeature(.avx)) .vmaxsd else .maxsd,
+                .min => if (self.hasFeature(.avx)) .vminsd else .minsd,
+                else => unreachable,
+            },
+            16, 80, 128 => null,
+            else => unreachable,
+        },
+        .Vector => switch (lhs_ty.childType().zigTypeTag()) {
+            else => null,
+            .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
+                32 => switch (lhs_ty.vectorLen()) {
+                    1 => switch (air_tag) {
+                        .add => if (self.hasFeature(.avx)) .vaddss else .addss,
+                        .sub => if (self.hasFeature(.avx)) .vsubss else .subss,
+                        .mul => if (self.hasFeature(.avx)) .vmulss else .mulss,
+                        .div_float,
+                        .div_trunc,
+                        .div_floor,
+                        .div_exact,
+                        => if (self.hasFeature(.avx)) .vdivss else .divss,
+                        .max => if (self.hasFeature(.avx)) .vmaxss else .maxss,
+                        .min => if (self.hasFeature(.avx)) .vminss else .minss,
+                        else => unreachable,
+                    },
+                    2...4 => switch (air_tag) {
+                        .add => if (self.hasFeature(.avx)) .vaddps else .addps,
+                        .sub => if (self.hasFeature(.avx)) .vsubps else .subps,
+                        .mul => if (self.hasFeature(.avx)) .vmulps else .mulps,
+                        .div_float,
+                        .div_trunc,
+                        .div_floor,
+                        .div_exact,
+                        => if (self.hasFeature(.avx)) .vdivps else .divps,
+                        .max => if (self.hasFeature(.avx)) .vmaxps else .maxps,
+                        .min => if (self.hasFeature(.avx)) .vminps else .minps,
+                        else => unreachable,
+                    },
+                    5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
+                        .add => .vaddps,
+                        .sub => .vsubps,
+                        .mul => .vmulps,
+                        .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
+                        .max => .vmaxps,
+                        .min => .vminps,
+                        else => unreachable,
+                    } else null,
+                    else => null,
+                },
+                64 => switch (lhs_ty.vectorLen()) {
+                    1 => switch (air_tag) {
+                        .add => if (self.hasFeature(.avx)) .vaddsd else .addsd,
+                        .sub => if (self.hasFeature(.avx)) .vsubsd else .subsd,
+                        .mul => if (self.hasFeature(.avx)) .vmulsd else .mulsd,
+                        .div_float,
+                        .div_trunc,
+                        .div_floor,
+                        .div_exact,
+                        => if (self.hasFeature(.avx)) .vdivsd else .divsd,
+                        .max => if (self.hasFeature(.avx)) .vmaxsd else .maxsd,
+                        .min => if (self.hasFeature(.avx)) .vminsd else .minsd,
+                        else => unreachable,
+                    },
+                    2 => switch (air_tag) {
+                        .add => if (self.hasFeature(.avx)) .vaddpd else .addpd,
+                        .sub => if (self.hasFeature(.avx)) .vsubpd else .subpd,
+                        .mul => if (self.hasFeature(.avx)) .vmulpd else .mulpd,
+                        .div_float,
+                        .div_trunc,
+                        .div_floor,
+                        .div_exact,
+                        => if (self.hasFeature(.avx)) .vdivpd else .divpd,
+                        .max => if (self.hasFeature(.avx)) .vmaxpd else .maxpd,
+                        .min => if (self.hasFeature(.avx)) .vminpd else .minpd,
+                        else => unreachable,
+                    },
+                    3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
+                        .add => .vaddpd,
+                        .sub => .vsubpd,
+                        .mul => .vmulpd,
+                        .div_float, .div_trunc, .div_floor, .div_exact => .vdivpd,
+                        .max => .vmaxpd,
+                        .min => .vminpd,
+                        else => unreachable,
+                    } else null,
+                    else => null,
+                },
+                16, 80, 128 => null,
+                else => unreachable,
+            },
+        },
+    })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
+        @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
+    });
+    const dst_alias = registerAlias(dst_mcv.getReg().?, abi_size);
+    if (self.hasFeature(.avx)) {
+        const src1_alias =
+            if (copied_to_dst) dst_alias else registerAlias(lhs_mcv.getReg().?, abi_size);
+        if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
+            mir_tag,
+            dst_alias,
+            src1_alias,
+            src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
+        ) else try self.asmRegisterRegisterRegister(
+            mir_tag,
+            dst_alias,
+            src1_alias,
+            registerAlias(if (src_mcv.isRegister())
+                src_mcv.getReg().?
+            else
+                try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
+        );
+    } else {
+        assert(copied_to_dst);
+        if (src_mcv.isMemory()) try self.asmRegisterMemory(
+            mir_tag,
+            dst_alias,
+            src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
+        ) else try self.asmRegisterRegister(
+            mir_tag,
+            dst_alias,
+            registerAlias(if (src_mcv.isRegister())
+                src_mcv.getReg().?
+            else
+                try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
+        );
+    }
+    switch (air_tag) {
+        .add, .sub, .mul, .div_float, .div_exact => {},
+        .div_trunc, .div_floor => if (self.hasFeature(.sse4_1)) {
+            const round_tag = if (@as(?Mir.Inst.Tag, switch (lhs_ty.zigTypeTag()) {
+                .Float => switch (lhs_ty.floatBits(self.target.*)) {
+                    32 => if (self.hasFeature(.avx)) .vroundss else .roundss,
+                    64 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
+                    16, 80, 128 => null,
+                    else => unreachable,
+                },
+                .Vector => switch (lhs_ty.childType().zigTypeTag()) {
+                    .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
+                        32 => switch (lhs_ty.vectorLen()) {
+                            1 => if (self.hasFeature(.avx)) .vroundss else .roundss,
+                            2...4 => if (self.hasFeature(.avx)) .vroundps else .roundps,
+                            5...8 => if (self.hasFeature(.avx)) .vroundps else null,
+                            else => null,
+                        },
+                        64 => switch (lhs_ty.vectorLen()) {
+                            1 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
+                            2 => if (self.hasFeature(.avx)) .vroundpd else .roundpd,
+                            3...4 => if (self.hasFeature(.avx)) .vroundpd else null,
+                            else => null,
+                        },
+                        16, 80, 128 => null,
+                        else => unreachable,
+                    },
+                    else => null,
+                },
+                else => unreachable,
+            })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
+                @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
+            });
+            const round_mode = Immediate.u(switch (air_tag) {
+                .div_trunc => 0b1_0_11,
+                .div_floor => 0b1_0_01,
+                else => unreachable,
+            });
+            switch (round_tag) {
+                .vroundss, .vroundsd => try self.asmRegisterRegisterRegisterImmediate(
+                    round_tag,
+                    dst_alias,
+                    dst_alias,
+                    dst_alias,
+                    round_mode,
+                ),
+                else => try self.asmRegisterRegisterImmediate(
+                    round_tag,
+                    dst_alias,
+                    dst_alias,
+                    round_mode,
+                ),
+            }
+        } else return self.fail("TODO implement genBinOp for {s} {} without sse4_1", .{
+            @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
+        }),
+        .max, .min => {}, // TODO: unordered select
         else => unreachable,
     }
     return dst_mcv;
@@ -6186,20 +6310,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
                 .register_overflow,
                 .reserved_frame,
                 => unreachable,
-                .register => |src_reg| switch (ty.zigTypeTag()) {
-                    .Float => {
-                        if (!Target.x86.featureSetHas(self.target.cpu.features, .sse))
-                            return self.fail("TODO genBinOpMir for {s} {} without sse", .{
-                                @tagName(mir_tag), ty.fmt(self.bin_file.options.module.?),
-                            });
-                        return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
-                    },
-                    else => try self.asmRegisterRegister(
-                        mir_tag,
-                        dst_alias,
-                        registerAlias(src_reg, abi_size),
-                    ),
-                },
+                .register => |src_reg| try self.asmRegisterRegister(
+                    mir_tag,
+                    dst_alias,
+                    registerAlias(src_reg, abi_size),
+                ),
                 .immediate => |imm| switch (self.regBitSize(ty)) {
                     8 => try self.asmRegisterImmediate(
                         mir_tag,
@@ -9646,7 +9761,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
         lock.* = self.register_manager.lockRegAssumeUnused(reg);
     }
 
-    const tag = if (@as(
+    const mir_tag = if (@as(
         ?Mir.Inst.Tag,
         if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or mem.eql(u2, &order, &.{ 3, 1, 2 }))
             switch (ty.zigTypeTag()) {
@@ -9741,20 +9856,17 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
     const abi_size = @intCast(u32, ty.abiSize(self.target.*));
     const mop1_reg = registerAlias(mops[0].getReg().?, abi_size);
     const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);
-    if (mops[2].isRegister())
-        try self.asmRegisterRegisterRegister(
-            tag,
-            mop1_reg,
-            mop2_reg,
-            registerAlias(mops[2].getReg().?, abi_size),
-        )
-    else
-        try self.asmRegisterRegisterMemory(
-            tag,
-            mop1_reg,
-            mop2_reg,
-            mops[2].mem(Memory.PtrSize.fromSize(abi_size)),
-        );
+    if (mops[2].isRegister()) try self.asmRegisterRegisterRegister(
+        mir_tag,
+        mop1_reg,
+        mop2_reg,
+        registerAlias(mops[2].getReg().?, abi_size),
+    ) else try self.asmRegisterRegisterMemory(
+        mir_tag,
+        mop1_reg,
+        mop2_reg,
+        mops[2].mem(Memory.PtrSize.fromSize(abi_size)),
+    );
     return self.finishAir(inst, mops[0], ops);
 }
 
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index b242c98bdc..b8ccc9efba 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -262,61 +262,69 @@ pub const Mnemonic = enum {
     // MMX
     movd,
     // SSE
-    addss,
+    addps, addss,
     andps,
     andnps,
     cmpss,
     cvtsi2ss,
-    divss,
-    maxss, minss,
+    divps, divss,
+    maxps, maxss,
+    minps, minss,
     movaps, movss, movups,
-    mulss,
+    mulps, mulss,
     orps,
     pextrw, pinsrw,
-    sqrtps,
-    sqrtss,
-    subss,
+    sqrtps, sqrtss,
+    subps, subss,
     ucomiss,
     xorps,
     // SSE2
-    addsd,
+    addpd, addsd,
     andpd,
     andnpd,
     //cmpsd,
     cvtsd2ss, cvtsi2sd, cvtss2sd,
-    divsd,
-    maxsd, minsd,
+    divpd, divsd,
+    maxpd, maxsd,
+    minpd, minsd,
     movapd,
     movq, //movd, movsd,
     movupd,
-    mulsd,
+    mulpd, mulsd,
     orpd,
     pshufhw, pshuflw,
     psrld, psrlq, psrlw,
     punpckhbw, punpckhdq, punpckhqdq, punpckhwd,
     punpcklbw, punpckldq, punpcklqdq, punpcklwd,
     sqrtpd, sqrtsd,
-    subsd,
+    subpd, subsd,
     ucomisd,
     xorpd,
     // SSE3
     movddup, movshdup, movsldup,
     // SSE4.1
-    roundsd, roundss,
+    roundpd, roundps, roundsd, roundss,
     // AVX
+    vaddpd, vaddps, vaddsd, vaddss,
     vcvtsd2ss, vcvtsi2sd, vcvtsi2ss, vcvtss2sd,
+    vdivpd, vdivps, vdivsd, vdivss,
+    vmaxpd, vmaxps, vmaxsd, vmaxss,
+    vminpd, vminps, vminsd, vminss,
     vmovapd, vmovaps,
     vmovddup,
     vmovsd,
     vmovshdup, vmovsldup,
     vmovss,
     vmovupd, vmovups,
+    vmulpd, vmulps, vmulsd, vmulss,
     vpextrw, vpinsrw,
     vpshufhw, vpshuflw,
     vpsrld, vpsrlq, vpsrlw,
     vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd,
     vpunpcklbw, vpunpckldq, vpunpcklqdq, vpunpcklwd,
+    vroundpd, vroundps, vroundsd, vroundss,
     vsqrtpd, vsqrtps, vsqrtsd, vsqrtss,
+    vsubpd, vsubps, vsubsd, vsubss,
     // F16C
     vcvtph2ps, vcvtps2ph,
     // FMA
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index 39ad2313e7..2cfa25ac84 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -124,27 +124,34 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .xchg,
         .xor,
 
+        .addps,
         .addss,
         .andnps,
         .andps,
         .cmpss,
         .cvtsi2ss,
+        .divps,
         .divss,
+        .maxps,
         .maxss,
+        .minps,
         .minss,
         .movaps,
         .movss,
         .movups,
+        .mulps,
         .mulss,
         .orps,
         .pextrw,
         .pinsrw,
         .sqrtps,
         .sqrtss,
+        .subps,
         .subss,
         .ucomiss,
         .xorps,
 
+        .addpd,
         .addsd,
         .andnpd,
         .andpd,
@@ -152,10 +159,14 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .cvtsd2ss,
         .cvtsi2sd,
         .cvtss2sd,
+        .divpd,
         .divsd,
+        .maxpd,
         .maxsd,
+        .minpd,
         .minsd,
         .movsd,
+        .mulpd,
         .mulsd,
         .orpd,
         .pshufhw,
@@ -173,6 +184,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .punpcklwd,
         .sqrtpd,
         .sqrtsd,
+        .subpd,
         .subsd,
         .ucomisd,
         .xorpd,
@@ -181,13 +193,31 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .movshdup,
         .movsldup,
 
+        .roundpd,
+        .roundps,
         .roundsd,
         .roundss,
 
+        .vaddpd,
+        .vaddps,
+        .vaddsd,
+        .vaddss,
         .vcvtsd2ss,
         .vcvtsi2sd,
         .vcvtsi2ss,
         .vcvtss2sd,
+        .vdivpd,
+        .vdivps,
+        .vdivsd,
+        .vdivss,
+        .vmaxpd,
+        .vmaxps,
+        .vmaxsd,
+        .vmaxss,
+        .vminpd,
+        .vminps,
+        .vminsd,
+        .vminss,
         .vmovapd,
         .vmovaps,
         .vmovddup,
@@ -197,6 +227,10 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .vmovss,
         .vmovupd,
         .vmovups,
+        .vmulpd,
+        .vmulps,
+        .vmulsd,
+        .vmulss,
         .vpextrw,
         .vpinsrw,
         .vpshufhw,
@@ -212,10 +246,18 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .vpunpckldq,
         .vpunpcklqdq,
         .vpunpcklwd,
+        .vroundpd,
+        .vroundps,
+        .vroundsd,
+        .vroundss,
         .vsqrtpd,
         .vsqrtps,
         .vsqrtsd,
         .vsqrtss,
+        .vsubpd,
+        .vsubps,
+        .vsubsd,
+        .vsubss,
 
         .vcvtph2ps,
         .vcvtps2ph,
@@ -304,6 +346,7 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
         .lock_mi_rip_s,
         => Immediate.s(@bitCast(i32, i)),
 
+        .rrri,
         .rri_u,
         .ri_u,
         .i_u,
@@ -429,6 +472,12 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .reg = inst.data.rrr.r2 },
             .{ .reg = inst.data.rrr.r3 },
         },
+        .rrri => &.{
+            .{ .reg = inst.data.rrri.r1 },
+            .{ .reg = inst.data.rrri.r2 },
+            .{ .reg = inst.data.rrri.r3 },
+            .{ .imm = lower.imm(inst.ops, inst.data.rrri.i) },
+        },
         .ri_s, .ri_u => &.{
             .{ .reg = inst.data.ri.r },
             .{ .imm = lower.imm(inst.ops, inst.data.ri.i) },
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index b6df0fff09..c0450406cf 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -166,7 +166,9 @@ pub const Inst = struct {
         /// Logical exclusive-or
         xor,
 
-        /// Add single precision floating point values
+        /// Add packed single-precision floating-point values
+        addps,
+        /// Add scalar single-precision floating-point values
         addss,
         /// Bitwise logical and of packed single precision floating-point values
         andps,
@@ -176,11 +178,17 @@ pub const Inst = struct {
         cmpss,
         /// Convert doubleword integer to scalar single-precision floating-point value
         cvtsi2ss,
+        /// Divide packed single-precision floating-point values
+        divps,
         /// Divide scalar single-precision floating-point values
         divss,
-        /// Return maximum single-precision floating-point value
+        /// Maximum of packed single-precision floating-point values
+        maxps,
+        /// Maximum of scalar single-precision floating-point values
         maxss,
-        /// Return minimum single-precision floating-point value
+        /// Minimum of packed single-precision floating-point values
+        minps,
+        /// Minimum of scalar single-precision floating-point values
         minss,
         /// Move aligned packed single-precision floating-point values
         movaps,
@@ -188,6 +196,8 @@ pub const Inst = struct {
         movss,
         /// Move unaligned packed single-precision floating-point values
         movups,
+        /// Multiply packed single-precision floating-point values
+        mulps,
         /// Multiply scalar single-precision floating-point values
         mulss,
         /// Bitwise logical or of packed single precision floating-point values
@@ -196,18 +206,22 @@ pub const Inst = struct {
         pextrw,
         /// Insert word
         pinsrw,
-        /// Square root of scalar single precision floating-point value
+        /// Square root of packed single-precision floating-point values
         sqrtps,
-        /// Subtract scalar single-precision floating-point values
+        /// Square root of scalar single-precision floating-point value
         sqrtss,
-        /// Square root of single precision floating-point values
+        /// Subtract packed single-precision floating-point values
+        subps,
+        /// Subtract scalar single-precision floating-point values
         subss,
         /// Unordered compare scalar single-precision floating-point values
         ucomiss,
         /// Bitwise logical xor of packed single precision floating-point values
         xorps,
 
-        /// Add double precision floating point values
+        /// Add packed double-precision floating-point values
+        addpd,
+        /// Add scalar double-precision floating-point values
         addsd,
         /// Bitwise logical and not of packed double precision floating-point values
         andnpd,
@@ -221,14 +235,22 @@ pub const Inst = struct {
         cvtsi2sd,
         /// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
         cvtss2sd,
+        /// Divide packed double-precision floating-point values
+        divpd,
         /// Divide scalar double-precision floating-point values
         divsd,
-        /// Return maximum double-precision floating-point value
+        /// Maximum of packed double-precision floating-point values
+        maxpd,
+        /// Maximum of scalar double-precision floating-point values
         maxsd,
-        /// Return minimum double-precision floating-point value
+        /// Minimum of packed double-precision floating-point values
+        minpd,
+        /// Minimum of scalar double-precision floating-point values
         minsd,
         /// Move scalar double-precision floating-point value
         movsd,
+        /// Multiply packed double-precision floating-point values
+        mulpd,
         /// Multiply scalar double-precision floating-point values
         mulsd,
         /// Bitwise logical or of packed double precision floating-point values
@@ -263,6 +285,8 @@ pub const Inst = struct {
         sqrtpd,
         /// Square root of scalar double precision floating-point value
         sqrtsd,
+        /// Subtract packed double-precision floating-point values
+        subpd,
         /// Subtract scalar double-precision floating-point values
         subsd,
         /// Unordered compare scalar double-precision floating-point values
@@ -277,11 +301,23 @@ pub const Inst = struct {
         /// Replicate single floating-point values
         movsldup,
 
-        /// Round scalar double-precision floating-point values
+        /// Round packed double-precision floating-point values
+        roundpd,
+        /// Round packed single-precision floating-point values
+        roundps,
+        /// Round scalar double-precision floating-point value
         roundsd,
-        /// Round scalar single-precision floating-point values
+        /// Round scalar single-precision floating-point value
         roundss,
 
+        /// Add packed double-precision floating-point values
+        vaddpd,
+        /// Add packed single-precision floating-point values
+        vaddps,
+        /// Add scalar double-precision floating-point values
+        vaddsd,
+        /// Add scalar single-precision floating-point values
+        vaddss,
         /// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
         vcvtsd2ss,
         /// Convert doubleword integer to scalar double-precision floating-point value
@@ -290,6 +326,30 @@ pub const Inst = struct {
         vcvtsi2ss,
         /// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
         vcvtss2sd,
+        /// Divide packed double-precision floating-point values
+        vdivpd,
+        /// Divide packed single-precision floating-point values
+        vdivps,
+        /// Divide scalar double-precision floating-point values
+        vdivsd,
+        /// Divide scalar single-precision floating-point values
+        vdivss,
+        /// Maximum of packed double-precision floating-point values
+        vmaxpd,
+        /// Maximum of packed single-precision floating-point values
+        vmaxps,
+        /// Maximum of scalar double-precision floating-point values
+        vmaxsd,
+        /// Maximum of scalar single-precision floating-point values
+        vmaxss,
+        /// Minimum of packed double-precision floating-point values
+        vminpd,
+        /// Minimum of packed single-precision floating-point values
+        vminps,
+        /// Minimum of scalar double-precision floating-point values
+        vminsd,
+        /// Minimum of scalar single-precision floating-point values
+        vminss,
         /// Move aligned packed double-precision floating-point values
         vmovapd,
         /// Move aligned packed single-precision floating-point values
@@ -308,6 +368,14 @@ pub const Inst = struct {
         vmovupd,
         /// Move unaligned packed single-precision floating-point values
         vmovups,
+        /// Multiply packed double-precision floating-point values
+        vmulpd,
+        /// Multiply packed single-precision floating-point values
+        vmulps,
+        /// Multiply scalar double-precision floating-point values
+        vmulsd,
+        /// Multiply scalar single-precision floating-point values
+        vmulss,
         /// Extract word
         vpextrw,
         /// Insert word
@@ -338,6 +406,14 @@ pub const Inst = struct {
         vpunpcklqdq,
         /// Unpack low data
         vpunpcklwd,
+        /// Round packed double-precision floating-point values
+        vroundpd,
+        /// Round packed single-precision floating-point values
+        vroundps,
+        /// Round scalar double-precision floating-point value
+        vroundsd,
+        /// Round scalar single-precision floating-point value
+        vroundss,
         /// Square root of packed double-precision floating-point value
         vsqrtpd,
         /// Square root of packed single-precision floating-point value
@@ -346,6 +422,14 @@ pub const Inst = struct {
         vsqrtsd,
         /// Square root of scalar single-precision floating-point value
         vsqrtss,
+        /// Subtract packed double-precision floating-point values
+        vsubpd,
+        /// Subtract packed single-precision floating-point values
+        vsubps,
+        /// Subtract scalar double-precision floating-point values
+        vsubsd,
+        /// Subtract scalar single-precision floating-point values
+        vsubss,
 
         /// Convert 16-bit floating-point values to single-precision floating-point values
         vcvtph2ps,
@@ -442,6 +526,9 @@ pub const Inst = struct {
         /// Register, register, register operands.
         /// Uses `rrr` payload.
         rrr,
+        /// Register, register, register, immediate (byte) operands.
+        /// Uses `rrri` payload.
+        rrri,
         /// Register, register, immediate (sign-extended) operands.
         /// Uses `rri`  payload.
         rri_s,
@@ -625,6 +712,12 @@ pub const Inst = struct {
             r2: Register,
             r3: Register,
         },
+        rrri: struct {
+            r1: Register,
+            r2: Register,
+            r3: Register,
+            i: u8,
+        },
         rri: struct {
             r1: Register,
             r2: Register,
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index 78bda4fc76..c41f0ea4e7 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -837,6 +837,8 @@ pub const table = [_]Entry{
     .{ .xor, .rm, &.{ .r64,  .rm64   }, &.{ 0x33 }, 0, .long,  .none },
 
     // SSE
+    .{ .addps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x58 }, 0, .none, .sse },
+
     .{ .addss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .none, .sse },
 
     .{ .andnps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x55 }, 0, .none, .sse },
@@ -848,10 +850,16 @@ pub const table = [_]Entry{
     .{ .cvtsi2ss, .rm, &.{ .xmm, .rm32 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .none, .sse },
     .{ .cvtsi2ss, .rm, &.{ .xmm, .rm64 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .long, .sse },
 
+    .{ .divps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5e }, 0, .none, .sse },
+
     .{ .divss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .none, .sse },
 
+    .{ .maxps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5f }, 0, .none, .sse },
+
     .{ .maxss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5f }, 0, .none, .sse },
 
+    .{ .minps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5d }, 0, .none, .sse },
+
     .{ .minss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5d }, 0, .none, .sse },
 
     .{ .movaps, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .none, .sse },
@@ -863,10 +871,14 @@ pub const table = [_]Entry{
     .{ .movups, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .none, .sse },
     .{ .movups, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x11 }, 0, .none, .sse },
 
+    .{ .mulps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x59 }, 0, .none, .sse },
+
     .{ .mulss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .none, .sse },
 
     .{ .orps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .none, .sse },
 
+    .{ .subps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5c }, 0, .none, .sse },
+
     .{ .subss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .none, .sse },
 
     .{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .none, .sse },
@@ -878,6 +890,8 @@ pub const table = [_]Entry{
     .{ .xorps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x57 }, 0, .none, .sse },
 
     // SSE2
+    .{ .addpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x58 }, 0, .none, .sse2 },
+
     .{ .addsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x58 }, 0, .none, .sse2 },
 
     .{ .andnpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x55 }, 0, .none, .sse2 },
@@ -893,10 +907,16 @@ pub const table = [_]Entry{
 
     .{ .cvtss2sd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5a }, 0, .none, .sse2 },
 
+    .{ .divpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5e }, 0, .none, .sse2 },
+
     .{ .divsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5e }, 0, .none, .sse2 },
 
+    .{ .maxpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5f }, 0, .none, .sse2 },
+
     .{ .maxsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5f }, 0, .none, .sse2 },
 
+    .{ .minpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5d }, 0, .none, .sse2 },
+
     .{ .minsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .none, .sse2 },
 
     .{ .movapd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .none, .sse2 },
@@ -914,6 +934,8 @@ pub const table = [_]Entry{
     .{ .movupd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .none, .sse2 },
     .{ .movupd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x11 }, 0, .none, .sse2 },
 
+    .{ .mulpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x59 }, 0, .none, .sse2 },
+
     .{ .mulsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x59 }, 0, .none, .sse2 },
 
     .{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .none, .sse2 },
@@ -947,6 +969,8 @@ pub const table = [_]Entry{
 
     .{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x51 }, 0, .none, .sse2 },
 
+    .{ .subpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5c }, 0, .none, .sse2 },
+
     .{ .subsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .none, .sse2 },
 
     .{ .movsd, .rm, &.{ .xmm,     .xmm_m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .none, .sse2 },
@@ -966,10 +990,25 @@ pub const table = [_]Entry{
     // SSE4.1
     .{ .pextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .none, .sse4_1 },
 
-    .{ .roundss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .none, .sse4_1 },
+    .{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },
+
+    .{ .roundps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .none, .sse4_1 },
+
     .{ .roundsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .none, .sse4_1 },
 
+    .{ .roundss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .none, .sse4_1 },
+
     // AVX
+    .{ .vaddpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_128_wig, .avx },
+    .{ .vaddpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_256_wig, .avx },
+
+    .{ .vaddps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x58 }, 0, .vex_128_wig, .avx },
+    .{ .vaddps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x58 }, 0, .vex_256_wig, .avx },
+
+    .{ .vaddsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x58 }, 0, .vex_lig_wig, .avx },
+
+    .{ .vaddss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .vex_lig_wig, .avx },
+
     .{ .vcvtsd2ss, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx },
 
     .{ .vcvtsi2sd, .rvm, &.{ .xmm, .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx },
@@ -980,6 +1019,36 @@ pub const table = [_]Entry{
 
     .{ .vcvtss2sd, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx },
 
+    .{ .vdivpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5e }, 0, .vex_128_wig, .avx },
+    .{ .vdivpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5e }, 0, .vex_256_wig, .avx },
+
+    .{ .vdivps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x5e }, 0, .vex_128_wig, .avx },
+    .{ .vdivps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x5e }, 0, .vex_256_wig, .avx },
+
+    .{ .vdivsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5e }, 0, .vex_lig_wig, .avx },
+
+    .{ .vdivss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .vex_lig_wig, .avx },
+
+    .{ .vmaxpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5f }, 0, .vex_128_wig, .avx },
+    .{ .vmaxpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5f }, 0, .vex_256_wig, .avx },
+
+    .{ .vmaxps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x5f }, 0, .vex_128_wig, .avx },
+    .{ .vmaxps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x5f }, 0, .vex_256_wig, .avx },
+
+    .{ .vmaxsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5f }, 0, .vex_lig_wig, .avx },
+
+    .{ .vmaxss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5f }, 0, .vex_lig_wig, .avx },
+
+    .{ .vminpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5d }, 0, .vex_128_wig, .avx },
+    .{ .vminpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5d }, 0, .vex_256_wig, .avx },
+
+    .{ .vminps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x5d }, 0, .vex_128_wig, .avx },
+    .{ .vminps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x5d }, 0, .vex_256_wig, .avx },
+
+    .{ .vminsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .vex_lig_wig, .avx },
+
+    .{ .vminss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5d }, 0, .vex_lig_wig, .avx },
+
     .{ .vmovapd, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .vex_128_wig, .avx },
     .{ .vmovapd, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x66, 0x0f, 0x29 }, 0, .vex_128_wig, .avx },
     .{ .vmovapd, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x66, 0x0f, 0x28 }, 0, .vex_256_wig, .avx },
@@ -1019,6 +1088,16 @@ pub const table = [_]Entry{
     .{ .vmovups, .rm, &.{ .ymm,      .ymm_m256 }, &.{ 0x0f, 0x10 }, 0, .vex_256_wig, .avx },
     .{ .vmovups, .mr, &.{ .ymm_m256, .ymm      }, &.{ 0x0f, 0x11 }, 0, .vex_256_wig, .avx },
 
+    .{ .vmulpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x59 }, 0, .vex_128_wig, .avx },
+    .{ .vmulpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x59 }, 0, .vex_256_wig, .avx },
+
+    .{ .vmulps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x59 }, 0, .vex_128_wig, .avx },
+    .{ .vmulps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x59 }, 0, .vex_256_wig, .avx },
+
+    .{ .vmulsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx },
+
+    .{ .vmulss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx },
+
     .{ .vpextrw, .rmi, &.{ .r32,     .xmm, .imm8 }, &.{ 0x66, 0x0f,       0x15 }, 0, .vex_128_wig, .avx },
     .{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_wig, .avx },
 
@@ -1041,6 +1120,16 @@ pub const table = [_]Entry{
     .{ .vpunpckldq,  .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .vex_128_wig, .avx },
     .{ .vpunpcklqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .vex_128_wig, .avx },
 
+    .{ .vroundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .vex_128_wig, .avx },
+    .{ .vroundpd, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .vex_256_wig, .avx },
+
+    .{ .vroundps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .vex_128_wig, .avx },
+    .{ .vroundps, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .vex_256_wig, .avx },
+
+    .{ .vroundsd, .rvmi, &.{ .xmm, .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .vex_lig_wig, .avx },
+
+    .{ .vroundss, .rvmi, &.{ .xmm, .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .vex_lig_wig, .avx },
+
     .{ .vsqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .vex_128_wig, .avx },
     .{ .vsqrtpd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x51 }, 0, .vex_256_wig, .avx },
 
@@ -1051,6 +1140,16 @@ pub const table = [_]Entry{
 
     .{ .vsqrtss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x51 }, 0, .vex_lig_wig, .avx },
 
+    .{ .vsubpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5c }, 0, .vex_128_wig, .avx },
+    .{ .vsubpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5c }, 0, .vex_256_wig, .avx },
+
+    .{ .vsubps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x5c }, 0, .vex_128_wig, .avx },
+    .{ .vsubps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x5c }, 0, .vex_256_wig, .avx },
+
+    .{ .vsubsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx },
+
+    .{ .vsubss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx },
+
     // F16C
     .{ .vcvtph2ps, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128_w0, .f16c },
     .{ .vcvtph2ps, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_256_w0, .f16c },

From f8708e2c4d93eece5b3e131fd2d1b5b210806cd6 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sun, 7 May 2023 10:04:56 -0400
Subject: [PATCH 133/725] x86_64: implement `@floor`, `@ceil`, and `@trunc` for
 float vectors

---
 src/arch/x86_64/CodeGen.zig | 186 ++++++++++++++++++------------------
 test/behavior/floatop.zig   |   9 +-
 2 files changed, 98 insertions(+), 97 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 8c6f14ec3a..3e2d418105 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -1587,9 +1587,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .round,
             => try self.airUnaryMath(inst),
 
-            .floor => try self.airRound(inst, Immediate.u(0b1_0_01)),
-            .ceil => try self.airRound(inst, Immediate.u(0b1_0_10)),
-            .trunc_float => try self.airRound(inst, Immediate.u(0b1_0_11)),
+            .floor => try self.airRound(inst, 0b1_0_01),
+            .ceil => try self.airRound(inst, 0b1_0_10),
+            .trunc_float => try self.airRound(inst, 0b1_0_11),
             .sqrt => try self.airSqrt(inst),
             .neg, .fabs => try self.airFloatSign(inst),
 
@@ -4509,51 +4509,93 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
 }
 
-fn airRound(self: *Self, inst: Air.Inst.Index, mode: Immediate) !void {
+fn airRound(self: *Self, inst: Air.Inst.Index, mode: u4) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     const ty = self.air.typeOf(un_op);
 
-    if (!self.hasFeature(.sse4_1))
-        return self.fail("TODO implement airRound without sse4_1 feature", .{});
-
     const src_mcv = try self.resolveInst(un_op);
     const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
         src_mcv
     else
         try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
-
-    const mir_tag: Mir.Inst.Tag = switch (ty.zigTypeTag()) {
-        .Float => switch (ty.floatBits(self.target.*)) {
-            32 => .roundss,
-            64 => .roundsd,
-            else => return self.fail("TODO implement airRound for {}", .{
-                ty.fmt(self.bin_file.options.module.?),
-            }),
-        },
-        else => return self.fail("TODO implement airRound for {}", .{
-            ty.fmt(self.bin_file.options.module.?),
-        }),
-    };
-    assert(dst_mcv.isRegister());
-    const abi_size = @intCast(u32, ty.abiSize(self.target.*));
-    const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
-    if (src_mcv.isRegister())
-        try self.asmRegisterRegisterImmediate(
-            mir_tag,
-            dst_reg,
-            registerAlias(src_mcv.getReg().?, abi_size),
-            mode,
-        )
-    else
-        try self.asmRegisterMemoryImmediate(
-            mir_tag,
-            dst_reg,
-            src_mcv.mem(Memory.PtrSize.fromSize(@intCast(u32, ty.abiSize(self.target.*)))),
-            mode,
-        );
+    const dst_reg = dst_mcv.getReg().?;
+    const dst_lock = self.register_manager.lockReg(dst_reg);
+    defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
+    try self.genRound(ty, dst_reg, src_mcv, mode);
     return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
 }
 
+fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4) !void {
+    if (!self.hasFeature(.sse4_1))
+        return self.fail("TODO implement genRound without sse4_1 feature", .{});
+
+    const mir_tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {
+        .Float => switch (ty.floatBits(self.target.*)) {
+            32 => if (self.hasFeature(.avx)) .vroundss else .roundss,
+            64 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
+            16, 80, 128 => null,
+            else => unreachable,
+        },
+        .Vector => switch (ty.childType().zigTypeTag()) {
+            .Float => switch (ty.childType().floatBits(self.target.*)) {
+                32 => switch (ty.vectorLen()) {
+                    1 => if (self.hasFeature(.avx)) .vroundss else .roundss,
+                    2...4 => if (self.hasFeature(.avx)) .vroundps else .roundps,
+                    5...8 => if (self.hasFeature(.avx)) .vroundps else null,
+                    else => null,
+                },
+                64 => switch (ty.vectorLen()) {
+                    1 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
+                    2 => if (self.hasFeature(.avx)) .vroundpd else .roundpd,
+                    3...4 => if (self.hasFeature(.avx)) .vroundpd else null,
+                    else => null,
+                },
+                16, 80, 128 => null,
+                else => unreachable,
+            },
+            else => null,
+        },
+        else => unreachable,
+    })) |tag| tag else return self.fail("TODO implement genRound for {}", .{
+        ty.fmt(self.bin_file.options.module.?),
+    });
+
+    const abi_size = @intCast(u32, ty.abiSize(self.target.*));
+    const dst_alias = registerAlias(dst_reg, abi_size);
+    switch (mir_tag) {
+        .vroundss, .vroundsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
+            mir_tag,
+            dst_alias,
+            dst_alias,
+            src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
+            Immediate.u(mode),
+        ) else try self.asmRegisterRegisterRegisterImmediate(
+            mir_tag,
+            dst_alias,
+            dst_alias,
+            registerAlias(if (src_mcv.isRegister())
+                src_mcv.getReg().?
+            else
+                try self.copyToTmpRegister(ty, src_mcv), abi_size),
+            Immediate.u(mode),
+        ),
+        else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
+            mir_tag,
+            dst_alias,
+            src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
+            Immediate.u(mode),
+        ) else try self.asmRegisterRegisterImmediate(
+            mir_tag,
+            dst_alias,
+            registerAlias(if (src_mcv.isRegister())
+                src_mcv.getReg().?
+            else
+                try self.copyToTmpRegister(ty, src_mcv), abi_size),
+            Immediate.u(mode),
+        ),
+    }
+}
+
 fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
     const un_op = self.air.instructions.items(.data)[inst].un_op;
     const ty = self.air.typeOf(un_op);
@@ -6188,18 +6230,18 @@ fn genBinOp(
     })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
         @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
     });
-    const dst_alias = registerAlias(dst_mcv.getReg().?, abi_size);
+    const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
     if (self.hasFeature(.avx)) {
         const src1_alias =
-            if (copied_to_dst) dst_alias else registerAlias(lhs_mcv.getReg().?, abi_size);
+            if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
         if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
             mir_tag,
-            dst_alias,
+            dst_reg,
             src1_alias,
             src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
         ) else try self.asmRegisterRegisterRegister(
             mir_tag,
-            dst_alias,
+            dst_reg,
             src1_alias,
             registerAlias(if (src_mcv.isRegister())
                 src_mcv.getReg().?
@@ -6210,11 +6252,11 @@ fn genBinOp(
         assert(copied_to_dst);
         if (src_mcv.isMemory()) try self.asmRegisterMemory(
             mir_tag,
-            dst_alias,
+            dst_reg,
             src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
         ) else try self.asmRegisterRegister(
             mir_tag,
-            dst_alias,
+            dst_reg,
             registerAlias(if (src_mcv.isRegister())
                 src_mcv.getReg().?
             else
@@ -6223,60 +6265,16 @@ fn genBinOp(
     }
     switch (air_tag) {
         .add, .sub, .mul, .div_float, .div_exact => {},
-        .div_trunc, .div_floor => if (self.hasFeature(.sse4_1)) {
-            const round_tag = if (@as(?Mir.Inst.Tag, switch (lhs_ty.zigTypeTag()) {
-                .Float => switch (lhs_ty.floatBits(self.target.*)) {
-                    32 => if (self.hasFeature(.avx)) .vroundss else .roundss,
-                    64 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
-                    16, 80, 128 => null,
-                    else => unreachable,
-                },
-                .Vector => switch (lhs_ty.childType().zigTypeTag()) {
-                    .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
-                        32 => switch (lhs_ty.vectorLen()) {
-                            1 => if (self.hasFeature(.avx)) .vroundss else .roundss,
-                            2...4 => if (self.hasFeature(.avx)) .vroundps else .roundps,
-                            5...8 => if (self.hasFeature(.avx)) .vroundps else null,
-                            else => null,
-                        },
-                        64 => switch (lhs_ty.vectorLen()) {
-                            1 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
-                            2 => if (self.hasFeature(.avx)) .vroundpd else .roundpd,
-                            3...4 => if (self.hasFeature(.avx)) .vroundpd else null,
-                            else => null,
-                        },
-                        16, 80, 128 => null,
-                        else => unreachable,
-                    },
-                    else => null,
-                },
-                else => unreachable,
-            })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
-                @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
-            });
-            const round_mode = Immediate.u(switch (air_tag) {
+        .div_trunc, .div_floor => try self.genRound(
+            lhs_ty,
+            dst_reg,
+            .{ .register = dst_reg },
+            switch (air_tag) {
                 .div_trunc => 0b1_0_11,
                 .div_floor => 0b1_0_01,
                 else => unreachable,
-            });
-            switch (round_tag) {
-                .vroundss, .vroundsd => try self.asmRegisterRegisterRegisterImmediate(
-                    round_tag,
-                    dst_alias,
-                    dst_alias,
-                    dst_alias,
-                    round_mode,
-                ),
-                else => try self.asmRegisterRegisterImmediate(
-                    round_tag,
-                    dst_alias,
-                    dst_alias,
-                    round_mode,
-                ),
-            }
-        } else return self.fail("TODO implement genBinOp for {s} {} without sse4_1", .{
-            @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
-        }),
+            },
+        ),
         .max, .min => {}, // TODO: unordered select
         else => unreachable,
     }
diff --git a/test/behavior/floatop.zig b/test/behavior/floatop.zig
index 3f407061f4..3d46c267d3 100644
--- a/test/behavior/floatop.zig
+++ b/test/behavior/floatop.zig
@@ -617,7 +617,8 @@ fn testFloor() !void {
 
 test "@floor with vectors" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
 
@@ -707,7 +708,8 @@ fn testCeil() !void {
 
 test "@ceil with vectors" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
 
@@ -797,7 +799,8 @@ fn testTrunc() !void {
 
 test "@trunc with vectors" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
 

From 6778da4516e68c271cb50fe9c252ab4084daf16b Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Sun, 7 May 2023 20:42:46 -0400
Subject: [PATCH 134/725] x86_64: implement binary operations for `f16` and
 `f16` vectors

---
 src/arch/x86_64/CodeGen.zig   | 261 +++++++++++++++++++++++++++++++---
 src/arch/x86_64/Encoding.zig  |  23 +--
 src/arch/x86_64/Lower.zig     |  22 +++
 src/arch/x86_64/Mir.zig       |  44 ++++++
 src/arch/x86_64/encodings.zig |  20 +++
 test/behavior/floatop.zig     |  22 +--
 test/behavior/muladd.zig      |   8 +-
 7 files changed, 354 insertions(+), 46 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 3e2d418105..154b909a21 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -4497,14 +4497,15 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
     const tag = self.air.instructions.items(.tag)[inst];
     try self.genBinOpMir(switch (ty_bits) {
         // No point using an extra prefix byte for *pd which performs the same operation.
-        32, 64 => switch (tag) {
+        16, 32, 64, 128 => switch (tag) {
             .neg => .xorps,
             .fabs => .andnps,
             else => unreachable,
         },
-        else => return self.fail("TODO implement airFloatSign for {}", .{
+        80 => return self.fail("TODO implement airFloatSign for {}", .{
             ty.fmt(self.bin_file.options.module.?),
         }),
+        else => unreachable,
     }, vec_ty, dst_mcv, sign_mcv);
     return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
 }
@@ -6112,9 +6113,53 @@ fn genBinOp(
         return dst_mcv;
     }
 
+    const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
     const mir_tag = if (@as(?Mir.Inst.Tag, switch (lhs_ty.zigTypeTag()) {
         else => unreachable,
         .Float => switch (lhs_ty.floatBits(self.target.*)) {
+            16 => if (self.hasFeature(.f16c)) {
+                const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
+                const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
+                defer self.register_manager.unlockReg(tmp_lock);
+
+                if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
+                    .vpinsrw,
+                    dst_reg,
+                    dst_reg,
+                    src_mcv.mem(.word),
+                    Immediate.u(1),
+                ) else try self.asmRegisterRegisterRegister(
+                    .vpunpcklwd,
+                    dst_reg,
+                    dst_reg,
+                    (if (src_mcv.isRegister())
+                        src_mcv.getReg().?
+                    else
+                        try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
+                );
+                try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
+                try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);
+                try self.asmRegisterRegisterRegister(
+                    switch (air_tag) {
+                        .add => .vaddss,
+                        .sub => .vsubss,
+                        .div_float, .div_trunc, .div_floor, .div_exact => .vdivss,
+                        .max => .vmaxss,
+                        .min => .vmaxss,
+                        else => unreachable,
+                    },
+                    dst_reg,
+                    dst_reg,
+                    tmp_reg,
+                );
+                try self.asmRegisterRegisterImmediate(
+                    .vcvtps2ph,
+                    dst_reg,
+                    dst_reg,
+                    Immediate.u(0b1_00),
+                );
+                return dst_mcv;
+            } else null,
             32 => switch (air_tag) {
                 .add => if (self.hasFeature(.avx)) .vaddss else .addss,
                 .sub => if (self.hasFeature(.avx)) .vsubss else .subss,
@@ -6141,12 +6186,178 @@ fn genBinOp(
                 .min => if (self.hasFeature(.avx)) .vminsd else .minsd,
                 else => unreachable,
             },
-            16, 80, 128 => null,
+            80, 128 => null,
             else => unreachable,
         },
         .Vector => switch (lhs_ty.childType().zigTypeTag()) {
             else => null,
             .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
+                16 => if (self.hasFeature(.f16c)) switch (lhs_ty.vectorLen()) {
+                    1 => {
+                        const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
+                        const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
+                        defer self.register_manager.unlockReg(tmp_lock);
+
+                        if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
+                            .vpinsrw,
+                            dst_reg,
+                            dst_reg,
+                            src_mcv.mem(.word),
+                            Immediate.u(1),
+                        ) else try self.asmRegisterRegisterRegister(
+                            .vpunpcklwd,
+                            dst_reg,
+                            dst_reg,
+                            (if (src_mcv.isRegister())
+                                src_mcv.getReg().?
+                            else
+                                try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
+                        );
+                        try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
+                        try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);
+                        try self.asmRegisterRegisterRegister(
+                            switch (air_tag) {
+                                .add => .vaddss,
+                                .sub => .vsubss,
+                                .div_float, .div_trunc, .div_floor, .div_exact => .vdivss,
+                                .max => .vmaxss,
+                                .min => .vmaxss,
+                                else => unreachable,
+                            },
+                            dst_reg,
+                            dst_reg,
+                            tmp_reg,
+                        );
+                        try self.asmRegisterRegisterImmediate(
+                            .vcvtps2ph,
+                            dst_reg,
+                            dst_reg,
+                            Immediate.u(0b1_00),
+                        );
+                        return dst_mcv;
+                    },
+                    2 => {
+                        const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
+                        const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
+                        defer self.register_manager.unlockReg(tmp_lock);
+
+                        if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
+                            .vpinsrd,
+                            dst_reg,
+                            src_mcv.mem(.dword),
+                            Immediate.u(1),
+                        ) else try self.asmRegisterRegisterRegister(
+                            .vunpcklps,
+                            dst_reg,
+                            dst_reg,
+                            (if (src_mcv.isRegister())
+                                src_mcv.getReg().?
+                            else
+                                try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
+                        );
+                        try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
+                        try self.asmRegisterRegisterRegister(.vmovhlps, tmp_reg, dst_reg, dst_reg);
+                        try self.asmRegisterRegisterRegister(
+                            switch (air_tag) {
+                                .add => .vaddps,
+                                .sub => .vsubps,
+                                .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
+                                .max => .vmaxps,
+                                .min => .vmaxps,
+                                else => unreachable,
+                            },
+                            dst_reg,
+                            dst_reg,
+                            tmp_reg,
+                        );
+                        try self.asmRegisterRegisterImmediate(
+                            .vcvtps2ph,
+                            dst_reg,
+                            dst_reg,
+                            Immediate.u(0b1_00),
+                        );
+                        return dst_mcv;
+                    },
+                    3...4 => {
+                        const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
+                        const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
+                        defer self.register_manager.unlockReg(tmp_lock);
+
+                        try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
+                        if (src_mcv.isMemory()) try self.asmRegisterMemory(
+                            .vcvtph2ps,
+                            tmp_reg,
+                            src_mcv.mem(.qword),
+                        ) else try self.asmRegisterRegister(
+                            .vcvtph2ps,
+                            tmp_reg,
+                            (if (src_mcv.isRegister())
+                                src_mcv.getReg().?
+                            else
+                                try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
+                        );
+                        try self.asmRegisterRegisterRegister(
+                            switch (air_tag) {
+                                .add => .vaddps,
+                                .sub => .vsubps,
+                                .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
+                                .max => .vmaxps,
+                                .min => .vmaxps,
+                                else => unreachable,
+                            },
+                            dst_reg,
+                            dst_reg,
+                            tmp_reg,
+                        );
+                        try self.asmRegisterRegisterImmediate(
+                            .vcvtps2ph,
+                            dst_reg,
+                            dst_reg,
+                            Immediate.u(0b1_00),
+                        );
+                        return dst_mcv;
+                    },
+                    5...8 => {
+                        const tmp_reg = (try self.register_manager.allocReg(null, sse)).to256();
+                        const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
+                        defer self.register_manager.unlockReg(tmp_lock);
+
+                        try self.asmRegisterRegister(.vcvtph2ps, dst_reg.to256(), dst_reg);
+                        if (src_mcv.isMemory()) try self.asmRegisterMemory(
+                            .vcvtph2ps,
+                            tmp_reg,
+                            src_mcv.mem(.xword),
+                        ) else try self.asmRegisterRegister(
+                            .vcvtph2ps,
+                            tmp_reg,
+                            (if (src_mcv.isRegister())
+                                src_mcv.getReg().?
+                            else
+                                try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
+                        );
+                        try self.asmRegisterRegisterRegister(
+                            switch (air_tag) {
+                                .add => .vaddps,
+                                .sub => .vsubps,
+                                .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
+                                .max => .vmaxps,
+                                .min => .vmaxps,
+                                else => unreachable,
+                            },
+                            dst_reg.to256(),
+                            dst_reg.to256(),
+                            tmp_reg,
+                        );
+                        try self.asmRegisterRegisterImmediate(
+                            .vcvtps2ph,
+                            dst_reg,
+                            dst_reg.to256(),
+                            Immediate.u(0b1_00),
+                        );
+                        return dst_mcv;
+                    },
+                    else => null,
+                } else null,
                 32 => switch (lhs_ty.vectorLen()) {
                     1 => switch (air_tag) {
                         .add => if (self.hasFeature(.avx)) .vaddss else .addss,
@@ -6223,14 +6434,13 @@ fn genBinOp(
                     } else null,
                     else => null,
                 },
-                16, 80, 128 => null,
+                80, 128 => null,
                 else => unreachable,
             },
         },
     })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
         @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
     });
-    const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
     if (self.hasFeature(.avx)) {
         const src1_alias =
             if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
@@ -7139,21 +7349,21 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
                         const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg);
                         defer self.register_manager.unlockReg(tmp2_lock);
 
-                        if (src_mcv.isRegister())
-                            try self.asmRegisterRegisterRegister(
-                                .vpunpcklwd,
-                                tmp1_reg,
-                                dst_reg.to128(),
-                                src_mcv.getReg().?.to128(),
-                            )
-                        else
-                            try self.asmRegisterRegisterMemoryImmediate(
-                                .vpinsrw,
-                                tmp1_reg,
-                                dst_reg.to128(),
-                                src_mcv.mem(.word),
-                                Immediate.u(1),
-                            );
+                        if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
+                            .vpinsrw,
+                            tmp1_reg,
+                            dst_reg.to128(),
+                            src_mcv.mem(.word),
+                            Immediate.u(1),
+                        ) else try self.asmRegisterRegisterRegister(
+                            .vpunpcklwd,
+                            tmp1_reg,
+                            dst_reg.to128(),
+                            (if (src_mcv.isRegister())
+                                src_mcv.getReg().?
+                            else
+                                try self.copyToTmpRegister(ty, src_mcv)).to128(),
+                        );
                         try self.asmRegisterRegister(.vcvtph2ps, tmp1_reg, tmp1_reg);
                         try self.asmRegisterRegister(.vmovshdup, tmp2_reg, tmp1_reg);
                         try self.genBinOpMir(.ucomiss, ty, tmp1_mcv, tmp2_mcv);
@@ -8139,7 +8349,16 @@ fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.Tag {
         },
         .Vector => switch (ty.childType().zigTypeTag()) {
             .Float => switch (ty.childType().floatBits(self.target.*)) {
-                16 => unreachable, // needs special handling
+                16 => switch (ty.vectorLen()) {
+                    1 => unreachable, // needs special handling
+                    2 => return if (self.hasFeature(.avx)) .vmovss else .movss,
+                    3...4 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,
+                    5...8 => return if (self.hasFeature(.avx))
+                        if (aligned) .vmovaps else .vmovups
+                    else if (aligned) .movaps else .movups,
+                    9...16 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,
+                    else => {},
+                },
                 32 => switch (ty.vectorLen()) {
                     1 => return if (self.hasFeature(.avx)) .vmovss else .movss,
                     2...4 => return if (self.hasFeature(.avx))
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index b8ccc9efba..3235b29358 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -270,7 +270,7 @@ pub const Mnemonic = enum {
     divps, divss,
     maxps, maxss,
     minps, minss,
-    movaps, movss, movups,
+    movaps, movhlps, movss, movups,
     mulps, mulss,
     orps,
     pextrw, pinsrw,
@@ -303,6 +303,8 @@ pub const Mnemonic = enum {
     // SSE3
     movddup, movshdup, movsldup,
     // SSE4.1
+    pextrb, pextrd, pextrq,
+    pinsrb, pinsrd, pinsrq,
     roundpd, roundps, roundsd, roundss,
     // AVX
     vaddpd, vaddps, vaddsd, vaddss,
@@ -311,13 +313,14 @@ pub const Mnemonic = enum {
     vmaxpd, vmaxps, vmaxsd, vmaxss,
     vminpd, vminps, vminsd, vminss,
     vmovapd, vmovaps,
-    vmovddup,
+    vmovddup, vmovhlps,
     vmovsd,
     vmovshdup, vmovsldup,
     vmovss,
     vmovupd, vmovups,
     vmulpd, vmulps, vmulsd, vmulss,
-    vpextrw, vpinsrw,
+    vpextrb, vpextrd, vpextrq, vpextrw,
+    vpinsrb, vpinsrd, vpinsrq, vpinsrw,
     vpshufhw, vpshuflw,
     vpsrld, vpsrlq, vpsrlw,
     vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd,
@@ -359,7 +362,7 @@ pub const Op = enum {
     cl,
     r8, r16, r32, r64,
     rm8, rm16, rm32, rm64,
-    r32_m16, r64_m16,
+    r32_m8, r32_m16, r64_m16,
     m8, m16, m32, m64, m80, m128, m256,
     rel8, rel16, rel32,
     m,
@@ -444,7 +447,7 @@ pub const Op = enum {
     pub fn immBitSize(op: Op) u64 {
         return switch (op) {
             .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
-            .al, .cl, .r8, .rm8 => unreachable,
+            .al, .cl, .r8, .rm8, .r32_m8 => unreachable,
             .ax, .r16, .rm16 => unreachable,
             .eax, .r32, .rm32, .r32_m16 => unreachable,
             .rax, .r64, .rm64, .r64_m16 => unreachable,
@@ -467,7 +470,7 @@ pub const Op = enum {
             .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
             .al, .cl, .r8, .rm8 => 8,
             .ax, .r16, .rm16 => 16,
-            .eax, .r32, .rm32, .r32_m16 => 32,
+            .eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,
             .rax, .r64, .rm64, .r64_m16 => 64,
             .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
             .ymm, .ymm_m256 => 256,
@@ -480,7 +483,7 @@ pub const Op = enum {
             .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
             .rel8, .rel16, .rel32 => unreachable,
             .al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64, .xmm, .ymm => unreachable,
-            .m8, .rm8 => 8,
+            .m8, .rm8, .r32_m8 => 8,
             .m16, .rm16, .r32_m16, .r64_m16 => 16,
             .m32, .rm32, .xmm_m32 => 32,
             .m64, .rm64, .xmm_m64 => 64,
@@ -509,7 +512,7 @@ pub const Op = enum {
             .al, .ax, .eax, .rax,
             .r8, .r16, .r32, .r64,
             .rm8, .rm16, .rm32, .rm64,
-            .r32_m16, .r64_m16,
+            .r32_m8, .r32_m16, .r64_m16,
             .xmm, .xmm_m32, .xmm_m64, .xmm_m128,
             .ymm, .ymm_m256,
             => true,
@@ -535,7 +538,7 @@ pub const Op = enum {
         // zig fmt: off
         return switch (op) {
             .rm8, .rm16, .rm32, .rm64,
-            .r32_m16, .r64_m16,
+            .r32_m8, .r32_m16, .r64_m16,
             .m8, .m16, .m32, .m64, .m80, .m128, .m256,
             .m,
             .xmm_m32, .xmm_m64, .xmm_m128,
@@ -559,7 +562,7 @@ pub const Op = enum {
             .al, .ax, .eax, .rax, .cl => .general_purpose,
             .r8, .r16, .r32, .r64 => .general_purpose,
             .rm8, .rm16, .rm32, .rm64 => .general_purpose,
-            .r32_m16, .r64_m16 => .general_purpose,
+            .r32_m8, .r32_m16, .r64_m16 => .general_purpose,
             .sreg => .segment,
             .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .floating_point,
             .ymm, .ymm_m256 => .floating_point,
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index 2cfa25ac84..5c079f4768 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -137,6 +137,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .minps,
         .minss,
         .movaps,
+        .movhlps,
         .movss,
         .movups,
         .mulps,
@@ -149,6 +150,8 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .subps,
         .subss,
         .ucomiss,
+        .unpckhps,
+        .unpcklps,
         .xorps,
 
         .addpd,
@@ -187,12 +190,20 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .subpd,
         .subsd,
         .ucomisd,
+        .unpckhpd,
+        .unpcklpd,
         .xorpd,
 
         .movddup,
         .movshdup,
         .movsldup,
 
+        .pextrb,
+        .pextrd,
+        .pextrq,
+        .pinsrb,
+        .pinsrd,
+        .pinsrq,
         .roundpd,
         .roundps,
         .roundsd,
@@ -221,6 +232,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .vmovapd,
         .vmovaps,
         .vmovddup,
+        .vmovhlps,
         .vmovsd,
         .vmovshdup,
         .vmovsldup,
@@ -231,7 +243,13 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .vmulps,
         .vmulsd,
         .vmulss,
+        .vpextrb,
+        .vpextrd,
+        .vpextrq,
         .vpextrw,
+        .vpinsrb,
+        .vpinsrd,
+        .vpinsrq,
         .vpinsrw,
         .vpshufhw,
         .vpshuflw,
@@ -258,6 +276,10 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
         .vsubps,
         .vsubsd,
         .vsubss,
+        .vunpckhpd,
+        .vunpckhps,
+        .vunpcklpd,
+        .vunpcklps,
 
         .vcvtph2ps,
         .vcvtps2ph,
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index c0450406cf..442cfabebb 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -192,6 +192,8 @@ pub const Inst = struct {
         minss,
         /// Move aligned packed single-precision floating-point values
         movaps,
+        /// Move packed single-precision floating-point values high to low
+        movhlps,
         /// Move scalar single-precision floating-point value
         movss,
         /// Move unaligned packed single-precision floating-point values
@@ -216,6 +218,10 @@ pub const Inst = struct {
         subss,
         /// Unordered compare scalar single-precision floating-point values
         ucomiss,
+        /// Unpack and interleave high packed single-precision floating-point values
+        unpckhps,
+        /// Unpack and interleave low packed single-precision floating-point values
+        unpcklps,
         /// Bitwise logical xor of packed single precision floating-point values
         xorps,
 
@@ -291,6 +297,10 @@ pub const Inst = struct {
         subsd,
         /// Unordered compare scalar double-precision floating-point values
         ucomisd,
+        /// Unpack and interleave high packed double-precision floating-point values
+        unpckhpd,
+        /// Unpack and interleave low packed double-precision floating-point values
+        unpcklpd,
         /// Bitwise logical xor of packed double precision floating-point values
         xorpd,
 
@@ -301,6 +311,18 @@ pub const Inst = struct {
         /// Replicate single floating-point values
         movsldup,
 
+        /// Extract Byte
+        pextrb,
+        /// Extract Doubleword
+        pextrd,
+        /// Extract Quadword
+        pextrq,
+        /// Insert Byte
+        pinsrb,
+        /// Insert Doubleword
+        pinsrd,
+        /// Insert Quadword
+        pinsrq,
         /// Round packed double-precision floating-point values
         roundpd,
         /// Round packed single-precision floating-point values
@@ -354,6 +376,8 @@ pub const Inst = struct {
         vmovapd,
         /// Move aligned packed single-precision floating-point values
         vmovaps,
+        /// Move packed single-precision floating-point values high to low
+        vmovhlps,
         /// Replicate double floating-point values
         vmovddup,
         /// Move or merge scalar double-precision floating-point value
@@ -376,8 +400,20 @@ pub const Inst = struct {
         vmulsd,
         /// Multiply scalar single-precision floating-point values
         vmulss,
+        /// Extract Byte
+        vpextrb,
+        /// Extract Doubleword
+        vpextrd,
+        /// Extract Quadword
+        vpextrq,
         /// Extract word
         vpextrw,
+        /// Insert Byte
+        vpinsrb,
+        /// Insert Doubleword
+        vpinsrd,
+        /// Insert Quadword
+        vpinsrq,
         /// Insert word
         vpinsrw,
         /// Shuffle packed high words
@@ -430,6 +466,14 @@ pub const Inst = struct {
         vsubsd,
         /// Subtract scalar single-precision floating-point values
         vsubss,
+        /// Unpack and interleave high packed double-precision floating-point values
+        vunpckhpd,
+        /// Unpack and interleave high packed single-precision floating-point values
+        vunpckhps,
+        /// Unpack and interleave low packed double-precision floating-point values
+        vunpcklpd,
+        /// Unpack and interleave low packed single-precision floating-point values
+        vunpcklps,
 
         /// Convert 16-bit floating-point values to single-precision floating-point values
         vcvtph2ps,
diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig
index c41f0ea4e7..2b9d530c1e 100644
--- a/src/arch/x86_64/encodings.zig
+++ b/src/arch/x86_64/encodings.zig
@@ -865,6 +865,8 @@ pub const table = [_]Entry{
     .{ .movaps, .rm, &.{ .xmm,      .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .none, .sse },
     .{ .movaps, .mr, &.{ .xmm_m128, .xmm      }, &.{ 0x0f, 0x29 }, 0, .none, .sse },
 
+    .{ .movhlps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .none, .sse },
+
     .{ .movss, .rm, &.{ .xmm,     .xmm_m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .none, .sse },
     .{ .movss, .mr, &.{ .xmm_m32, .xmm     }, &.{ 0xf3, 0x0f, 0x11 }, 0, .none, .sse },
 
@@ -988,8 +990,16 @@ pub const table = [_]Entry{
     .{ .movsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .none, .sse3 },
 
     // SSE4.1
+    .{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },
+    .{ .pextrd, .mri, &.{ .rm32,   .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .none, .sse4_1 },
+    .{ .pextrq, .mri, &.{ .rm64,   .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .long, .sse4_1 },
+
     .{ .pextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .none, .sse4_1 },
 
+    .{ .pinsrb, .rmi, &.{ .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .none, .sse4_1 },
+    .{ .pinsrd, .rmi, &.{ .xmm, .rm32,   .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .none, .sse4_1 },
+    .{ .pinsrq, .rmi, &.{ .xmm, .rm64,   .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .long, .sse4_1 },
+
     .{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },
 
     .{ .roundps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .none, .sse4_1 },
@@ -1062,6 +1072,8 @@ pub const table = [_]Entry{
     .{ .vmovddup, .rm, &.{ .xmm, .xmm_m64  }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
     .{ .vmovddup, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_256_wig, .avx },
 
+    .{ .vmovhlps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
+
     .{ .vmovsd, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
     .{ .vmovsd, .rm,  &.{       .xmm, .m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
     .{ .vmovsd, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
@@ -1098,9 +1110,17 @@ pub const table = [_]Entry{
 
     .{ .vmulss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx },
 
+    .{ .vpextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .vex_128_w0, .avx },
+    .{ .vpextrd, .mri, &.{ .rm32,   .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w0, .avx },
+    .{ .vpextrq, .mri, &.{ .rm64,   .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w1, .avx },
+
     .{ .vpextrw, .rmi, &.{ .r32,     .xmm, .imm8 }, &.{ 0x66, 0x0f,       0x15 }, 0, .vex_128_wig, .avx },
     .{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_wig, .avx },
 
+    .{ .vpinsrb, .rmi, &.{ .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .vex_128_w0, .avx },
+    .{ .vpinsrd, .rmi, &.{ .xmm, .rm32,   .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w0, .avx },
+    .{ .vpinsrq, .rmi, &.{ .xmm, .rm64,   .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w1, .avx },
+
     .{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_wig, .avx },
 
     .{ .vpsrlw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .vex_128_wig, .avx },
diff --git a/test/behavior/floatop.zig b/test/behavior/floatop.zig
index 3d46c267d3..242c8dabe5 100644
--- a/test/behavior/floatop.zig
+++ b/test/behavior/floatop.zig
@@ -8,6 +8,8 @@ const has_f80_rt = switch (builtin.cpu.arch) {
     .x86_64, .x86 => true,
     else => false,
 };
+const no_x86_64_hardware_f16_support = builtin.zig_backend == .stage2_x86_64 and
+    !std.Target.x86.featureSetHas(builtin.cpu.features, .f16c);
 
 const epsilon_16 = 0.001;
 const epsilon = 0.000001;
@@ -52,8 +54,7 @@ fn testFloatComparisons() !void {
 }
 
 test "different sized float comparisons" {
-    if (builtin.zig_backend == .stage2_x86_64 and
-        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .f16c)) return error.SkipZigTest; // TODO
+    if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -152,7 +153,7 @@ fn testSqrtWithVectors() !void {
 }
 
 test "more @sqrt f16 tests" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -202,7 +203,7 @@ fn testSqrtLegacy(comptime T: type, x: T) !void {
 }
 
 test "@sin" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -241,7 +242,7 @@ fn testSinWithVectors() !void {
 }
 
 test "@cos" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -280,7 +281,7 @@ fn testCosWithVectors() !void {
 }
 
 test "@exp" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -318,7 +319,7 @@ fn testExpWithVectors() !void {
 }
 
 test "@exp2" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -403,7 +404,7 @@ test "@log with @vectors" {
 }
 
 test "@log2" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -445,7 +446,7 @@ fn testLog2WithVectors() !void {
 }
 
 test "@log10" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -881,7 +882,7 @@ fn testTruncLegacy(comptime T: type, x: T) !void {
 }
 
 test "negation f16" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
+    if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -1040,7 +1041,6 @@ test "comptime_float zero divided by zero produces zero" {
 }
 
 test "nan negation f16" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
diff --git a/test/behavior/muladd.zig b/test/behavior/muladd.zig
index bfb94de270..199f117e7b 100644
--- a/test/behavior/muladd.zig
+++ b/test/behavior/muladd.zig
@@ -2,11 +2,11 @@ const std = @import("std");
 const builtin = @import("builtin");
 const expect = std.testing.expect;
 
-const stage2_x86_64_without_hardware_fma_support = builtin.zig_backend == .stage2_x86_64 and
+const no_x86_64_hardware_fma_support = builtin.zig_backend == .stage2_x86_64 and
     !std.Target.x86.featureSetHas(builtin.cpu.features, .fma);
 
 test "@mulAdd" {
-    if (stage2_x86_64_without_hardware_fma_support) return error.SkipZigTest; // TODO
+    if (no_x86_64_hardware_fma_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -120,7 +120,7 @@ fn vector32() !void {
 
 test "vector f32" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (stage2_x86_64_without_hardware_fma_support) return error.SkipZigTest; // TODO
+    if (no_x86_64_hardware_fma_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
@@ -143,7 +143,7 @@ fn vector64() !void {
 
 test "vector f64" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (stage2_x86_64_without_hardware_fma_support) return error.SkipZigTest; // TODO
+    if (no_x86_64_hardware_fma_support) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

From 6c14eb2863c7c00f809c5e447ceb8186b55f2eef Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Mon, 8 May 2023 06:50:18 -0400
Subject: [PATCH 135/725] x86_64: optimize mir tag usage

This moves all pseudo-instructions to a single `Mir.Inst.Tag` tag and
prepares to start coalescing similar mnemonics. 239 tags left in use.
---
 src/arch/x86_64/CodeGen.zig  | 427 +++++++++++++--------
 src/arch/x86_64/Emit.zig     |  65 ++--
 src/arch/x86_64/Encoding.zig |   2 +-
 src/arch/x86_64/Lower.zig    | 717 ++++++++++-------------------------
 src/arch/x86_64/Mir.zig      | 517 ++++++++++++++++---------
 src/arch/x86_64/bits.zig     |   3 -
 6 files changed, 853 insertions(+), 878 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 154b909a21..3ac05c95ac 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -973,14 +973,14 @@ fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
     try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
     const result_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);
     self.mir_instructions.appendAssumeCapacity(inst);
-    switch (inst.tag) {
-        else => wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)}),
-        .dbg_line,
-        .dbg_prologue_end,
-        .dbg_epilogue_begin,
-        .dead,
-        => {},
-    }
+    if (inst.tag != .pseudo or switch (inst.ops) {
+        else => true,
+        .pseudo_dbg_prologue_end_none,
+        .pseudo_dbg_line_line_column,
+        .pseudo_dbg_epilogue_begin_none,
+        .pseudo_dead_none,
+        => false,
+    }) wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});
     return result_index;
 }
 
@@ -1003,98 +1003,166 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
     return result;
 }
 
+/// A `cc` of `.z_and_np` clobbers `reg2`!
+fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
+    _ = try self.addInst(.{
+        .tag = switch (cc) {
+            else => .cmov,
+            .z_and_np, .nz_or_p => .pseudo,
+        },
+        .ops = switch (cc) {
+            else => .rr,
+            .z_and_np => .pseudo_cmov_z_and_np_rr,
+            .nz_or_p => .pseudo_cmov_nz_or_p_rr,
+        },
+        .data = .{ .rr = .{
+            .fixes = switch (cc) {
+                else => Mir.Inst.Fixes.fromCondition(cc),
+                .z_and_np, .nz_or_p => ._,
+            },
+            .r1 = reg1,
+            .r2 = reg2,
+        } },
+    });
+}
+
+/// A `cc` of `.z_and_np` is not supported by this encoding!
+fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
+    _ = try self.addInst(.{
+        .tag = switch (cc) {
+            else => .cmov,
+            .z_and_np => unreachable,
+            .nz_or_p => .pseudo,
+        },
+        .ops = switch (cc) {
+            else => switch (m) {
+                .sib => .rm_sib,
+                .rip => .rm_rip,
+                else => unreachable,
+            },
+            .z_and_np => unreachable,
+            .nz_or_p => switch (m) {
+                .sib => .pseudo_cmov_nz_or_p_rm_sib,
+                .rip => .pseudo_cmov_nz_or_p_rm_rip,
+                else => unreachable,
+            },
+        },
+        .data = .{ .rx = .{
+            .fixes = switch (cc) {
+                else => Mir.Inst.Fixes.fromCondition(cc),
+                .z_and_np => unreachable,
+                .nz_or_p => ._,
+            },
+            .r1 = reg,
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
+        } },
+    });
+}
+
 fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
     _ = try self.addInst(.{
-        .tag = .setcc,
-        .ops = .r_cc,
-        .data = .{ .r_cc = .{
-            .r = reg,
-            .scratch = if (cc == .z_and_np or cc == .nz_or_p)
-                (try self.register_manager.allocReg(null, gp)).to8()
-            else
-                .none,
-            .cc = cc,
-        } },
+        .tag = switch (cc) {
+            else => .set,
+            .z_and_np, .nz_or_p => .pseudo,
+        },
+        .ops = switch (cc) {
+            else => .r,
+            .z_and_np => .pseudo_set_z_and_np_r,
+            .nz_or_p => .pseudo_set_nz_or_p_r,
+        },
+        .data = switch (cc) {
+            else => .{ .r = .{
+                .fixes = Mir.Inst.Fixes.fromCondition(cc),
+                .r1 = reg,
+            } },
+            .z_and_np, .nz_or_p => .{ .r_scratch = .{
+                .r1 = reg,
+                .scratch_reg = (try self.register_manager.allocReg(null, gp)).to8(),
+            } },
+        },
     });
 }
 
 fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
+    const payload = switch (m) {
+        .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+        .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+        else => unreachable,
+    };
     _ = try self.addInst(.{
-        .tag = .setcc,
-        .ops = switch (m) {
-            .sib => .m_sib_cc,
-            .rip => .m_rip_cc,
-            else => unreachable,
+        .tag = switch (cc) {
+            else => .set,
+            .z_and_np, .nz_or_p => .pseudo,
         },
-        .data = .{ .x_cc = .{
-            .scratch = if (cc == .z_and_np or cc == .nz_or_p)
-                (try self.register_manager.allocReg(null, gp)).to8()
-            else
-                .none,
-            .cc = cc,
-            .payload = switch (m) {
-                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
-                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+        .ops = switch (cc) {
+            else => switch (m) {
+                .sib => .m_sib,
+                .rip => .m_rip,
                 else => unreachable,
             },
-        } },
-    });
-}
-
-/// A `cc` of `.z_and_np` clobbers `reg2`!
-fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
-    _ = try self.addInst(.{
-        .tag = .cmovcc,
-        .ops = .rr_cc,
-        .data = .{ .rr_cc = .{
-            .r1 = reg1,
-            .r2 = reg2,
-            .cc = cc,
-        } },
-    });
-}
-
-fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
-    assert(cc != .z_and_np); // not supported
-    _ = try self.addInst(.{
-        .tag = .cmovcc,
-        .ops = switch (m) {
-            .sib => .rm_sib_cc,
-            .rip => .rm_rip_cc,
-            else => unreachable,
-        },
-        .data = .{ .rx_cc = .{
-            .r = reg,
-            .cc = cc,
-            .payload = switch (m) {
-                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
-                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+            .z_and_np => switch (m) {
+                .sib => .pseudo_set_z_and_np_m_sib,
+                .rip => .pseudo_set_z_and_np_m_rip,
                 else => unreachable,
             },
-        } },
+            .nz_or_p => switch (m) {
+                .sib => .pseudo_set_nz_or_p_m_sib,
+                .rip => .pseudo_set_nz_or_p_m_rip,
+                else => unreachable,
+            },
+        },
+        .data = switch (cc) {
+            else => .{ .x = .{
+                .fixes = Mir.Inst.Fixes.fromCondition(cc),
+                .payload = payload,
+            } },
+            .z_and_np, .nz_or_p => .{ .x_scratch = .{
+                .scratch_reg = (try self.register_manager.allocReg(null, gp)).to8(),
+                .payload = payload,
+            } },
+        },
     });
 }
 
 fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
     return self.addInst(.{
-        .tag = .jmp_reloc,
-        .ops = undefined,
-        .data = .{ .inst = target },
+        .tag = .jmp,
+        .ops = .inst,
+        .data = .{ .inst = .{
+            .inst = target,
+        } },
     });
 }
 
 fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Inst.Index {
     return self.addInst(.{
-        .tag = .jcc,
-        .ops = .inst_cc,
-        .data = .{ .inst_cc = .{ .inst = target, .cc = cc } },
+        .tag = switch (cc) {
+            else => .j,
+            .z_and_np, .nz_or_p => .pseudo,
+        },
+        .ops = switch (cc) {
+            else => .inst,
+            .z_and_np => .pseudo_j_z_and_np_inst,
+            .nz_or_p => .pseudo_j_nz_or_p_inst,
+        },
+        .data = .{ .inst = .{
+            .fixes = switch (cc) {
+                else => Mir.Inst.Fixes.fromCondition(cc),
+                .z_and_np, .nz_or_p => ._,
+            },
+            .inst = target,
+        } },
     });
 }
 
 fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
     return self.addInst(.{
-        .tag = .dead,
-        .ops = undefined,
+        .tag = .pseudo,
+        .ops = .pseudo_dead_none,
         .data = undefined,
     });
 }
@@ -1107,11 +1175,19 @@ fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {
     });
 }
 
+fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
+    _ = try self.addInst(.{
+        .tag = .pseudo,
+        .ops = ops,
+        .data = undefined,
+    });
+}
+
 fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {
     _ = try self.addInst(.{
         .tag = tag,
         .ops = .r,
-        .data = .{ .r = reg },
+        .data = .{ .r = .{ .r1 = reg } },
     });
 }
 
@@ -1122,9 +1198,11 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
             .signed => .i_s,
             .unsigned => .i_u,
         },
-        .data = .{ .i = switch (imm) {
-            .signed => |s| @bitCast(u32, s),
-            .unsigned => |u| @intCast(u32, u),
+        .data = .{ .i = .{
+            .i = switch (imm) {
+                .signed => |s| @bitCast(u32, s),
+                .unsigned => |u| @intCast(u32, u),
+            },
         } },
     });
 }
@@ -1147,14 +1225,14 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
         .ops = ops,
         .data = switch (ops) {
             .ri_s, .ri_u => .{ .ri = .{
-                .r = reg,
+                .r1 = reg,
                 .i = switch (imm) {
                     .signed => |s| @bitCast(u32, s),
                     .unsigned => |u| @intCast(u32, u),
                 },
             } },
             .ri64 => .{ .rx = .{
-                .r = reg,
+                .r1 = reg,
                 .payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
             } },
             else => unreachable,
@@ -1249,10 +1327,12 @@ fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
             .rip => .m_rip,
             else => unreachable,
         },
-        .data = .{ .payload = switch (m) {
-            .sib => try self.addExtra(Mir.MemorySib.encode(m)),
-            .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
-            else => unreachable,
+        .data = .{ .x = .{
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
         } },
     });
 }
@@ -1266,7 +1346,7 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !
             else => unreachable,
         },
         .data = .{ .rx = .{
-            .r = reg,
+            .r1 = reg,
             .payload = switch (m) {
                 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
                 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
@@ -1291,7 +1371,7 @@ fn asmRegisterMemoryImmediate(
             else => unreachable,
         },
         .data = .{ .rix = .{
-            .r = reg,
+            .r1 = reg,
             .i = @intCast(u8, imm.unsigned),
             .payload = switch (m) {
                 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
@@ -1339,7 +1419,7 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !
             else => unreachable,
         },
         .data = .{ .rx = .{
-            .r = reg,
+            .r1 = reg,
             .payload = switch (m) {
                 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
                 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
@@ -1413,11 +1493,15 @@ fn asmMemoryRegisterImmediate(
             .rip => .mri_rip,
             else => unreachable,
         },
-        .data = .{ .rix = .{ .r = reg, .i = @intCast(u8, imm.unsigned), .payload = switch (m) {
-            .sib => try self.addExtra(Mir.MemorySib.encode(m)),
-            .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
-            else => unreachable,
-        } } },
+        .data = .{ .rix = .{
+            .r1 = reg,
+            .i = @intCast(u8, imm.unsigned),
+            .payload = switch (m) {
+                .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+                .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+                else => unreachable,
+            },
+        } },
     });
 }
 
@@ -1450,7 +1534,7 @@ fn gen(self: *Self) InnerError!void {
             else => unreachable,
         }
 
-        try self.asmOpOnly(.dbg_prologue_end);
+        try self.asmPseudo(.pseudo_dbg_prologue_end_none);
 
         try self.genBody(self.air.getMainBody());
 
@@ -1462,11 +1546,11 @@ fn gen(self: *Self) InnerError!void {
         // }
         // Eliding the reloc will cause a miscompilation in this case.
         for (self.exitlude_jump_relocs.items) |jmp_reloc| {
-            self.mir_instructions.items(.data)[jmp_reloc].inst =
+            self.mir_instructions.items(.data)[jmp_reloc].inst.inst =
                 @intCast(u32, self.mir_instructions.len);
         }
 
-        try self.asmOpOnly(.dbg_epilogue_begin);
+        try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
         const backpatch_stack_dealloc = try self.asmPlaceholder();
         const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
         try self.asmRegister(.pop, .rbp);
@@ -1480,46 +1564,54 @@ fn gen(self: *Self) InnerError!void {
             self.mir_instructions.set(backpatch_frame_align, .{
                 .tag = .@"and",
                 .ops = .ri_s,
-                .data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_mask } },
+                .data = .{ .ri = .{
+                    .r1 = .rsp,
+                    .i = frame_layout.stack_mask,
+                } },
             });
         }
         if (need_stack_adjust) {
             self.mir_instructions.set(backpatch_stack_alloc, .{
                 .tag = .sub,
                 .ops = .ri_s,
-                .data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_adjust } },
+                .data = .{ .ri = .{
+                    .r1 = .rsp,
+                    .i = frame_layout.stack_adjust,
+                } },
             });
         }
         if (need_frame_align or need_stack_adjust) {
             self.mir_instructions.set(backpatch_stack_dealloc, .{
                 .tag = .mov,
                 .ops = .rr,
-                .data = .{ .rr = .{ .r1 = .rsp, .r2 = .rbp } },
+                .data = .{ .rr = .{
+                    .r1 = .rsp,
+                    .r2 = .rbp,
+                } },
             });
         }
         if (need_save_reg) {
-            const save_reg_list = frame_layout.save_reg_list.asInt();
             self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
-                .tag = .push_regs,
-                .ops = undefined,
-                .data = .{ .payload = save_reg_list },
+                .tag = .pseudo,
+                .ops = .pseudo_push_reg_list,
+                .data = .{ .reg_list = frame_layout.save_reg_list },
             });
             self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{
-                .tag = .pop_regs,
-                .ops = undefined,
-                .data = .{ .payload = save_reg_list },
+                .tag = .pseudo,
+                .ops = .pseudo_pop_reg_list,
+                .data = .{ .reg_list = frame_layout.save_reg_list },
             });
         }
     } else {
-        try self.asmOpOnly(.dbg_prologue_end);
+        try self.asmPseudo(.pseudo_dbg_prologue_end_none);
         try self.genBody(self.air.getMainBody());
-        try self.asmOpOnly(.dbg_epilogue_begin);
+        try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
     }
 
     // Drop them off at the rbrace.
     _ = try self.addInst(.{
-        .tag = .dbg_line,
-        .ops = undefined,
+        .tag = .pseudo,
+        .ops = .pseudo_dbg_line_line_column,
         .data = .{ .line_column = .{
             .line = self.end_di_line,
             .column = self.end_di_column,
@@ -2446,11 +2538,11 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
         .register => |dst_reg| {
             const min_abi_size = @min(dst_abi_size, src_abi_size);
             const tag: Mir.Inst.Tag = switch (signedness) {
-                .signed => .movsx,
-                .unsigned => if (min_abi_size > 2) .mov else .movzx,
+                .signed => if (min_abi_size >= 4) .movsxd else .movsx,
+                .unsigned => if (min_abi_size >= 4) .mov else .movzx,
             };
             const dst_alias = switch (tag) {
-                .movsx => dst_reg.to64(),
+                .movsx, .movsxd => dst_reg.to64(),
                 .mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(),
                 else => unreachable,
             };
@@ -5247,7 +5339,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
                 const field_byte_size = @intCast(u32, field_ty.abiSize(self.target.*));
                 if (signedness == .signed and field_byte_size < 8) {
                     try self.asmRegisterRegister(
-                        .movsx,
+                        if (field_byte_size >= 4) .movsxd else .movsx,
                         dst_mcv.register,
                         registerAlias(dst_mcv.register, field_byte_size),
                     );
@@ -7194,10 +7286,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
                 const atom_index = try self.owner.getSymbolIndex(self);
                 const sym_index = try coff_file.getGlobalSymbol(decl_name, lib_name);
                 _ = try self.addInst(.{
-                    .tag = .mov_linker,
+                    .tag = .mov,
                     .ops = .import_reloc,
                     .data = .{ .rx = .{
-                        .r = .rax,
+                        .r1 = .rax,
                         .payload = try self.addExtra(Mir.Reloc{
                             .atom_index = atom_index,
                             .sym_index = sym_index,
@@ -7209,9 +7301,9 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
                 const atom_index = try self.owner.getSymbolIndex(self);
                 const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);
                 _ = try self.addInst(.{
-                    .tag = .call_extern,
-                    .ops = undefined,
-                    .data = .{ .relocation = .{
+                    .tag = .call,
+                    .ops = .extern_fn_reloc,
+                    .data = .{ .reloc = .{
                         .atom_index = atom_index,
                         .sym_index = sym_index,
                     } },
@@ -7489,8 +7581,8 @@ fn genTry(
 fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
     const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
     _ = try self.addInst(.{
-        .tag = .dbg_line,
-        .ops = undefined,
+        .tag = .pseudo,
+        .ops = .pseudo_dbg_line_line_column,
         .data = .{ .line_column = .{
             .line = dbg_stmt.line,
             .column = dbg_stmt.column,
@@ -8021,14 +8113,14 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
 fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
     const next_inst = @intCast(u32, self.mir_instructions.len);
     switch (self.mir_instructions.items(.tag)[reloc]) {
-        .jcc => {
-            self.mir_instructions.items(.data)[reloc].inst_cc.inst = next_inst;
-        },
-        .jmp_reloc => {
-            self.mir_instructions.items(.data)[reloc].inst = next_inst;
+        .j, .jmp => {},
+        .pseudo => switch (self.mir_instructions.items(.ops)[reloc]) {
+            .pseudo_j_z_and_np_inst, .pseudo_j_nz_or_p_inst => {},
+            else => unreachable,
         },
         else => unreachable,
     }
+    self.mir_instructions.items(.data)[reloc].inst.inst = next_inst;
 }
 
 fn airBr(self: *Self, inst: Air.Inst.Index) !void {
@@ -8577,10 +8669,10 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                 .load_direct => |sym_index| if (!ty.isRuntimeFloat()) {
                     const atom_index = try self.owner.getSymbolIndex(self);
                     _ = try self.addInst(.{
-                        .tag = .mov_linker,
+                        .tag = .mov,
                         .ops = .direct_reloc,
                         .data = .{ .rx = .{
-                            .r = dst_reg.to64(),
+                            .r1 = dst_reg.to64(),
                             .payload = try self.addExtra(Mir.Reloc{
                                 .atom_index = atom_index,
                                 .sym_index = sym_index,
@@ -8618,8 +8710,8 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             const atom_index = try self.owner.getSymbolIndex(self);
             _ = try self.addInst(.{
                 .tag = switch (src_mcv) {
-                    .lea_direct => .lea_linker,
-                    .lea_got => .mov_linker,
+                    .lea_direct => .lea,
+                    .lea_got => .mov,
                     else => unreachable,
                 },
                 .ops = switch (src_mcv) {
@@ -8628,7 +8720,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                     else => unreachable,
                 },
                 .data = .{ .rx = .{
-                    .r = dst_reg.to64(),
+                    .r1 = dst_reg.to64(),
                     .payload = try self.addExtra(Mir.Reloc{
                         .atom_index = atom_index,
                         .sym_index = sym_index,
@@ -8640,10 +8732,10 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             const atom_index = try self.owner.getSymbolIndex(self);
             if (self.bin_file.cast(link.File.MachO)) |_| {
                 _ = try self.addInst(.{
-                    .tag = .lea_linker,
+                    .tag = .lea,
                     .ops = .tlv_reloc,
                     .data = .{ .rx = .{
-                        .r = .rdi,
+                        .r1 = .rdi,
                         .payload = try self.addExtra(Mir.Reloc{
                             .atom_index = atom_index,
                             .sym_index = sym_index,
@@ -8847,9 +8939,9 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue
     try self.genSetReg(.rsi, Type.usize, src_ptr);
     try self.genSetReg(.rcx, Type.usize, len);
     _ = try self.addInst(.{
-        .tag = .movs,
-        .ops = .string,
-        .data = .{ .string = .{ .repeat = .rep, .width = .b } },
+        .tag = .mov,
+        .ops = .none,
+        .data = .{ .none = .{ .fixes = .@"rep _sb" } },
     });
 }
 
@@ -8859,9 +8951,9 @@ fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue)
     try self.genSetReg(.al, Type.u8, value);
     try self.genSetReg(.rcx, Type.usize, len);
     _ = try self.addInst(.{
-        .tag = .stos,
-        .ops = .string,
-        .data = .{ .string = .{ .repeat = .rep, .width = .b } },
+        .tag = .sto,
+        .ops = .none,
+        .data = .{ .none = .{ .fixes = .@"rep _sb" } },
     });
 }
 
@@ -9135,22 +9227,22 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
     defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
 
     try self.spillEflagsIfOccupied();
-    if (val_abi_size <= 8) {
-        _ = try self.addInst(.{
-            .tag = .cmpxchg,
-            .ops = .lock_mr_sib,
-            .data = .{ .rx = .{
-                .r = registerAlias(new_reg.?, val_abi_size),
-                .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
-            } },
-        });
-    } else {
-        _ = try self.addInst(.{
-            .tag = .cmpxchgb,
-            .ops = .lock_m_sib,
-            .data = .{ .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)) },
-        });
-    }
+    _ = try self.addInst(if (val_abi_size <= 8) .{
+        .tag = .cmpxchg,
+        .ops = .mr_sib,
+        .data = .{ .rx = .{
+            .fixes = .@"lock _",
+            .r1 = registerAlias(new_reg.?, val_abi_size),
+            .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
+        } },
+    } else .{
+        .tag = .cmpxchg,
+        .ops = .m_sib,
+        .data = .{ .x = .{
+            .fixes = .@"lock _16b",
+            .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
+        } },
+    });
 
     const result: MCValue = result: {
         if (self.liveness.isUnused(inst)) break :result .unreach;
@@ -9252,13 +9344,14 @@ fn atomicOp(
             }
             _ = try self.addInst(.{
                 .tag = tag,
-                .ops = switch (tag) {
-                    .mov, .xchg => .mr_sib,
-                    .xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
-                    else => unreachable,
-                },
+                .ops = .mr_sib,
                 .data = .{ .rx = .{
-                    .r = registerAlias(dst_reg, val_abi_size),
+                    .fixes = switch (tag) {
+                        .mov, .xchg => ._,
+                        .xadd, .add, .sub, .@"and", .@"or", .xor => .@"lock _",
+                        else => unreachable,
+                    },
+                    .r1 = registerAlias(dst_reg, val_abi_size),
                     .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
                 } },
             });
@@ -9330,9 +9423,10 @@ fn atomicOp(
             };
             _ = try self.addInst(.{
                 .tag = .cmpxchg,
-                .ops = .lock_mr_sib,
+                .ops = .mr_sib,
                 .data = .{ .rx = .{
-                    .r = registerAlias(tmp_reg, val_abi_size),
+                    .fixes = .@"lock _",
+                    .r1 = registerAlias(tmp_reg, val_abi_size),
                     .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
                 } },
             });
@@ -9397,9 +9491,14 @@ fn atomicOp(
                     val_ty.fmt(self.bin_file.options.module.?), @tagName(op),
                 }),
             };
-            _ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{
-                .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
-            } });
+            _ = try self.addInst(.{
+                .tag = .cmpxchg,
+                .ops = .m_sib,
+                .data = .{ .x = .{
+                    .fixes = .@"lock _16b",
+                    .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
+                } },
+            });
             _ = try self.asmJccReloc(loop, .ne);
 
             if (unused) return .unreach;
diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig
index 3574d52878..506092ff17 100644
--- a/src/arch/x86_64/Emit.zig
+++ b/src/arch/x86_64/Emit.zig
@@ -41,7 +41,7 @@ pub fn emitMir(emit: *Emit) Error!void {
                     .offset = end_offset - 4,
                     .length = @intCast(u5, end_offset - start_offset),
                 }),
-                .@"extern" => |symbol| if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
+                .linker_extern_fn => |symbol| if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
                     // Add relocation to the decl.
                     const atom_index = macho_file.getAtomIndexForSymbol(
                         .{ .sym_index = symbol.atom_index, .file = null },
@@ -129,36 +129,39 @@ pub fn emitMir(emit: *Emit) Error!void {
             const mir_inst = emit.lower.mir.instructions.get(mir_index);
             switch (mir_inst.tag) {
                 else => unreachable,
-                .dead => {},
-                .dbg_line => try emit.dbgAdvancePCAndLine(
-                    mir_inst.data.line_column.line,
-                    mir_inst.data.line_column.column,
-                ),
-                .dbg_prologue_end => {
-                    switch (emit.debug_output) {
-                        .dwarf => |dw| {
-                            try dw.setPrologueEnd();
-                            log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
-                                emit.prev_di_line, emit.prev_di_column,
-                            });
-                            try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
-                        },
-                        .plan9 => {},
-                        .none => {},
-                    }
-                },
-                .dbg_epilogue_begin => {
-                    switch (emit.debug_output) {
-                        .dwarf => |dw| {
-                            try dw.setEpilogueBegin();
-                            log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
-                                emit.prev_di_line, emit.prev_di_column,
-                            });
-                            try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
-                        },
-                        .plan9 => {},
-                        .none => {},
-                    }
+                .pseudo => switch (mir_inst.ops) {
+                    else => unreachable,
+                    .pseudo_dbg_prologue_end_none => {
+                        switch (emit.debug_output) {
+                            .dwarf => |dw| {
+                                try dw.setPrologueEnd();
+                                log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
+                                    emit.prev_di_line, emit.prev_di_column,
+                                });
+                                try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
+                            },
+                            .plan9 => {},
+                            .none => {},
+                        }
+                    },
+                    .pseudo_dbg_line_line_column => try emit.dbgAdvancePCAndLine(
+                        mir_inst.data.line_column.line,
+                        mir_inst.data.line_column.column,
+                    ),
+                    .pseudo_dbg_epilogue_begin_none => {
+                        switch (emit.debug_output) {
+                            .dwarf => |dw| {
+                                try dw.setEpilogueBegin();
+                                log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
+                                    emit.prev_di_line, emit.prev_di_column,
+                                });
+                                try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
+                            },
+                            .plan9 => {},
+                            .none => {},
+                        }
+                    },
+                    .pseudo_dead_none => {},
                 },
             }
         }
diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig
index 3235b29358..b6b49e8939 100644
--- a/src/arch/x86_64/Encoding.zig
+++ b/src/arch/x86_64/Encoding.zig
@@ -705,7 +705,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
 }
 
 const mnemonic_to_encodings_map = init: {
-    @setEvalBranchQuota(100_000);
+    @setEvalBranchQuota(20_000);
     const encodings = @import("encodings.zig");
     var entries = encodings.table;
     std.sort.sort(encodings.Entry, &entries, {}, struct {
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index 5c079f4768..2d7fa4b4fd 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -35,7 +35,7 @@ pub const Reloc = struct {
 
     const Target = union(enum) {
         inst: Mir.Inst.Index,
-        @"extern": Mir.Reloc,
+        linker_extern_fn: Mir.Reloc,
         linker_got: Mir.Reloc,
         linker_direct: Mir.Reloc,
         linker_import: Mir.Reloc,
@@ -59,280 +59,119 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
 
     const inst = lower.mir.instructions.get(index);
     switch (inst.tag) {
-        .adc,
-        .add,
-        .@"and",
-        .bsf,
-        .bsr,
-        .bswap,
-        .bt,
-        .btc,
-        .btr,
-        .bts,
-        .call,
-        .cbw,
-        .cwde,
-        .cdqe,
-        .cwd,
-        .cdq,
-        .cqo,
-        .cmp,
-        .cmpxchg,
-        .div,
-        .fisttp,
-        .fld,
-        .idiv,
-        .imul,
-        .int3,
-        .jmp,
-        .lea,
-        .lfence,
-        .lzcnt,
-        .mfence,
-        .mov,
-        .movbe,
-        .movd,
-        .movq,
-        .movzx,
-        .mul,
-        .neg,
-        .nop,
-        .not,
-        .@"or",
-        .pop,
-        .popcnt,
-        .push,
-        .rcl,
-        .rcr,
-        .ret,
-        .rol,
-        .ror,
-        .sal,
-        .sar,
-        .sbb,
-        .sfence,
-        .shl,
-        .shld,
-        .shr,
-        .shrd,
-        .sub,
-        .syscall,
-        .@"test",
-        .tzcnt,
-        .ud2,
-        .xadd,
-        .xchg,
-        .xor,
+        else => try lower.generic(inst),
+        .pseudo => switch (inst.ops) {
+            .pseudo_cmov_z_and_np_rr => {
+                try lower.emit(.none, .cmovnz, &.{
+                    .{ .reg = inst.data.rr.r2 },
+                    .{ .reg = inst.data.rr.r1 },
+                });
+                try lower.emit(.none, .cmovnp, &.{
+                    .{ .reg = inst.data.rr.r1 },
+                    .{ .reg = inst.data.rr.r2 },
+                });
+            },
+            .pseudo_cmov_nz_or_p_rr => {
+                try lower.emit(.none, .cmovnz, &.{
+                    .{ .reg = inst.data.rr.r1 },
+                    .{ .reg = inst.data.rr.r2 },
+                });
+                try lower.emit(.none, .cmovp, &.{
+                    .{ .reg = inst.data.rr.r1 },
+                    .{ .reg = inst.data.rr.r2 },
+                });
+            },
+            .pseudo_cmov_nz_or_p_rm_sib,
+            .pseudo_cmov_nz_or_p_rm_rip,
+            => {
+                try lower.emit(.none, .cmovnz, &.{
+                    .{ .reg = inst.data.rx.r1 },
+                    .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
+                });
+                try lower.emit(.none, .cmovp, &.{
+                    .{ .reg = inst.data.rx.r1 },
+                    .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
+                });
+            },
+            .pseudo_set_z_and_np_r => {
+                try lower.emit(.none, .setz, &.{
+                    .{ .reg = inst.data.r_scratch.r1 },
+                });
+                try lower.emit(.none, .setnp, &.{
+                    .{ .reg = inst.data.r_scratch.scratch_reg },
+                });
+                try lower.emit(.none, .@"and", &.{
+                    .{ .reg = inst.data.r_scratch.r1 },
+                    .{ .reg = inst.data.r_scratch.scratch_reg },
+                });
+            },
+            .pseudo_set_z_and_np_m_sib,
+            .pseudo_set_z_and_np_m_rip,
+            => {
+                try lower.emit(.none, .setz, &.{
+                    .{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
+                });
+                try lower.emit(.none, .setnp, &.{
+                    .{ .reg = inst.data.x_scratch.scratch_reg },
+                });
+                try lower.emit(.none, .@"and", &.{
+                    .{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
+                    .{ .reg = inst.data.x_scratch.scratch_reg },
+                });
+            },
+            .pseudo_set_nz_or_p_r => {
+                try lower.emit(.none, .setnz, &.{
+                    .{ .reg = inst.data.r_scratch.r1 },
+                });
+                try lower.emit(.none, .setp, &.{
+                    .{ .reg = inst.data.r_scratch.scratch_reg },
+                });
+                try lower.emit(.none, .@"or", &.{
+                    .{ .reg = inst.data.r_scratch.r1 },
+                    .{ .reg = inst.data.r_scratch.scratch_reg },
+                });
+            },
+            .pseudo_set_nz_or_p_m_sib,
+            .pseudo_set_nz_or_p_m_rip,
+            => {
+                try lower.emit(.none, .setnz, &.{
+                    .{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
+                });
+                try lower.emit(.none, .setp, &.{
+                    .{ .reg = inst.data.x_scratch.scratch_reg },
+                });
+                try lower.emit(.none, .@"or", &.{
+                    .{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
+                    .{ .reg = inst.data.x_scratch.scratch_reg },
+                });
+            },
+            .pseudo_j_z_and_np_inst => {
+                try lower.emit(.none, .jnz, &.{
+                    .{ .imm = lower.reloc(.{ .inst = index + 1 }) },
+                });
+                try lower.emit(.none, .jnp, &.{
+                    .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
+                });
+            },
+            .pseudo_j_nz_or_p_inst => {
+                try lower.emit(.none, .jnz, &.{
+                    .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
+                });
+                try lower.emit(.none, .jp, &.{
+                    .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
+                });
+            },
 
-        .addps,
-        .addss,
-        .andnps,
-        .andps,
-        .cmpss,
-        .cvtsi2ss,
-        .divps,
-        .divss,
-        .maxps,
-        .maxss,
-        .minps,
-        .minss,
-        .movaps,
-        .movhlps,
-        .movss,
-        .movups,
-        .mulps,
-        .mulss,
-        .orps,
-        .pextrw,
-        .pinsrw,
-        .sqrtps,
-        .sqrtss,
-        .subps,
-        .subss,
-        .ucomiss,
-        .unpckhps,
-        .unpcklps,
-        .xorps,
+            .pseudo_push_reg_list => try lower.pushPopRegList(.push, inst),
+            .pseudo_pop_reg_list => try lower.pushPopRegList(.pop, inst),
 
-        .addpd,
-        .addsd,
-        .andnpd,
-        .andpd,
-        .cmpsd,
-        .cvtsd2ss,
-        .cvtsi2sd,
-        .cvtss2sd,
-        .divpd,
-        .divsd,
-        .maxpd,
-        .maxsd,
-        .minpd,
-        .minsd,
-        .movsd,
-        .mulpd,
-        .mulsd,
-        .orpd,
-        .pshufhw,
-        .pshuflw,
-        .psrld,
-        .psrlq,
-        .psrlw,
-        .punpckhbw,
-        .punpckhdq,
-        .punpckhqdq,
-        .punpckhwd,
-        .punpcklbw,
-        .punpckldq,
-        .punpcklqdq,
-        .punpcklwd,
-        .sqrtpd,
-        .sqrtsd,
-        .subpd,
-        .subsd,
-        .ucomisd,
-        .unpckhpd,
-        .unpcklpd,
-        .xorpd,
-
-        .movddup,
-        .movshdup,
-        .movsldup,
-
-        .pextrb,
-        .pextrd,
-        .pextrq,
-        .pinsrb,
-        .pinsrd,
-        .pinsrq,
-        .roundpd,
-        .roundps,
-        .roundsd,
-        .roundss,
-
-        .vaddpd,
-        .vaddps,
-        .vaddsd,
-        .vaddss,
-        .vcvtsd2ss,
-        .vcvtsi2sd,
-        .vcvtsi2ss,
-        .vcvtss2sd,
-        .vdivpd,
-        .vdivps,
-        .vdivsd,
-        .vdivss,
-        .vmaxpd,
-        .vmaxps,
-        .vmaxsd,
-        .vmaxss,
-        .vminpd,
-        .vminps,
-        .vminsd,
-        .vminss,
-        .vmovapd,
-        .vmovaps,
-        .vmovddup,
-        .vmovhlps,
-        .vmovsd,
-        .vmovshdup,
-        .vmovsldup,
-        .vmovss,
-        .vmovupd,
-        .vmovups,
-        .vmulpd,
-        .vmulps,
-        .vmulsd,
-        .vmulss,
-        .vpextrb,
-        .vpextrd,
-        .vpextrq,
-        .vpextrw,
-        .vpinsrb,
-        .vpinsrd,
-        .vpinsrq,
-        .vpinsrw,
-        .vpshufhw,
-        .vpshuflw,
-        .vpsrld,
-        .vpsrlq,
-        .vpsrlw,
-        .vpunpckhbw,
-        .vpunpckhdq,
-        .vpunpckhqdq,
-        .vpunpckhwd,
-        .vpunpcklbw,
-        .vpunpckldq,
-        .vpunpcklqdq,
-        .vpunpcklwd,
-        .vroundpd,
-        .vroundps,
-        .vroundsd,
-        .vroundss,
-        .vsqrtpd,
-        .vsqrtps,
-        .vsqrtsd,
-        .vsqrtss,
-        .vsubpd,
-        .vsubps,
-        .vsubsd,
-        .vsubss,
-        .vunpckhpd,
-        .vunpckhps,
-        .vunpcklpd,
-        .vunpcklps,
-
-        .vcvtph2ps,
-        .vcvtps2ph,
-
-        .vfmadd132pd,
-        .vfmadd213pd,
-        .vfmadd231pd,
-        .vfmadd132ps,
-        .vfmadd213ps,
-        .vfmadd231ps,
-        .vfmadd132sd,
-        .vfmadd213sd,
-        .vfmadd231sd,
-        .vfmadd132ss,
-        .vfmadd213ss,
-        .vfmadd231ss,
-        => try lower.mirGeneric(inst),
-
-        .cmps,
-        .lods,
-        .movs,
-        .scas,
-        .stos,
-        => try lower.mirString(inst),
-
-        .cmpxchgb => try lower.mirCmpxchgBytes(inst),
-
-        .jmp_reloc => try lower.emitInstWithReloc(.none, .jmp, &.{
-            .{ .imm = Immediate.s(0) },
-        }, .{ .inst = inst.data.inst }),
-
-        .call_extern => try lower.emitInstWithReloc(.none, .call, &.{
-            .{ .imm = Immediate.s(0) },
-        }, .{ .@"extern" = inst.data.relocation }),
-
-        .lea_linker => try lower.mirLinker(.lea, inst),
-        .mov_linker => try lower.mirLinker(.mov, inst),
-
-        .mov_moffs => try lower.mirMovMoffs(inst),
-
-        .movsx => try lower.mirMovsx(inst),
-        .cmovcc => try lower.mirCmovcc(inst),
-        .setcc => try lower.mirSetcc(inst),
-        .jcc => try lower.mirJcc(index, inst),
-
-        .push_regs => try lower.mirRegisterList(.push, inst),
-        .pop_regs => try lower.mirRegisterList(.pop, inst),
-
-        .dbg_line,
-        .dbg_prologue_end,
-        .dbg_epilogue_begin,
-        .dead,
-        => {},
+            .pseudo_dbg_prologue_end_none,
+            .pseudo_dbg_line_line_column,
+            .pseudo_dbg_epilogue_begin_none,
+            .pseudo_dead_none,
+            => {},
+            else => unreachable,
+        },
     }
 
     return .{
@@ -348,15 +187,6 @@ pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
     return error.LowerFail;
 }
 
-fn mnem_cc(comptime base: @Type(.EnumLiteral), cc: bits.Condition) Mnemonic {
-    return switch (cc) {
-        inline else => |c| if (@hasField(Mnemonic, @tagName(base) ++ @tagName(c)))
-            @field(Mnemonic, @tagName(base) ++ @tagName(c))
-        else
-            unreachable,
-    };
-}
-
 fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
     return switch (ops) {
         .rri_s,
@@ -364,8 +194,6 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
         .i_s,
         .mi_sib_s,
         .mi_rip_s,
-        .lock_mi_sib_s,
-        .lock_mi_rip_s,
         => Immediate.s(@bitCast(i32, i)),
 
         .rrri,
@@ -374,8 +202,6 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
         .i_u,
         .mi_sib_u,
         .mi_rip_u,
-        .lock_mi_sib_u,
-        .lock_mi_rip_u,
         .rmi_sib,
         .rmi_rip,
         .mri_sib,
@@ -395,10 +221,8 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
 fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
     return lower.mir.resolveFrameLoc(switch (ops) {
         .rm_sib,
-        .rm_sib_cc,
         .rmi_sib,
         .m_sib,
-        .m_sib_cc,
         .mi_sib_u,
         .mi_sib_s,
         .mr_sib,
@@ -406,17 +230,15 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
         .mri_sib,
         .rrm_sib,
         .rrmi_sib,
-        .lock_m_sib,
-        .lock_mi_sib_u,
-        .lock_mi_sib_s,
-        .lock_mr_sib,
+
+        .pseudo_cmov_nz_or_p_rm_sib,
+        .pseudo_set_z_and_np_m_sib,
+        .pseudo_set_nz_or_p_m_sib,
         => lower.mir.extraData(Mir.MemorySib, payload).data.decode(),
 
         .rm_rip,
-        .rm_rip_cc,
         .rmi_rip,
         .m_rip,
-        .m_rip_cc,
         .mi_rip_u,
         .mi_rip_s,
         .mr_rip,
@@ -424,66 +246,83 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
         .mri_rip,
         .rrm_rip,
         .rrmi_rip,
-        .lock_m_rip,
-        .lock_mi_rip_u,
-        .lock_mi_rip_s,
-        .lock_mr_rip,
+
+        .pseudo_cmov_nz_or_p_rm_rip,
+        .pseudo_set_z_and_np_m_rip,
+        .pseudo_set_nz_or_p_m_rip,
         => lower.mir.extraData(Mir.MemoryRip, payload).data.decode(),
 
         .rax_moffs,
         .moffs_rax,
-        .lock_moffs_rax,
         => lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),
 
         else => unreachable,
     });
 }
 
-fn emitInst(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
-    lower.result_insts[lower.result_insts_len] = try Instruction.new(prefix, mnemonic, ops);
-    lower.result_insts_len += 1;
-}
-
-fn emitInstWithReloc(
-    lower: *Lower,
-    prefix: Prefix,
-    mnemonic: Mnemonic,
-    ops: []const Operand,
-    target: Reloc.Target,
-) Error!void {
+fn reloc(lower: *Lower, target: Reloc.Target) Immediate {
     lower.result_relocs[lower.result_relocs_len] = .{
         .lowered_inst_index = lower.result_insts_len,
         .target = target,
     };
     lower.result_relocs_len += 1;
-    try lower.emitInst(prefix, mnemonic, ops);
+    return Immediate.s(0);
 }
 
-fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
-    try lower.emitInst(switch (inst.ops) {
-        else => .none,
-        .lock_m_sib,
-        .lock_m_rip,
-        .lock_mi_sib_u,
-        .lock_mi_rip_u,
-        .lock_mi_sib_s,
-        .lock_mi_rip_s,
-        .lock_mr_sib,
-        .lock_mr_rip,
-        .lock_moffs_rax,
-        => .lock,
-    }, switch (inst.tag) {
-        inline else => |tag| if (@hasField(Mnemonic, @tagName(tag)))
-            @field(Mnemonic, @tagName(tag))
+fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
+    lower.result_insts[lower.result_insts_len] = try Instruction.new(prefix, mnemonic, ops);
+    lower.result_insts_len += 1;
+}
+
+fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
+    const fixes = switch (inst.ops) {
+        .none => inst.data.none.fixes,
+        .inst => inst.data.inst.fixes,
+        .i_s, .i_u => inst.data.i.fixes,
+        .r => inst.data.r.fixes,
+        .rr => inst.data.rr.fixes,
+        .rrr => inst.data.rrr.fixes,
+        .rrri => inst.data.rrri.fixes,
+        .rri_s, .rri_u => inst.data.rri.fixes,
+        .ri_s, .ri_u => inst.data.ri.fixes,
+        .ri64, .rm_sib, .rm_rip, .mr_sib, .mr_rip => inst.data.rx.fixes,
+        .mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => ._,
+        .mrr_sib, .mrr_rip, .rrm_sib, .rrm_rip => inst.data.rrx.fixes,
+        .rmi_sib, .rmi_rip, .mri_sib, .mri_rip => inst.data.rix.fixes,
+        .rrmi_sib, .rrmi_rip => inst.data.rrix.fixes,
+        .m_sib, .m_rip, .rax_moffs, .moffs_rax => inst.data.x.fixes,
+        .extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._,
+        else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}),
+    };
+    try lower.emit(switch (fixes) {
+        inline else => |tag| comptime if (std.mem.indexOfScalar(u8, @tagName(tag), ' ')) |space|
+            @field(Prefix, @tagName(tag)[0..space])
         else
-            unreachable,
+            .none,
+    }, mnemonic: {
+        comptime var max_len = 0;
+        inline for (@typeInfo(Mnemonic).Enum.fields) |field| max_len = @max(field.name.len, max_len);
+        var buf: [max_len]u8 = undefined;
+
+        const fixes_name = @tagName(fixes);
+        const pattern = fixes_name[if (std.mem.indexOfScalar(u8, fixes_name, ' ')) |i| i + 1 else 0..];
+        const wildcard_i = std.mem.indexOfScalar(u8, pattern, '_').?;
+        const parts = .{ pattern[0..wildcard_i], @tagName(inst.tag), pattern[wildcard_i + 1 ..] };
+        const err_msg = "unsupported mnemonic: ";
+        const mnemonic = std.fmt.bufPrint(&buf, "{s}{s}{s}", parts) catch
+            return lower.fail(err_msg ++ "'{s}{s}{s}'", parts);
+        break :mnemonic std.meta.stringToEnum(Mnemonic, mnemonic) orelse
+            return lower.fail(err_msg ++ "'{s}'", .{mnemonic});
     }, switch (inst.ops) {
         .none => &.{},
+        .inst => &.{
+            .{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
+        },
         .i_s, .i_u => &.{
-            .{ .imm = lower.imm(inst.ops, inst.data.i) },
+            .{ .imm = lower.imm(inst.ops, inst.data.i.i) },
         },
         .r => &.{
-            .{ .reg = inst.data.r },
+            .{ .reg = inst.data.r.r1 },
         },
         .rr => &.{
             .{ .reg = inst.data.rr.r1 },
@@ -501,11 +340,11 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .imm = lower.imm(inst.ops, inst.data.rrri.i) },
         },
         .ri_s, .ri_u => &.{
-            .{ .reg = inst.data.ri.r },
+            .{ .reg = inst.data.ri.r1 },
             .{ .imm = lower.imm(inst.ops, inst.data.ri.i) },
         },
         .ri64 => &.{
-            .{ .reg = inst.data.rx.r },
+            .{ .reg = inst.data.rx.r1 },
             .{ .imm = lower.imm(inst.ops, inst.data.rx.payload) },
         },
         .rri_s, .rri_u => &.{
@@ -513,33 +352,25 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .reg = inst.data.rri.r2 },
             .{ .imm = lower.imm(inst.ops, inst.data.rri.i) },
         },
-        .m_sib, .lock_m_sib, .m_rip, .lock_m_rip => &.{
-            .{ .mem = lower.mem(inst.ops, inst.data.payload) },
+        .m_sib, .m_rip => &.{
+            .{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
         },
-        .mi_sib_s,
-        .lock_mi_sib_s,
-        .mi_sib_u,
-        .lock_mi_sib_u,
-        .mi_rip_u,
-        .lock_mi_rip_u,
-        .mi_rip_s,
-        .lock_mi_rip_s,
-        => &.{
+        .mi_sib_s, .mi_sib_u, .mi_rip_u, .mi_rip_s => &.{
             .{ .mem = lower.mem(inst.ops, inst.data.ix.payload) },
             .{ .imm = lower.imm(inst.ops, inst.data.ix.i) },
         },
         .rm_sib, .rm_rip => &.{
-            .{ .reg = inst.data.rx.r },
+            .{ .reg = inst.data.rx.r1 },
             .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
         },
         .rmi_sib, .rmi_rip => &.{
-            .{ .reg = inst.data.rix.r },
+            .{ .reg = inst.data.rix.r1 },
             .{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
             .{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
         },
-        .mr_sib, .lock_mr_sib, .mr_rip, .lock_mr_rip => &.{
+        .mr_sib, .mr_rip => &.{
             .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
-            .{ .reg = inst.data.rx.r },
+            .{ .reg = inst.data.rx.r1 },
         },
         .mrr_sib, .mrr_rip => &.{
             .{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },
@@ -548,7 +379,7 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
         },
         .mri_sib, .mri_rip => &.{
             .{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
-            .{ .reg = inst.data.rix.r },
+            .{ .reg = inst.data.rix.r1 },
             .{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
         },
         .rrm_sib, .rrm_rip => &.{
@@ -562,180 +393,46 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .mem = lower.mem(inst.ops, inst.data.rrix.payload) },
             .{ .imm = lower.imm(inst.ops, inst.data.rrix.i) },
         },
-        else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
-    });
-}
-
-fn mirString(lower: *Lower, inst: Mir.Inst) Error!void {
-    switch (inst.ops) {
-        .string => try lower.emitInst(switch (inst.data.string.repeat) {
-            inline else => |repeat| @field(Prefix, @tagName(repeat)),
-        }, switch (inst.tag) {
-            inline .cmps, .lods, .movs, .scas, .stos => |tag| switch (inst.data.string.width) {
-                inline else => |width| @field(Mnemonic, @tagName(tag) ++ @tagName(width)),
-            },
-            else => unreachable,
-        }, &.{}),
-        else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
-    }
-}
-
-fn mirCmpxchgBytes(lower: *Lower, inst: Mir.Inst) Error!void {
-    const ops: [1]Operand = switch (inst.ops) {
-        .m_sib, .lock_m_sib, .m_rip, .lock_m_rip => .{
-            .{ .mem = lower.mem(inst.ops, inst.data.payload) },
-        },
-        else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
-    };
-    try lower.emitInst(switch (inst.ops) {
-        .m_sib, .m_rip => .none,
-        .lock_m_sib, .lock_m_rip => .lock,
-        else => unreachable,
-    }, switch (@divExact(ops[0].bitSize(), 8)) {
-        8 => .cmpxchg8b,
-        16 => .cmpxchg16b,
-        else => return lower.fail("invalid operand for {s}", .{@tagName(inst.tag)}),
-    }, &ops);
-}
-
-fn mirMovMoffs(lower: *Lower, inst: Mir.Inst) Error!void {
-    try lower.emitInst(switch (inst.ops) {
-        .rax_moffs, .moffs_rax => .none,
-        .lock_moffs_rax => .lock,
-        else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
-    }, .mov, switch (inst.ops) {
         .rax_moffs => &.{
             .{ .reg = .rax },
-            .{ .mem = lower.mem(inst.ops, inst.data.payload) },
+            .{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
         },
-        .moffs_rax, .lock_moffs_rax => &.{
-            .{ .mem = lower.mem(inst.ops, inst.data.payload) },
+        .moffs_rax => &.{
+            .{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
             .{ .reg = .rax },
         },
-        else => unreachable,
+        .extern_fn_reloc => &.{
+            .{ .imm = lower.reloc(.{ .linker_extern_fn = inst.data.reloc }) },
+        },
+        .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ops: {
+            const reg = inst.data.rx.r1;
+            const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
+            _ = lower.reloc(switch (inst.ops) {
+                .got_reloc => .{ .linker_got = extra },
+                .direct_reloc => .{ .linker_direct = extra },
+                .import_reloc => .{ .linker_import = extra },
+                .tlv_reloc => .{ .linker_tlv = extra },
+                else => unreachable,
+            });
+            break :ops &.{
+                .{ .reg = reg },
+                .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
+            };
+        },
+        else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
     });
 }
 
-fn mirMovsx(lower: *Lower, inst: Mir.Inst) Error!void {
-    const ops: [2]Operand = switch (inst.ops) {
-        .rr => .{
-            .{ .reg = inst.data.rr.r1 },
-            .{ .reg = inst.data.rr.r2 },
-        },
-        .rm_sib, .rm_rip => .{
-            .{ .reg = inst.data.rx.r },
-            .{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
-        },
-        else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
-    };
-    try lower.emitInst(.none, switch (ops[0].bitSize()) {
-        32, 64 => switch (ops[1].bitSize()) {
-            32 => .movsxd,
-            else => .movsx,
-        },
-        else => .movsx,
-    }, &ops);
-}
-
-fn mirCmovcc(lower: *Lower, inst: Mir.Inst) Error!void {
-    const data: struct { cc: bits.Condition, ops: [2]Operand } = switch (inst.ops) {
-        .rr_cc => .{ .cc = inst.data.rr_cc.cc, .ops = .{
-            .{ .reg = inst.data.rr_cc.r1 },
-            .{ .reg = inst.data.rr_cc.r2 },
-        } },
-        .rm_sib_cc, .rm_rip_cc => .{ .cc = inst.data.rx_cc.cc, .ops = .{
-            .{ .reg = inst.data.rx_cc.r },
-            .{ .mem = lower.mem(inst.ops, inst.data.rx_cc.payload) },
-        } },
-        else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
-    };
-    switch (data.cc) {
-        else => |cc| try lower.emitInst(.none, mnem_cc(.cmov, cc), &data.ops),
-        .z_and_np => {
-            try lower.emitInst(.none, mnem_cc(.cmov, .nz), &.{ data.ops[1], data.ops[0] });
-            try lower.emitInst(.none, mnem_cc(.cmov, .np), &data.ops);
-        },
-        .nz_or_p => {
-            try lower.emitInst(.none, mnem_cc(.cmov, .nz), &data.ops);
-            try lower.emitInst(.none, mnem_cc(.cmov, .p), &data.ops);
-        },
-    }
-}
-
-fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {
-    const data: struct { cc: bits.Condition, ops: [2]Operand } = switch (inst.ops) {
-        .r_cc => .{ .cc = inst.data.r_cc.cc, .ops = .{
-            .{ .reg = inst.data.r_cc.r },
-            .{ .reg = inst.data.r_cc.scratch },
-        } },
-        .m_sib_cc, .m_rip_cc => .{ .cc = inst.data.x_cc.cc, .ops = .{
-            .{ .mem = lower.mem(inst.ops, inst.data.x_cc.payload) },
-            .{ .reg = inst.data.x_cc.scratch },
-        } },
-        else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
-    };
-    switch (data.cc) {
-        else => |cc| try lower.emitInst(.none, mnem_cc(.set, cc), data.ops[0..1]),
-        .z_and_np => {
-            try lower.emitInst(.none, mnem_cc(.set, .z), data.ops[0..1]);
-            try lower.emitInst(.none, mnem_cc(.set, .np), data.ops[1..2]);
-            try lower.emitInst(.none, .@"and", data.ops[0..2]);
-        },
-        .nz_or_p => {
-            try lower.emitInst(.none, mnem_cc(.set, .nz), data.ops[0..1]);
-            try lower.emitInst(.none, mnem_cc(.set, .p), data.ops[1..2]);
-            try lower.emitInst(.none, .@"or", data.ops[0..2]);
-        },
-    }
-}
-
-fn mirJcc(lower: *Lower, index: Mir.Inst.Index, inst: Mir.Inst) Error!void {
-    switch (inst.data.inst_cc.cc) {
-        else => |cc| try lower.emitInstWithReloc(.none, mnem_cc(.j, cc), &.{
-            .{ .imm = Immediate.s(0) },
-        }, .{ .inst = inst.data.inst_cc.inst }),
-        .z_and_np => {
-            try lower.emitInstWithReloc(.none, mnem_cc(.j, .nz), &.{
-                .{ .imm = Immediate.s(0) },
-            }, .{ .inst = index + 1 });
-            try lower.emitInstWithReloc(.none, mnem_cc(.j, .np), &.{
-                .{ .imm = Immediate.s(0) },
-            }, .{ .inst = inst.data.inst_cc.inst });
-        },
-        .nz_or_p => {
-            try lower.emitInstWithReloc(.none, mnem_cc(.j, .nz), &.{
-                .{ .imm = Immediate.s(0) },
-            }, .{ .inst = inst.data.inst_cc.inst });
-            try lower.emitInstWithReloc(.none, mnem_cc(.j, .p), &.{
-                .{ .imm = Immediate.s(0) },
-            }, .{ .inst = inst.data.inst_cc.inst });
-        },
-    }
-}
-
-fn mirRegisterList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
-    const reg_list = Mir.RegisterList.fromInt(inst.data.payload);
+fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
     const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);
-    var it = reg_list.iterator(.{ .direction = switch (mnemonic) {
+    var it = inst.data.reg_list.iterator(.{ .direction = switch (mnemonic) {
         .push => .reverse,
         .pop => .forward,
         else => unreachable,
     } });
-    while (it.next()) |i| try lower.emitInst(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});
-}
-
-fn mirLinker(lower: *Lower, mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
-    const reloc = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
-    try lower.emitInstWithReloc(.none, mnemonic, &.{
-        .{ .reg = inst.data.rx.r },
-        .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(inst.data.rx.r.bitSize()), 0) },
-    }, switch (inst.ops) {
-        .got_reloc => .{ .linker_got = reloc },
-        .direct_reloc => .{ .linker_direct = reloc },
-        .import_reloc => .{ .linker_import = reloc },
-        .tlv_reloc => .{ .linker_tlv = reloc },
-        else => unreachable,
-    });
+    while (it.next()) |i| try lower.emit(.none, mnemonic, &.{.{
+        .reg = callee_preserved_regs[i],
+    }});
 }
 
 const abi = @import("abi.zig");
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index 442cfabebb..951a0c5d4d 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -32,6 +32,210 @@ pub const Inst = struct {
 
     pub const Index = u32;
 
+    pub const Fixes = enum(u8) {
+        /// ___
+        @"_",
+
+        /// ___ Above
+        _a,
+        /// ___ Above Or Equal
+        _ae,
+        /// ___ Below
+        _b,
+        /// ___ Below Or Equal
+        _be,
+        /// ___ Carry
+        _c,
+        /// ___ Equal
+        _e,
+        /// ___ Greater
+        _g,
+        /// ___ Greater Or Equal
+        _ge,
+        /// ___ Less
+        _l,
+        /// ___ Less Or Equal
+        _le,
+        /// ___ Not Above
+        _na,
+        /// ___ Not Above Or Equal
+        _nae,
+        /// ___ Not Below
+        _nb,
+        /// ___ Not Below Or Equal
+        _nbe,
+        /// ___ Not Carry
+        _nc,
+        /// ___ Not Equal
+        _ne,
+        /// ___ Not Greater
+        _ng,
+        /// ___ Not Greater Or Equal
+        _nge,
+        /// ___ Not Less
+        _nl,
+        /// ___ Not Less Or Equal
+        _nle,
+        /// ___ Not Overflow
+        _no,
+        /// ___ Not Parity
+        _np,
+        /// ___ Not Sign
+        _ns,
+        /// ___ Not Zero
+        _nz,
+        /// ___ Overflow
+        _o,
+        /// ___ Parity
+        _p,
+        /// ___ Parity Even
+        _pe,
+        /// ___ Parity Odd
+        _po,
+        /// ___ Sign
+        _s,
+        /// ___ Zero
+        _z,
+
+        /// ___ String
+        //_s,
+        /// ___ String Byte
+        _sb,
+        /// ___ String Word
+        _sw,
+        /// ___ String Doubleword
+        _sd,
+        /// ___ String Quadword
+        _sq,
+
+        /// Repeat ___ String
+        @"rep _s",
+        /// Repeat ___ String Byte
+        @"rep _sb",
+        /// Repeat ___ String Word
+        @"rep _sw",
+        /// Repeat ___ String Doubleword
+        @"rep _sd",
+        /// Repeat ___ String Quadword
+        @"rep _sq",
+
+        /// Repeat Equal ___ String
+        @"repe _s",
+        /// Repeat Equal ___ String Byte
+        @"repe _sb",
+        /// Repeat Equal ___ String Word
+        @"repe _sw",
+        /// Repeat Equal ___ String Doubleword
+        @"repe _sd",
+        /// Repeat Equal ___ String Quadword
+        @"repe _sq",
+
+        /// Repeat Not Equal ___ String
+        @"repne _s",
+        /// Repeat Not Equal ___ String Byte
+        @"repne _sb",
+        /// Repeat Not Equal ___ String Word
+        @"repne _sw",
+        /// Repeat Not Equal ___ String Doubleword
+        @"repne _sd",
+        /// Repeat Not Equal ___ String Quadword
+        @"repne _sq",
+
+        /// Repeat Not Zero ___ String
+        @"repnz _s",
+        /// Repeat Not Zero ___ String Byte
+        @"repnz _sb",
+        /// Repeat Not Zero ___ String Word
+        @"repnz _sw",
+        /// Repeat Not Zero ___ String Doubleword
+        @"repnz _sd",
+        /// Repeat Not Zero ___ String Quadword
+        @"repnz _sq",
+
+        /// Repeat Zero ___ String
+        @"repz _s",
+        /// Repeat Zero ___ String Byte
+        @"repz _sb",
+        /// Repeat Zero ___ String Word
+        @"repz _sw",
+        /// Repeat Zero ___ String Doubleword
+        @"repz _sd",
+        /// Repeat Zero ___ String Quadword
+        @"repz _sq",
+
+        /// Locked ___
+        @"lock _",
+        /// ___ 8 Bytes
+        _8b,
+        /// Locked ___ 8 Bytes
+        @"lock _8b",
+        /// ___ 16 Bytes
+        _16b,
+        /// Locked ___ 16 Bytes
+        @"lock _16b",
+
+        /// Packed ___
+        p_,
+        /// Packed ___ Byte
+        p_b,
+        /// Packed ___ Word
+        p_w,
+        /// Packed ___ Doubleword
+        p_d,
+        /// Packed ___ Quadword
+        p_q,
+        /// Packed ___ Double Quadword
+        p_dq,
+
+        /// ___ Scalar Single-Precision Values
+        _ss,
+        /// ___ Packed Single-Precision Values
+        _ps,
+        /// ___ Scalar Double-Precision Values
+        //_sd,
+        /// ___ Packed Double-Precision Values
+        _pd,
+
+        /// VEX-Encoded ___
+        v_,
+        /// VEX-Encoded Packed ___
+        vp_,
+        /// VEX-Encoded Packed ___ Byte
+        vp_b,
+        /// VEX-Encoded Packed ___ Word
+        vp_w,
+        /// VEX-Encoded Packed ___ Doubleword
+        vp_d,
+        /// VEX-Encoded Packed ___ Quadword
+        vp_q,
+        /// VEX-Encoded Packed ___ Double Quadword
+        vp_dq,
+        /// VEX-Encoded ___ Scalar Single-Precision Values
+        v_ss,
+        /// VEX-Encoded ___ Packed Single-Precision Values
+        v_ps,
+        /// VEX-Encoded ___ Scalar Double-Precision Values
+        v_sd,
+        /// VEX-Encoded ___ Packed Double-Precision Values
+        v_pd,
+
+        /// Mask ___ Byte
+        k_b,
+        /// Mask ___ Word
+        k_w,
+        /// Mask ___ Doubleword
+        k_d,
+        /// Mask ___ Quadword
+        k_q,
+
+        pub fn fromCondition(cc: bits.Condition) Fixes {
+            return switch (cc) {
+                inline else => |cc_tag| @field(Fixes, "_" ++ @tagName(cc_tag)),
+                .z_and_np, .nz_or_p => unreachable,
+            };
+        }
+    };
+
     pub const Tag = enum(u8) {
         /// Add with carry
         adc,
@@ -57,22 +261,24 @@ pub const Inst = struct {
         call,
         /// Convert byte to word
         cbw,
-        /// Convert word to doubleword
-        cwde,
-        /// Convert doubleword to quadword
-        cdqe,
-        /// Convert word to doubleword
-        cwd,
         /// Convert doubleword to quadword
         cdq,
         /// Convert doubleword to quadword
-        cqo,
+        cdqe,
+        /// Conditional move
+        cmov,
         /// Logical compare
+        /// Compare string
         cmp,
         /// Compare and exchange
-        cmpxchg,
         /// Compare and exchange bytes
-        cmpxchgb,
+        cmpxchg,
+        /// Convert doubleword to quadword
+        cqo,
+        /// Convert word to doubleword
+        cwd,
+        /// Convert word to doubleword
+        cwde,
         /// Unsigned division
         div,
         /// Store integer with truncation
@@ -85,10 +291,14 @@ pub const Inst = struct {
         imul,
         ///
         int3,
+        /// Conditional jump
+        j,
         /// Jump
         jmp,
         /// Load effective address
         lea,
+        /// Load string
+        lod,
         /// Load fence
         lfence,
         /// Count the number of leading zero bits
@@ -96,6 +306,7 @@ pub const Inst = struct {
         /// Memory fence
         mfence,
         /// Move
+        /// Move data from string to string
         mov,
         /// Move data after swapping bytes
         movbe,
@@ -105,6 +316,8 @@ pub const Inst = struct {
         movq,
         /// Move with sign extension
         movsx,
+        /// Move with sign extension
+        movsxd,
         /// Move with zero extension
         movzx,
         /// Multiply
@@ -139,6 +352,10 @@ pub const Inst = struct {
         sar,
         /// Integer subtraction with borrow
         sbb,
+        /// Scan string
+        sca,
+        /// Set byte on condition
+        set,
         /// Store fence
         sfence,
         /// Logical shift left
@@ -151,6 +368,8 @@ pub const Inst = struct {
         shrd,
         /// Subtract
         sub,
+        /// Store string
+        sto,
         /// Syscall
         syscall,
         /// Test condition
@@ -505,57 +724,10 @@ pub const Inst = struct {
         /// Fused multiply-add of scalar single-precision floating-point values
         vfmadd231ss,
 
-        /// Compare string operands
-        cmps,
-        /// Load string
-        lods,
-        /// Move data from string to string
-        movs,
-        /// Scan string
-        scas,
-        /// Store string
-        stos,
-
-        /// Conditional move
-        cmovcc,
-        /// Conditional jump
-        jcc,
-        /// Set byte on condition
-        setcc,
-
-        /// Mov absolute to/from memory wrt segment register to/from rax
-        mov_moffs,
-
-        /// Jump with relocation to another local MIR instruction
-        /// Uses `inst` payload.
-        jmp_reloc,
-
-        /// Call to an extern symbol via linker relocation.
-        /// Uses `relocation` payload.
-        call_extern,
-
-        /// Load effective address of a symbol not yet allocated in VM.
-        lea_linker,
-        /// Move address of a symbol not yet allocated in VM.
-        mov_linker,
-
-        /// End of prologue
-        dbg_prologue_end,
-        /// Start of epilogue
-        dbg_epilogue_begin,
-        /// Update debug line
-        /// Uses `line_column` payload containing the line and column.
-        dbg_line,
-        /// Push registers
-        /// Uses `payload` payload containing `RegisterList.asInt` directly.
-        push_regs,
-        /// Pop registers
-        /// Uses `payload` payload containing `RegisterList.asInt` directly.
-        pop_regs,
-
-        /// Tombstone
-        /// Emitter should skip this instruction.
-        dead,
+        /// A pseudo instruction that requires special lowering.
+        /// This should be the only tag in this enum that doesn't
+        /// directly correspond to one or more instruction mnemonics.
+        pseudo,
     };
 
     pub const Ops = enum(u8) {
@@ -579,12 +751,6 @@ pub const Inst = struct {
         /// Register, register, immediate (unsigned) operands.
         /// Uses `rri`  payload.
         rri_u,
-        /// Register with condition code (CC).
-        /// Uses `r_cc` payload.
-        r_cc,
-        /// Register, register with condition code (CC).
-        /// Uses `rr_cc` payload.
-        rr_cc,
         /// Register, immediate (sign-extended) operands.
         /// Uses `ri` payload.
         ri_s,
@@ -609,12 +775,6 @@ pub const Inst = struct {
         /// Register, memory (RIP) operands.
         /// Uses `rx` payload.
         rm_rip,
-        /// Register, memory (SIB) operands with condition code (CC).
-        /// Uses `rx_cc` payload.
-        rm_sib_cc,
-        /// Register, memory (RIP) operands with condition code (CC).
-        /// Uses `rx_cc` payload.
-        rm_rip_cc,
         /// Register, memory (SIB), immediate (byte) operands.
         /// Uses `rix` payload with extra data of type `MemorySib`.
         rmi_sib,
@@ -634,17 +794,11 @@ pub const Inst = struct {
         /// Uses `rix` payload with extra data of type `MemoryRip`.
         rmi_rip,
         /// Single memory (SIB) operand.
-        /// Uses `payload` with extra data of type `MemorySib`.
+        /// Uses `x` with extra data of type `MemorySib`.
         m_sib,
         /// Single memory (RIP) operand.
-        /// Uses `payload` with extra data of type `MemoryRip`.
+        /// Uses `x` with extra data of type `MemoryRip`.
         m_rip,
-        /// Single memory (SIB) operand with condition code (CC).
-        /// Uses `x_cc` with extra data of type `MemorySib`.
-        m_sib_cc,
-        /// Single memory (RIP) operand with condition code (CC).
-        /// Uses `x_cc` with extra data of type `MemoryRip`.
-        m_rip_cc,
         /// Memory (SIB), immediate (unsigned) operands.
         /// Uses `ix` payload with extra data of type `MemorySib`.
         mi_sib_u,
@@ -676,49 +830,17 @@ pub const Inst = struct {
         /// Uses `rix` payload with extra data of type `MemoryRip`.
         mri_rip,
         /// Rax, Memory moffs.
-        /// Uses `payload` with extra data of type `MemoryMoffs`.
+        /// Uses `x` with extra data of type `MemoryMoffs`.
         rax_moffs,
         /// Memory moffs, rax.
-        /// Uses `payload` with extra data of type `MemoryMoffs`.
+        /// Uses `x` with extra data of type `MemoryMoffs`.
         moffs_rax,
-        /// Single memory (SIB) operand with lock prefix.
-        /// Uses `payload` with extra data of type `MemorySib`.
-        lock_m_sib,
-        /// Single memory (RIP) operand with lock prefix.
-        /// Uses `payload` with extra data of type `MemoryRip`.
-        lock_m_rip,
-        /// Memory (SIB), immediate (unsigned) operands with lock prefix.
-        /// Uses `xi` payload with extra data of type `MemorySib`.
-        lock_mi_sib_u,
-        /// Memory (RIP), immediate (unsigned) operands with lock prefix.
-        /// Uses `xi` payload with extra data of type `MemoryRip`.
-        lock_mi_rip_u,
-        /// Memory (SIB), immediate (sign-extend) operands with lock prefix.
-        /// Uses `xi` payload with extra data of type `MemorySib`.
-        lock_mi_sib_s,
-        /// Memory (RIP), immediate (sign-extend) operands with lock prefix.
-        /// Uses `xi` payload with extra data of type `MemoryRip`.
-        lock_mi_rip_s,
-        /// Memory (SIB), register operands with lock prefix.
-        /// Uses `rx` payload with extra data of type `MemorySib`.
-        lock_mr_sib,
-        /// Memory (RIP), register operands with lock prefix.
-        /// Uses `rx` payload with extra data of type `MemoryRip`.
-        lock_mr_rip,
-        /// Memory moffs, rax with lock prefix.
-        /// Uses `payload` with extra data of type `MemoryMoffs`.
-        lock_moffs_rax,
         /// References another Mir instruction directly.
         /// Uses `inst` payload.
         inst,
-        /// References another Mir instruction directly with condition code (CC).
-        /// Uses `inst_cc` payload.
-        inst_cc,
-        /// String repeat and width
-        /// Uses `string` payload.
-        string,
+        /// Linker relocation - external function.
         /// Uses `reloc` payload.
-        reloc,
+        extern_fn_reloc,
         /// Linker relocation - GOT indirection.
         /// Uses `rx` payload with extra data of type `Reloc`.
         got_reloc,
@@ -731,74 +853,125 @@ pub const Inst = struct {
         /// Linker relocation - threadlocal variable via GOT indirection.
         /// Uses `rx` payload with extra data of type `Reloc`.
         tlv_reloc,
+
+        // Pseudo instructions:
+
+        /// Conditional move if zero flag set and parity flag not set
+        /// Clobbers the source operand!
+        /// Uses `rr` payload.
+        pseudo_cmov_z_and_np_rr,
+        /// Conditional move if zero flag not set or parity flag set
+        /// Uses `rr` payload.
+        pseudo_cmov_nz_or_p_rr,
+        /// Conditional move if zero flag not set or parity flag set
+        /// Uses `rx` payload.
+        pseudo_cmov_nz_or_p_rm_sib,
+        /// Conditional move if zero flag not set or parity flag set
+        /// Uses `rx` payload.
+        pseudo_cmov_nz_or_p_rm_rip,
+        /// Set byte if zero flag set and parity flag not set
+        /// Requires a scratch register!
+        /// Uses `r_scratch` payload.
+        pseudo_set_z_and_np_r,
+        /// Set byte if zero flag set and parity flag not set
+        /// Requires a scratch register!
+        /// Uses `x_scratch` payload.
+        pseudo_set_z_and_np_m_sib,
+        /// Set byte if zero flag set and parity flag not set
+        /// Requires a scratch register!
+        /// Uses `x_scratch` payload.
+        pseudo_set_z_and_np_m_rip,
+        /// Set byte if zero flag not set or parity flag set
+        /// Requires a scratch register!
+        /// Uses `r_scratch` payload.
+        pseudo_set_nz_or_p_r,
+        /// Set byte if zero flag not set or parity flag set
+        /// Requires a scratch register!
+        /// Uses `x_scratch` payload.
+        pseudo_set_nz_or_p_m_sib,
+        /// Set byte if zero flag not set or parity flag set
+        /// Requires a scratch register!
+        /// Uses `x_scratch` payload.
+        pseudo_set_nz_or_p_m_rip,
+        /// Jump if zero flag set and parity flag not set
+        /// Uses `inst` payload.
+        pseudo_j_z_and_np_inst,
+        /// Jump if zero flag not set or parity flag set
+        /// Uses `inst` payload.
+        pseudo_j_nz_or_p_inst,
+
+        /// Push registers
+        /// Uses `reg_list` payload.
+        pseudo_push_reg_list,
+        /// Pop registers
+        /// Uses `reg_list` payload.
+        pseudo_pop_reg_list,
+
+        /// End of prologue
+        pseudo_dbg_prologue_end_none,
+        /// Update debug line
+        /// Uses `line_column` payload.
+        pseudo_dbg_line_line_column,
+        /// Start of epilogue
+        pseudo_dbg_epilogue_begin_none,
+
+        /// Tombstone
+        /// Emitter should skip this instruction.
+        pseudo_dead_none,
     };
 
     pub const Data = union {
+        none: struct {
+            fixes: Fixes = ._,
+        },
         /// References another Mir instruction.
-        inst: Index,
-        /// Another instruction with condition code (CC).
-        /// Used by `jcc`.
-        inst_cc: struct {
-            /// Another instruction.
+        inst: struct {
+            fixes: Fixes = ._,
             inst: Index,
-            /// A condition code for use with EFLAGS register.
-            cc: bits.Condition,
         },
         /// A 32-bit immediate value.
-        i: u32,
-        r: Register,
+        i: struct {
+            fixes: Fixes = ._,
+            i: u32,
+        },
+        r: struct {
+            fixes: Fixes = ._,
+            r1: Register,
+        },
         rr: struct {
+            fixes: Fixes = ._,
             r1: Register,
             r2: Register,
         },
         rrr: struct {
+            fixes: Fixes = ._,
             r1: Register,
             r2: Register,
             r3: Register,
         },
         rrri: struct {
+            fixes: Fixes = ._,
             r1: Register,
             r2: Register,
             r3: Register,
             i: u8,
         },
         rri: struct {
+            fixes: Fixes = ._,
             r1: Register,
             r2: Register,
             i: u32,
         },
-        /// Condition code (CC), followed by custom payload found in extra.
-        x_cc: struct {
-            scratch: Register,
-            cc: bits.Condition,
-            payload: u32,
-        },
-        /// Register with condition code (CC).
-        r_cc: struct {
-            r: Register,
-            scratch: Register,
-            cc: bits.Condition,
-        },
-        /// Register, register with condition code (CC).
-        rr_cc: struct {
-            r1: Register,
-            r2: Register,
-            cc: bits.Condition,
-        },
         /// Register, immediate.
         ri: struct {
-            r: Register,
+            fixes: Fixes = ._,
+            r1: Register,
             i: u32,
         },
         /// Register, followed by custom payload found in extra.
         rx: struct {
-            r: Register,
-            payload: u32,
-        },
-        /// Register with condition code (CC), followed by custom payload found in extra.
-        rx_cc: struct {
-            r: Register,
-            cc: bits.Condition,
+            fixes: Fixes = ._,
+            r1: Register,
             payload: u32,
         },
         /// Immediate, followed by Custom payload found in extra.
@@ -808,39 +981,54 @@ pub const Inst = struct {
         },
         /// Register, register, followed by Custom payload found in extra.
         rrx: struct {
+            fixes: Fixes = ._,
             r1: Register,
             r2: Register,
             payload: u32,
         },
         /// Register, byte immediate, followed by Custom payload found in extra.
         rix: struct {
-            r: Register,
+            fixes: Fixes = ._,
+            r1: Register,
             i: u8,
             payload: u32,
         },
         /// Register, register, byte immediate, followed by Custom payload found in extra.
         rrix: struct {
+            fixes: Fixes = ._,
             r1: Register,
             r2: Register,
             i: u8,
             payload: u32,
         },
-        /// String instruction prefix and width.
-        string: struct {
-            repeat: bits.StringRepeat,
-            width: bits.StringWidth,
+        /// Register, scratch register
+        r_scratch: struct {
+            fixes: Fixes = ._,
+            r1: Register,
+            scratch_reg: Register,
+        },
+        /// Scratch register, followed by Custom payload found in extra.
+        x_scratch: struct {
+            fixes: Fixes = ._,
+            scratch_reg: Register,
+            payload: u32,
+        },
+        /// Custom payload found in extra.
+        x: struct {
+            fixes: Fixes = ._,
+            payload: u32,
         },
         /// Relocation for the linker where:
         /// * `atom_index` is the index of the source
         /// * `sym_index` is the index of the target
-        relocation: Reloc,
+        reloc: Reloc,
         /// Debug line and column position
         line_column: struct {
             line: u32,
             column: u32,
         },
-        /// Index into `extra`. Meaning of what can be found there is context-dependent.
-        payload: u32,
+        /// Register list
+        reg_list: RegisterList,
     };
 
     // Make sure we don't accidentally make instructions bigger than expected.
@@ -852,6 +1040,7 @@ pub const Inst = struct {
     }
 };
 
+/// A linker symbol not yet allocated in VM.
 pub const Reloc = struct {
     /// Index of the containing atom.
     atom_index: u32,
@@ -887,16 +1076,6 @@ pub const RegisterList = struct {
         return self.bitset.iterator(options);
     }
 
-    pub fn asInt(self: Self) u32 {
-        return self.bitset.mask;
-    }
-
-    pub fn fromInt(mask: u32) Self {
-        return .{
-            .bitset = BitSet{ .mask = @intCast(BitSet.MaskInt, mask) },
-        };
-    }
-
     pub fn count(self: Self) u32 {
         return @intCast(u32, self.bitset.count());
     }
diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig
index b73a37d6cb..3343f280b9 100644
--- a/src/arch/x86_64/bits.zig
+++ b/src/arch/x86_64/bits.zig
@@ -6,9 +6,6 @@ const Allocator = std.mem.Allocator;
 const ArrayList = std.ArrayList;
 const DW = std.dwarf;
 
-pub const StringRepeat = enum(u3) { none, rep, repe, repz, repne, repnz };
-pub const StringWidth = enum(u2) { b, w, d, q };
-
 /// EFLAGS condition codes
 pub const Condition = enum(u5) {
     /// above

From ecb5feaf94bf49dc4c180f09c170223d6c1898b3 Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Mon, 8 May 2023 06:51:05 -0400
Subject: [PATCH 136/725] x86_64: continue to optimize mir tag usage

Migrate mnemonic literals to tuples that represent the compressed
storage.  225 tags left in use, many tags left to compress.
---
 src/arch/x86_64/CodeGen.zig | 1489 +++++++++++++++++++----------------
 src/arch/x86_64/Lower.zig   |    9 +-
 src/arch/x86_64/Mir.zig     |  100 ++-
 3 files changed, 880 insertions(+), 718 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 3ac05c95ac..147be62e28 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -1167,11 +1167,13 @@ fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
     });
 }
 
-fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {
+fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = .none,
-        .data = undefined,
+        .data = .{ .none = .{
+            .fixes = tag[0],
+        } },
     });
 }
 
@@ -1183,22 +1185,26 @@ fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
     });
 }
 
-fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {
+fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = .r,
-        .data = .{ .r = .{ .r1 = reg } },
+        .data = .{ .r = .{
+            .fixes = tag[0],
+            .r1 = reg,
+        } },
     });
 }
 
-fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
+fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = switch (imm) {
             .signed => .i_s,
             .unsigned => .i_u,
         },
         .data = .{ .i = .{
+            .fixes = tag[0],
             .i = switch (imm) {
                 .signed => |s| @bitCast(u32, s),
                 .unsigned => |u| @intCast(u32, u),
@@ -1207,24 +1213,29 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
     });
 }
 
-fn asmRegisterRegister(self: *Self, tag: Mir.Inst.Tag, reg1: Register, reg2: Register) !void {
+fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = .rr,
-        .data = .{ .rr = .{ .r1 = reg1, .r2 = reg2 } },
+        .data = .{ .rr = .{
+            .fixes = tag[0],
+            .r1 = reg1,
+            .r2 = reg2,
+        } },
     });
 }
 
-fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Immediate) !void {
+fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
     const ops: Mir.Inst.Ops = switch (imm) {
         .signed => .ri_s,
         .unsigned => |u| if (math.cast(u32, u)) |_| .ri_u else .ri64,
     };
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = ops,
         .data = switch (ops) {
             .ri_s, .ri_u => .{ .ri = .{
+                .fixes = tag[0],
                 .r1 = reg,
                 .i = switch (imm) {
                     .signed => |s| @bitCast(u32, s),
@@ -1232,6 +1243,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
                 },
             } },
             .ri64 => .{ .rx = .{
+                .fixes = tag[0],
                 .r1 = reg,
                 .payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
             } },
@@ -1242,47 +1254,59 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
 
 fn asmRegisterRegisterRegister(
     self: *Self,
-    tag: Mir.Inst.Tag,
+    tag: Mir.Inst.FixedTag,
     reg1: Register,
     reg2: Register,
     reg3: Register,
 ) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = .rrr,
-        .data = .{ .rrr = .{ .r1 = reg1, .r2 = reg2, .r3 = reg3 } },
+        .data = .{ .rrr = .{
+            .fixes = tag[0],
+            .r1 = reg1,
+            .r2 = reg2,
+            .r3 = reg3,
+        } },
     });
 }
 
 fn asmRegisterRegisterRegisterImmediate(
     self: *Self,
-    tag: Mir.Inst.Tag,
+    tag: Mir.Inst.FixedTag,
     reg1: Register,
     reg2: Register,
     reg3: Register,
     imm: Immediate,
 ) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = .rrri,
-        .data = .{ .rrri = .{ .r1 = reg1, .r2 = reg2, .r3 = reg3, .i = @intCast(u8, imm.unsigned) } },
+        .data = .{ .rrri = .{
+            .fixes = tag[0],
+            .r1 = reg1,
+            .r2 = reg2,
+            .r3 = reg3,
+            .i = @intCast(u8, imm.unsigned),
+        } },
     });
 }
 
 fn asmRegisterRegisterImmediate(
     self: *Self,
-    tag: Mir.Inst.Tag,
+    tag: Mir.Inst.FixedTag,
     reg1: Register,
     reg2: Register,
     imm: Immediate,
 ) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = switch (imm) {
             .signed => .rri_s,
             .unsigned => .rri_u,
         },
         .data = .{ .rri = .{
+            .fixes = tag[0],
             .r1 = reg1,
             .r2 = reg2,
             .i = switch (imm) {
@@ -1295,19 +1319,20 @@ fn asmRegisterRegisterImmediate(
 
 fn asmRegisterRegisterMemory(
     self: *Self,
-    tag: Mir.Inst.Tag,
+    tag: Mir.Inst.FixedTag,
     reg1: Register,
     reg2: Register,
     m: Memory,
 ) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = switch (m) {
             .sib => .rrm_sib,
             .rip => .rrm_rip,
             else => unreachable,
         },
         .data = .{ .rrx = .{
+            .fixes = tag[0],
             .r1 = reg1,
             .r2 = reg2,
             .payload = switch (m) {
@@ -1319,15 +1344,16 @@ fn asmRegisterRegisterMemory(
     });
 }
 
-fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
+fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = switch (m) {
             .sib => .m_sib,
             .rip => .m_rip,
             else => unreachable,
         },
         .data = .{ .x = .{
+            .fixes = tag[0],
             .payload = switch (m) {
                 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
                 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
@@ -1337,15 +1363,16 @@ fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
     });
 }
 
-fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !void {
+fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = switch (m) {
             .sib => .rm_sib,
             .rip => .rm_rip,
             else => unreachable,
         },
         .data = .{ .rx = .{
+            .fixes = tag[0],
             .r1 = reg,
             .payload = switch (m) {
                 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
@@ -1358,19 +1385,20 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !
 
 fn asmRegisterMemoryImmediate(
     self: *Self,
-    tag: Mir.Inst.Tag,
+    tag: Mir.Inst.FixedTag,
     reg: Register,
     m: Memory,
     imm: Immediate,
 ) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = switch (m) {
             .sib => .rmi_sib,
             .rip => .rmi_rip,
             else => unreachable,
         },
         .data = .{ .rix = .{
+            .fixes = tag[0],
             .r1 = reg,
             .i = @intCast(u8, imm.unsigned),
             .payload = switch (m) {
@@ -1384,20 +1412,21 @@ fn asmRegisterMemoryImmediate(
 
 fn asmRegisterRegisterMemoryImmediate(
     self: *Self,
-    tag: Mir.Inst.Tag,
+    tag: Mir.Inst.FixedTag,
     reg1: Register,
     reg2: Register,
     m: Memory,
     imm: Immediate,
 ) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = switch (m) {
             .sib => .rrmi_sib,
             .rip => .rrmi_rip,
             else => unreachable,
         },
         .data = .{ .rrix = .{
+            .fixes = tag[0],
             .r1 = reg1,
             .r2 = reg2,
             .i = @intCast(u8, imm.unsigned),
@@ -1410,15 +1439,16 @@ fn asmRegisterRegisterMemoryImmediate(
     });
 }
 
-fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !void {
+fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = switch (m) {
             .sib => .mr_sib,
             .rip => .mr_rip,
             else => unreachable,
         },
         .data = .{ .rx = .{
+            .fixes = tag[0],
             .r1 = reg,
             .payload = switch (m) {
                 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
@@ -1429,9 +1459,9 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !
     });
 }
 
-fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate) !void {
+fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = switch (m) {
             .sib => switch (imm) {
                 .signed => .mi_sib_s,
@@ -1443,11 +1473,39 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate)
             },
             else => unreachable,
         },
-        .data = .{ .ix = .{
-            .i = switch (imm) {
+        .data = .{ .x = .{
+            .fixes = tag[0],
+            .payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
                 .signed => |s| @bitCast(u32, s),
                 .unsigned => |u| @intCast(u32, u),
-            },
+            } }),
+        } },
+    });
+    _ = switch (m) {
+        .sib => try self.addExtra(Mir.MemorySib.encode(m)),
+        .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
+        else => unreachable,
+    };
+}
+
+fn asmMemoryRegisterRegister(
+    self: *Self,
+    tag: Mir.Inst.FixedTag,
+    m: Memory,
+    reg1: Register,
+    reg2: Register,
+) !void {
+    _ = try self.addInst(.{
+        .tag = tag[1],
+        .ops = switch (m) {
+            .sib => .mrr_sib,
+            .rip => .mrr_rip,
+            else => unreachable,
+        },
+        .data = .{ .rrx = .{
+            .fixes = tag[0],
+            .r1 = reg1,
+            .r2 = reg2,
             .payload = switch (m) {
                 .sib => try self.addExtra(Mir.MemorySib.encode(m)),
                 .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
@@ -1457,43 +1515,22 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate)
     });
 }
 
-fn asmMemoryRegisterRegister(
-    self: *Self,
-    tag: Mir.Inst.Tag,
-    m: Memory,
-    reg1: Register,
-    reg2: Register,
-) !void {
-    _ = try self.addInst(.{
-        .tag = tag,
-        .ops = switch (m) {
-            .sib => .mrr_sib,
-            .rip => .mrr_rip,
-            else => unreachable,
-        },
-        .data = .{ .rrx = .{ .r1 = reg1, .r2 = reg2, .payload = switch (m) {
-            .sib => try self.addExtra(Mir.MemorySib.encode(m)),
-            .rip => try self.addExtra(Mir.MemoryRip.encode(m)),
-            else => unreachable,
-        } } },
-    });
-}
-
 fn asmMemoryRegisterImmediate(
     self: *Self,
-    tag: Mir.Inst.Tag,
+    tag: Mir.Inst.FixedTag,
     m: Memory,
     reg: Register,
     imm: Immediate,
 ) !void {
     _ = try self.addInst(.{
-        .tag = tag,
+        .tag = tag[1],
         .ops = switch (m) {
             .sib => .mri_sib,
             .rip => .mri_rip,
             else => unreachable,
         },
         .data = .{ .rix = .{
+            .fixes = tag[0],
             .r1 = reg,
             .i = @intCast(u8, imm.unsigned),
             .payload = switch (m) {
@@ -1508,9 +1545,9 @@ fn asmMemoryRegisterImmediate(
 fn gen(self: *Self) InnerError!void {
     const cc = self.fn_type.fnCallingConvention();
     if (cc != .Naked) {
-        try self.asmRegister(.push, .rbp);
+        try self.asmRegister(.{ ._, .push }, .rbp);
         const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
-        try self.asmRegisterRegister(.mov, .rbp, .rsp);
+        try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);
         const backpatch_frame_align = try self.asmPlaceholder();
         const backpatch_stack_alloc = try self.asmPlaceholder();
 
@@ -1553,8 +1590,8 @@ fn gen(self: *Self) InnerError!void {
         try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
         const backpatch_stack_dealloc = try self.asmPlaceholder();
         const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
-        try self.asmRegister(.pop, .rbp);
-        try self.asmOpOnly(.ret);
+        try self.asmRegister(.{ ._, .pop }, .rbp);
+        try self.asmOpOnly(.{ ._, .ret });
 
         const frame_layout = try self.computeFrameLayout();
         const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);
@@ -1927,7 +1964,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
                 };
                 const tag_val = Value.initPayload(&tag_pl.base);
                 const tag_mcv = try self.genTypedValue(.{ .ty = enum_ty, .val = tag_val });
-                try self.genBinOpMir(.cmp, enum_ty, enum_mcv, tag_mcv);
+                try self.genBinOpMir(.{ ._, .cmp }, enum_ty, enum_mcv, tag_mcv);
                 const skip_reloc = try self.asmJccReloc(undefined, .ne);
 
                 try self.genSetMem(
@@ -1947,7 +1984,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
             try self.airTrap();
 
             for (exitlude_jump_relocs) |reloc| try self.performReloc(reloc);
-            try self.asmOpOnly(.ret);
+            try self.asmOpOnly(.{ ._, .ret });
         },
         else => return self.fail(
             "TODO implement {s} for {}",
@@ -2406,7 +2443,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
                 else
                     try self.copyToTmpRegister(src_ty, src_mcv);
                 try self.asmRegisterRegisterImmediate(
-                    .vcvtps2ph,
+                    .{ ._, .vcvtps2ph },
                     dst_reg,
                     mat_src_reg.to128(),
                     Immediate.u(0b1_00),
@@ -2418,12 +2455,12 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
         }
     } else if (src_bits == 64 and dst_bits == 32) {
         if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
-            .vcvtsd2ss,
+            .{ ._, .vcvtsd2ss },
             dst_reg,
             dst_reg,
             src_mcv.mem(.qword),
         ) else try self.asmRegisterRegisterRegister(
-            .vcvtsd2ss,
+            .{ ._, .vcvtsd2ss },
             dst_reg,
             dst_reg,
             (if (src_mcv.isRegister())
@@ -2431,11 +2468,11 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
             else
                 try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
         ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
-            .cvtsd2ss,
+            .{ ._, .cvtsd2ss },
             dst_reg,
             src_mcv.mem(.qword),
         ) else try self.asmRegisterRegister(
-            .cvtsd2ss,
+            .{ ._, .cvtsd2ss },
             dst_reg,
             (if (src_mcv.isRegister())
                 src_mcv.getReg().?
@@ -2469,22 +2506,22 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
             src_mcv.getReg().?
         else
             try self.copyToTmpRegister(src_ty, src_mcv);
-        try self.asmRegisterRegister(.vcvtph2ps, dst_reg, mat_src_reg.to128());
+        try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, mat_src_reg.to128());
         switch (dst_bits) {
             32 => {},
-            64 => try self.asmRegisterRegisterRegister(.vcvtss2sd, dst_reg, dst_reg, dst_reg),
+            64 => try self.asmRegisterRegisterRegister(.{ ._, .vcvtss2sd }, dst_reg, dst_reg, dst_reg),
             else => return self.fail("TODO implement airFpext from {} to {}", .{
                 src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
             }),
         }
     } else if (src_bits == 32 and dst_bits == 64) {
         if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
-            .vcvtss2sd,
+            .{ ._, .vcvtss2sd },
             dst_reg,
             dst_reg,
             src_mcv.mem(.dword),
         ) else try self.asmRegisterRegisterRegister(
-            .vcvtss2sd,
+            .{ ._, .vcvtss2sd },
             dst_reg,
             dst_reg,
             (if (src_mcv.isRegister())
@@ -2492,11 +2529,11 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
             else
                 try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
         ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
-            .cvtss2sd,
+            .{ ._, .cvtss2sd },
             dst_reg,
             src_mcv.mem(.dword),
         ) else try self.asmRegisterRegister(
-            .cvtss2sd,
+            .{ ._, .cvtss2sd },
             dst_reg,
             (if (src_mcv.isRegister())
                 src_mcv.getReg().?
@@ -2537,12 +2574,12 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
     switch (dst_mcv) {
         .register => |dst_reg| {
             const min_abi_size = @min(dst_abi_size, src_abi_size);
-            const tag: Mir.Inst.Tag = switch (signedness) {
-                .signed => if (min_abi_size >= 4) .movsxd else .movsx,
-                .unsigned => if (min_abi_size >= 4) .mov else .movzx,
+            const tag: Mir.Inst.FixedTag = switch (signedness) {
+                .signed => if (min_abi_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx },
+                .unsigned => if (min_abi_size >= 4) .{ ._, .mov } else .{ ._, .movzx },
             };
-            const dst_alias = switch (tag) {
-                .movsx, .movsxd => dst_reg.to64(),
+            const dst_alias = switch (tag[1]) {
+                .movsx => dst_reg.to64(),
                 .mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(),
                 else => unreachable,
             };
@@ -2570,14 +2607,24 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
             try self.genCopy(min_ty, dst_mcv, src_mcv);
             const extra = dst_abi_size * 8 - dst_int_info.bits;
             if (extra > 0) {
-                try self.genShiftBinOpMir(switch (signedness) {
-                    .signed => .sal,
-                    .unsigned => .shl,
-                }, dst_ty, dst_mcv, .{ .immediate = extra });
-                try self.genShiftBinOpMir(switch (signedness) {
-                    .signed => .sar,
-                    .unsigned => .shr,
-                }, dst_ty, dst_mcv, .{ .immediate = extra });
+                try self.genShiftBinOpMir(
+                    switch (signedness) {
+                        .signed => .{ ._l, .sa },
+                        .unsigned => .{ ._l, .sh },
+                    },
+                    dst_ty,
+                    dst_mcv,
+                    .{ .immediate = extra },
+                );
+                try self.genShiftBinOpMir(
+                    switch (signedness) {
+                        .signed => .{ ._r, .sa },
+                        .unsigned => .{ ._r, .sh },
+                    },
+                    dst_ty,
+                    dst_mcv,
+                    .{ .immediate = extra },
+                );
             }
         },
     }
@@ -2762,8 +2809,8 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
     const reg_bits = self.regBitSize(ty);
     const cc: Condition = if (ty.isSignedInt()) cc: {
         try self.genSetReg(limit_reg, ty, dst_mcv);
-        try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
-        try self.genBinOpMir(.xor, ty, limit_mcv, .{
+        try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
+        try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
             .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
         });
         break :cc .o;
@@ -2773,7 +2820,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
         });
         break :cc .c;
     };
-    try self.genBinOpMir(.add, ty, dst_mcv, rhs_mcv);
+    try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
 
     const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
     try self.asmCmovccRegisterRegister(
@@ -2813,8 +2860,8 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
     const reg_bits = self.regBitSize(ty);
     const cc: Condition = if (ty.isSignedInt()) cc: {
         try self.genSetReg(limit_reg, ty, dst_mcv);
-        try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
-        try self.genBinOpMir(.xor, ty, limit_mcv, .{
+        try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
+        try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
             .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
         });
         break :cc .o;
@@ -2822,7 +2869,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
         try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });
         break :cc .c;
     };
-    try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);
+    try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
 
     const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
     try self.asmCmovccRegisterRegister(
@@ -2864,9 +2911,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
     const reg_bits = self.regBitSize(ty);
     const cc: Condition = if (ty.isSignedInt()) cc: {
         try self.genSetReg(limit_reg, ty, lhs_mcv);
-        try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);
-        try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
-        try self.genBinOpMir(.xor, ty, limit_mcv, .{
+        try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
+        try self.genShiftBinOpMir(.{ ._, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
+        try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
             .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
         });
         break :cc .o;
@@ -2979,7 +3026,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
                 };
                 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
 
-                try self.genBinOpMir(.cmp, lhs_ty, tmp_mcv, lhs);
+                try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, tmp_mcv, lhs);
                 const cc = Condition.ne;
 
                 const tuple_ty = self.air.typeOfIndex(inst);
@@ -3066,12 +3113,17 @@ fn genSetFrameTruncatedOverflowCompare(
         src_mcv;
     try self.genSetReg(scratch_reg, hi_limb_ty, hi_limb_mcv);
     try self.truncateRegister(hi_limb_ty, scratch_reg);
-    try self.genBinOpMir(.cmp, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
+    try self.genBinOpMir(.{ ._, .cmp }, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
 
     const eq_reg = temp_regs[2];
     if (overflow_cc) |_| {
         try self.asmSetccRegister(eq_reg.to8(), .ne);
-        try self.genBinOpMir(.@"or", Type.u8, .{ .register = overflow_reg }, .{ .register = eq_reg });
+        try self.genBinOpMir(
+            .{ ._, .@"or" },
+            Type.u8,
+            .{ .register = overflow_reg },
+            .{ .register = eq_reg },
+        );
     }
 
     const payload_off = @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*));
@@ -3200,28 +3252,25 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
 /// Generates signed or unsigned integer multiplication/division.
 /// Clobbers .rax and .rdx registers.
 /// Quotient is saved in .rax and remainder in .rdx.
-fn genIntMulDivOpMir(
-    self: *Self,
-    tag: Mir.Inst.Tag,
-    ty: Type,
-    lhs: MCValue,
-    rhs: MCValue,
-) !void {
+fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
     const abi_size = @intCast(u32, ty.abiSize(self.target.*));
     if (abi_size > 8) {
         return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});
     }
 
     try self.genSetReg(.rax, ty, lhs);
-    switch (tag) {
+    switch (tag[1]) {
         else => unreachable,
-        .mul, .imul => {},
-        .div => try self.asmRegisterRegister(.xor, .edx, .edx),
-        .idiv => switch (self.regBitSize(ty)) {
-            8 => try self.asmOpOnly(.cbw),
-            16 => try self.asmOpOnly(.cwd),
-            32 => try self.asmOpOnly(.cdq),
-            64 => try self.asmOpOnly(.cqo),
+        .mul => {},
+        .div => switch (tag[0]) {
+            ._ => try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx),
+            .i_ => switch (self.regBitSize(ty)) {
+                8 => try self.asmOpOnly(.{ ._, .cbw }),
+                16 => try self.asmOpOnly(.{ ._, .cwd }),
+                32 => try self.asmOpOnly(.{ ._, .cdq }),
+                64 => try self.asmOpOnly(.{ ._, .cqo }),
+                else => unreachable,
+            },
             else => unreachable,
         },
     }
@@ -3259,23 +3308,28 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
     const divisor_lock = self.register_manager.lockReg(divisor);
     defer if (divisor_lock) |lock| self.register_manager.unlockReg(lock);
 
-    try self.genIntMulDivOpMir(switch (int_info.signedness) {
-        .signed => .idiv,
-        .unsigned => .div,
-    }, ty, .{ .register = dividend }, .{ .register = divisor });
+    try self.genIntMulDivOpMir(
+        switch (int_info.signedness) {
+            .signed => .{ .i_, .div },
+            .unsigned => .{ ._, .div },
+        },
+        ty,
+        .{ .register = dividend },
+        .{ .register = divisor },
+    );
 
     try self.asmRegisterRegister(
-        .xor,
+        .{ ._, .xor },
         registerAlias(divisor, abi_size),
         registerAlias(dividend, abi_size),
     );
     try self.asmRegisterImmediate(
-        .sar,
+        .{ ._r, .sa },
         registerAlias(divisor, abi_size),
         Immediate.u(int_info.bits - 1),
     );
     try self.asmRegisterRegister(
-        .@"test",
+        .{ ._, .@"test" },
         registerAlias(.rdx, abi_size),
         registerAlias(.rdx, abi_size),
     );
@@ -3284,7 +3338,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
         registerAlias(.rdx, abi_size),
         .z,
     );
-    try self.genBinOpMir(.add, ty, .{ .register = divisor }, .{ .register = .rax });
+    try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });
     return MCValue{ .register = divisor };
 }
 
@@ -3406,7 +3460,12 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
                 const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
                 if (err_off > 0) {
                     const shift = @intCast(u6, err_off * 8);
-                    try self.genShiftBinOpMir(.shr, err_union_ty, result, .{ .immediate = shift });
+                    try self.genShiftBinOpMir(
+                        .{ ._r, .sh },
+                        err_union_ty,
+                        result,
+                        .{ .immediate = shift },
+                    );
                 } else {
                     try self.truncateRegister(Type.anyerror, result.register);
                 }
@@ -3458,7 +3517,12 @@ fn genUnwrapErrorUnionPayloadMir(
                     .{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
                 if (payload_off > 0) {
                     const shift = @intCast(u6, payload_off * 8);
-                    try self.genShiftBinOpMir(.shr, err_union_ty, result_mcv, .{ .immediate = shift });
+                    try self.genShiftBinOpMir(
+                        .{ ._r, .sh },
+                        err_union_ty,
+                        result_mcv,
+                        .{ .immediate = shift },
+                    );
                 } else {
                     try self.truncateRegister(payload_ty, result_mcv.register);
                 }
@@ -3495,7 +3559,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
     const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
     const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
     try self.asmRegisterMemory(
-        .mov,
+        .{ ._, .mov },
         registerAlias(dst_reg, err_abi_size),
         Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
             .base = .{ .reg = src_reg },
@@ -3533,7 +3597,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
     const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
     const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
     try self.asmRegisterMemory(
-        .lea,
+        .{ ._, .lea },
         registerAlias(dst_reg, dst_abi_size),
         Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
     );
@@ -3559,7 +3623,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
         const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
         const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
         try self.asmMemoryImmediate(
-            .mov,
+            .{ ._, .mov },
             Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
                 .base = .{ .reg = src_reg },
                 .disp = err_off,
@@ -3580,7 +3644,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
         const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
         const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
         try self.asmRegisterMemory(
-            .lea,
+            .{ ._, .lea },
             registerAlias(dst_reg, dst_abi_size),
             Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
         );
@@ -3631,13 +3695,13 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
                 else => unreachable,
 
                 .register => |opt_reg| try self.asmRegisterImmediate(
-                    .bts,
+                    .{ ._s, .bt },
                     opt_reg,
                     Immediate.u(@intCast(u6, pl_abi_size * 8)),
                 ),
 
                 .load_frame => |frame_addr| try self.asmMemoryImmediate(
-                    .mov,
+                    .{ ._, .mov },
                     Memory.sib(.byte, .{
                         .base = .{ .frame = frame_addr.index },
                         .disp = frame_addr.off + pl_abi_size,
@@ -3749,7 +3813,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
 
     const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
     try self.asmRegisterMemory(
-        .lea,
+        .{ ._, .lea },
         registerAlias(dst_reg, dst_abi_size),
         Memory.sib(.qword, .{
             .base = .{ .reg = src_reg },
@@ -3823,7 +3887,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
     try self.genSetReg(addr_reg, Type.usize, slice_mcv);
     // TODO we could allocate register here, but need to expect addr register and potentially
     // offset register.
-    try self.genBinOpMir(.add, slice_ptr_field_type, .{ .register = addr_reg }, .{
+    try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{
         .register = offset_reg,
     });
     return MCValue{ .register = addr_reg.to64() };
@@ -3881,13 +3945,13 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
             const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, self.target.*));
             try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array);
             try self.asmRegisterMemory(
-                .lea,
+                .{ ._, .lea },
                 addr_reg,
                 Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
             );
         },
         .load_frame => |frame_addr| try self.asmRegisterMemory(
-            .lea,
+            .{ ._, .lea },
             addr_reg,
             Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),
         ),
@@ -3903,7 +3967,12 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
     // TODO we could allocate register here, but need to expect addr register and potentially
     // offset register.
     const dst_mcv = try self.allocRegOrMem(inst, false);
-    try self.genBinOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
+    try self.genBinOpMir(
+        .{ ._, .add },
+        Type.usize,
+        .{ .register = addr_reg },
+        .{ .register = offset_reg },
+    );
     try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
 
     return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
@@ -3937,7 +4006,11 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
         try self.copyToTmpRegister(ptr_ty, ptr_mcv);
     const elem_ptr_lock = self.register_manager.lockRegAssumeUnused(elem_ptr_reg);
     defer self.register_manager.unlockReg(elem_ptr_lock);
-    try self.asmRegisterRegister(.add, elem_ptr_reg, offset_reg);
+    try self.asmRegisterRegister(
+        .{ ._, .add },
+        elem_ptr_reg,
+        offset_reg,
+    );
 
     const dst_mcv = try self.allocRegOrMem(inst, true);
     const dst_lock = switch (dst_mcv) {
@@ -3977,7 +4050,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
     defer self.register_manager.unlockReg(offset_reg_lock);
 
     const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
-    try self.genBinOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });
+    try self.genBinOpMir(.{ ._, .add }, ptr_ty, dst_mcv, .{ .register = offset_reg });
 
     return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
 }
@@ -4010,7 +4083,12 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
     const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
         // TODO reusing the operand
         const reg = try self.copyToTmpRegister(ptr_union_ty, ptr);
-        try self.genBinOpMir(.add, ptr_union_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });
+        try self.genBinOpMir(
+            .{ ._, .add },
+            ptr_union_ty,
+            .{ .register = reg },
+            .{ .immediate = layout.payload_size },
+        );
         break :blk MCValue{ .register = reg };
     } else ptr;
 
@@ -4063,7 +4141,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
                 else
                     0;
                 const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
-                try self.genShiftBinOpMir(.shr, Type.usize, result, .{ .immediate = shift });
+                try self.genShiftBinOpMir(.{ ._r, .sh }, Type.usize, result, .{ .immediate = shift });
                 break :blk MCValue{
                     .register = registerAlias(result.register, @intCast(u32, layout.tag_size)),
                 };
@@ -4100,11 +4178,11 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
         const src_bits = src_ty.bitSize(self.target.*);
         if (self.hasFeature(.lzcnt)) {
             if (src_bits <= 64) {
-                try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
+                try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);
 
                 const extra_bits = self.regExtraBits(src_ty);
                 if (extra_bits > 0) {
-                    try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });
+                    try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits });
                 }
             } else if (src_bits <= 128) {
                 const tmp_reg = try self.register_manager.allocReg(null, gp);
@@ -4112,13 +4190,23 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
                 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
                 defer self.register_manager.unlockReg(tmp_lock);
 
-                try self.genBinOpMir(.lzcnt, Type.u64, dst_mcv, mat_src_mcv);
-                try self.genBinOpMir(.add, dst_ty, dst_mcv, .{ .immediate = 64 });
-                try self.genBinOpMir(.lzcnt, Type.u64, tmp_mcv, mat_src_mcv.address().offset(8).deref());
+                try self.genBinOpMir(.{ ._, .lzcnt }, Type.u64, dst_mcv, mat_src_mcv);
+                try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
+                try self.genBinOpMir(
+                    .{ ._, .lzcnt },
+                    Type.u64,
+                    tmp_mcv,
+                    mat_src_mcv.address().offset(8).deref(),
+                );
                 try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
 
                 if (src_bits < 128) {
-                    try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = 128 - src_bits });
+                    try self.genBinOpMir(
+                        .{ ._, .sub },
+                        dst_ty,
+                        dst_mcv,
+                        .{ .immediate = 128 - src_bits },
+                    );
                 }
             } else return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
             break :result dst_mcv;
@@ -4130,7 +4218,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
             const imm_reg = try self.copyToTmpRegister(dst_ty, .{
                 .immediate = src_bits ^ (src_bits - 1),
             });
-            try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
+            try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
 
             const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
             try self.asmCmovccRegisterRegister(
@@ -4139,12 +4227,12 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
                 .z,
             );
 
-            try self.genBinOpMir(.xor, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
+            try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
         } else {
             const imm_reg = try self.copyToTmpRegister(dst_ty, .{
                 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - self.regBitSize(dst_ty)),
             });
-            try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
+            try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
 
             const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
             try self.asmCmovccRegisterRegister(
@@ -4154,7 +4242,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
             );
 
             try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });
-            try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });
+            try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .register = imm_reg });
         }
         break :result dst_mcv;
     };
@@ -4195,7 +4283,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
                         break :tmp dst_mcv;
                     };
                     try self.genBinOpMir(
-                        .@"or",
+                        .{ ._, .@"or" },
                         src_ty,
                         tmp_mcv,
                         .{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - extra_bits)) <<
@@ -4203,7 +4291,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
                     );
                     break :masked tmp_mcv;
                 } else mat_src_mcv;
-                try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);
+                try self.genBinOpMir(.{ ._, .tzcnt }, src_ty, dst_mcv, masked_mcv);
             } else if (src_bits <= 128) {
                 const tmp_reg = try self.register_manager.allocReg(null, gp);
                 const tmp_mcv = MCValue{ .register = tmp_reg };
@@ -4213,16 +4301,16 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
                 const masked_mcv = if (src_bits < 128) masked: {
                     try self.genCopy(Type.u64, dst_mcv, mat_src_mcv.address().offset(8).deref());
                     try self.genBinOpMir(
-                        .@"or",
+                        .{ ._, .@"or" },
                         Type.u64,
                         dst_mcv,
                         .{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(u6, src_bits - 64) },
                     );
                     break :masked dst_mcv;
                 } else mat_src_mcv.address().offset(8).deref();
-                try self.genBinOpMir(.tzcnt, Type.u64, dst_mcv, masked_mcv);
-                try self.genBinOpMir(.add, dst_ty, dst_mcv, .{ .immediate = 64 });
-                try self.genBinOpMir(.tzcnt, Type.u64, tmp_mcv, mat_src_mcv);
+                try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, dst_mcv, masked_mcv);
+                try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
+                try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, mat_src_mcv);
                 try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
             } else return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
             break :result dst_mcv;
@@ -4232,7 +4320,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
             return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
 
         const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
-        try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
+        try self.genBinOpMir(.{ ._, .bsf }, src_ty, dst_mcv, mat_src_mcv);
 
         const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
         try self.asmCmovccRegisterRegister(
@@ -4270,7 +4358,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
                 .{ .register = try self.register_manager.allocReg(inst, gp) };
 
             const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16;
-            try self.genBinOpMir(.popcnt, popcnt_ty, dst_mcv, mat_src_mcv);
+            try self.genBinOpMir(.{ ._, .popcnt }, popcnt_ty, dst_mcv, mat_src_mcv);
             break :result dst_mcv;
         }
 
@@ -4301,54 +4389,54 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
                 undefined;
 
             // dst = operand
-            try self.asmRegisterRegister(.mov, tmp, dst);
+            try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
             // tmp = operand
-            try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));
+            try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
             // tmp = operand >> 1
             if (src_abi_size > 4) {
-                try self.asmRegisterImmediate(.mov, imm, imm_0_1);
-                try self.asmRegisterRegister(.@"and", tmp, imm);
-            } else try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);
+                try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
+                try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
+            } else try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1);
             // tmp = (operand >> 1) & 0x55...55
-            try self.asmRegisterRegister(.sub, dst, tmp);
+            try self.asmRegisterRegister(.{ ._, .sub }, dst, tmp);
             // dst = temp1 = operand - ((operand >> 1) & 0x55...55)
-            try self.asmRegisterRegister(.mov, tmp, dst);
+            try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
             // tmp = temp1
-            try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));
+            try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
             // dst = temp1 >> 2
             if (src_abi_size > 4) {
-                try self.asmRegisterImmediate(.mov, imm, imm_00_11);
-                try self.asmRegisterRegister(.@"and", tmp, imm);
-                try self.asmRegisterRegister(.@"and", dst, imm);
+                try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
+                try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
+                try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
             } else {
-                try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);
-                try self.asmRegisterImmediate(.@"and", dst, imm_00_11);
+                try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11);
+                try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11);
             }
             // tmp = temp1 & 0x33...33
             // dst = (temp1 >> 2) & 0x33...33
-            try self.asmRegisterRegister(.add, tmp, dst);
+            try self.asmRegisterRegister(.{ ._, .add }, tmp, dst);
             // tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
-            try self.asmRegisterRegister(.mov, dst, tmp);
+            try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
             // dst = temp2
-            try self.asmRegisterImmediate(.shr, tmp, Immediate.u(4));
+            try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(4));
             // tmp = temp2 >> 4
-            try self.asmRegisterRegister(.add, dst, tmp);
+            try self.asmRegisterRegister(.{ ._, .add }, dst, tmp);
             // dst = temp2 + (temp2 >> 4)
             if (src_abi_size > 4) {
-                try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);
-                try self.asmRegisterImmediate(.mov, tmp, imm_0000_0001);
-                try self.asmRegisterRegister(.@"and", dst, imm);
-                try self.asmRegisterRegister(.imul, dst, tmp);
+                try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
+                try self.asmRegisterImmediate(.{ ._, .mov }, tmp, imm_0000_0001);
+                try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
+                try self.asmRegisterRegister(.{ .i_, .mul }, dst, tmp);
             } else {
-                try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);
+                try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
                 if (src_abi_size > 1) {
-                    try self.asmRegisterRegisterImmediate(.imul, dst, dst, imm_0000_0001);
+                    try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst, dst, imm_0000_0001);
                 }
             }
             // dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
             // dst = temp3 * 0x01...01
             if (src_abi_size > 1) {
-                try self.asmRegisterImmediate(.shr, dst, Immediate.u((src_abi_size - 1) * 8));
+                try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u((src_abi_size - 1) * 8));
             }
             // dst = (temp3 * 0x01...01) >> (bits - 8)
         }
@@ -4377,11 +4465,11 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
         16 => if ((mem_ok or src_mcv.isRegister()) and
             self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
         {
-            try self.genBinOpMir(.rol, src_ty, src_mcv, .{ .immediate = 8 });
+            try self.genBinOpMir(.{ ._l, .ro }, src_ty, src_mcv, .{ .immediate = 8 });
             return src_mcv;
         },
         32, 64 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
-            try self.genUnOpMir(.bswap, src_ty, src_mcv);
+            try self.genUnOpMir(.{ ._, .bswap }, src_ty, src_mcv);
             return src_mcv;
         },
     }
@@ -4398,10 +4486,10 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
             try self.genSetReg(dst_mcv.register, src_ty, src_mcv);
             switch (src_bits) {
                 else => unreachable,
-                16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),
-                32, 64 => try self.genUnOpMir(.bswap, src_ty, dst_mcv),
+                16 => try self.genBinOpMir(.{ ._l, .ro }, src_ty, dst_mcv, .{ .immediate = 8 }),
+                32, 64 => try self.genUnOpMir(.{ ._, .bswap }, src_ty, dst_mcv),
             }
-        } else try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);
+        } else try self.genBinOpMir(.{ ._, .movbe }, src_ty, dst_mcv, src_mcv);
         return dst_mcv;
     }
 
@@ -4410,7 +4498,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
     const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
     defer self.register_manager.unlockReg(dst_lock);
 
-    try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);
+    try self.genBinOpMir(.{ ._, .movbe }, src_ty, dst_mcv, src_mcv);
     return dst_mcv;
 }
 
@@ -4424,7 +4512,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
     switch (self.regExtraBits(src_ty)) {
         0 => {},
         else => |extra| try self.genBinOpMir(
-            if (src_ty.isSignedInt()) .sar else .shr,
+            if (src_ty.isSignedInt()) .{ ._r, .sa } else .{ ._r, .sh },
             src_ty,
             dst_mcv,
             .{ .immediate = extra },
@@ -4464,40 +4552,40 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
         const imm_0_1 = Immediate.u(mask / 0b1_1);
 
         // dst = temp1 = bswap(operand)
-        try self.asmRegisterRegister(.mov, tmp, dst);
+        try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
         // tmp = temp1
-        try self.asmRegisterImmediate(.shr, dst, Immediate.u(4));
+        try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(4));
         // dst = temp1 >> 4
         if (src_abi_size > 4) {
-            try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);
-            try self.asmRegisterRegister(.@"and", tmp, imm);
-            try self.asmRegisterRegister(.@"and", dst, imm);
+            try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
+            try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
+            try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
         } else {
-            try self.asmRegisterImmediate(.@"and", tmp, imm_0000_1111);
-            try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);
+            try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0000_1111);
+            try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
         }
         // tmp = temp1 & 0x0F...0F
         // dst = (temp1 >> 4) & 0x0F...0F
-        try self.asmRegisterImmediate(.shl, tmp, Immediate.u(4));
+        try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, Immediate.u(4));
         // tmp = (temp1 & 0x0F...0F) << 4
-        try self.asmRegisterRegister(.@"or", dst, tmp);
+        try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);
         // dst = temp2 = ((temp1 >> 4) & 0x0F...0F) | ((temp1 & 0x0F...0F) << 4)
-        try self.asmRegisterRegister(.mov, tmp, dst);
+        try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
         // tmp = temp2
-        try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));
+        try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
         // dst = temp2 >> 2
         if (src_abi_size > 4) {
-            try self.asmRegisterImmediate(.mov, imm, imm_00_11);
-            try self.asmRegisterRegister(.@"and", tmp, imm);
-            try self.asmRegisterRegister(.@"and", dst, imm);
+            try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
+            try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
+            try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
         } else {
-            try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);
-            try self.asmRegisterImmediate(.@"and", dst, imm_00_11);
+            try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11);
+            try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11);
         }
         // tmp = temp2 & 0x33...33
         // dst = (temp2 >> 2) & 0x33...33
         try self.asmRegisterMemory(
-            .lea,
+            .{ ._, .lea },
             if (src_abi_size > 4) tmp.to64() else tmp.to32(),
             Memory.sib(.qword, .{
                 .base = .{ .reg = dst.to64() },
@@ -4505,22 +4593,22 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
             }),
         );
         // tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
-        try self.asmRegisterRegister(.mov, dst, tmp);
+        try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
         // dst = temp3
-        try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));
+        try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
         // tmp = temp3 >> 1
         if (src_abi_size > 4) {
-            try self.asmRegisterImmediate(.mov, imm, imm_0_1);
-            try self.asmRegisterRegister(.@"and", dst, imm);
-            try self.asmRegisterRegister(.@"and", tmp, imm);
+            try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
+            try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
+            try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
         } else {
-            try self.asmRegisterImmediate(.@"and", dst, imm_0_1);
-            try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);
+            try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0_1);
+            try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1);
         }
         // dst = temp3 & 0x55...55
         // tmp = (temp3 >> 1) & 0x55...55
         try self.asmRegisterMemory(
-            .lea,
+            .{ ._, .lea },
             if (src_abi_size > 4) dst.to64() else dst.to32(),
             Memory.sib(.qword, .{
                 .base = .{ .reg = tmp.to64() },
@@ -4533,7 +4621,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
     switch (self.regExtraBits(src_ty)) {
         0 => {},
         else => |extra| try self.genBinOpMir(
-            if (src_ty.isSignedInt()) .sar else .shr,
+            if (src_ty.isSignedInt()) .{ ._r, .sa } else .{ ._r, .sh },
             src_ty,
             dst_mcv,
             .{ .immediate = extra },
@@ -4590,8 +4678,8 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
     try self.genBinOpMir(switch (ty_bits) {
         // No point using an extra prefix byte for *pd which performs the same operation.
         16, 32, 64, 128 => switch (tag) {
-            .neg => .xorps,
-            .fabs => .andnps,
+            .neg => .{ ._, .xorps },
+            .fabs => .{ ._, .andnps },
             else => unreachable,
         },
         80 => return self.fail("TODO implement airFloatSign for {}", .{
@@ -4622,25 +4710,25 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
     if (!self.hasFeature(.sse4_1))
         return self.fail("TODO implement genRound without sse4_1 feature", .{});
 
-    const mir_tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {
+    const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag()) {
         .Float => switch (ty.floatBits(self.target.*)) {
-            32 => if (self.hasFeature(.avx)) .vroundss else .roundss,
-            64 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
+            32 => if (self.hasFeature(.avx)) .{ ._, .vroundss } else .{ ._, .roundss },
+            64 => if (self.hasFeature(.avx)) .{ ._, .vroundsd } else .{ ._, .roundsd },
             16, 80, 128 => null,
             else => unreachable,
         },
         .Vector => switch (ty.childType().zigTypeTag()) {
             .Float => switch (ty.childType().floatBits(self.target.*)) {
                 32 => switch (ty.vectorLen()) {
-                    1 => if (self.hasFeature(.avx)) .vroundss else .roundss,
-                    2...4 => if (self.hasFeature(.avx)) .vroundps else .roundps,
-                    5...8 => if (self.hasFeature(.avx)) .vroundps else null,
+                    1 => if (self.hasFeature(.avx)) .{ ._, .vroundss } else .{ ._, .roundss },
+                    2...4 => if (self.hasFeature(.avx)) .{ ._, .vroundps } else .{ ._, .roundps },
+                    5...8 => if (self.hasFeature(.avx)) .{ ._, .vroundps } else null,
                     else => null,
                 },
                 64 => switch (ty.vectorLen()) {
-                    1 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
-                    2 => if (self.hasFeature(.avx)) .vroundpd else .roundpd,
-                    3...4 => if (self.hasFeature(.avx)) .vroundpd else null,
+                    1 => if (self.hasFeature(.avx)) .{ ._, .vroundsd } else .{ ._, .roundsd },
+                    2 => if (self.hasFeature(.avx)) .{ ._, .vroundpd } else .{ ._, .roundpd },
+                    3...4 => if (self.hasFeature(.avx)) .{ ._, .vroundpd } else null,
                     else => null,
                 },
                 16, 80, 128 => null,
@@ -4655,7 +4743,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
 
     const abi_size = @intCast(u32, ty.abiSize(self.target.*));
     const dst_alias = registerAlias(dst_reg, abi_size);
-    switch (mir_tag) {
+    switch (mir_tag[1]) {
         .vroundss, .vroundsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
             mir_tag,
             dst_alias,
@@ -4704,25 +4792,25 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
     defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
 
     const result: MCValue = result: {
-        const mir_tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {
+        const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag()) {
             .Float => switch (ty.floatBits(self.target.*)) {
                 16 => if (self.hasFeature(.f16c)) {
                     const mat_src_reg = if (src_mcv.isRegister())
                         src_mcv.getReg().?
                     else
                         try self.copyToTmpRegister(ty, src_mcv);
-                    try self.asmRegisterRegister(.vcvtph2ps, dst_reg, mat_src_reg.to128());
-                    try self.asmRegisterRegisterRegister(.vsqrtss, dst_reg, dst_reg, dst_reg);
+                    try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, mat_src_reg.to128());
+                    try self.asmRegisterRegisterRegister(.{ ._, .vsqrtss }, dst_reg, dst_reg, dst_reg);
                     try self.asmRegisterRegisterImmediate(
-                        .vcvtps2ph,
+                        .{ ._, .vcvtps2ph },
                         dst_reg,
                         dst_reg,
                         Immediate.u(0b1_00),
                     );
                     break :result dst_mcv;
                 } else null,
-                32 => if (self.hasFeature(.avx)) .vsqrtss else .sqrtss,
-                64 => if (self.hasFeature(.avx)) .vsqrtsd else .sqrtsd,
+                32 => if (self.hasFeature(.avx)) .{ ._, .vsqrtss } else .{ ._, .sqrtss },
+                64 => if (self.hasFeature(.avx)) .{ ._, .vsqrtsd } else .{ ._, .sqrtsd },
                 80, 128 => null,
                 else => unreachable,
             },
@@ -4731,16 +4819,21 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
                     16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen()) {
                         1 => {
                             try self.asmRegisterRegister(
-                                .vcvtph2ps,
+                                .{ ._, .vcvtph2ps },
                                 dst_reg,
                                 (if (src_mcv.isRegister())
                                     src_mcv.getReg().?
                                 else
                                     try self.copyToTmpRegister(ty, src_mcv)).to128(),
                             );
-                            try self.asmRegisterRegisterRegister(.vsqrtss, dst_reg, dst_reg, dst_reg);
+                            try self.asmRegisterRegisterRegister(
+                                .{ ._, .vsqrtss },
+                                dst_reg,
+                                dst_reg,
+                                dst_reg,
+                            );
                             try self.asmRegisterRegisterImmediate(
-                                .vcvtps2ph,
+                                .{ ._, .vcvtps2ph },
                                 dst_reg,
                                 dst_reg,
                                 Immediate.u(0b1_00),
@@ -4750,22 +4843,22 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
                         2...8 => {
                             const wide_reg = registerAlias(dst_reg, abi_size * 2);
                             if (src_mcv.isMemory()) try self.asmRegisterMemory(
-                                .vcvtph2ps,
+                                .{ ._, .vcvtph2ps },
                                 wide_reg,
                                 src_mcv.mem(Memory.PtrSize.fromSize(
                                     @intCast(u32, @divExact(wide_reg.bitSize(), 16)),
                                 )),
                             ) else try self.asmRegisterRegister(
-                                .vcvtph2ps,
+                                .{ ._, .vcvtph2ps },
                                 wide_reg,
                                 (if (src_mcv.isRegister())
                                     src_mcv.getReg().?
                                 else
                                     try self.copyToTmpRegister(ty, src_mcv)).to128(),
                             );
-                            try self.asmRegisterRegister(.vsqrtps, wide_reg, wide_reg);
+                            try self.asmRegisterRegister(.{ ._, .vsqrtps }, wide_reg, wide_reg);
                             try self.asmRegisterRegisterImmediate(
-                                .vcvtps2ph,
+                                .{ ._, .vcvtps2ph },
                                 dst_reg,
                                 wide_reg,
                                 Immediate.u(0b1_00),
@@ -4775,15 +4868,15 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
                         else => null,
                     } else null,
                     32 => switch (ty.vectorLen()) {
-                        1 => if (self.hasFeature(.avx)) .vsqrtss else .sqrtss,
-                        2...4 => if (self.hasFeature(.avx)) .vsqrtps else .sqrtps,
-                        5...8 => if (self.hasFeature(.avx)) .vsqrtps else null,
+                        1 => if (self.hasFeature(.avx)) .{ ._, .vsqrtss } else .{ ._, .sqrtss },
+                        2...4 => if (self.hasFeature(.avx)) .{ ._, .vsqrtps } else .{ ._, .sqrtps },
+                        5...8 => if (self.hasFeature(.avx)) .{ ._, .vsqrtps } else null,
                         else => null,
                     },
                     64 => switch (ty.vectorLen()) {
-                        1 => if (self.hasFeature(.avx)) .vsqrtsd else .sqrtsd,
-                        2 => if (self.hasFeature(.avx)) .vsqrtpd else .sqrtpd,
-                        3...4 => if (self.hasFeature(.avx)) .vsqrtpd else null,
+                        1 => if (self.hasFeature(.avx)) .{ ._, .vsqrtsd } else .{ ._, .sqrtsd },
+                        2 => if (self.hasFeature(.avx)) .{ ._, .vsqrtpd } else .{ ._, .sqrtpd },
+                        3...4 => if (self.hasFeature(.avx)) .{ ._, .vsqrtpd } else null,
                         else => null,
                     },
                     80, 128 => null,
@@ -4795,7 +4888,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
         })) |tag| tag else return self.fail("TODO implement airSqrt for {}", .{
             ty.fmt(self.bin_file.options.module.?),
         });
-        switch (mir_tag) {
+        switch (mir_tag[1]) {
             .vsqrtss, .vsqrtsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
                 mir_tag,
                 dst_reg,
@@ -4911,14 +5004,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
     if (load_abi_size <= 8) {
         const load_reg = registerAlias(dst_reg, load_abi_size);
         try self.asmRegisterMemory(
-            .mov,
+            .{ ._, .mov },
             load_reg,
             Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
                 .base = .{ .reg = ptr_reg },
                 .disp = val_byte_off,
             }),
         );
-        try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off));
+        try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
     } else {
         const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);
         const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
@@ -4926,7 +5019,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
 
         const dst_alias = registerAlias(dst_reg, val_abi_size);
         try self.asmRegisterMemory(
-            .mov,
+            .{ ._, .mov },
             dst_alias,
             Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
                 .base = .{ .reg = ptr_reg },
@@ -4934,14 +5027,19 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
             }),
         );
         try self.asmRegisterMemory(
-            .mov,
+            .{ ._, .mov },
             tmp_reg,
             Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
                 .base = .{ .reg = ptr_reg },
                 .disp = val_byte_off + 1,
             }),
         );
-        try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off));
+        try self.asmRegisterRegisterImmediate(
+            .{ ._rd, .sh },
+            dst_alias,
+            tmp_reg,
+            Immediate.u(val_bit_off),
+        );
     }
 
     if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
@@ -5047,13 +5145,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
         const part_mask_not = part_mask ^
             (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_abi_bits));
         if (limb_abi_size <= 4) {
-            try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.u(part_mask_not));
+            try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.u(part_mask_not));
         } else if (math.cast(i32, @bitCast(i64, part_mask_not))) |small| {
-            try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.s(small));
+            try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.s(small));
         } else {
             const part_mask_reg = try self.register_manager.allocReg(null, gp);
-            try self.asmRegisterImmediate(.mov, part_mask_reg, Immediate.u(part_mask_not));
-            try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg);
+            try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, Immediate.u(part_mask_not));
+            try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg);
         }
 
         if (src_bit_size <= 64) {
@@ -5064,14 +5162,26 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
 
             try self.genSetReg(tmp_reg, src_ty, src_mcv);
             switch (limb_i) {
-                0 => try self.genShiftBinOpMir(.shl, src_ty, tmp_mcv, .{ .immediate = src_bit_off }),
-                1 => try self.genShiftBinOpMir(.shr, src_ty, tmp_mcv, .{
-                    .immediate = limb_abi_bits - src_bit_off,
-                }),
+                0 => try self.genShiftBinOpMir(
+                    .{ ._l, .sh },
+                    src_ty,
+                    tmp_mcv,
+                    .{ .immediate = src_bit_off },
+                ),
+                1 => try self.genShiftBinOpMir(
+                    .{ ._r, .sh },
+                    src_ty,
+                    tmp_mcv,
+                    .{ .immediate = limb_abi_bits - src_bit_off },
+                ),
                 else => unreachable,
             }
-            try self.genBinOpMir(.@"and", src_ty, tmp_mcv, .{ .immediate = part_mask });
-            try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size));
+            try self.genBinOpMir(.{ ._, .@"and" }, src_ty, tmp_mcv, .{ .immediate = part_mask });
+            try self.asmMemoryRegister(
+                .{ ._, .@"or" },
+                limb_mem,
+                registerAlias(tmp_reg, limb_abi_size),
+            );
         } else return self.fail("TODO: implement packed store of {}", .{
             src_ty.fmt(self.bin_file.options.module.?),
         });
@@ -5171,7 +5281,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32
                     .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
                     else => mcv,
                 });
-                try self.genBinOpMir(.add, Type.usize, dst_mcv, .{ .register = offset_reg });
+                try self.genBinOpMir(.{ ._, .add }, Type.usize, dst_mcv, .{ .register = offset_reg });
                 break :result dst_mcv;
             },
             .indirect => |reg_off| break :result .{ .indirect = .{
@@ -5255,14 +5365,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
                 if (load_abi_size <= 8) {
                     const load_reg = registerAlias(dst_reg, load_abi_size);
                     try self.asmRegisterMemory(
-                        .mov,
+                        .{ ._, .mov },
                         load_reg,
                         Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
                             .base = .{ .frame = frame_addr.index },
                             .disp = frame_addr.off + field_byte_off,
                         }),
                     );
-                    try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(field_bit_off));
+                    try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
                 } else {
                     const tmp_reg = registerAlias(
                         try self.register_manager.allocReg(null, gp),
@@ -5273,7 +5383,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
 
                     const dst_alias = registerAlias(dst_reg, field_abi_size);
                     try self.asmRegisterMemory(
-                        .mov,
+                        .{ ._, .mov },
                         dst_alias,
                         Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
                             .base = .{ .frame = frame_addr.index },
@@ -5281,7 +5391,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
                         }),
                     );
                     try self.asmRegisterMemory(
-                        .mov,
+                        .{ ._, .mov },
                         tmp_reg,
                         Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
                             .base = .{ .frame = frame_addr.index },
@@ -5289,7 +5399,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
                         }),
                     );
                     try self.asmRegisterRegisterImmediate(
-                        .shrd,
+                        .{ ._rd, .sh },
                         dst_alias,
                         tmp_reg,
                         Immediate.u(field_bit_off),
@@ -5325,21 +5435,26 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
                 defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock);
 
                 // Shift by struct_field_offset.
-                try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv, .{ .immediate = field_off });
+                try self.genShiftBinOpMir(
+                    .{ ._r, .sh },
+                    Type.usize,
+                    dst_mcv,
+                    .{ .immediate = field_off },
+                );
 
                 // Mask to field_bit_size bits
                 const field_bit_size = field_ty.bitSize(self.target.*);
                 const mask = ~@as(u64, 0) >> @intCast(u6, 64 - field_bit_size);
 
                 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
-                try self.genBinOpMir(.@"and", Type.usize, dst_mcv, .{ .register = tmp_reg });
+                try self.genBinOpMir(.{ ._, .@"and" }, Type.usize, dst_mcv, .{ .register = tmp_reg });
 
                 const signedness =
                     if (field_ty.isAbiInt()) field_ty.intInfo(self.target.*).signedness else .unsigned;
                 const field_byte_size = @intCast(u32, field_ty.abiSize(self.target.*));
                 if (signedness == .signed and field_byte_size < 8) {
                     try self.asmRegisterRegister(
-                        if (field_byte_size >= 4) .movsxd else .movsx,
+                        if (field_byte_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx },
                         dst_mcv.register,
                         registerAlias(dst_mcv.register, field_byte_size),
                     );
@@ -5451,17 +5566,17 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
 
                 if (limb_pl.base.tag == .int_unsigned and self.regExtraBits(limb_ty) > 0) {
                     const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_pl.data);
-                    try self.genBinOpMir(.xor, limb_ty, limb_mcv, .{ .immediate = mask });
-                } else try self.genUnOpMir(.not, limb_ty, limb_mcv);
+                    try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });
+                } else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
             }
         },
-        .neg => try self.genUnOpMir(.neg, src_ty, dst_mcv),
+        .neg => try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv),
         else => unreachable,
     }
     return dst_mcv;
 }
 
-fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {
+fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
     const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
     if (abi_size > 8) return self.fail("TODO implement {} for {}", .{
         mir_tag,
@@ -5504,7 +5619,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue
 /// Clobbers .rcx for non-immediate shift value.
 fn genShiftBinOpMir(
     self: *Self,
-    tag: Mir.Inst.Tag,
+    tag: Mir.Inst.FixedTag,
     ty: Type,
     lhs_mcv: MCValue,
     shift_mcv: MCValue,
@@ -5589,16 +5704,16 @@ fn genShiftBinOpMir(
         const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
         defer self.register_manager.unlockReg(tmp_lock);
 
-        const info: struct { offsets: [2]i32, double_tag: Mir.Inst.Tag } = switch (tag) {
-            .shl, .sal => .{ .offsets = .{ 0, 8 }, .double_tag = .shld },
-            .shr, .sar => .{ .offsets = .{ 8, 0 }, .double_tag = .shrd },
+        const info: struct { offsets: [2]i32, double_tag: Mir.Inst.FixedTag } = switch (tag[0]) {
+            ._l => .{ .offsets = .{ 0, 8 }, .double_tag = .{ ._ld, .sh } },
+            ._r => .{ .offsets = .{ 8, 0 }, .double_tag = .{ ._rd, .sh } },
             else => unreachable,
         };
         switch (lhs_mcv) {
             .load_frame => |dst_frame_addr| switch (rhs_mcv) {
                 .immediate => |rhs_imm| if (rhs_imm == 0) {} else if (rhs_imm < 64) {
                     try self.asmRegisterMemory(
-                        .mov,
+                        .{ ._, .mov },
                         tmp_reg,
                         Memory.sib(.qword, .{
                             .base = .{ .frame = dst_frame_addr.index },
@@ -5625,7 +5740,7 @@ fn genShiftBinOpMir(
                 } else {
                     assert(rhs_imm < 128);
                     try self.asmRegisterMemory(
-                        .mov,
+                        .{ ._, .mov },
                         tmp_reg,
                         Memory.sib(.qword, .{
                             .base = .{ .frame = dst_frame_addr.index },
@@ -5636,34 +5751,30 @@ fn genShiftBinOpMir(
                         try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));
                     }
                     try self.asmMemoryRegister(
-                        .mov,
+                        .{ ._, .mov },
                         Memory.sib(.qword, .{
                             .base = .{ .frame = dst_frame_addr.index },
                             .disp = dst_frame_addr.off + info.offsets[1],
                         }),
                         tmp_reg,
                     );
-                    switch (tag) {
-                        .shl, .sal, .shr => {
-                            try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());
-                            try self.asmMemoryRegister(
-                                .mov,
-                                Memory.sib(.qword, .{
-                                    .base = .{ .frame = dst_frame_addr.index },
-                                    .disp = dst_frame_addr.off + info.offsets[0],
-                                }),
-                                tmp_reg,
-                            );
-                        },
-                        .sar => try self.asmMemoryImmediate(
-                            tag,
+                    if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryImmediate(
+                        tag,
+                        Memory.sib(.qword, .{
+                            .base = .{ .frame = dst_frame_addr.index },
+                            .disp = dst_frame_addr.off + info.offsets[0],
+                        }),
+                        Immediate.u(63),
+                    ) else {
+                        try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());
+                        try self.asmMemoryRegister(
+                            .{ ._, .mov },
                             Memory.sib(.qword, .{
                                 .base = .{ .frame = dst_frame_addr.index },
                                 .disp = dst_frame_addr.off + info.offsets[0],
                             }),
-                            Immediate.u(63),
-                        ),
-                        else => unreachable,
+                            tmp_reg,
+                        );
                     }
                 },
                 else => {
@@ -5677,7 +5788,7 @@ fn genShiftBinOpMir(
 
                     try self.genSetReg(.cl, Type.u8, rhs_mcv);
                     try self.asmRegisterMemory(
-                        .mov,
+                        .{ ._, .mov },
                         first_reg,
                         Memory.sib(.qword, .{
                             .base = .{ .frame = dst_frame_addr.index },
@@ -5685,32 +5796,28 @@ fn genShiftBinOpMir(
                         }),
                     );
                     try self.asmRegisterMemory(
-                        .mov,
+                        .{ ._, .mov },
                         second_reg,
                         Memory.sib(.qword, .{
                             .base = .{ .frame = dst_frame_addr.index },
                             .disp = dst_frame_addr.off + info.offsets[1],
                         }),
                     );
-                    switch (tag) {
-                        .shl, .sal, .shr => try self.asmRegisterRegister(
-                            .xor,
-                            tmp_reg.to32(),
-                            tmp_reg.to32(),
-                        ),
-                        .sar => {
-                            try self.asmRegisterRegister(.mov, tmp_reg, first_reg);
-                            try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
-                        },
-                        else => unreachable,
-                    }
+                    if (tag[0] == ._r and tag[1] == .sa) {
+                        try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);
+                        try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
+                    } else try self.asmRegisterRegister(
+                        .{ ._, .xor },
+                        tmp_reg.to32(),
+                        tmp_reg.to32(),
+                    );
                     try self.asmRegisterRegisterRegister(info.double_tag, second_reg, first_reg, .cl);
                     try self.asmRegisterRegister(tag, first_reg, .cl);
-                    try self.asmRegisterImmediate(.cmp, .cl, Immediate.u(64));
+                    try self.asmRegisterImmediate(.{ ._, .cmp }, .cl, Immediate.u(64));
                     try self.asmCmovccRegisterRegister(second_reg, first_reg, .ae);
                     try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);
                     try self.asmMemoryRegister(
-                        .mov,
+                        .{ ._, .mov },
                         Memory.sib(.qword, .{
                             .base = .{ .frame = dst_frame_addr.index },
                             .disp = dst_frame_addr.off + info.offsets[1],
@@ -5718,7 +5825,7 @@ fn genShiftBinOpMir(
                         second_reg,
                     );
                     try self.asmMemoryRegister(
-                        .mov,
+                        .{ ._, .mov },
                         Memory.sib(.qword, .{
                             .base = .{ .frame = dst_frame_addr.index },
                             .disp = dst_frame_addr.off + info.offsets[0],
@@ -5743,7 +5850,7 @@ fn genShiftBinOpMir(
 /// Asserts .rcx is free.
 fn genShiftBinOp(
     self: *Self,
-    tag: Air.Inst.Tag,
+    air_tag: Air.Inst.Tag,
     maybe_inst: ?Air.Inst.Index,
     lhs_mcv: MCValue,
     rhs_mcv: MCValue,
@@ -5788,14 +5895,14 @@ fn genShiftBinOp(
     };
 
     const signedness = lhs_ty.intInfo(self.target.*).signedness;
-    try self.genShiftBinOpMir(switch (tag) {
+    try self.genShiftBinOpMir(switch (air_tag) {
         .shl, .shl_exact => switch (signedness) {
-            .signed => .sal,
-            .unsigned => .shl,
+            .signed => .{ ._l, .sa },
+            .unsigned => .{ ._l, .sh },
         },
         .shr, .shr_exact => switch (signedness) {
-            .signed => .sar,
-            .unsigned => .shr,
+            .signed => .{ ._r, .sa },
+            .unsigned => .{ ._r, .sh },
         },
         else => unreachable,
     }, lhs_ty, dst_mcv, rhs_mcv);
@@ -5855,20 +5962,18 @@ fn genMulDivBinOp(
             try self.register_manager.getReg(.rax, track_inst_rax);
             try self.register_manager.getReg(.rdx, track_inst_rdx);
 
-            const mir_tag: Mir.Inst.Tag = switch (signedness) {
+            try self.genIntMulDivOpMir(switch (signedness) {
                 .signed => switch (tag) {
-                    .mul, .mulwrap => .imul,
-                    .div_trunc, .div_exact, .rem => .idiv,
+                    .mul, .mulwrap => .{ .i_, .mul },
+                    .div_trunc, .div_exact, .rem => .{ .i_, .div },
                     else => unreachable,
                 },
                 .unsigned => switch (tag) {
-                    .mul, .mulwrap => .mul,
-                    .div_trunc, .div_exact, .rem => .div,
+                    .mul, .mulwrap => .{ ._, .mul },
+                    .div_trunc, .div_exact, .rem => .{ ._, .div },
                     else => unreachable,
                 },
-            };
-
-            try self.genIntMulDivOpMir(mir_tag, ty, lhs, rhs);
+            }, ty, lhs, rhs);
 
             if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) {
                 .mul, .mulwrap, .div_trunc, .div_exact => .rax,
@@ -5878,7 +5983,7 @@ fn genMulDivBinOp(
 
             const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);
             try self.asmMemoryRegister(
-                .mov,
+                .{ ._, .mov },
                 Memory.sib(.qword, .{
                     .base = .{ .frame = dst_mcv.load_frame.index },
                     .disp = dst_mcv.load_frame.off,
@@ -5886,7 +5991,7 @@ fn genMulDivBinOp(
                 .rax,
             );
             try self.asmMemoryRegister(
-                .mov,
+                .{ ._, .mov },
                 Memory.sib(.qword, .{
                     .base = .{ .frame = dst_mcv.load_frame.index },
                     .disp = dst_mcv.load_frame.off + 8,
@@ -5927,12 +6032,12 @@ fn genMulDivBinOp(
                         try self.copyToRegisterWithInstTracking(inst, ty, lhs)
                     else
                         .{ .register = try self.copyToTmpRegister(ty, lhs) };
-                    try self.genBinOpMir(.sub, ty, result, div_floor);
+                    try self.genBinOpMir(.{ ._, .sub }, ty, result, div_floor);
 
                     return result;
                 },
                 .unsigned => {
-                    try self.genIntMulDivOpMir(.div, ty, lhs, rhs);
+                    try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs, rhs);
                     return .{ .register = registerAlias(.rdx, abi_size) };
                 },
             }
@@ -5974,7 +6079,7 @@ fn genMulDivBinOp(
             switch (signedness) {
                 .signed => return try self.genInlineIntDivFloor(ty, lhs, actual_rhs),
                 .unsigned => {
-                    try self.genIntMulDivOpMir(.div, ty, lhs, actual_rhs);
+                    try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs, actual_rhs);
                     return .{ .register = registerAlias(.rax, abi_size) };
                 },
             }
@@ -6072,11 +6177,11 @@ fn genBinOp(
         switch (air_tag) {
             .add,
             .addwrap,
-            => try self.genBinOpMir(.add, lhs_ty, dst_mcv, src_mcv),
+            => try self.genBinOpMir(.{ ._, .add }, lhs_ty, dst_mcv, src_mcv),
 
             .sub,
             .subwrap,
-            => try self.genBinOpMir(.sub, lhs_ty, dst_mcv, src_mcv),
+            => try self.genBinOpMir(.{ ._, .sub }, lhs_ty, dst_mcv, src_mcv),
 
             .ptr_add,
             .ptr_sub,
@@ -6088,22 +6193,27 @@ fn genBinOp(
 
                 const elem_size = lhs_ty.elemType2().abiSize(self.target.*);
                 try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size });
-                try self.genBinOpMir(switch (air_tag) {
-                    .ptr_add => .add,
-                    .ptr_sub => .sub,
-                    else => unreachable,
-                }, lhs_ty, dst_mcv, tmp_mcv);
+                try self.genBinOpMir(
+                    switch (air_tag) {
+                        .ptr_add => .{ ._, .add },
+                        .ptr_sub => .{ ._, .sub },
+                        else => unreachable,
+                    },
+                    lhs_ty,
+                    dst_mcv,
+                    tmp_mcv,
+                );
             },
 
             .bool_or,
             .bit_or,
-            => try self.genBinOpMir(.@"or", lhs_ty, dst_mcv, src_mcv),
+            => try self.genBinOpMir(.{ ._, .@"or" }, lhs_ty, dst_mcv, src_mcv),
 
             .bool_and,
             .bit_and,
-            => try self.genBinOpMir(.@"and", lhs_ty, dst_mcv, src_mcv),
+            => try self.genBinOpMir(.{ ._, .@"and" }, lhs_ty, dst_mcv, src_mcv),
 
-            .xor => try self.genBinOpMir(.xor, lhs_ty, dst_mcv, src_mcv),
+            .xor => try self.genBinOpMir(.{ ._, .xor }, lhs_ty, dst_mcv, src_mcv),
 
             .min,
             .max,
@@ -6129,7 +6239,7 @@ fn genBinOp(
                 };
                 defer if (mat_mcv_lock) |lock| self.register_manager.unlockReg(lock);
 
-                try self.genBinOpMir(.cmp, lhs_ty, dst_mcv, mat_src_mcv);
+                try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, dst_mcv, mat_src_mcv);
 
                 const int_info = lhs_ty.intInfo(self.target.*);
                 const cc: Condition = switch (int_info.signedness) {
@@ -6206,7 +6316,7 @@ fn genBinOp(
     }
 
     const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
-    const mir_tag = if (@as(?Mir.Inst.Tag, switch (lhs_ty.zigTypeTag()) {
+    const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag()) {
         else => unreachable,
         .Float => switch (lhs_ty.floatBits(self.target.*)) {
             16 => if (self.hasFeature(.f16c)) {
@@ -6215,13 +6325,13 @@ fn genBinOp(
                 defer self.register_manager.unlockReg(tmp_lock);
 
                 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
-                    .vpinsrw,
+                    .{ ._, .vpinsrw },
                     dst_reg,
                     dst_reg,
                     src_mcv.mem(.word),
                     Immediate.u(1),
                 ) else try self.asmRegisterRegisterRegister(
-                    .vpunpcklwd,
+                    .{ ._, .vpunpcklwd },
                     dst_reg,
                     dst_reg,
                     (if (src_mcv.isRegister())
@@ -6229,15 +6339,15 @@ fn genBinOp(
                     else
                         try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
                 );
-                try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
-                try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);
+                try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
+                try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp_reg, dst_reg);
                 try self.asmRegisterRegisterRegister(
                     switch (air_tag) {
-                        .add => .vaddss,
-                        .sub => .vsubss,
-                        .div_float, .div_trunc, .div_floor, .div_exact => .vdivss,
-                        .max => .vmaxss,
-                        .min => .vmaxss,
+                        .add => .{ ._, .vaddss },
+                        .sub => .{ ._, .vsubss },
+                        .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivss },
+                        .max => .{ ._, .vmaxss },
+                        .min => .{ ._, .vmaxss },
                         else => unreachable,
                     },
                     dst_reg,
@@ -6245,7 +6355,7 @@ fn genBinOp(
                     tmp_reg,
                 );
                 try self.asmRegisterRegisterImmediate(
-                    .vcvtps2ph,
+                    .{ ._, .vcvtps2ph },
                     dst_reg,
                     dst_reg,
                     Immediate.u(0b1_00),
@@ -6253,29 +6363,29 @@ fn genBinOp(
                 return dst_mcv;
             } else null,
             32 => switch (air_tag) {
-                .add => if (self.hasFeature(.avx)) .vaddss else .addss,
-                .sub => if (self.hasFeature(.avx)) .vsubss else .subss,
-                .mul => if (self.hasFeature(.avx)) .vmulss else .mulss,
+                .add => if (self.hasFeature(.avx)) .{ ._, .vaddss } else .{ ._, .addss },
+                .sub => if (self.hasFeature(.avx)) .{ ._, .vsubss } else .{ ._, .subss },
+                .mul => if (self.hasFeature(.avx)) .{ ._, .vmulss } else .{ ._, .mulss },
                 .div_float,
                 .div_trunc,
                 .div_floor,
                 .div_exact,
-                => if (self.hasFeature(.avx)) .vdivss else .divss,
-                .max => if (self.hasFeature(.avx)) .vmaxss else .maxss,
-                .min => if (self.hasFeature(.avx)) .vminss else .minss,
+                => if (self.hasFeature(.avx)) .{ ._, .vdivss } else .{ ._, .divss },
+                .max => if (self.hasFeature(.avx)) .{ ._, .vmaxss } else .{ ._, .maxss },
+                .min => if (self.hasFeature(.avx)) .{ ._, .vminss } else .{ ._, .minss },
                 else => unreachable,
             },
             64 => switch (air_tag) {
-                .add => if (self.hasFeature(.avx)) .vaddsd else .addsd,
-                .sub => if (self.hasFeature(.avx)) .vsubsd else .subsd,
-                .mul => if (self.hasFeature(.avx)) .vmulsd else .mulsd,
+                .add => if (self.hasFeature(.avx)) .{ ._, .vaddsd } else .{ ._, .addsd },
+                .sub => if (self.hasFeature(.avx)) .{ ._, .vsubsd } else .{ ._, .subsd },
+                .mul => if (self.hasFeature(.avx)) .{ ._, .vmulsd } else .{ ._, .mulsd },
                 .div_float,
                 .div_trunc,
                 .div_floor,
                 .div_exact,
-                => if (self.hasFeature(.avx)) .vdivsd else .divsd,
-                .max => if (self.hasFeature(.avx)) .vmaxsd else .maxsd,
-                .min => if (self.hasFeature(.avx)) .vminsd else .minsd,
+                => if (self.hasFeature(.avx)) .{ ._, .vdivsd } else .{ ._, .divsd },
+                .max => if (self.hasFeature(.avx)) .{ ._, .vmaxsd } else .{ ._, .maxsd },
+                .min => if (self.hasFeature(.avx)) .{ ._, .vminsd } else .{ ._, .minsd },
                 else => unreachable,
             },
             80, 128 => null,
@@ -6291,13 +6401,13 @@ fn genBinOp(
                         defer self.register_manager.unlockReg(tmp_lock);
 
                         if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
-                            .vpinsrw,
+                            .{ ._, .vpinsrw },
                             dst_reg,
                             dst_reg,
                             src_mcv.mem(.word),
                             Immediate.u(1),
                         ) else try self.asmRegisterRegisterRegister(
-                            .vpunpcklwd,
+                            .{ ._, .vpunpcklwd },
                             dst_reg,
                             dst_reg,
                             (if (src_mcv.isRegister())
@@ -6305,15 +6415,15 @@ fn genBinOp(
                             else
                                 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
                         );
-                        try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
-                        try self.asmRegisterRegister(.vmovshdup, tmp_reg, dst_reg);
+                        try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
+                        try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp_reg, dst_reg);
                         try self.asmRegisterRegisterRegister(
                             switch (air_tag) {
-                                .add => .vaddss,
-                                .sub => .vsubss,
-                                .div_float, .div_trunc, .div_floor, .div_exact => .vdivss,
-                                .max => .vmaxss,
-                                .min => .vmaxss,
+                                .add => .{ ._, .vaddss },
+                                .sub => .{ ._, .vsubss },
+                                .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivss },
+                                .max => .{ ._, .vmaxss },
+                                .min => .{ ._, .vmaxss },
                                 else => unreachable,
                             },
                             dst_reg,
@@ -6321,7 +6431,7 @@ fn genBinOp(
                             tmp_reg,
                         );
                         try self.asmRegisterRegisterImmediate(
-                            .vcvtps2ph,
+                            .{ ._, .vcvtps2ph },
                             dst_reg,
                             dst_reg,
                             Immediate.u(0b1_00),
@@ -6334,12 +6444,12 @@ fn genBinOp(
                         defer self.register_manager.unlockReg(tmp_lock);
 
                         if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
-                            .vpinsrd,
+                            .{ ._, .vpinsrd },
                             dst_reg,
                             src_mcv.mem(.dword),
                             Immediate.u(1),
                         ) else try self.asmRegisterRegisterRegister(
-                            .vunpcklps,
+                            .{ ._, .vunpcklps },
                             dst_reg,
                             dst_reg,
                             (if (src_mcv.isRegister())
@@ -6347,15 +6457,20 @@ fn genBinOp(
                             else
                                 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
                         );
-                        try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
-                        try self.asmRegisterRegisterRegister(.vmovhlps, tmp_reg, dst_reg, dst_reg);
+                        try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
+                        try self.asmRegisterRegisterRegister(
+                            .{ ._, .vmovhlps },
+                            tmp_reg,
+                            dst_reg,
+                            dst_reg,
+                        );
                         try self.asmRegisterRegisterRegister(
                             switch (air_tag) {
-                                .add => .vaddps,
-                                .sub => .vsubps,
-                                .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
-                                .max => .vmaxps,
-                                .min => .vmaxps,
+                                .add => .{ ._, .vaddps },
+                                .sub => .{ ._, .vsubps },
+                                .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
+                                .max => .{ ._, .vmaxps },
+                                .min => .{ ._, .vmaxps },
                                 else => unreachable,
                             },
                             dst_reg,
@@ -6363,7 +6478,7 @@ fn genBinOp(
                             tmp_reg,
                         );
                         try self.asmRegisterRegisterImmediate(
-                            .vcvtps2ph,
+                            .{ ._, .vcvtps2ph },
                             dst_reg,
                             dst_reg,
                             Immediate.u(0b1_00),
@@ -6375,13 +6490,13 @@ fn genBinOp(
                         const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
                         defer self.register_manager.unlockReg(tmp_lock);
 
-                        try self.asmRegisterRegister(.vcvtph2ps, dst_reg, dst_reg);
+                        try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
                         if (src_mcv.isMemory()) try self.asmRegisterMemory(
-                            .vcvtph2ps,
+                            .{ ._, .vcvtph2ps },
                             tmp_reg,
                             src_mcv.mem(.qword),
                         ) else try self.asmRegisterRegister(
-                            .vcvtph2ps,
+                            .{ ._, .vcvtph2ps },
                             tmp_reg,
                             (if (src_mcv.isRegister())
                                 src_mcv.getReg().?
@@ -6390,11 +6505,11 @@ fn genBinOp(
                         );
                         try self.asmRegisterRegisterRegister(
                             switch (air_tag) {
-                                .add => .vaddps,
-                                .sub => .vsubps,
-                                .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
-                                .max => .vmaxps,
-                                .min => .vmaxps,
+                                .add => .{ ._, .vaddps },
+                                .sub => .{ ._, .vsubps },
+                                .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
+                                .max => .{ ._, .vmaxps },
+                                .min => .{ ._, .vmaxps },
                                 else => unreachable,
                             },
                             dst_reg,
@@ -6402,7 +6517,7 @@ fn genBinOp(
                             tmp_reg,
                         );
                         try self.asmRegisterRegisterImmediate(
-                            .vcvtps2ph,
+                            .{ ._, .vcvtps2ph },
                             dst_reg,
                             dst_reg,
                             Immediate.u(0b1_00),
@@ -6414,13 +6529,13 @@ fn genBinOp(
                         const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
                         defer self.register_manager.unlockReg(tmp_lock);
 
-                        try self.asmRegisterRegister(.vcvtph2ps, dst_reg.to256(), dst_reg);
+                        try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg.to256(), dst_reg);
                         if (src_mcv.isMemory()) try self.asmRegisterMemory(
-                            .vcvtph2ps,
+                            .{ ._, .vcvtph2ps },
                             tmp_reg,
                             src_mcv.mem(.xword),
                         ) else try self.asmRegisterRegister(
-                            .vcvtph2ps,
+                            .{ ._, .vcvtph2ps },
                             tmp_reg,
                             (if (src_mcv.isRegister())
                                 src_mcv.getReg().?
@@ -6429,11 +6544,11 @@ fn genBinOp(
                         );
                         try self.asmRegisterRegisterRegister(
                             switch (air_tag) {
-                                .add => .vaddps,
-                                .sub => .vsubps,
-                                .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
-                                .max => .vmaxps,
-                                .min => .vmaxps,
+                                .add => .{ ._, .vaddps },
+                                .sub => .{ ._, .vsubps },
+                                .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
+                                .max => .{ ._, .vmaxps },
+                                .min => .{ ._, .vmaxps },
                                 else => unreachable,
                             },
                             dst_reg.to256(),
@@ -6441,7 +6556,7 @@ fn genBinOp(
                             tmp_reg,
                         );
                         try self.asmRegisterRegisterImmediate(
-                            .vcvtps2ph,
+                            .{ ._, .vcvtps2ph },
                             dst_reg,
                             dst_reg.to256(),
                             Immediate.u(0b1_00),
@@ -6452,76 +6567,76 @@ fn genBinOp(
                 } else null,
                 32 => switch (lhs_ty.vectorLen()) {
                     1 => switch (air_tag) {
-                        .add => if (self.hasFeature(.avx)) .vaddss else .addss,
-                        .sub => if (self.hasFeature(.avx)) .vsubss else .subss,
-                        .mul => if (self.hasFeature(.avx)) .vmulss else .mulss,
+                        .add => if (self.hasFeature(.avx)) .{ ._, .vaddss } else .{ ._, .addss },
+                        .sub => if (self.hasFeature(.avx)) .{ ._, .vsubss } else .{ ._, .subss },
+                        .mul => if (self.hasFeature(.avx)) .{ ._, .vmulss } else .{ ._, .mulss },
                         .div_float,
                         .div_trunc,
                         .div_floor,
                         .div_exact,
-                        => if (self.hasFeature(.avx)) .vdivss else .divss,
-                        .max => if (self.hasFeature(.avx)) .vmaxss else .maxss,
-                        .min => if (self.hasFeature(.avx)) .vminss else .minss,
+                        => if (self.hasFeature(.avx)) .{ ._, .vdivss } else .{ ._, .divss },
+                        .max => if (self.hasFeature(.avx)) .{ ._, .vmaxss } else .{ ._, .maxss },
+                        .min => if (self.hasFeature(.avx)) .{ ._, .vminss } else .{ ._, .minss },
                         else => unreachable,
                     },
                     2...4 => switch (air_tag) {
-                        .add => if (self.hasFeature(.avx)) .vaddps else .addps,
-                        .sub => if (self.hasFeature(.avx)) .vsubps else .subps,
-                        .mul => if (self.hasFeature(.avx)) .vmulps else .mulps,
+                        .add => if (self.hasFeature(.avx)) .{ ._, .vaddps } else .{ ._, .addps },
+                        .sub => if (self.hasFeature(.avx)) .{ ._, .vsubps } else .{ ._, .subps },
+                        .mul => if (self.hasFeature(.avx)) .{ ._, .vmulps } else .{ ._, .mulps },
                         .div_float,
                         .div_trunc,
                         .div_floor,
                         .div_exact,
-                        => if (self.hasFeature(.avx)) .vdivps else .divps,
-                        .max => if (self.hasFeature(.avx)) .vmaxps else .maxps,
-                        .min => if (self.hasFeature(.avx)) .vminps else .minps,
+                        => if (self.hasFeature(.avx)) .{ ._, .vdivps } else .{ ._, .divps },
+                        .max => if (self.hasFeature(.avx)) .{ ._, .vmaxps } else .{ ._, .maxps },
+                        .min => if (self.hasFeature(.avx)) .{ ._, .vminps } else .{ ._, .minps },
                         else => unreachable,
                     },
                     5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
-                        .add => .vaddps,
-                        .sub => .vsubps,
-                        .mul => .vmulps,
-                        .div_float, .div_trunc, .div_floor, .div_exact => .vdivps,
-                        .max => .vmaxps,
-                        .min => .vminps,
+                        .add => .{ ._, .vaddps },
+                        .sub => .{ ._, .vsubps },
+                        .mul => .{ ._, .vmulps },
+                        .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
+                        .max => .{ ._, .vmaxps },
+                        .min => .{ ._, .vminps },
                         else => unreachable,
                     } else null,
                     else => null,
                 },
                 64 => switch (lhs_ty.vectorLen()) {
                     1 => switch (air_tag) {
-                        .add => if (self.hasFeature(.avx)) .vaddsd else .addsd,
-                        .sub => if (self.hasFeature(.avx)) .vsubsd else .subsd,
-                        .mul => if (self.hasFeature(.avx)) .vmulsd else .mulsd,
+                        .add => if (self.hasFeature(.avx)) .{ ._, .vaddsd } else .{ ._, .addsd },
+                        .sub => if (self.hasFeature(.avx)) .{ ._, .vsubsd } else .{ ._, .subsd },
+                        .mul => if (self.hasFeature(.avx)) .{ ._, .vmulsd } else .{ ._, .mulsd },
                         .div_float,
                         .div_trunc,
                         .div_floor,
                         .div_exact,
-                        => if (self.hasFeature(.avx)) .vdivsd else .divsd,
-                        .max => if (self.hasFeature(.avx)) .vmaxsd else .maxsd,
-                        .min => if (self.hasFeature(.avx)) .vminsd else .minsd,
+                        => if (self.hasFeature(.avx)) .{ ._, .vdivsd } else .{ ._, .divsd },
+                        .max => if (self.hasFeature(.avx)) .{ ._, .vmaxsd } else .{ ._, .maxsd },
+                        .min => if (self.hasFeature(.avx)) .{ ._, .vminsd } else .{ ._, .minsd },
                         else => unreachable,
                     },
                     2 => switch (air_tag) {
-                        .add => if (self.hasFeature(.avx)) .vaddpd else .addpd,
-                        .sub => if (self.hasFeature(.avx)) .vsubpd else .subpd,
-                        .mul => if (self.hasFeature(.avx)) .vmulpd else .mulpd,
+                        .add => if (self.hasFeature(.avx)) .{ ._, .vaddpd } else .{ ._, .addpd },
+                        .sub => if (self.hasFeature(.avx)) .{ ._, .vsubpd } else .{ ._, .subpd },
+                        .mul => if (self.hasFeature(.avx)) .{ ._, .vmulpd } else .{ ._, .mulpd },
                         .div_float,
                         .div_trunc,
                         .div_floor,
                         .div_exact,
-                        => if (self.hasFeature(.avx)) .vdivpd else .divpd,
-                        .max => if (self.hasFeature(.avx)) .vmaxpd else .maxpd,
-                        .min => if (self.hasFeature(.avx)) .vminpd else .minpd,
+                        => if (self.hasFeature(.avx)) .{ ._, .vdivpd } else .{ ._, .divpd },
+                        .max => if (self.hasFeature(.avx)) .{ ._, .vmaxpd } else .{ ._, .maxpd },
+                        .min => if (self.hasFeature(.avx)) .{ ._, .vminpd } else .{ ._, .minpd },
                         else => unreachable,
                     },
                     3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
-                        .add => .vaddpd,
-                        .sub => .vsubpd,
-                        .mul => .vmulpd,
-                        .div_float, .div_trunc, .div_floor, .div_exact => .vdivpd,
-                        .max => .vmaxpd,
-                        .min => .vminpd,
+                        .add => .{ ._, .vaddpd },
+                        .sub => .{ ._, .vsubpd },
+                        .mul => .{ ._, .vmulpd },
+                        .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivpd },
+                        .max => .{ ._, .vmaxpd },
+                        .min => .{ ._, .vminpd },
                         else => unreachable,
                     } else null,
                     else => null,
@@ -6583,7 +6698,13 @@ fn genBinOp(
     return dst_mcv;
 }
 
-fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
+fn genBinOpMir(
+    self: *Self,
+    mir_tag: Mir.Inst.FixedTag,
+    ty: Type,
+    dst_mcv: MCValue,
+    src_mcv: MCValue,
+) !void {
     const abi_size = @intCast(u32, ty.abiSize(self.target.*));
     switch (dst_mcv) {
         .none,
@@ -6788,14 +6909,14 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
             };
             var off: i32 = 0;
             while (off < abi_size) : (off += 8) {
-                const mir_limb_tag = switch (off) {
+                const mir_limb_tag: Mir.Inst.FixedTag = switch (off) {
                     0 => mir_tag,
-                    else => switch (mir_tag) {
-                        .add => .adc,
-                        .sub, .cmp => .sbb,
+                    else => switch (mir_tag[1]) {
+                        .add => .{ ._, .adc },
+                        .sub, .cmp => .{ ._, .sbb },
                         .@"or", .@"and", .xor => mir_tag,
                         else => return self.fail("TODO genBinOpMir implement large ABI for {s}", .{
-                            @tagName(mir_tag),
+                            @tagName(mir_tag[1]),
                         }),
                     },
                 };
@@ -6967,14 +7088,14 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
                 .reserved_frame,
                 => unreachable,
                 .register => |src_reg| try self.asmRegisterRegister(
-                    .imul,
+                    .{ .i_, .mul },
                     dst_alias,
                     registerAlias(src_reg, abi_size),
                 ),
                 .immediate => |imm| {
                     if (math.cast(i32, imm)) |small| {
                         try self.asmRegisterRegisterImmediate(
-                            .imul,
+                            .{ .i_, .mul },
                             dst_alias,
                             dst_alias,
                             Immediate.s(small),
@@ -6994,19 +7115,19 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
                 .lea_tlv,
                 .lea_frame,
                 => try self.asmRegisterRegister(
-                    .imul,
+                    .{ .i_, .mul },
                     dst_alias,
                     registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
                 ),
                 .memory, .indirect, .load_frame => try self.asmRegisterMemory(
-                    .imul,
+                    .{ .i_, .mul },
                     dst_alias,
                     Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
                         .memory => |addr| .{
                             .base = .{ .reg = .ds },
                             .disp = math.cast(i32, @bitCast(i64, addr)) orelse
                                 return self.asmRegisterRegister(
-                                .imul,
+                                .{ .i_, .mul },
                                 dst_alias,
                                 registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
                             ),
@@ -7131,12 +7252,12 @@ fn genVarDbgInfo(
 }
 
 fn airTrap(self: *Self) !void {
-    try self.asmOpOnly(.ud2);
+    try self.asmOpOnly(.{ ._, .ud2 });
     return self.finishAirBookkeeping();
 }
 
 fn airBreakpoint(self: *Self) !void {
-    try self.asmOpOnly(.int3);
+    try self.asmOpOnly(.{ ._, .int3 });
     return self.finishAirBookkeeping();
 }
 
@@ -7157,7 +7278,7 @@ fn airFence(self: *Self, inst: Air.Inst.Index) !void {
     switch (order) {
         .Unordered, .Monotonic => unreachable,
         .Acquire, .Release, .AcqRel => {},
-        .SeqCst => try self.asmOpOnly(.mfence),
+        .SeqCst => try self.asmOpOnly(.{ ._, .mfence }),
     }
     return self.finishAirBookkeeping();
 }
@@ -7251,7 +7372,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
                 const atom = elf_file.getAtom(atom_index);
                 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
                 const got_addr = atom.getOffsetTableAddress(elf_file);
-                try self.asmMemory(.call, Memory.sib(.qword, .{
+                try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
                     .base = .{ .reg = .ds },
                     .disp = @intCast(i32, got_addr),
                 }));
@@ -7259,12 +7380,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
                 const atom = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
                 const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
                 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
-                try self.asmRegister(.call, .rax);
+                try self.asmRegister(.{ ._, .call }, .rax);
             } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
                 const atom = try macho_file.getOrCreateAtomForDecl(func.owner_decl);
                 const sym_index = macho_file.getAtom(atom).getSymbolIndex().?;
                 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
-                try self.asmRegister(.call, .rax);
+                try self.asmRegister(.{ ._, .call }, .rax);
             } else if (self.bin_file.cast(link.File.Plan9)) |p9| {
                 const decl_block_index = try p9.seeDecl(func.owner_decl);
                 const decl_block = p9.getDeclBlock(decl_block_index);
@@ -7273,7 +7394,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
                 const got_addr = p9.bases.data;
                 const got_index = decl_block.got_index.?;
                 const fn_got_addr = got_addr + got_index * ptr_bytes;
-                try self.asmMemory(.call, Memory.sib(.qword, .{
+                try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
                     .base = .{ .reg = .ds },
                     .disp = @intCast(i32, fn_got_addr),
                 }));
@@ -7296,7 +7417,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
                         }),
                     } },
                 });
-                try self.asmRegister(.call, .rax);
+                try self.asmRegister(.{ ._, .call }, .rax);
             } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
                 const atom_index = try self.owner.getSymbolIndex(self);
                 const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);
@@ -7318,7 +7439,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
         assert(ty.zigTypeTag() == .Pointer);
         const mcv = try self.resolveInst(callee);
         try self.genSetReg(.rax, Type.usize, mcv);
-        try self.asmRegister(.call, .rax);
+        try self.asmRegister(.{ ._, .call }, .rax);
     }
 
     var bt = self.liveness.iterateBigTomb(inst);
@@ -7408,7 +7529,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
                 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
                 const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
 
-                try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv);
+                try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, src_mcv);
                 break :result Condition.fromCompareOperator(
                     if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned,
                     if (flipped) op.reverse() else op,
@@ -7442,13 +7563,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
                         defer self.register_manager.unlockReg(tmp2_lock);
 
                         if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
-                            .vpinsrw,
+                            .{ ._, .vpinsrw },
                             tmp1_reg,
                             dst_reg.to128(),
                             src_mcv.mem(.word),
                             Immediate.u(1),
                         ) else try self.asmRegisterRegisterRegister(
-                            .vpunpcklwd,
+                            .{ ._, .vpunpcklwd },
                             tmp1_reg,
                             dst_reg.to128(),
                             (if (src_mcv.isRegister())
@@ -7456,14 +7577,24 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
                             else
                                 try self.copyToTmpRegister(ty, src_mcv)).to128(),
                         );
-                        try self.asmRegisterRegister(.vcvtph2ps, tmp1_reg, tmp1_reg);
-                        try self.asmRegisterRegister(.vmovshdup, tmp2_reg, tmp1_reg);
-                        try self.genBinOpMir(.ucomiss, ty, tmp1_mcv, tmp2_mcv);
+                        try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, tmp1_reg, tmp1_reg);
+                        try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp2_reg, tmp1_reg);
+                        try self.genBinOpMir(.{ ._, .ucomiss }, ty, tmp1_mcv, tmp2_mcv);
                     } else return self.fail("TODO implement airCmp for {}", .{
                         ty.fmt(self.bin_file.options.module.?),
                     }),
-                    32 => try self.genBinOpMir(.ucomiss, ty, .{ .register = dst_reg }, src_mcv),
-                    64 => try self.genBinOpMir(.ucomisd, ty, .{ .register = dst_reg }, src_mcv),
+                    32 => try self.genBinOpMir(
+                        .{ ._, .ucomiss },
+                        ty,
+                        .{ .register = dst_reg },
+                        src_mcv,
+                    ),
+                    64 => try self.genBinOpMir(
+                        .{ ._, .ucomisd },
+                        ty,
+                        .{ .register = dst_reg },
+                        src_mcv,
+                    ),
                     else => return self.fail("TODO implement airCmp for {}", .{
                         ty.fmt(self.bin_file.options.module.?),
                     }),
@@ -7507,7 +7638,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
         else => try self.copyToTmpRegister(op_ty, op_mcv),
     };
     try self.asmRegisterMemory(
-        .cmp,
+        .{ ._, .cmp },
         registerAlias(dst_reg, op_abi_size),
         Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),
     );
@@ -7627,7 +7758,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
         },
         .register => |reg| {
             try self.spillEflagsIfOccupied();
-            try self.asmRegisterImmediate(.@"test", reg, Immediate.u(1));
+            try self.asmRegisterImmediate(.{ ._, .@"test" }, reg, Immediate.u(1));
             return self.asmJccReloc(undefined, .e);
         },
         .immediate,
@@ -7730,13 +7861,13 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
                 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
                 const alias_reg = registerAlias(opt_reg, some_abi_size);
                 assert(some_abi_size * 8 == alias_reg.bitSize());
-                try self.asmRegisterRegister(.@"test", alias_reg, alias_reg);
+                try self.asmRegisterRegister(.{ ._, .@"test" }, alias_reg, alias_reg);
                 return .{ .eflags = .z };
             }
             assert(some_info.ty.tag() == .bool);
             const opt_abi_size = @intCast(u32, opt_ty.abiSize(self.target.*));
             try self.asmRegisterImmediate(
-                .bt,
+                .{ ._, .bt },
                 registerAlias(opt_reg, opt_abi_size),
                 Immediate.u(@intCast(u6, some_info.off * 8)),
             );
@@ -7755,7 +7886,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
             try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());
             const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
             try self.asmMemoryImmediate(
-                .cmp,
+                .{ ._, .cmp },
                 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
                     .base = .{ .reg = addr_reg },
                     .disp = some_info.off,
@@ -7768,7 +7899,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
         .indirect, .load_frame => {
             const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
             try self.asmMemoryImmediate(
-                .cmp,
+                .{ ._, .cmp },
                 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {
                     .indirect => |reg_off| .{
                         .base = .{ .reg = reg_off.reg },
@@ -7810,7 +7941,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
 
     const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
     try self.asmMemoryImmediate(
-        .cmp,
+        .{ ._, .cmp },
         Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
             .base = .{ .reg = ptr_reg },
             .disp = some_info.off,
@@ -7841,14 +7972,24 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
             const tmp_reg = try self.copyToTmpRegister(ty, operand);
             if (err_off > 0) {
                 const shift = @intCast(u6, err_off * 8);
-                try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{ .immediate = shift });
+                try self.genShiftBinOpMir(
+                    .{ ._r, .sh },
+                    ty,
+                    .{ .register = tmp_reg },
+                    .{ .immediate = shift },
+                );
             } else {
                 try self.truncateRegister(Type.anyerror, tmp_reg);
             }
-            try self.genBinOpMir(.cmp, Type.anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });
+            try self.genBinOpMir(
+                .{ ._, .cmp },
+                Type.anyerror,
+                .{ .register = tmp_reg },
+                .{ .immediate = 0 },
+            );
         },
         .load_frame => |frame_addr| try self.genBinOpMir(
-            .cmp,
+            .{ ._, .cmp },
             Type.anyerror,
             .{ .load_frame = .{
                 .index = frame_addr.index,
@@ -8073,7 +8214,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
         try self.spillEflagsIfOccupied();
         for (items, relocs, 0..) |item, *reloc, i| {
             const item_mcv = try self.resolveInst(item);
-            try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);
+            try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, item_mcv);
             reloc.* = try self.asmJccReloc(undefined, if (i < relocs.len - 1) .e else .ne);
         }
 
@@ -8284,7 +8425,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
             .qword
         else
             null;
-        const mnem = mnem: {
+        const mnem_tag = Mir.Inst.FixedTag{ ._, mnem: {
             if (mnem_size) |_| {
                 if (std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])) |mnem| {
                     break :mnem mnem;
@@ -8292,7 +8433,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
             }
             break :mnem std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse
                 return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});
-        };
+        } };
 
         var op_it = mem.tokenize(u8, mnem_it.rest(), ",");
         var ops = [1]encoder.Instruction.Operand{.none} ** 4;
@@ -8343,51 +8484,51 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
         } else if (op_it.next()) |op_str| return self.fail("Extra operand: '{s}'", .{op_str});
 
         (switch (ops[0]) {
-            .none => self.asmOpOnly(mnem),
+            .none => self.asmOpOnly(mnem_tag),
             .reg => |reg0| switch (ops[1]) {
-                .none => self.asmRegister(mnem, reg0),
+                .none => self.asmRegister(mnem_tag, reg0),
                 .reg => |reg1| switch (ops[2]) {
-                    .none => self.asmRegisterRegister(mnem, reg1, reg0),
+                    .none => self.asmRegisterRegister(mnem_tag, reg1, reg0),
                     .reg => |reg2| switch (ops[3]) {
-                        .none => self.asmRegisterRegisterRegister(mnem, reg2, reg1, reg0),
+                        .none => self.asmRegisterRegisterRegister(mnem_tag, reg2, reg1, reg0),
                         else => error.InvalidInstruction,
                     },
                     .mem => |mem2| switch (ops[3]) {
-                        .none => self.asmMemoryRegisterRegister(mnem, mem2, reg1, reg0),
+                        .none => self.asmMemoryRegisterRegister(mnem_tag, mem2, reg1, reg0),
                         else => error.InvalidInstruction,
                     },
                     else => error.InvalidInstruction,
                 },
                 .mem => |mem1| switch (ops[2]) {
-                    .none => self.asmMemoryRegister(mnem, mem1, reg0),
+                    .none => self.asmMemoryRegister(mnem_tag, mem1, reg0),
                     else => error.InvalidInstruction,
                 },
                 else => error.InvalidInstruction,
             },
             .mem => |mem0| switch (ops[1]) {
-                .none => self.asmMemory(mnem, mem0),
+                .none => self.asmMemory(mnem_tag, mem0),
                 .reg => |reg1| switch (ops[2]) {
-                    .none => self.asmRegisterMemory(mnem, reg1, mem0),
+                    .none => self.asmRegisterMemory(mnem_tag, reg1, mem0),
                     else => error.InvalidInstruction,
                 },
                 else => error.InvalidInstruction,
             },
             .imm => |imm0| switch (ops[1]) {
-                .none => self.asmImmediate(mnem, imm0),
+                .none => self.asmImmediate(mnem_tag, imm0),
                 .reg => |reg1| switch (ops[2]) {
-                    .none => self.asmRegisterImmediate(mnem, reg1, imm0),
+                    .none => self.asmRegisterImmediate(mnem_tag, reg1, imm0),
                     .reg => |reg2| switch (ops[3]) {
-                        .none => self.asmRegisterRegisterImmediate(mnem, reg2, reg1, imm0),
+                        .none => self.asmRegisterRegisterImmediate(mnem_tag, reg2, reg1, imm0),
                         else => error.InvalidInstruction,
                     },
                     .mem => |mem2| switch (ops[3]) {
-                        .none => self.asmMemoryRegisterImmediate(mnem, mem2, reg1, imm0),
+                        .none => self.asmMemoryRegisterImmediate(mnem_tag, mem2, reg1, imm0),
                         else => error.InvalidInstruction,
                     },
                     else => error.InvalidInstruction,
                 },
                 .mem => |mem1| switch (ops[2]) {
-                    .none => self.asmMemoryImmediate(mnem, mem1, imm0),
+                    .none => self.asmMemoryImmediate(mnem_tag, mem1, imm0),
                     else => error.InvalidInstruction,
                 },
                 else => error.InvalidInstruction,
@@ -8396,7 +8537,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
             error.InvalidInstruction => return self.fail(
                 "Invalid instruction: '{s} {s} {s} {s} {s}'",
                 .{
-                    @tagName(mnem),
+                    @tagName(mnem_tag[1]),
                     @tagName(ops[0]),
                     @tagName(ops[1]),
                     @tagName(ops[2]),
@@ -8427,44 +8568,47 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
     return self.finishAirResult(inst, result);
 }
 
-fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.Tag {
+fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.FixedTag {
     switch (ty.zigTypeTag()) {
-        else => return .mov,
+        else => return .{ ._, .mov },
         .Float => switch (ty.floatBits(self.target.*)) {
             16 => unreachable, // needs special handling
-            32 => return if (self.hasFeature(.avx)) .vmovss else .movss,
-            64 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,
+            32 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
+            64 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
             128 => return if (self.hasFeature(.avx))
-                if (aligned) .vmovaps else .vmovups
-            else if (aligned) .movaps else .movups,
+                if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
+            else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
             else => {},
         },
         .Vector => switch (ty.childType().zigTypeTag()) {
             .Float => switch (ty.childType().floatBits(self.target.*)) {
                 16 => switch (ty.vectorLen()) {
                     1 => unreachable, // needs special handling
-                    2 => return if (self.hasFeature(.avx)) .vmovss else .movss,
-                    3...4 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,
+                    2 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
+                    3...4 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
                     5...8 => return if (self.hasFeature(.avx))
-                        if (aligned) .vmovaps else .vmovups
-                    else if (aligned) .movaps else .movups,
-                    9...16 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,
+                        if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
+                    else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
+                    9...16 => if (self.hasFeature(.avx))
+                        return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
                     else => {},
                 },
                 32 => switch (ty.vectorLen()) {
-                    1 => return if (self.hasFeature(.avx)) .vmovss else .movss,
+                    1 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
                     2...4 => return if (self.hasFeature(.avx))
-                        if (aligned) .vmovaps else .vmovups
-                    else if (aligned) .movaps else .movups,
-                    5...8 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,
+                        if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
+                    else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
+                    5...8 => if (self.hasFeature(.avx))
+                        return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
                     else => {},
                 },
                 64 => switch (ty.vectorLen()) {
-                    1 => return if (self.hasFeature(.avx)) .vmovsd else .movsd,
+                    1 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
                     2 => return if (self.hasFeature(.avx))
-                        if (aligned) .vmovaps else .vmovups
-                    else if (aligned) .movaps else .movups,
-                    3...4 => if (self.hasFeature(.avx)) return if (aligned) .vmovaps else .vmovups,
+                        if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
+                    else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
+                    3...4 => if (self.hasFeature(.avx))
+                        return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
                     else => {},
                 },
                 else => {},
@@ -8558,19 +8702,19 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             if (imm == 0) {
                 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
                 // register is the fastest way to zero a register.
-                try self.asmRegisterRegister(.xor, dst_reg.to32(), dst_reg.to32());
+                try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
             } else if (abi_size > 4 and math.cast(u32, imm) != null) {
                 // 32-bit moves zero-extend to 64-bit.
-                try self.asmRegisterImmediate(.mov, dst_reg.to32(), Immediate.u(imm));
+                try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(imm));
             } else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
                 try self.asmRegisterImmediate(
-                    .mov,
+                    .{ ._, .mov },
                     registerAlias(dst_reg, abi_size),
                     Immediate.s(@intCast(i32, @bitCast(i64, imm))),
                 );
             } else {
                 try self.asmRegisterImmediate(
-                    .mov,
+                    .{ ._, .mov },
                     registerAlias(dst_reg, abi_size),
                     Immediate.u(imm),
                 );
@@ -8579,18 +8723,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
         .register => |src_reg| if (dst_reg.id() != src_reg.id()) try self.asmRegisterRegister(
             if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))
                 switch (ty.zigTypeTag()) {
-                    else => .mov,
-                    .Float, .Vector => .movaps,
+                    else => .{ ._, .mov },
+                    .Float, .Vector => .{ ._, .movaps },
                 }
             else switch (abi_size) {
                 2 => return try self.asmRegisterRegisterImmediate(
-                    if (dst_reg.class() == .floating_point) .pinsrw else .pextrw,
+                    if (dst_reg.class() == .floating_point) .{ ._, .pinsrw } else .{ ._, .pextrw },
                     registerAlias(dst_reg, 4),
                     registerAlias(src_reg, 4),
                     Immediate.u(0),
                 ),
-                4 => .movd,
-                8 => .movq,
+                4 => .{ ._d, .mov },
+                8 => .{ ._q, .mov },
                 else => return self.fail(
                     "unsupported register copy from {s} to {s}",
                     .{ @tagName(src_reg), @tagName(dst_reg) },
@@ -8617,7 +8761,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             });
             if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
                 try self.asmRegisterMemoryImmediate(
-                    .pinsrw,
+                    .{ ._, .pinsrw },
                     registerAlias(dst_reg, abi_size),
                     src_mem,
                     Immediate.u(0),
@@ -8627,14 +8771,14 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                     switch (src_mcv) {
                         .register_offset => |reg_off| switch (reg_off.off) {
                             0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
-                            else => .lea,
+                            else => .{ ._, .lea },
                         },
                         .indirect => try self.movMirTag(ty, false),
                         .load_frame => |frame_addr| try self.movMirTag(
                             ty,
                             self.getFrameAddrAlignment(frame_addr) >= ty.abiAlignment(self.target.*),
                         ),
-                        .lea_frame => .lea,
+                        .lea_frame => .{ ._, .lea },
                         else => unreachable,
                     },
                     registerAlias(dst_reg, abi_size),
@@ -8650,7 +8794,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                     });
                     return if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
                         self.asmRegisterMemoryImmediate(
-                            .pinsrw,
+                            .{ ._, .pinsrw },
                             registerAlias(dst_reg, abi_size),
                             src_mem,
                             Immediate.u(0),
@@ -8694,7 +8838,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             });
             if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
                 try self.asmRegisterMemoryImmediate(
-                    .pinsrw,
+                    .{ ._, .pinsrw },
                     registerAlias(dst_reg, abi_size),
                     src_mem,
                     Immediate.u(0),
@@ -8743,7 +8887,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                     } },
                 });
                 // TODO: spill registers before calling
-                try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
+                try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
                 try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });
             } else return self.fail("TODO emit ptr to TLV sequence on {s}", .{
                 @tagName(self.bin_file.tag),
@@ -8770,7 +8914,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
                 else
                     Immediate.u(@intCast(u32, imm));
                 try self.asmMemoryImmediate(
-                    .mov,
+                    .{ ._, .mov },
                     Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
                     immediate,
                 );
@@ -8778,14 +8922,14 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
             3, 5...7 => unreachable,
             else => if (math.cast(i32, @bitCast(i64, imm))) |small| {
                 try self.asmMemoryImmediate(
-                    .mov,
+                    .{ ._, .mov },
                     Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
                     Immediate.s(small),
                 );
             } else {
                 var offset: i32 = 0;
                 while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
-                    .mov,
+                    .{ ._, .mov },
                     Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),
                     if (ty.isSignedInt())
                         Immediate.s(@truncate(
@@ -8808,7 +8952,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
             );
             if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
                 try self.asmMemoryRegisterImmediate(
-                    .pextrw,
+                    .{ ._, .pextrw },
                     dst_mem,
                     src_reg.to128(),
                     Immediate.u(0),
@@ -8904,7 +9048,7 @@ fn genInlineMemcpyRegisterRegister(
         while (remainder > 0) {
             const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
             try self.asmMemoryRegister(
-                .mov,
+                .{ ._, .mov },
                 Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
                     .base = dst_reg,
                     .disp = -next_offset,
@@ -8913,7 +9057,7 @@ fn genInlineMemcpyRegisterRegister(
             );
 
             if (nearest_power_of_two > 1) {
-                try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{
+                try self.genShiftBinOpMir(.{ ._r, .sh }, ty, .{ .register = tmp_reg }, .{
                     .immediate = nearest_power_of_two * 8,
                 });
             }
@@ -8924,8 +9068,8 @@ fn genInlineMemcpyRegisterRegister(
     } else {
         try self.asmMemoryRegister(
             switch (src_reg.class()) {
-                .general_purpose, .segment => .mov,
-                .floating_point => .movss,
+                .general_purpose, .segment => .{ ._, .mov },
+                .floating_point => .{ ._, .movss },
             },
             Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),
             registerAlias(src_reg, abi_size),
@@ -8938,11 +9082,7 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue
     try self.genSetReg(.rdi, Type.usize, dst_ptr);
     try self.genSetReg(.rsi, Type.usize, src_ptr);
     try self.genSetReg(.rcx, Type.usize, len);
-    _ = try self.addInst(.{
-        .tag = .mov,
-        .ops = .none,
-        .data = .{ .none = .{ .fixes = .@"rep _sb" } },
-    });
+    try self.asmOpOnly(.{ .@"rep _sb", .mov });
 }
 
 fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue) InnerError!void {
@@ -8950,11 +9090,7 @@ fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue)
     try self.genSetReg(.rdi, Type.usize, dst_ptr);
     try self.genSetReg(.al, Type.u8, value);
     try self.genSetReg(.rcx, Type.usize, len);
-    _ = try self.addInst(.{
-        .tag = .sto,
-        .ops = .none,
-        .data = .{ .none = .{ .fixes = .@"rep _sb" } },
-    });
+    try self.asmOpOnly(.{ .@"rep _sb", .sto });
 }
 
 fn genLazySymbolRef(
@@ -8972,14 +9108,14 @@ fn genLazySymbolRef(
         const got_mem =
             Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) });
         switch (tag) {
-            .lea, .mov => try self.asmRegisterMemory(.mov, reg.to64(), got_mem),
-            .call => try self.asmMemory(.call, got_mem),
+            .lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), got_mem),
+            .call => try self.asmMemory(.{ ._, .call }, got_mem),
             else => unreachable,
         }
         switch (tag) {
             .lea, .call => {},
             .mov => try self.asmRegisterMemory(
-                tag,
+                .{ ._, tag },
                 reg.to64(),
                 Memory.sib(.qword, .{ .base = .{ .reg = reg.to64() } }),
             ),
@@ -8996,7 +9132,7 @@ fn genLazySymbolRef(
         }
         switch (tag) {
             .lea, .mov => {},
-            .call => try self.asmRegister(.call, reg),
+            .call => try self.asmRegister(.{ ._, .call }, reg),
             else => unreachable,
         }
     } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
@@ -9010,7 +9146,7 @@ fn genLazySymbolRef(
         }
         switch (tag) {
             .lea, .mov => {},
-            .call => try self.asmRegister(.call, reg),
+            .call => try self.asmRegister(.{ ._, .call }, reg),
             else => unreachable,
         }
     } else {
@@ -9115,13 +9251,13 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
 
     try self.asmRegisterRegister(switch (dst_ty.floatBits(self.target.*)) {
         32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
-            .cvtsi2ss
+            .{ ._, .cvtsi2ss }
         else
             return self.fail("TODO implement airIntToFloat from {} to {} without sse", .{
                 src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
             }),
         64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
-            .cvtsi2sd
+            .{ ._, .cvtsi2sd }
         else
             return self.fail("TODO implement airIntToFloat from {} to {} without sse2", .{
                 src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
@@ -9161,7 +9297,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
         },
     };
     try self.asmMemory(
-        .fld,
+        .{ .f_, .ld },
         Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
             .base = .{ .frame = frame_addr.index },
             .disp = frame_addr.off,
@@ -9171,7 +9307,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
     // convert
     const stack_dst = try self.allocRegOrMem(inst, false);
     try self.asmMemory(
-        .fisttp,
+        .{ .f_p, .istt },
         Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
             .base = .{ .frame = stack_dst.load_frame.index },
             .disp = stack_dst.load_frame.off,
@@ -9227,22 +9363,11 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
     defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
 
     try self.spillEflagsIfOccupied();
-    _ = try self.addInst(if (val_abi_size <= 8) .{
-        .tag = .cmpxchg,
-        .ops = .mr_sib,
-        .data = .{ .rx = .{
-            .fixes = .@"lock _",
-            .r1 = registerAlias(new_reg.?, val_abi_size),
-            .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
-        } },
-    } else .{
-        .tag = .cmpxchg,
-        .ops = .m_sib,
-        .data = .{ .x = .{
-            .fixes = .@"lock _16b",
-            .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
-        } },
-    });
+    if (val_abi_size <= 8) try self.asmMemoryRegister(
+        .{ .@"lock _", .cmpxchg },
+        ptr_mem,
+        registerAlias(new_reg.?, val_abi_size),
+    ) else try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
 
     const result: MCValue = result: {
         if (self.liveness.isUnused(inst)) break :result .unreach;
@@ -9340,21 +9465,17 @@ fn atomicOp(
 
             try self.genSetReg(dst_reg, val_ty, val_mcv);
             if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
-                try self.genUnOpMir(.neg, val_ty, dst_mcv);
+                try self.genUnOpMir(.{ ._, .neg }, val_ty, dst_mcv);
             }
-            _ = try self.addInst(.{
-                .tag = tag,
-                .ops = .mr_sib,
-                .data = .{ .rx = .{
-                    .fixes = switch (tag) {
-                        .mov, .xchg => ._,
-                        .xadd, .add, .sub, .@"and", .@"or", .xor => .@"lock _",
-                        else => unreachable,
-                    },
-                    .r1 = registerAlias(dst_reg, val_abi_size),
-                    .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
-                } },
-            });
+            try self.asmMemoryRegister(
+                switch (tag) {
+                    .mov, .xchg => .{ ._, tag },
+                    .xadd, .add, .sub, .@"and", .@"or", .xor => .{ .@"lock _", tag },
+                    else => unreachable,
+                },
+                ptr_mem,
+                registerAlias(dst_reg, val_abi_size),
+            );
 
             return if (unused) .unreach else dst_mcv;
         },
@@ -9364,22 +9485,22 @@ fn atomicOp(
             const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
             defer self.register_manager.unlockReg(tmp_lock);
 
-            try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);
+            try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);
             const loop = @intCast(u32, self.mir_instructions.len);
             if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
                 try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
             }
             if (rmw_op) |op| switch (op) {
                 .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),
-                .Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),
-                .Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),
-                .And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),
+                .Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv),
+                .Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv),
+                .And => try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv),
                 .Nand => {
-                    try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv);
-                    try self.genUnOpMir(.not, val_ty, tmp_mcv);
+                    try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv);
+                    try self.genUnOpMir(.{ ._, .not }, val_ty, tmp_mcv);
                 },
-                .Or => try self.genBinOpMir(.@"or", val_ty, tmp_mcv, val_mcv),
-                .Xor => try self.genBinOpMir(.xor, val_ty, tmp_mcv, val_mcv),
+                .Or => try self.genBinOpMir(.{ ._, .@"or" }, val_ty, tmp_mcv, val_mcv),
+                .Xor => try self.genBinOpMir(.{ ._, .xor }, val_ty, tmp_mcv, val_mcv),
                 .Min, .Max => {
                     const cc: Condition = switch (if (val_ty.isAbiInt())
                         val_ty.intInfo(self.target.*).signedness
@@ -9397,7 +9518,7 @@ fn atomicOp(
                         },
                     };
 
-                    try self.genBinOpMir(.cmp, val_ty, tmp_mcv, val_mcv);
+                    try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
                     const cmov_abi_size = @max(val_abi_size, 2);
                     switch (val_mcv) {
                         .register => |val_reg| try self.asmCmovccRegisterRegister(
@@ -9421,24 +9542,20 @@ fn atomicOp(
                     }
                 },
             };
-            _ = try self.addInst(.{
-                .tag = .cmpxchg,
-                .ops = .mr_sib,
-                .data = .{ .rx = .{
-                    .fixes = .@"lock _",
-                    .r1 = registerAlias(tmp_reg, val_abi_size),
-                    .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
-                } },
-            });
+            try self.asmMemoryRegister(
+                .{ .@"lock _", .cmpxchg },
+                ptr_mem,
+                registerAlias(tmp_reg, val_abi_size),
+            );
             _ = try self.asmJccReloc(loop, .ne);
             return if (unused) .unreach else .{ .register = .rax };
         } else {
-            try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{
+            try self.asmRegisterMemory(.{ ._, .mov }, .rax, Memory.sib(.qword, .{
                 .base = ptr_mem.sib.base,
                 .scale_index = ptr_mem.scaleIndex(),
                 .disp = ptr_mem.sib.disp + 0,
             }));
-            try self.asmRegisterMemory(.mov, .rdx, Memory.sib(.qword, .{
+            try self.asmRegisterMemory(.{ ._, .mov }, .rdx, Memory.sib(.qword, .{
                 .base = ptr_mem.sib.base,
                 .scale_index = ptr_mem.scaleIndex(),
                 .disp = ptr_mem.sib.disp + 8,
@@ -9453,58 +9570,51 @@ fn atomicOp(
             const val_lo_mem = val_mem_mcv.mem(.qword);
             const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword);
             if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
-                try self.asmRegisterRegister(.mov, .rbx, .rax);
-                try self.asmRegisterRegister(.mov, .rcx, .rdx);
+                try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
+                try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
             }
             if (rmw_op) |op| switch (op) {
                 .Xchg => {
-                    try self.asmRegisterMemory(.mov, .rbx, val_lo_mem);
-                    try self.asmRegisterMemory(.mov, .rcx, val_hi_mem);
+                    try self.asmRegisterMemory(.{ ._, .mov }, .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.{ ._, .mov }, .rcx, val_hi_mem);
                 },
                 .Add => {
-                    try self.asmRegisterMemory(.add, .rbx, val_lo_mem);
-                    try self.asmRegisterMemory(.adc, .rcx, val_hi_mem);
+                    try self.asmRegisterMemory(.{ ._, .add }, .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.{ ._, .adc }, .rcx, val_hi_mem);
                 },
                 .Sub => {
-                    try self.asmRegisterMemory(.sub, .rbx, val_lo_mem);
-                    try self.asmRegisterMemory(.sbb, .rcx, val_hi_mem);
+                    try self.asmRegisterMemory(.{ ._, .sub }, .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.{ ._, .sbb }, .rcx, val_hi_mem);
                 },
                 .And => {
-                    try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);
-                    try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);
+                    try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem);
                 },
                 .Nand => {
-                    try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);
-                    try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);
-                    try self.asmRegister(.not, .rbx);
-                    try self.asmRegister(.not, .rcx);
+                    try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem);
+                    try self.asmRegister(.{ ._, .not }, .rbx);
+                    try self.asmRegister(.{ ._, .not }, .rcx);
                 },
                 .Or => {
-                    try self.asmRegisterMemory(.@"or", .rbx, val_lo_mem);
-                    try self.asmRegisterMemory(.@"or", .rcx, val_hi_mem);
+                    try self.asmRegisterMemory(.{ ._, .@"or" }, .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.{ ._, .@"or" }, .rcx, val_hi_mem);
                 },
                 .Xor => {
-                    try self.asmRegisterMemory(.xor, .rbx, val_lo_mem);
-                    try self.asmRegisterMemory(.xor, .rcx, val_hi_mem);
+                    try self.asmRegisterMemory(.{ ._, .xor }, .rbx, val_lo_mem);
+                    try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem);
                 },
                 else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{
                     val_ty.fmt(self.bin_file.options.module.?), @tagName(op),
                 }),
             };
-            _ = try self.addInst(.{
-                .tag = .cmpxchg,
-                .ops = .m_sib,
-                .data = .{ .x = .{
-                    .fixes = .@"lock _16b",
-                    .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
-                } },
-            });
+            try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
             _ = try self.asmJccReloc(loop, .ne);
 
             if (unused) return .unreach;
             const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
             try self.asmMemoryRegister(
-                .mov,
+                .{ ._, .mov },
                 Memory.sib(.qword, .{
                     .base = .{ .frame = dst_mcv.load_frame.index },
                     .disp = dst_mcv.load_frame.off + 0,
@@ -9512,7 +9622,7 @@ fn atomicOp(
                 .rax,
             );
             try self.asmMemoryRegister(
-                .mov,
+                .{ ._, .mov },
                 Memory.sib(.qword, .{
                     .base = .{ .frame = dst_mcv.load_frame.index },
                     .disp = dst_mcv.load_frame.off + 8,
@@ -9664,8 +9774,13 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
                 .off = elem_abi_size,
             } });
 
-            try self.genBinOpMir(.sub, Type.usize, len_mcv, .{ .immediate = 1 });
-            try self.asmRegisterRegisterImmediate(.imul, len_reg, len_reg, Immediate.u(elem_abi_size));
+            try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 });
+            try self.asmRegisterRegisterImmediate(
+                .{ .i_, .mul },
+                len_reg,
+                len_reg,
+                Immediate.u(elem_abi_size),
+            );
             try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);
 
             try self.performReloc(skip_reloc);
@@ -9803,7 +9918,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
     try self.truncateRegister(err_ty, err_reg.to32());
 
     try self.asmRegisterMemory(
-        .mov,
+        .{ ._, .mov },
         start_reg.to32(),
         Memory.sib(.dword, .{
             .base = .{ .reg = addr_reg.to64() },
@@ -9812,7 +9927,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
         }),
     );
     try self.asmRegisterMemory(
-        .mov,
+        .{ ._, .mov },
         end_reg.to32(),
         Memory.sib(.dword, .{
             .base = .{ .reg = addr_reg.to64() },
@@ -9820,9 +9935,9 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
             .disp = 8,
         }),
     );
-    try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());
+    try self.asmRegisterRegister(.{ ._, .sub }, end_reg.to32(), start_reg.to32());
     try self.asmRegisterMemory(
-        .lea,
+        .{ ._, .lea },
         start_reg.to64(),
         Memory.sib(.byte, .{
             .base = .{ .reg = addr_reg.to64() },
@@ -9831,7 +9946,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
         }),
     );
     try self.asmRegisterMemory(
-        .lea,
+        .{ ._, .lea },
         end_reg.to32(),
         Memory.sib(.byte, .{
             .base = .{ .reg = end_reg.to64() },
@@ -9841,7 +9956,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
 
     const dst_mcv = try self.allocRegOrMem(inst, false);
     try self.asmMemoryRegister(
-        .mov,
+        .{ ._, .mov },
         Memory.sib(.qword, .{
             .base = .{ .frame = dst_mcv.load_frame.index },
             .disp = dst_mcv.load_frame.off,
@@ -9849,7 +9964,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
         start_reg.to64(),
     );
     try self.asmMemoryRegister(
-        .mov,
+        .{ ._, .mov },
         Memory.sib(.qword, .{
             .base = .{ .frame = dst_mcv.load_frame.index },
             .disp = dst_mcv.load_frame.off + 8,
@@ -9945,13 +10060,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
                             try self.truncateRegister(elem_ty, elem_reg);
                         }
                         if (elem_bit_off > 0) try self.genShiftBinOpMir(
-                            .shl,
+                            .{ ._l, .sh },
                             elem_ty,
                             .{ .register = elem_reg },
                             .{ .immediate = elem_bit_off },
                         );
                         try self.genBinOpMir(
-                            .@"or",
+                            .{ ._, .@"or" },
                             elem_ty,
                             .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
                             .{ .register = elem_reg },
@@ -9962,13 +10077,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
                                 try self.truncateRegister(elem_ty, registerAlias(reg, elem_abi_size));
                             }
                             try self.genShiftBinOpMir(
-                                .shr,
+                                .{ ._r, .sh },
                                 elem_ty,
                                 .{ .register = reg },
                                 .{ .immediate = elem_abi_bits - elem_bit_off },
                             );
                             try self.genBinOpMir(
-                                .@"or",
+                                .{ ._, .@"or" },
                                 elem_ty,
                                 .{ .load_frame = .{
                                     .index = frame_index,
@@ -10078,25 +10193,25 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
     }
 
     const mir_tag = if (@as(
-        ?Mir.Inst.Tag,
+        ?Mir.Inst.FixedTag,
         if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or mem.eql(u2, &order, &.{ 3, 1, 2 }))
             switch (ty.zigTypeTag()) {
                 .Float => switch (ty.floatBits(self.target.*)) {
-                    32 => .vfmadd132ss,
-                    64 => .vfmadd132sd,
+                    32 => .{ ._, .vfmadd132ss },
+                    64 => .{ ._, .vfmadd132sd },
                     16, 80, 128 => null,
                     else => unreachable,
                 },
                 .Vector => switch (ty.childType().zigTypeTag()) {
                     .Float => switch (ty.childType().floatBits(self.target.*)) {
                         32 => switch (ty.vectorLen()) {
-                            1 => .vfmadd132ss,
-                            2...8 => .vfmadd132ps,
+                            1 => .{ ._, .vfmadd132ss },
+                            2...8 => .{ ._, .vfmadd132ps },
                             else => null,
                         },
                         64 => switch (ty.vectorLen()) {
-                            1 => .vfmadd132sd,
-                            2...4 => .vfmadd132pd,
+                            1 => .{ ._, .vfmadd132sd },
+                            2...4 => .{ ._, .vfmadd132pd },
                             else => null,
                         },
                         16, 80, 128 => null,
@@ -10109,21 +10224,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
         else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
             switch (ty.zigTypeTag()) {
                 .Float => switch (ty.floatBits(self.target.*)) {
-                    32 => .vfmadd213ss,
-                    64 => .vfmadd213sd,
+                    32 => .{ ._, .vfmadd213ss },
+                    64 => .{ ._, .vfmadd213sd },
                     16, 80, 128 => null,
                     else => unreachable,
                 },
                 .Vector => switch (ty.childType().zigTypeTag()) {
                     .Float => switch (ty.childType().floatBits(self.target.*)) {
                         32 => switch (ty.vectorLen()) {
-                            1 => .vfmadd213ss,
-                            2...8 => .vfmadd213ps,
+                            1 => .{ ._, .vfmadd213ss },
+                            2...8 => .{ ._, .vfmadd213ps },
                             else => null,
                         },
                         64 => switch (ty.vectorLen()) {
-                            1 => .vfmadd213sd,
-                            2...4 => .vfmadd213pd,
+                            1 => .{ ._, .vfmadd213sd },
+                            2...4 => .{ ._, .vfmadd213pd },
                             else => null,
                         },
                         16, 80, 128 => null,
@@ -10136,21 +10251,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
         else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
             switch (ty.zigTypeTag()) {
                 .Float => switch (ty.floatBits(self.target.*)) {
-                    32 => .vfmadd231ss,
-                    64 => .vfmadd231sd,
+                    32 => .{ ._, .vfmadd231ss },
+                    64 => .{ ._, .vfmadd231sd },
                     16, 80, 128 => null,
                     else => unreachable,
                 },
                 .Vector => switch (ty.childType().zigTypeTag()) {
                     .Float => switch (ty.childType().floatBits(self.target.*)) {
                         32 => switch (ty.vectorLen()) {
-                            1 => .vfmadd231ss,
-                            2...8 => .vfmadd231ps,
+                            1 => .{ ._, .vfmadd231ss },
+                            2...8 => .{ ._, .vfmadd231ps },
                             else => null,
                         },
                         64 => switch (ty.vectorLen()) {
-                            1 => .vfmadd231sd,
-                            2...4 => .vfmadd231pd,
+                            1 => .{ ._, .vfmadd231sd },
+                            2...4 => .{ ._, .vfmadd231pd },
                             else => null,
                         },
                         16, 80, 128 => null,
@@ -10522,17 +10637,37 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
     switch (int_info.signedness) {
         .signed => {
             const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
-            try self.genShiftBinOpMir(.sal, Type.isize, .{ .register = reg }, .{ .immediate = shift });
-            try self.genShiftBinOpMir(.sar, Type.isize, .{ .register = reg }, .{ .immediate = shift });
+            try self.genShiftBinOpMir(
+                .{ ._l, .sa },
+                Type.isize,
+                .{ .register = reg },
+                .{ .immediate = shift },
+            );
+            try self.genShiftBinOpMir(
+                .{ ._r, .sa },
+                Type.isize,
+                .{ .register = reg },
+                .{ .immediate = shift },
+            );
         },
         .unsigned => {
             const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
             const mask = (~@as(u64, 0)) >> shift;
             if (int_info.bits <= 32) {
-                try self.genBinOpMir(.@"and", Type.u32, .{ .register = reg }, .{ .immediate = mask });
+                try self.genBinOpMir(
+                    .{ ._, .@"and" },
+                    Type.u32,
+                    .{ .register = reg },
+                    .{ .immediate = mask },
+                );
             } else {
                 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
-                try self.genBinOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .register = tmp_reg });
+                try self.genBinOpMir(
+                    .{ ._, .@"and" },
+                    Type.usize,
+                    .{ .register = reg },
+                    .{ .register = tmp_reg },
+                );
             }
         },
     }
diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig
index 2d7fa4b4fd..c32e7fc974 100644
--- a/src/arch/x86_64/Lower.zig
+++ b/src/arch/x86_64/Lower.zig
@@ -286,10 +286,10 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
         .rri_s, .rri_u => inst.data.rri.fixes,
         .ri_s, .ri_u => inst.data.ri.fixes,
         .ri64, .rm_sib, .rm_rip, .mr_sib, .mr_rip => inst.data.rx.fixes,
-        .mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => ._,
         .mrr_sib, .mrr_rip, .rrm_sib, .rrm_rip => inst.data.rrx.fixes,
         .rmi_sib, .rmi_rip, .mri_sib, .mri_rip => inst.data.rix.fixes,
         .rrmi_sib, .rrmi_rip => inst.data.rrix.fixes,
+        .mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => inst.data.x.fixes,
         .m_sib, .m_rip, .rax_moffs, .moffs_rax => inst.data.x.fixes,
         .extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._,
         else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}),
@@ -356,8 +356,11 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
         },
         .mi_sib_s, .mi_sib_u, .mi_rip_u, .mi_rip_s => &.{
-            .{ .mem = lower.mem(inst.ops, inst.data.ix.payload) },
-            .{ .imm = lower.imm(inst.ops, inst.data.ix.i) },
+            .{ .mem = lower.mem(inst.ops, inst.data.x.payload + 1) },
+            .{ .imm = lower.imm(
+                inst.ops,
+                lower.mir.extraData(Mir.Imm32, inst.data.x.payload).data.imm,
+            ) },
         },
         .rm_sib, .rm_rip => &.{
             .{ .reg = inst.data.rx.r1 },
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index 951a0c5d4d..6b5e2bded7 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -36,6 +36,18 @@ pub const Inst = struct {
         /// ___
         @"_",
 
+        /// Integer __
+        i_,
+
+        /// ___ Left
+        _l,
+        /// ___ Left Double
+        _ld,
+        /// ___ Right
+        _r,
+        /// ___ Right Double
+        _rd,
+
         /// ___ Above
         _a,
         /// ___ Above Or Equal
@@ -53,7 +65,7 @@ pub const Inst = struct {
         /// ___ Greater Or Equal
         _ge,
         /// ___ Less
-        _l,
+        //_l,
         /// ___ Less Or Equal
         _le,
         /// ___ Not Above
@@ -97,6 +109,15 @@ pub const Inst = struct {
         /// ___ Zero
         _z,
 
+        /// ___ Byte
+        //_b,
+        /// ___ Word
+        _w,
+        /// ___ Doubleword
+        _d,
+        /// ___ QuadWord
+        _q,
+
         /// ___ String
         //_s,
         /// ___ String Byte
@@ -165,6 +186,18 @@ pub const Inst = struct {
 
         /// Locked ___
         @"lock _",
+        /// ___ And Complement
+        //_c,
+        /// Locked ___ And Complement
+        @"lock _c",
+        /// ___ And Reset
+        //_r,
+        /// Locked ___ And Reset
+        @"lock _r",
+        /// ___ And Set
+        //_s,
+        /// Locked ___ And Set
+        @"lock _s",
         /// ___ 8 Bytes
         _8b,
         /// Locked ___ 8 Bytes
@@ -174,6 +207,11 @@ pub const Inst = struct {
         /// Locked ___ 16 Bytes
         @"lock _16b",
 
+        /// Float ___
+        f_,
+        /// Float ___ Pop
+        f_p,
+
         /// Packed ___
         p_,
         /// Packed ___ Byte
@@ -250,13 +288,10 @@ pub const Inst = struct {
         /// Byte swap
         bswap,
         /// Bit test
-        bt,
         /// Bit test and complement
-        btc,
         /// Bit test and reset
-        btr,
         /// Bit test and set
-        bts,
+        bt,
         /// Call
         call,
         /// Convert byte to word
@@ -280,21 +315,18 @@ pub const Inst = struct {
         /// Convert word to doubleword
         cwde,
         /// Unsigned division
-        div,
-        /// Store integer with truncation
-        fisttp,
-        /// Load floating-point value
-        fld,
         /// Signed division
-        idiv,
-        /// Signed multiplication
-        imul,
+        div,
         ///
         int3,
+        /// Store integer with truncation
+        istt,
         /// Conditional jump
         j,
         /// Jump
         jmp,
+        /// Load floating-point value
+        ld,
         /// Load effective address
         lea,
         /// Load string
@@ -307,20 +339,17 @@ pub const Inst = struct {
         mfence,
         /// Move
         /// Move data from string to string
+        /// Move doubleword
+        /// Move quadword
         mov,
         /// Move data after swapping bytes
         movbe,
-        /// Move doubleword
-        movd,
-        /// Move quadword
-        movq,
         /// Move with sign extension
         movsx,
-        /// Move with sign extension
-        movsxd,
         /// Move with zero extension
         movzx,
         /// Multiply
+        /// Signed multiplication
         mul,
         /// Two's complement negation
         neg,
@@ -337,19 +366,16 @@ pub const Inst = struct {
         /// Push
         push,
         /// Rotate left through carry
-        rcl,
         /// Rotate right through carry
-        rcr,
+        rc,
         /// Return
         ret,
         /// Rotate left
-        rol,
         /// Rotate right
-        ror,
+        ro,
         /// Arithmetic shift left
-        sal,
         /// Arithmetic shift right
-        sar,
+        sa,
         /// Integer subtraction with borrow
         sbb,
         /// Scan string
@@ -359,13 +385,10 @@ pub const Inst = struct {
         /// Store fence
         sfence,
         /// Logical shift left
-        shl,
         /// Double precision shift left
-        shld,
         /// Logical shift right
-        shr,
         /// Double precision shift right
-        shrd,
+        sh,
         /// Subtract
         sub,
         /// Store string
@@ -730,6 +753,8 @@ pub const Inst = struct {
         pseudo,
     };
 
+    pub const FixedTag = struct { Fixes, Tag };
+
     pub const Ops = enum(u8) {
         /// No data associated with this instruction (only mnemonic is used).
         none,
@@ -800,16 +825,16 @@ pub const Inst = struct {
         /// Uses `x` with extra data of type `MemoryRip`.
         m_rip,
         /// Memory (SIB), immediate (unsigned) operands.
-        /// Uses `ix` payload with extra data of type `MemorySib`.
+        /// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
         mi_sib_u,
         /// Memory (RIP), immediate (unsigned) operands.
-        /// Uses `ix` payload with extra data of type `MemoryRip`.
+        /// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
         mi_rip_u,
         /// Memory (SIB), immediate (sign-extend) operands.
-        /// Uses `ix` payload with extra data of type `MemorySib`.
+        /// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
         mi_sib_s,
         /// Memory (RIP), immediate (sign-extend) operands.
-        /// Uses `ix` payload with extra data of type `MemoryRip`.
+        /// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
         mi_rip_s,
         /// Memory (SIB), register operands.
         /// Uses `rx` payload with extra data of type `MemorySib`.
@@ -974,11 +999,6 @@ pub const Inst = struct {
             r1: Register,
             payload: u32,
         },
-        /// Immediate, followed by Custom payload found in extra.
-        ix: struct {
-            i: u32,
-            payload: u32,
-        },
         /// Register, register, followed by Custom payload found in extra.
         rrx: struct {
             fixes: Fixes = ._,
@@ -1081,6 +1101,10 @@ pub const RegisterList = struct {
     }
 };
 
+pub const Imm32 = struct {
+    imm: u32,
+};
+
 pub const Imm64 = struct {
     msb: u32,
     lsb: u32,

From 1f5aa7747f5710e281cd2190508ce562a4bfd35f Mon Sep 17 00:00:00 2001
From: Jacob Young 
Date: Mon, 8 May 2023 07:35:31 -0400
Subject: [PATCH 137/725] x86_64: finish optimizing mir tag usage

Final tag count is 95.
---
 src/arch/x86_64/CodeGen.zig | 368 ++++++++++++++++++------------------
 src/arch/x86_64/Mir.zig     | 364 ++++++++---------------------------
 2 files changed, 266 insertions(+), 466 deletions(-)

diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig
index 147be62e28..2dc1cc8ee4 100644
--- a/src/arch/x86_64/CodeGen.zig
+++ b/src/arch/x86_64/CodeGen.zig
@@ -2443,7 +2443,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
                 else
                     try self.copyToTmpRegister(src_ty, src_mcv);
                 try self.asmRegisterRegisterImmediate(
-                    .{ ._, .vcvtps2ph },
+                    .{ .v_, .cvtps2ph },
                     dst_reg,
                     mat_src_reg.to128(),
                     Immediate.u(0b1_00),
@@ -2455,12 +2455,12 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
         }
     } else if (src_bits == 64 and dst_bits == 32) {
         if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
-            .{ ._, .vcvtsd2ss },
+            .{ .v_, .cvtsd2ss },
             dst_reg,
             dst_reg,
             src_mcv.mem(.qword),
         ) else try self.asmRegisterRegisterRegister(
-            .{ ._, .vcvtsd2ss },
+            .{ .v_, .cvtsd2ss },
             dst_reg,
             dst_reg,
             (if (src_mcv.isRegister())
@@ -2506,22 +2506,22 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
             src_mcv.getReg().?
         else
             try self.copyToTmpRegister(src_ty, src_mcv);
-        try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, mat_src_reg.to128());
+        try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, mat_src_reg.to128());
         switch (dst_bits) {
             32 => {},
-            64 => try self.asmRegisterRegisterRegister(.{ ._, .vcvtss2sd }, dst_reg, dst_reg, dst_reg),
+            64 => try self.asmRegisterRegisterRegister(.{ .v_, .cvtss2sd }, dst_reg, dst_reg, dst_reg),
             else => return self.fail("TODO implement airFpext from {} to {}", .{
                 src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
             }),
         }
     } else if (src_bits == 32 and dst_bits == 64) {
         if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
-            .{ ._, .vcvtss2sd },
+            .{ .v_, .cvtss2sd },
             dst_reg,
             dst_reg,
             src_mcv.mem(.dword),
         ) else try self.asmRegisterRegisterRegister(
-            .{ ._, .vcvtss2sd },
+            .{ .v_, .cvtss2sd },
             dst_reg,
             dst_reg,
             (if (src_mcv.isRegister())
@@ -4678,8 +4678,8 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
     try self.genBinOpMir(switch (ty_bits) {
         // No point using an extra prefix byte for *pd which performs the same operation.
         16, 32, 64, 128 => switch (tag) {
-            .neg => .{ ._, .xorps },
-            .fabs => .{ ._, .andnps },
+            .neg => .{ ._ps, .xor },
+            .fabs => .{ ._ps, .andn },
             else => unreachable,
         },
         80 => return self.fail("TODO implement airFloatSign for {}", .{
@@ -4712,23 +4712,23 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
 
     const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag()) {
         .Float => switch (ty.floatBits(self.target.*)) {
-            32 => if (self.hasFeature(.avx)) .{ ._, .vroundss } else .{ ._, .roundss },
-            64 => if (self.hasFeature(.avx)) .{ ._, .vroundsd } else .{ ._, .roundsd },
+            32 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round },
+            64 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round },
             16, 80, 128 => null,
             else => unreachable,
         },
         .Vector => switch (ty.childType().zigTypeTag()) {
             .Float => switch (ty.childType().floatBits(self.target.*)) {
                 32 => switch (ty.vectorLen()) {
-                    1 => if (self.hasFeature(.avx)) .{ ._, .vroundss } else .{ ._, .roundss },
-                    2...4 => if (self.hasFeature(.avx)) .{ ._, .vroundps } else .{ ._, .roundps },
-                    5...8 => if (self.hasFeature(.avx)) .{ ._, .vroundps } else null,
+                    1 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round },
+                    2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else .{ ._ps, .round },
+                    5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else null,
                     else => null,
                 },
                 64 => switch (ty.vectorLen()) {
-                    1 => if (self.hasFeature(.avx)) .{ ._, .vroundsd } else .{ ._, .roundsd },
-                    2 => if (self.hasFeature(.avx)) .{ ._, .vroundpd } else .{ ._, .roundpd },
-                    3...4 => if (self.hasFeature(.avx)) .{ ._, .vroundpd } else null,
+                    1 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round },
+                    2 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else .{ ._pd, .round },
+                    3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else null,
                     else => null,
                 },
                 16, 80, 128 => null,
@@ -4743,8 +4743,8 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4
 
     const abi_size = @intCast(u32, ty.abiSize(self.target.*));
     const dst_alias = registerAlias(dst_reg, abi_size);
-    switch (mir_tag[1]) {
-        .vroundss, .vroundsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
+    switch (mir_tag[0]) {
+        .v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
             mir_tag,
             dst_alias,
             dst_alias,
@@ -4799,18 +4799,18 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
                         src_mcv.getReg().?
                     else
                         try self.copyToTmpRegister(ty, src_mcv);
-                    try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, mat_src_reg.to128());
-                    try self.asmRegisterRegisterRegister(.{ ._, .vsqrtss }, dst_reg, dst_reg, dst_reg);
+                    try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, mat_src_reg.to128());
+                    try self.asmRegisterRegisterRegister(.{ .v_ss, .sqrt }, dst_reg, dst_reg, dst_reg);
                     try self.asmRegisterRegisterImmediate(
-                        .{ ._, .vcvtps2ph },
+                        .{ .v_, .cvtps2ph },
                         dst_reg,
                         dst_reg,
                         Immediate.u(0b1_00),
                     );
                     break :result dst_mcv;
                 } else null,
-                32 => if (self.hasFeature(.avx)) .{ ._, .vsqrtss } else .{ ._, .sqrtss },
-                64 => if (self.hasFeature(.avx)) .{ ._, .vsqrtsd } else .{ ._, .sqrtsd },
+                32 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt },
+                64 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt },
                 80, 128 => null,
                 else => unreachable,
             },
@@ -4819,7 +4819,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
                     16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen()) {
                         1 => {
                             try self.asmRegisterRegister(
-                                .{ ._, .vcvtph2ps },
+                                .{ .v_, .cvtph2ps },
                                 dst_reg,
                                 (if (src_mcv.isRegister())
                                     src_mcv.getReg().?
@@ -4827,13 +4827,13 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
                                     try self.copyToTmpRegister(ty, src_mcv)).to128(),
                             );
                             try self.asmRegisterRegisterRegister(
-                                .{ ._, .vsqrtss },
+                                .{ .v_ss, .sqrt },
                                 dst_reg,
                                 dst_reg,
                                 dst_reg,
                             );
                             try self.asmRegisterRegisterImmediate(
-                                .{ ._, .vcvtps2ph },
+                                .{ .v_, .cvtps2ph },
                                 dst_reg,
                                 dst_reg,
                                 Immediate.u(0b1_00),
@@ -4843,22 +4843,22 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
                         2...8 => {
                             const wide_reg = registerAlias(dst_reg, abi_size * 2);
                             if (src_mcv.isMemory()) try self.asmRegisterMemory(
-                                .{ ._, .vcvtph2ps },
+                                .{ .v_, .cvtph2ps },
                                 wide_reg,
                                 src_mcv.mem(Memory.PtrSize.fromSize(
                                     @intCast(u32, @divExact(wide_reg.bitSize(), 16)),
                                 )),
                             ) else try self.asmRegisterRegister(
-                                .{ ._, .vcvtph2ps },
+                                .{ .v_, .cvtph2ps },
                                 wide_reg,
                                 (if (src_mcv.isRegister())
                                     src_mcv.getReg().?
                                 else
                                     try self.copyToTmpRegister(ty, src_mcv)).to128(),
                             );
-                            try self.asmRegisterRegister(.{ ._, .vsqrtps }, wide_reg, wide_reg);
+                            try self.asmRegisterRegister(.{ .v_ps, .sqrt }, wide_reg, wide_reg);
                             try self.asmRegisterRegisterImmediate(
-                                .{ ._, .vcvtps2ph },
+                                .{ .v_, .cvtps2ph },
                                 dst_reg,
                                 wide_reg,
                                 Immediate.u(0b1_00),
@@ -4868,15 +4868,15 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
                         else => null,
                     } else null,
                     32 => switch (ty.vectorLen()) {
-                        1 => if (self.hasFeature(.avx)) .{ ._, .vsqrtss } else .{ ._, .sqrtss },
-                        2...4 => if (self.hasFeature(.avx)) .{ ._, .vsqrtps } else .{ ._, .sqrtps },
-                        5...8 => if (self.hasFeature(.avx)) .{ ._, .vsqrtps } else null,
+                        1 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt },
+                        2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else .{ ._ps, .sqrt },
+                        5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else null,
                         else => null,
                     },
                     64 => switch (ty.vectorLen()) {
-                        1 => if (self.hasFeature(.avx)) .{ ._, .vsqrtsd } else .{ ._, .sqrtsd },
-                        2 => if (self.hasFeature(.avx)) .{ ._, .vsqrtpd } else .{ ._, .sqrtpd },
-                        3...4 => if (self.hasFeature(.avx)) .{ ._, .vsqrtpd } else null,
+                        1 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt },
+                        2 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else .{ ._pd, .sqrt },
+                        3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else null,
                         else => null,
                     },
                     80, 128 => null,
@@ -4888,8 +4888,8 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
         })) |tag| tag else return self.fail("TODO implement airSqrt for {}", .{
             ty.fmt(self.bin_file.options.module.?),
         });
-        switch (mir_tag[1]) {
-            .vsqrtss, .vsqrtsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
+        switch (mir_tag[0]) {
+            .v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
                 mir_tag,
                 dst_reg,
                 dst_reg,
@@ -6325,13 +6325,13 @@ fn genBinOp(
                 defer self.register_manager.unlockReg(tmp_lock);
 
                 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
-                    .{ ._, .vpinsrw },
+                    .{ .vp_w, .insr },
                     dst_reg,
                     dst_reg,
                     src_mcv.mem(.word),
                     Immediate.u(1),
                 ) else try self.asmRegisterRegisterRegister(
-                    .{ ._, .vpunpcklwd },
+                    .{ .vp_, .unpcklwd },
                     dst_reg,
                     dst_reg,
                     (if (src_mcv.isRegister())
@@ -6339,15 +6339,15 @@ fn genBinOp(
                     else
                         try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
                 );
-                try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
-                try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp_reg, dst_reg);
+                try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
+                try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
                 try self.asmRegisterRegisterRegister(
                     switch (air_tag) {
-                        .add => .{ ._, .vaddss },
-                        .sub => .{ ._, .vsubss },
-                        .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivss },
-                        .max => .{ ._, .vmaxss },
-                        .min => .{ ._, .vmaxss },
+                        .add => .{ .v_ss, .add },
+                        .sub => .{ .v_ss, .sub },
+                        .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div },
+                        .max => .{ .v_ss, .max },
+                        .min => .{ .v_ss, .max },
                         else => unreachable,
                     },
                     dst_reg,
@@ -6355,7 +6355,7 @@ fn genBinOp(
                     tmp_reg,
                 );
                 try self.asmRegisterRegisterImmediate(
-                    .{ ._, .vcvtps2ph },
+                    .{ .v_, .cvtps2ph },
                     dst_reg,
                     dst_reg,
                     Immediate.u(0b1_00),
@@ -6363,29 +6363,29 @@ fn genBinOp(
                 return dst_mcv;
             } else null,
             32 => switch (air_tag) {
-                .add => if (self.hasFeature(.avx)) .{ ._, .vaddss } else .{ ._, .addss },
-                .sub => if (self.hasFeature(.avx)) .{ ._, .vsubss } else .{ ._, .subss },
-                .mul => if (self.hasFeature(.avx)) .{ ._, .vmulss } else .{ ._, .mulss },
+                .add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add },
+                .sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub },
+                .mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul },
                 .div_float,
                 .div_trunc,
                 .div_floor,
                 .div_exact,
-                => if (self.hasFeature(.avx)) .{ ._, .vdivss } else .{ ._, .divss },
-                .max => if (self.hasFeature(.avx)) .{ ._, .vmaxss } else .{ ._, .maxss },
-                .min => if (self.hasFeature(.avx)) .{ ._, .vminss } else .{ ._, .minss },
+                => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
+                .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
+                .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
                 else => unreachable,
             },
             64 => switch (air_tag) {
-                .add => if (self.hasFeature(.avx)) .{ ._, .vaddsd } else .{ ._, .addsd },
-                .sub => if (self.hasFeature(.avx)) .{ ._, .vsubsd } else .{ ._, .subsd },
-                .mul => if (self.hasFeature(.avx)) .{ ._, .vmulsd } else .{ ._, .mulsd },
+                .add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add },
+                .sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub },
+                .mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul },
                 .div_float,
                 .div_trunc,
                 .div_floor,
                 .div_exact,
-                => if (self.hasFeature(.avx)) .{ ._, .vdivsd } else .{ ._, .divsd },
-                .max => if (self.hasFeature(.avx)) .{ ._, .vmaxsd } else .{ ._, .maxsd },
-                .min => if (self.hasFeature(.avx)) .{ ._, .vminsd } else .{ ._, .minsd },
+                => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
+                .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
+                .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
                 else => unreachable,
             },
             80, 128 => null,
@@ -6401,13 +6401,13 @@ fn genBinOp(
                         defer self.register_manager.unlockReg(tmp_lock);
 
                         if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
-                            .{ ._, .vpinsrw },
+                            .{ .vp_w, .insr },
                             dst_reg,
                             dst_reg,
                             src_mcv.mem(.word),
                             Immediate.u(1),
                         ) else try self.asmRegisterRegisterRegister(
-                            .{ ._, .vpunpcklwd },
+                            .{ .vp_, .unpcklwd },
                             dst_reg,
                             dst_reg,
                             (if (src_mcv.isRegister())
@@ -6415,15 +6415,15 @@ fn genBinOp(
                             else
                                 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
                         );
-                        try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
-                        try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp_reg, dst_reg);
+                        try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
+                        try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
                         try self.asmRegisterRegisterRegister(
                             switch (air_tag) {
-                                .add => .{ ._, .vaddss },
-                                .sub => .{ ._, .vsubss },
-                                .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivss },
-                                .max => .{ ._, .vmaxss },
-                                .min => .{ ._, .vmaxss },
+                                .add => .{ .v_ss, .add },
+                                .sub => .{ .v_ss, .sub },
+                                .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div },
+                                .max => .{ .v_ss, .max },
+                                .min => .{ .v_ss, .max },
                                 else => unreachable,
                             },
                             dst_reg,
@@ -6431,7 +6431,7 @@ fn genBinOp(
                             tmp_reg,
                         );
                         try self.asmRegisterRegisterImmediate(
-                            .{ ._, .vcvtps2ph },
+                            .{ .v_, .cvtps2ph },
                             dst_reg,
                             dst_reg,
                             Immediate.u(0b1_00),
@@ -6444,12 +6444,12 @@ fn genBinOp(
                         defer self.register_manager.unlockReg(tmp_lock);
 
                         if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
-                            .{ ._, .vpinsrd },
+                            .{ .vp_d, .insr },
                             dst_reg,
                             src_mcv.mem(.dword),
                             Immediate.u(1),
                         ) else try self.asmRegisterRegisterRegister(
-                            .{ ._, .vunpcklps },
+                            .{ .v_ps, .unpckl },
                             dst_reg,
                             dst_reg,
                             (if (src_mcv.isRegister())
@@ -6457,20 +6457,20 @@ fn genBinOp(
                             else
                                 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
                         );
-                        try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
+                        try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
                         try self.asmRegisterRegisterRegister(
-                            .{ ._, .vmovhlps },
+                            .{ .v_ps, .movhl },
                             tmp_reg,
                             dst_reg,
                             dst_reg,
                         );
                         try self.asmRegisterRegisterRegister(
                             switch (air_tag) {
-                                .add => .{ ._, .vaddps },
-                                .sub => .{ ._, .vsubps },
-                                .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
-                                .max => .{ ._, .vmaxps },
-                                .min => .{ ._, .vmaxps },
+                                .add => .{ .v_ps, .add },
+                                .sub => .{ .v_ps, .sub },
+                                .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
+                                .max => .{ .v_ps, .max },
+                                .min => .{ .v_ps, .max },
                                 else => unreachable,
                             },
                             dst_reg,
@@ -6478,7 +6478,7 @@ fn genBinOp(
                             tmp_reg,
                         );
                         try self.asmRegisterRegisterImmediate(
-                            .{ ._, .vcvtps2ph },
+                            .{ .v_, .cvtps2ph },
                             dst_reg,
                             dst_reg,
                             Immediate.u(0b1_00),
@@ -6490,13 +6490,13 @@ fn genBinOp(
                         const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
                         defer self.register_manager.unlockReg(tmp_lock);
 
-                        try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg, dst_reg);
+                        try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
                         if (src_mcv.isMemory()) try self.asmRegisterMemory(
-                            .{ ._, .vcvtph2ps },
+                            .{ .v_, .cvtph2ps },
                             tmp_reg,
                             src_mcv.mem(.qword),
                         ) else try self.asmRegisterRegister(
-                            .{ ._, .vcvtph2ps },
+                            .{ .v_, .cvtph2ps },
                             tmp_reg,
                             (if (src_mcv.isRegister())
                                 src_mcv.getReg().?
@@ -6505,11 +6505,11 @@ fn genBinOp(
                         );
                         try self.asmRegisterRegisterRegister(
                             switch (air_tag) {
-                                .add => .{ ._, .vaddps },
-                                .sub => .{ ._, .vsubps },
-                                .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
-                                .max => .{ ._, .vmaxps },
-                                .min => .{ ._, .vmaxps },
+                                .add => .{ .v_ps, .add },
+                                .sub => .{ .v_ps, .sub },
+                                .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
+                                .max => .{ .v_ps, .max },
+                                .min => .{ .v_ps, .max },
                                 else => unreachable,
                             },
                             dst_reg,
@@ -6517,7 +6517,7 @@ fn genBinOp(
                             tmp_reg,
                         );
                         try self.asmRegisterRegisterImmediate(
-                            .{ ._, .vcvtps2ph },
+                            .{ .v_, .cvtps2ph },
                             dst_reg,
                             dst_reg,
                             Immediate.u(0b1_00),
@@ -6529,13 +6529,13 @@ fn genBinOp(
                         const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
                         defer self.register_manager.unlockReg(tmp_lock);
 
-                        try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, dst_reg.to256(), dst_reg);
+                        try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg.to256(), dst_reg);
                         if (src_mcv.isMemory()) try self.asmRegisterMemory(
-                            .{ ._, .vcvtph2ps },
+                            .{ .v_, .cvtph2ps },
                             tmp_reg,
                             src_mcv.mem(.xword),
                         ) else try self.asmRegisterRegister(
-                            .{ ._, .vcvtph2ps },
+                            .{ .v_, .cvtph2ps },
                             tmp_reg,
                             (if (src_mcv.isRegister())
                                 src_mcv.getReg().?
@@ -6544,11 +6544,11 @@ fn genBinOp(
                         );
                         try self.asmRegisterRegisterRegister(
                             switch (air_tag) {
-                                .add => .{ ._, .vaddps },
-                                .sub => .{ ._, .vsubps },
-                                .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
-                                .max => .{ ._, .vmaxps },
-                                .min => .{ ._, .vmaxps },
+                                .add => .{ .v_ps, .add },
+                                .sub => .{ .v_ps, .sub },
+                                .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
+                                .max => .{ .v_ps, .max },
+                                .min => .{ .v_ps, .max },
                                 else => unreachable,
                             },
                             dst_reg.to256(),
@@ -6556,7 +6556,7 @@ fn genBinOp(
                             tmp_reg,
                         );
                         try self.asmRegisterRegisterImmediate(
-                            .{ ._, .vcvtps2ph },
+                            .{ .v_, .cvtps2ph },
                             dst_reg,
                             dst_reg.to256(),
                             Immediate.u(0b1_00),
@@ -6567,76 +6567,76 @@ fn genBinOp(
                 } else null,
                 32 => switch (lhs_ty.vectorLen()) {
                     1 => switch (air_tag) {
-                        .add => if (self.hasFeature(.avx)) .{ ._, .vaddss } else .{ ._, .addss },
-                        .sub => if (self.hasFeature(.avx)) .{ ._, .vsubss } else .{ ._, .subss },
-                        .mul => if (self.hasFeature(.avx)) .{ ._, .vmulss } else .{ ._, .mulss },
+                        .add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add },
+                        .sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub },
+                        .mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul },
                         .div_float,
                         .div_trunc,
                         .div_floor,
                         .div_exact,
-                        => if (self.hasFeature(.avx)) .{ ._, .vdivss } else .{ ._, .divss },
-                        .max => if (self.hasFeature(.avx)) .{ ._, .vmaxss } else .{ ._, .maxss },
-                        .min => if (self.hasFeature(.avx)) .{ ._, .vminss } else .{ ._, .minss },
+                        => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
+                        .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
+                        .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
                         else => unreachable,
                     },
                     2...4 => switch (air_tag) {
-                        .add => if (self.hasFeature(.avx)) .{ ._, .vaddps } else .{ ._, .addps },
-                        .sub => if (self.hasFeature(.avx)) .{ ._, .vsubps } else .{ ._, .subps },
-                        .mul => if (self.hasFeature(.avx)) .{ ._, .vmulps } else .{ ._, .mulps },
+                        .add => if (self.hasFeature(.avx)) .{ .v_ps, .add } else .{ ._ps, .add },
+                        .sub => if (self.hasFeature(.avx)) .{ .v_ps, .sub } else .{ ._ps, .sub },
+                        .mul => if (self.hasFeature(.avx)) .{ .v_ps, .mul } else .{ ._ps, .mul },
                         .div_float,
                         .div_trunc,
                         .div_floor,
                         .div_exact,
-                        => if (self.hasFeature(.avx)) .{ ._, .vdivps } else .{ ._, .divps },
-                        .max => if (self.hasFeature(.avx)) .{ ._, .vmaxps } else .{ ._, .maxps },
-                        .min => if (self.hasFeature(.avx)) .{ ._, .vminps } else .{ ._, .minps },
+                        => if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div },
+                        .max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max },
+                        .min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min },
                         else => unreachable,
                     },
                     5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
-                        .add => .{ ._, .vaddps },
-                        .sub => .{ ._, .vsubps },
-                        .mul => .{ ._, .vmulps },
-                        .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivps },
-                        .max => .{ ._, .vmaxps },
-                        .min => .{ ._, .vminps },
+                        .add => .{ .v_ps, .add },
+                        .sub => .{ .v_ps, .sub },
+                        .mul => .{ .v_ps, .mul },
+                        .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
+                        .max => .{ .v_ps, .max },
+                        .min => .{ .v_ps, .min },
                         else => unreachable,
                     } else null,
                     else => null,
                 },
                 64 => switch (lhs_ty.vectorLen()) {
                     1 => switch (air_tag) {
-                        .add => if (self.hasFeature(.avx)) .{ ._, .vaddsd } else .{ ._, .addsd },
-                        .sub => if (self.hasFeature(.avx)) .{ ._, .vsubsd } else .{ ._, .subsd },
-                        .mul => if (self.hasFeature(.avx)) .{ ._, .vmulsd } else .{ ._, .mulsd },
+                        .add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add },
+                        .sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub },
+                        .mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul },
                         .div_float,
                         .div_trunc,
                         .div_floor,
                         .div_exact,
-                        => if (self.hasFeature(.avx)) .{ ._, .vdivsd } else .{ ._, .divsd },
-                        .max => if (self.hasFeature(.avx)) .{ ._, .vmaxsd } else .{ ._, .maxsd },
-                        .min => if (self.hasFeature(.avx)) .{ ._, .vminsd } else .{ ._, .minsd },
+                        => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
+                        .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
+                        .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
                         else => unreachable,
                     },
                     2 => switch (air_tag) {
-                        .add => if (self.hasFeature(.avx)) .{ ._, .vaddpd } else .{ ._, .addpd },
-                        .sub => if (self.hasFeature(.avx)) .{ ._, .vsubpd } else .{ ._, .subpd },
-                        .mul => if (self.hasFeature(.avx)) .{ ._, .vmulpd } else .{ ._, .mulpd },
+                        .add => if (self.hasFeature(.avx)) .{ .v_pd, .add } else .{ ._pd, .add },
+                        .sub => if (self.hasFeature(.avx)) .{ .v_pd, .sub } else .{ ._pd, .sub },
+                        .mul => if (self.hasFeature(.avx)) .{ .v_pd, .mul } else .{ ._pd, .mul },
                         .div_float,
                         .div_trunc,
                         .div_floor,
                         .div_exact,
-                        => if (self.hasFeature(.avx)) .{ ._, .vdivpd } else .{ ._, .divpd },
-                        .max => if (self.hasFeature(.avx)) .{ ._, .vmaxpd } else .{ ._, .maxpd },
-                        .min => if (self.hasFeature(.avx)) .{ ._, .vminpd } else .{ ._, .minpd },
+                        => if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div },
+                        .max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max },
+                        .min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min },
                         else => unreachable,
                     },
                     3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
-                        .add => .{ ._, .vaddpd },
-                        .sub => .{ ._, .vsubpd },
-                        .mul => .{ ._, .vmulpd },
-                        .div_float, .div_trunc, .div_floor, .div_exact => .{ ._, .vdivpd },
-                        .max => .{ ._, .vmaxpd },
-                        .min => .{ ._, .vminpd },
+                        .add => .{ .v_pd, .add },
+                        .sub => .{ .v_pd, .sub },
+                        .mul => .{ .v_pd, .mul },
+                        .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div },
+                        .max => .{ .v_pd, .max },
+                        .min => .{ .v_pd, .min },
                         else => unreachable,
                     } else null,
                     else => null,
@@ -7563,13 +7563,13 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
                         defer self.register_manager.unlockReg(tmp2_lock);
 
                         if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
-                            .{ ._, .vpinsrw },
+                            .{ .vp_w, .insr },
                             tmp1_reg,
                             dst_reg.to128(),
                             src_mcv.mem(.word),
                             Immediate.u(1),
                         ) else try self.asmRegisterRegisterRegister(
-                            .{ ._, .vpunpcklwd },
+                            .{ .vp_, .unpcklwd },
                             tmp1_reg,
                             dst_reg.to128(),
                             (if (src_mcv.isRegister())
@@ -7577,20 +7577,20 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
                             else
                                 try self.copyToTmpRegister(ty, src_mcv)).to128(),
                         );
-                        try self.asmRegisterRegister(.{ ._, .vcvtph2ps }, tmp1_reg, tmp1_reg);
-                        try self.asmRegisterRegister(.{ ._, .vmovshdup }, tmp2_reg, tmp1_reg);
-                        try self.genBinOpMir(.{ ._, .ucomiss }, ty, tmp1_mcv, tmp2_mcv);
+                        try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, tmp1_reg, tmp1_reg);
+                        try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp2_reg, tmp1_reg);
+                        try self.genBinOpMir(.{ ._ss, .ucomi }, ty, tmp1_mcv, tmp2_mcv);
                     } else return self.fail("TODO implement airCmp for {}", .{
                         ty.fmt(self.bin_file.options.module.?),
                     }),
                     32 => try self.genBinOpMir(
-                        .{ ._, .ucomiss },
+                        .{ ._ss, .ucomi },
                         ty,
                         .{ .register = dst_reg },
                         src_mcv,
                     ),
                     64 => try self.genBinOpMir(
-                        .{ ._, .ucomisd },
+                        .{ ._sd, .ucomi },
                         ty,
                         .{ .register = dst_reg },
                         src_mcv,
@@ -8573,42 +8573,42 @@ fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.FixedTag {
         else => return .{ ._, .mov },
         .Float => switch (ty.floatBits(self.target.*)) {
             16 => unreachable, // needs special handling
-            32 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
-            64 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
+            32 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
+            64 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
             128 => return if (self.hasFeature(.avx))
-                if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
-            else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
+                if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
+            else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
             else => {},
         },
         .Vector => switch (ty.childType().zigTypeTag()) {
             .Float => switch (ty.childType().floatBits(self.target.*)) {
                 16 => switch (ty.vectorLen()) {
                     1 => unreachable, // needs special handling
-                    2 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
-                    3...4 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
+                    2 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
+                    3...4 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
                     5...8 => return if (self.hasFeature(.avx))
-                        if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
-                    else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
+                        if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
+                    else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
                     9...16 => if (self.hasFeature(.avx))
-                        return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
+                        return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
                     else => {},
                 },
                 32 => switch (ty.vectorLen()) {
-                    1 => return if (self.hasFeature(.avx)) .{ ._, .vmovss } else .{ ._, .movss },
+                    1 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
                     2...4 => return if (self.hasFeature(.avx))
-                        if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
-                    else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
+                        if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
+                    else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
                     5...8 => if (self.hasFeature(.avx))
-                        return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
+                        return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
                     else => {},
                 },
                 64 => switch (ty.vectorLen()) {
-                    1 => return if (self.hasFeature(.avx)) .{ ._, .vmovsd } else .{ ._, .movsd },
+                    1 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
                     2 => return if (self.hasFeature(.avx))
-                        if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups }
-                    else if (aligned) .{ ._, .movaps } else .{ ._, .movups },
+                        if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
+                    else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
                     3...4 => if (self.hasFeature(.avx))
-                        return if (aligned) .{ ._, .vmovaps } else .{ ._, .vmovups },
+                        return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
                     else => {},
                 },
                 else => {},
@@ -8724,11 +8724,11 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))
                 switch (ty.zigTypeTag()) {
                     else => .{ ._, .mov },
-                    .Float, .Vector => .{ ._, .movaps },
+                    .Float, .Vector => .{ ._ps, .mova },
                 }
             else switch (abi_size) {
                 2 => return try self.asmRegisterRegisterImmediate(
-                    if (dst_reg.class() == .floating_point) .{ ._, .pinsrw } else .{ ._, .pextrw },
+                    if (dst_reg.class() == .floating_point) .{ .p_w, .insr } else .{ .p_w, .extr },
                     registerAlias(dst_reg, 4),
                     registerAlias(src_reg, 4),
                     Immediate.u(0),
@@ -8761,7 +8761,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             });
             if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
                 try self.asmRegisterMemoryImmediate(
-                    .{ ._, .pinsrw },
+                    .{ .p_w, .insr },
                     registerAlias(dst_reg, abi_size),
                     src_mem,
                     Immediate.u(0),
@@ -8794,7 +8794,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
                     });
                     return if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
                         self.asmRegisterMemoryImmediate(
-                            .{ ._, .pinsrw },
+                            .{ .p_w, .insr },
                             registerAlias(dst_reg, abi_size),
                             src_mem,
                             Immediate.u(0),
@@ -8838,7 +8838,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
             });
             if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
                 try self.asmRegisterMemoryImmediate(
-                    .{ ._, .pinsrw },
+                    .{ .p_w, .insr },
                     registerAlias(dst_reg, abi_size),
                     src_mem,
                     Immediate.u(0),
@@ -8952,7 +8952,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
             );
             if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
                 try self.asmMemoryRegisterImmediate(
-                    .{ ._, .pextrw },
+                    .{ .p_w, .extr },
                     dst_mem,
                     src_reg.to128(),
                     Immediate.u(0),
@@ -9069,7 +9069,7 @@ fn genInlineMemcpyRegisterRegister(
         try self.asmMemoryRegister(
             switch (src_reg.class()) {
                 .general_purpose, .segment => .{ ._, .mov },
-                .floating_point => .{ ._, .movss },
+                .floating_point => .{ ._ss, .mov },
             },
             Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),
             registerAlias(src_reg, abi_size),
@@ -10197,21 +10197,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
         if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or mem.eql(u2, &order, &.{ 3, 1, 2 }))
             switch (ty.zigTypeTag()) {
                 .Float => switch (ty.floatBits(self.target.*)) {
-                    32 => .{ ._, .vfmadd132ss },
-                    64 => .{ ._, .vfmadd132sd },
+                    32 => .{ .v_ss, .fmadd132 },
+                    64 => .{ .v_sd, .fmadd132 },
                     16, 80, 128 => null,
                     else => unreachable,
                 },
                 .Vector => switch (ty.childType().zigTypeTag()) {
                     .Float => switch (ty.childType().floatBits(self.target.*)) {
                         32 => switch (ty.vectorLen()) {
-                            1 => .{ ._, .vfmadd132ss },
-                            2...8 => .{ ._, .vfmadd132ps },
+                            1 => .{ .v_ss, .fmadd132 },
+                            2...8 => .{ .v_ps, .fmadd132 },
                             else => null,
                         },
                         64 => switch (ty.vectorLen()) {
-                            1 => .{ ._, .vfmadd132sd },
-                            2...4 => .{ ._, .vfmadd132pd },
+                            1 => .{ .v_sd, .fmadd132 },
+                            2...4 => .{ .v_pd, .fmadd132 },
                             else => null,
                         },
                         16, 80, 128 => null,
@@ -10224,21 +10224,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
         else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
             switch (ty.zigTypeTag()) {
                 .Float => switch (ty.floatBits(self.target.*)) {
-                    32 => .{ ._, .vfmadd213ss },
-                    64 => .{ ._, .vfmadd213sd },
+                    32 => .{ .v_ss, .fmadd213 },
+                    64 => .{ .v_sd, .fmadd213 },
                     16, 80, 128 => null,
                     else => unreachable,
                 },
                 .Vector => switch (ty.childType().zigTypeTag()) {
                     .Float => switch (ty.childType().floatBits(self.target.*)) {
                         32 => switch (ty.vectorLen()) {
-                            1 => .{ ._, .vfmadd213ss },
-                            2...8 => .{ ._, .vfmadd213ps },
+                            1 => .{ .v_ss, .fmadd213 },
+                            2...8 => .{ .v_ps, .fmadd213 },
                             else => null,
                         },
                         64 => switch (ty.vectorLen()) {
-                            1 => .{ ._, .vfmadd213sd },
-                            2...4 => .{ ._, .vfmadd213pd },
+                            1 => .{ .v_sd, .fmadd213 },
+                            2...4 => .{ .v_pd, .fmadd213 },
                             else => null,
                         },
                         16, 80, 128 => null,
@@ -10251,21 +10251,21 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
         else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
             switch (ty.zigTypeTag()) {
                 .Float => switch (ty.floatBits(self.target.*)) {
-                    32 => .{ ._, .vfmadd231ss },
-                    64 => .{ ._, .vfmadd231sd },
+                    32 => .{ .v_ss, .fmadd231 },
+                    64 => .{ .v_sd, .fmadd231 },
                     16, 80, 128 => null,
                     else => unreachable,
                 },
                 .Vector => switch (ty.childType().zigTypeTag()) {
                     .Float => switch (ty.childType().floatBits(self.target.*)) {
                         32 => switch (ty.vectorLen()) {
-                            1 => .{ ._, .vfmadd231ss },
-                            2...8 => .{ ._, .vfmadd231ps },
+                            1 => .{ .v_ss, .fmadd231 },
+                            2...8 => .{ .v_ps, .fmadd231 },
                             else => null,
                         },
                         64 => switch (ty.vectorLen()) {
-                            1 => .{ ._, .vfmadd231sd },
-                            2...4 => .{ ._, .vfmadd231pd },
+                            1 => .{ .v_sd, .fmadd231 },
+                            2...4 => .{ .v_pd, .fmadd231 },
                             else => null,
                         },
                         16, 80, 128 => null,
diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig
index 6b5e2bded7..0a7b5597b3 100644
--- a/src/arch/x86_64/Mir.zig
+++ b/src/arch/x86_64/Mir.zig
@@ -278,8 +278,14 @@ pub const Inst = struct {
         /// Add with carry
         adc,
         /// Add
+        /// Add packed single-precision floating-point values
+        /// Add scalar single-precision floating-point values
+        /// Add packed double-precision floating-point values
+        /// Add scalar double-precision floating-point values
         add,
         /// Logical and
+        /// Bitwise logical and of packed single-precision floating-point values
+        /// Bitwise logical and of packed double-precision floating-point values
         @"and",
         /// Bit scan forward
         bsf,
@@ -304,6 +310,8 @@ pub const Inst = struct {
         cmov,
         /// Logical compare
         /// Compare string
+        /// Compare scalar single-precision floating-point values
+        /// Compare scalar double-precision floating-point values
         cmp,
         /// Compare and exchange
         /// Compare and exchange bytes
@@ -316,6 +324,10 @@ pub const Inst = struct {
         cwde,
         /// Unsigned division
         /// Signed division
+        /// Divide packed single-precision floating-point values
+        /// Divide scalar single-precision floating-point values
+        /// Divide packed double-precision floating-point values
+        /// Divide scalar double-precision floating-point values
         div,
         ///
         int3,
@@ -339,6 +351,8 @@ pub const Inst = struct {
         mfence,
         /// Move
         /// Move data from string to string
+        /// Move scalar single-precision floating-point value
+        /// Move scalar double-precision floating-point value
         /// Move doubleword
         /// Move quadword
         mov,
@@ -350,6 +364,10 @@ pub const Inst = struct {
         movzx,
         /// Multiply
         /// Signed multiplication
+        /// Multiply packed single-precision floating-point values
+        /// Multiply scalar single-precision floating-point values
+        /// Multiply packed double-precision floating-point values
+        /// Multiply scalar double-precision floating-point values
         mul,
         /// Two's complement negation
         neg,
@@ -358,6 +376,8 @@ pub const Inst = struct {
         /// One's complement negation
         not,
         /// Logical or
+        /// Bitwise logical or of packed single-precision floating-point values
+        /// Bitwise logical or of packed double-precision floating-point values
         @"or",
         /// Pop
         pop,
@@ -390,6 +410,10 @@ pub const Inst = struct {
         /// Double precision shift right
         sh,
         /// Subtract
+        /// Subtract packed single-precision floating-point values
+        /// Subtract scalar single-precision floating-point values
+        /// Subtract packed double-precision floating-point values
+        /// Subtract scalar double-precision floating-point values
         sub,
         /// Store string
         sto,
@@ -406,145 +430,88 @@ pub const Inst = struct {
         /// Exchange register/memory with register
         xchg,
         /// Logical exclusive-or
+        /// Bitwise logical xor of packed single-precision floating-point values
+        /// Bitwise logical xor of packed double-precision floating-point values
         xor,
 
-        /// Add packed single-precision floating-point values
-        addps,
-        /// Add scalar single-precision floating-point values
-        addss,
-        /// Bitwise logical and of packed single precision floating-point values
-        andps,
-        /// Bitwise logical and not of packed single precision floating-point values
-        andnps,
-        /// Compare scalar single-precision floating-point values
-        cmpss,
+        /// Bitwise logical and not of packed single-precision floating-point values
+        /// Bitwise logical and not of packed double-precision floating-point values
+        andn,
         /// Convert doubleword integer to scalar single-precision floating-point value
         cvtsi2ss,
-        /// Divide packed single-precision floating-point values
-        divps,
-        /// Divide scalar single-precision floating-point values
-        divss,
         /// Maximum of packed single-precision floating-point values
-        maxps,
         /// Maximum of scalar single-precision floating-point values
-        maxss,
+        /// Maximum of packed double-precision floating-point values
+        /// Maximum of scalar double-precision floating-point values
+        max,
         /// Minimum of packed single-precision floating-point values
-        minps,
         /// Minimum of scalar single-precision floating-point values
-        minss,
+        /// Minimum of packed double-precision floating-point values
+        /// Minimum of scalar double-precision floating-point values
+        min,
         /// Move aligned packed single-precision floating-point values
-        movaps,
+        /// Move aligned packed double-precision floating-point values
+        mova,
         /// Move packed single-precision floating-point values high to low
-        movhlps,
-        /// Move scalar single-precision floating-point value
-        movss,
+        movhl,
         /// Move unaligned packed single-precision floating-point values
-        movups,
-        /// Multiply packed single-precision floating-point values
-        mulps,
-        /// Multiply scalar single-precision floating-point values
-        mulss,
-        /// Bitwise logical or of packed single precision floating-point values
-        orps,
+        /// Move unaligned packed double-precision floating-point values
+        movu,
+        /// Extract byte
         /// Extract word
-        pextrw,
+        /// Extract doubleword
+        /// Extract quadword
+        extr,
+        /// Insert byte
         /// Insert word
-        pinsrw,
+        /// Insert doubleword
+        /// Insert quadword
+        insr,
         /// Square root of packed single-precision floating-point values
-        sqrtps,
         /// Square root of scalar single-precision floating-point value
-        sqrtss,
-        /// Subtract packed single-precision floating-point values
-        subps,
-        /// Subtract scalar single-precision floating-point values
-        subss,
+        /// Square root of packed double-precision floating-point values
+        /// Square root of scalar double-precision floating-point value
+        sqrt,
         /// Unordered compare scalar single-precision floating-point values
-        ucomiss,
+        /// Unordered compare scalar double-precision floating-point values
+        ucomi,
         /// Unpack and interleave high packed single-precision floating-point values
-        unpckhps,
+        /// Unpack and interleave high packed double-precision floating-point values
+        unpckh,
         /// Unpack and interleave low packed single-precision floating-point values
-        unpcklps,
-        /// Bitwise logical xor of packed single precision floating-point values
-        xorps,
+        /// Unpack and interleave low packed double-precision floating-point values
+        unpckl,
 
-        /// Add packed double-precision floating-point values
-        addpd,
-        /// Add scalar double-precision floating-point values
-        addsd,
-        /// Bitwise logical and not of packed double precision floating-point values
-        andnpd,
-        /// Bitwise logical and of packed double precision floating-point values
-        andpd,
-        /// Compare scalar double-precision floating-point values
-        cmpsd,
         /// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
         cvtsd2ss,
         /// Convert doubleword integer to scalar double-precision floating-point value
         cvtsi2sd,
         /// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
         cvtss2sd,
-        /// Divide packed double-precision floating-point values
-        divpd,
-        /// Divide scalar double-precision floating-point values
-        divsd,
-        /// Maximum of packed double-precision floating-point values
-        maxpd,
-        /// Maximum of scalar double-precision floating-point values
-        maxsd,
-        /// Minimum of packed double-precision floating-point values
-        minpd,
-        /// Minimum of scalar double-precision floating-point values
-        minsd,
-        /// Move scalar double-precision floating-point value
-        movsd,
-        /// Multiply packed double-precision floating-point values
-        mulpd,
-        /// Multiply scalar double-precision floating-point values
-        mulsd,
-        /// Bitwise logical or of packed double precision floating-point values
-        orpd,
         /// Shuffle packed high words
-        pshufhw,
+        shufh,
         /// Shuffle packed low words
-        pshuflw,
+        shufl,
         /// Shift packed data right logical
-        psrld,
         /// Shift packed data right logical
-        psrlq,
         /// Shift packed data right logical
-        psrlw,
+        srl,
         /// Unpack high data
-        punpckhbw,
+        unpckhbw,
         /// Unpack high data
-        punpckhdq,
+        unpckhdq,
         /// Unpack high data
-        punpckhqdq,
+        unpckhqdq,
         /// Unpack high data
-        punpckhwd,
+        unpckhwd,
         /// Unpack low data
-        punpcklbw,
+        unpcklbw,
         /// Unpack low data
-        punpckldq,
+        unpckldq,
         /// Unpack low data
-        punpcklqdq,
+        unpcklqdq,
         /// Unpack low data
-        punpcklwd,
-        /// Square root of double precision floating-point values
-        sqrtpd,
-        /// Square root of scalar double precision floating-point value
-        sqrtsd,
-        /// Subtract packed double-precision floating-point values
-        subpd,
-        /// Subtract scalar double-precision floating-point values
-        subsd,
-        /// Unordered compare scalar double-precision floating-point values
-        ucomisd,
-        /// Unpack and interleave high packed double-precision floating-point values
-        unpckhpd,
-        /// Unpack and interleave low packed double-precision floating-point values
-        unpcklpd,
-        /// Bitwise logical xor of packed double precision floating-point values
-        xorpd,
+        unpcklwd,
 
         /// Replicate double floating-point values
         movddup,
@@ -553,199 +520,32 @@ pub const Inst = struct {
         /// Replicate single floating-point values
         movsldup,
 
-        /// Extract Byte
-        pextrb,
-        /// Extract Doubleword
-        pextrd,
-        /// Extract Quadword
-        pextrq,
-        /// Insert Byte
-        pinsrb,
-        /// Insert Doubleword
-        pinsrd,
-        /// Insert Quadword
-        pinsrq,
-        /// Round packed double-precision floating-point values
-        roundpd,
         /// Round packed single-precision floating-point values
-        roundps,
-        /// Round scalar double-precision floating-point value
-        roundsd,
         /// Round scalar single-precision floating-point value
-        roundss,
-
-        /// Add packed double-precision floating-point values
-        vaddpd,
-        /// Add packed single-precision floating-point values
-        vaddps,
-        /// Add scalar double-precision floating-point values
-        vaddsd,
-        /// Add scalar single-precision floating-point values
-        vaddss,
-        /// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
-        vcvtsd2ss,
-        /// Convert doubleword integer to scalar double-precision floating-point value
-        vcvtsi2sd,
-        /// Convert doubleword integer to scalar single-precision floating-point value
-        vcvtsi2ss,
-        /// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
-        vcvtss2sd,
-        /// Divide packed double-precision floating-point values
-        vdivpd,
-        /// Divide packed single-precision floating-point values
-        vdivps,
-        /// Divide scalar double-precision floating-point values
-        vdivsd,
-        /// Divide scalar single-precision floating-point values
-        vdivss,
-        /// Maximum of packed double-precision floating-point values
-        vmaxpd,
-        /// Maximum of packed single-precision floating-point values
-        vmaxps,
-        /// Maximum of scalar double-precision floating-point values
-        vmaxsd,
-        /// Maximum of scalar single-precision floating-point values
-        vmaxss,
-        /// Minimum of packed double-precision floating-point values
-        vminpd,
-        /// Minimum of packed single-precision floating-point values
-        vminps,
-        /// Minimum of scalar double-precision floating-point values
-        vminsd,
-        /// Minimum of scalar single-precision floating-point values
-        vminss,
-        /// Move aligned packed double-precision floating-point values
-        vmovapd,
-        /// Move aligned packed single-precision floating-point values
-        vmovaps,
-        /// Move packed single-precision floating-point values high to low
-        vmovhlps,
-        /// Replicate double floating-point values
-        vmovddup,
-        /// Move or merge scalar double-precision floating-point value
-        vmovsd,
-        /// Replicate single floating-point values
-        vmovshdup,
-        /// Replicate single floating-point values
-        vmovsldup,
-        /// Move or merge scalar single-precision floating-point value
-        vmovss,
-        /// Move unaligned packed double-precision floating-point values
-        vmovupd,
-        /// Move unaligned packed single-precision floating-point values
-        vmovups,
-        /// Multiply packed double-precision floating-point values
-        vmulpd,
-        /// Multiply packed single-precision floating-point values
-        vmulps,
-        /// Multiply scalar double-precision floating-point values
-        vmulsd,
-        /// Multiply scalar single-precision floating-point values
-        vmulss,
-        /// Extract Byte
-        vpextrb,
-        /// Extract Doubleword
-        vpextrd,
-        /// Extract Quadword
-        vpextrq,
-        /// Extract word
-        vpextrw,
-        /// Insert Byte
-        vpinsrb,
-        /// Insert Doubleword
-        vpinsrd,
-        /// Insert Quadword
-        vpinsrq,
-        /// Insert word
-        vpinsrw,
-        /// Shuffle packed high words
-        vpshufhw,
-        /// Shuffle packed low words
-        vpshuflw,
-        /// Shift packed data right logical
-        vpsrld,
-        /// Shift packed data right logical
-        vpsrlq,
-        /// Shift packed data right logical
-        vpsrlw,
-        /// Unpack high data
-        vpunpckhbw,
-        /// Unpack high data
-        vpunpckhdq,
-        /// Unpack high data
-        vpunpckhqdq,
-        /// Unpack high data
-        vpunpckhwd,
-        /// Unpack low data
-        vpunpcklbw,
-        /// Unpack low data
-        vpunpckldq,
-        /// Unpack low data
-        vpunpcklqdq,
-        /// Unpack low data
-        vpunpcklwd,
         /// Round packed double-precision floating-point values
-        vroundpd,
-        /// Round packed single-precision floating-point values
-        vroundps,
         /// Round scalar double-precision floating-point value
-        vroundsd,
-        /// Round scalar single-precision floating-point value
-        vroundss,
-        /// Square root of packed double-precision floating-point value
-        vsqrtpd,
-        /// Square root of packed single-precision floating-point value
-        vsqrtps,
-        /// Square root of scalar double-precision floating-point value
-        vsqrtsd,
-        /// Square root of scalar single-precision floating-point value
-        vsqrtss,
-        /// Subtract packed double-precision floating-point values
-        vsubpd,
-        /// Subtract packed single-precision floating-point values
-        vsubps,
-        /// Subtract scalar double-precision floating-point values
-        vsubsd,
-        /// Subtract scalar single-precision floating-point values
-        vsubss,
-        /// Unpack and interleave high packed double-precision floating-point values
-        vunpckhpd,
-        /// Unpack and interleave high packed single-precision floating-point values
-        vunpckhps,
-        /// Unpack and interleave low packed double-precision floating-point values
-        vunpcklpd,
-        /// Unpack and interleave low packed single-precision floating-point values
-        vunpcklps,
+        round,
 
         /// Convert 16-bit floating-point values to single-precision floating-point values
-        vcvtph2ps,
+        cvtph2ps,
         /// Convert single-precision floating-point values to 16-bit floating-point values
-        vcvtps2ph,
+        cvtps2ph,
 
-        /// Fused multiply-add of packed double-precision floating-point values
-        vfmadd132pd,
-        /// Fused multiply-add of packed double-precision floating-point values
-        vfmadd213pd,
-        /// Fused multiply-add of packed double-precision floating-point values
-        vfmadd231pd,
         /// Fused multiply-add of packed single-precision floating-point values
-        vfmadd132ps,
+        /// Fused multiply-add of scalar single-precision floating-point values
+        /// Fused multiply-add of packed double-precision floating-point values
+        /// Fused multiply-add of scalar double-precision floating-point values
+        fmadd132,
         /// Fused multiply-add of packed single-precision floating-point values
-        vfmadd213ps,
+        /// Fused multiply-add of scalar single-precision floating-point values
+        /// Fused multiply-add of packed double-precision floating-point values
+        /// Fused multiply-add of scalar double-precision floating-point values
+        fmadd213,
         /// Fused multiply-add of packed single-precision floating-point values
-        vfmadd231ps,
-        /// Fused multiply-add of scalar double-precision floating-point values
-        vfmadd132sd,
-        /// Fused multiply-add of scalar double-precision floating-point values
-        vfmadd213sd,
-        /// Fused multiply-add of scalar double-precision floating-point values
-        vfmadd231sd,
         /// Fused multiply-add of scalar single-precision floating-point values
-        vfmadd132ss,
-        /// Fused multiply-add of scalar single-precision floating-point values
-        vfmadd213ss,
-        /// Fused multiply-add of scalar single-precision floating-point values
-        vfmadd231ss,
+        /// Fused multiply-add of packed double-precision floating-point values
+        /// Fused multiply-add of scalar double-precision floating-point values
+        fmadd231,
 
         /// A pseudo instruction that requires special lowering.
         /// This should be the only tag in this enum that doesn't

From 16314e0e199051289075c9e3dab425ce3fb52719 Mon Sep 17 00:00:00 2001
From: jcalabro 
Date: Mon, 8 May 2023 11:49:52 -0400
Subject: [PATCH 138/725] Fix langref typo

---
 doc/langref.html.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/langref.html.in b/doc/langref.html.in
index 69ad0624c4..057bb91f71 100644
--- a/doc/langref.html.in
+++ b/doc/langref.html.in
@@ -2983,7 +2983,7 @@ test "using slices for strings" {
     const world: []const u8 = "世界";
 
     var all_together: [100]u8 = undefined;
-    // You can use slice syntax with at least one runtime-know index on an
+    // You can use slice syntax with at least one runtime-known index on an
     // array to convert an array into a slice.
     var start : usize = 0;
     const all_together_slice = all_together[start..];

From f2aa7c86980df7bcbf0405be63cf62c88ed22005 Mon Sep 17 00:00:00 2001
From: Krzysztof Wolicki 
Date: Mon, 8 May 2023 22:29:20 +0200
Subject: [PATCH 139/725] autodoc: Remove rendering "tuple" as part of the name
 in exprName

---
 lib/docs/main.js | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/lib/docs/main.js b/lib/docs/main.js
index 9b650643e9..1e037673e1 100644
--- a/lib/docs/main.js
+++ b/lib/docs/main.js
@@ -1845,17 +1845,9 @@ const NAV_MODES = {
             let structObj = typeObj;
             let name = "";
             if (opts.wantHtml) {
-              if (structObj.is_tuple) {
-                name = "tuple { ";
-              } else {
-                name = "struct { ";
-              }
+              name = "struct { ";
             } else {
-              if (structObj.is_tuple) {
-                name = "tuple { ";
-              } else {
-                name = "struct { ";
-              }
+              name = "struct { ";
             }
             if (structObj.field_types.length > 1 && opts.wantHtml) { name += "
"; }
             let indent = "";

From b488f97e92115581e12ac1a5aee6362fc4f12563 Mon Sep 17 00:00:00 2001
From: Krzysztof Wolicki 
Date: Mon, 8 May 2023 22:34:46 +0200
Subject: [PATCH 140/725] autodoc: Analyze and expose backing integer types for
 packed structs

---
 lib/docs/main.js | 16 +++++++++++++---
 src/Autodoc.zig  | 12 +++++++++++-
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/lib/docs/main.js b/lib/docs/main.js
index 1e037673e1..a7448c18ff 100644
--- a/lib/docs/main.js
+++ b/lib/docs/main.js
@@ -1849,6 +1849,15 @@ const NAV_MODES = {
             } else {
               name = "struct { ";
             }
+            if (structObj.backing_int !== null) {
+              if (opts.wantHtml) {
+                name = "packed " + name;
+              } else {
+                name = "packed " + name;
+              }
+              name += " (" + exprName(structObj.backing_int, opts) + ")";
+            }
+            name += " { ";
             if (structObj.field_types.length > 1 && opts.wantHtml) { name += "
"; }
             let indent = "";
             if (structObj.field_types.length > 1 && opts.wantHtml) {
@@ -4486,9 +4495,10 @@ function addDeclToSearchResults(decl, declIndex, modNames, item, list, stack) {
           pubDecls: ty[4],
           field_types: ty[5],
           field_defaults: ty[6],
-          is_tuple: ty[7],
-          line_number: ty[8],
-          parent_container: ty[9],
+          backing_int: ty[7],
+          is_tuple: ty[8],
+          line_number: ty[9],
+          parent_container: ty[10],
         };
       case 10: // ComptimeExpr
       case 11: // ComptimeFloat
diff --git a/src/Autodoc.zig b/src/Autodoc.zig
index 00fef3c768..b0ea618e70 100644
--- a/src/Autodoc.zig
+++ b/src/Autodoc.zig
@@ -604,6 +604,7 @@ const DocData = struct {
             pubDecls: []usize = &.{}, // index into decls
             field_types: []Expr = &.{}, // (use src->fields to find names)
             field_defaults: []?Expr = &.{}, // default values is specified
+            backing_int: ?Expr = null, // backing integer if specified
             is_tuple: bool,
             line_number: usize,
             parent_container: ?usize, // index into `types`
@@ -2854,13 +2855,21 @@ fn walkInstruction(
                         break :blk fields_len;
                     } else 0;
 
-                    // TODO: Expose explicit backing integer types in some way.
+                    var backing_int: ?DocData.Expr = null;
                     if (small.has_backing_int) {
                         const backing_int_body_len = file.zir.extra[extra_index];
                         extra_index += 1; // backing_int_body_len
                         if (backing_int_body_len == 0) {
+                            const backing_int_ref = @intToEnum(Ref, file.zir.extra[extra_index]);
+                            const backing_int_res = try self.walkRef(file, &scope, src_info, backing_int_ref, true);
+                            backing_int = backing_int_res.expr;
                             extra_index += 1; // backing_int_ref
                         } else {
+                            const backing_int_body = file.zir.extra[extra_index..][0..backing_int_body_len];
+                            const break_inst = backing_int_body[backing_int_body.len - 1];
+                            const operand = data[break_inst].@"break".operand;
+                            const backing_int_res = try self.walkRef(file, &scope, src_info, operand, true);
+                            backing_int = backing_int_res.expr;
                             extra_index += backing_int_body_len; // backing_int_body_inst
                         }
                     }
@@ -2903,6 +2912,7 @@ fn walkInstruction(
                             .field_types = field_type_refs.items,
                             .field_defaults = field_default_refs.items,
                             .is_tuple = small.is_tuple,
+                            .backing_int = backing_int,
                             .line_number = self.ast_nodes.items[self_ast_node_index].line,
                             .parent_container = parent_scope.enclosing_type,
                         },

From 9c8b9719cb0757eb1035af0692ab89b0f2c13579 Mon Sep 17 00:00:00 2001
From: Krzysztof Wolicki 
Date: Tue, 9 May 2023 10:51:19 +0200
Subject: [PATCH 141/725] autodoc: Fix exprName for struct and union types

---
 lib/docs/main.js | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/docs/main.js b/lib/docs/main.js
index a7448c18ff..f9bb9b9fb8 100644
--- a/lib/docs/main.js
+++ b/lib/docs/main.js
@@ -1845,9 +1845,9 @@ const NAV_MODES = {
             let structObj = typeObj;
             let name = "";
             if (opts.wantHtml) {
-              name = "struct { ";
+              name = "struct";
             } else {
-              name = "struct { ";
+              name = "struct";
             }
             if (structObj.backing_int !== null) {
               if (opts.wantHtml) {
@@ -1983,11 +1983,11 @@ const NAV_MODES = {
               name += " (" + exprName(unionObj.tag, opts) + ")";
             }
             name += " { ";
-            if (unionObj.fields.length > 1 && opts.wantHtml) {
+            if (unionObj.field_types.length > 1 && opts.wantHtml) {
               name += "
";
             }
             let indent = "";
-            if (unionObj.fields.length > 1 && opts.wantHtml) {
+            if (unionObj.field_types.length > 1 && opts.wantHtml) {
               indent = "    "
             }
             if (opts.indent) {
@@ -1995,17 +1995,17 @@ const NAV_MODES = {
             }
             let unionNode = getAstNode(unionObj.src);
             let field_end = ",";
-            if (unionObj.fields.length > 1 && opts.wantHtml) {
+            if (unionObj.field_types.length > 1 && opts.wantHtml) {
               field_end += "
";
             } else {
               field_end += " ";
             }
-            for (let i = 0; i < unionObj.fields.length; i += 1) {
+            for (let i = 0; i < unionObj.field_types.length; i += 1) {
               let fieldNode = getAstNode(unionNode.fields[i]);
               let fieldName = fieldNode.name;
               let html = indent + escapeHtml(fieldName);
 
-              let fieldTypeExpr = unionObj.fields[i];
+              let fieldTypeExpr = unionObj.field_types[i];
               html += ": ";
 
               html += exprName(fieldTypeExpr, { ...opts, indent: indent });

From a714f07022cb8a5d98e700090990111a7d1e234a Mon Sep 17 00:00:00 2001
From: Krzysztof Wolicki 
Date: Tue, 9 May 2023 10:52:51 +0200
Subject: [PATCH 142/725] autodoc: Fix tag type for unions; Fix backing int for
 packed structs

---
 src/Autodoc.zig | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/Autodoc.zig b/src/Autodoc.zig
index b0ea618e70..e9fb8ab630 100644
--- a/src/Autodoc.zig
+++ b/src/Autodoc.zig
@@ -2594,12 +2594,12 @@ fn walkInstruction(
 
                     // We delay analysis because union tags can refer to
                     // decls defined inside the union itself.
-                    const tag_type_ref: Ref = if (small.has_tag_type) blk: {
+                    const tag_type_ref: ?Ref = if (small.has_tag_type) blk: {
                         const tag_type = file.zir.extra[extra_index];
                         extra_index += 1;
                         const tag_ref = @intToEnum(Ref, tag_type);
                         break :blk tag_ref;
-                    } else .none;
+                    } else null;
 
                     const body_len = if (small.has_body_len) blk: {
                         const body_len = file.zir.extra[extra_index];
@@ -2626,13 +2626,13 @@ fn walkInstruction(
                     );
 
                     // Analyze the tag once all decls have been analyzed
-                    const tag_type = try self.walkRef(
+                    const tag_type = if (tag_type_ref) |tt_ref| (try self.walkRef(
                         file,
                         &scope,
                         parent_src,
-                        tag_type_ref,
+                        tt_ref,
                         false,
-                    );
+                    )).expr else null;
 
                     // Fields
                     extra_index += body_len;
@@ -2664,7 +2664,7 @@ fn walkInstruction(
                             .privDecls = priv_decl_indexes.items,
                             .pubDecls = decl_indexes.items,
                             .fields = field_type_refs.items,
-                            .tag = tag_type.expr,
+                            .tag = tag_type,
                             .auto_enum = small.auto_enum_tag,
                             .parent_container = parent_scope.enclosing_type,
                         },
@@ -2855,6 +2855,9 @@ fn walkInstruction(
                         break :blk fields_len;
                     } else 0;
 
+                    // We don't care about decls yet
+                    if (small.has_decls_len) extra_index += 1;
+
                     var backing_int: ?DocData.Expr = null;
                     if (small.has_backing_int) {
                         const backing_int_body_len = file.zir.extra[extra_index];

From 297b5d1074008b52a8d08a32c9f3281542105bdd Mon Sep 17 00:00:00 2001
From: r00ster91 
Date: Mon, 1 May 2023 19:15:51 +0200
Subject: [PATCH 143/725] fix `[x]u65529` and above overflowing

```
$ cat overflow.zig
test {
    var a: [1]u65535 = undefined;
    _ = a;
}
$ zig-out/bin/zig test overflow.zig
thread 290266 panic: integer overflow
zig/src/type.zig:3604:55: 0xada43d in intAbiAlignment (zig)
            std.math.ceilPowerOfTwoPromote(u16, (bits + 7) / 8),
                                                      ^
zig/src/type.zig:3598:42: 0xadd4ea in intAbiSize (zig)
        const alignment = intAbiAlignment(bits, target);
                                         ^
zig/src/type.zig:3500:61: 0x92be91 in abiSizeAdvanced (zig)
                return AbiSizeAdvanced{ .scalar = intAbiSize(bits, target) };
                                                            ^
zig/src/type.zig:3385:62: 0x928933 in abiSizeAdvanced (zig)
                switch (try payload.elem_type.abiSizeAdvanced(target, strat)) {
                                                             ^
zig/src/type.zig:3268:32: 0x92c012 in abiSize (zig)
        return (abiSizeAdvanced(ty, target, .eager) catch unreachable).scalar;
                               ^
```
This is only noticed in a debug build of zig and silently does the wrong
thing and overflows in release builds.

This happened to `[x]u65529` and above because of the ` + 7` on a `u16`.
---
 src/type.zig            |  4 ++--
 test/behavior/basic.zig | 21 +++++++++++++++++++++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/type.zig b/src/type.zig
index c9a6f49d3e..6122afda62 100644
--- a/src/type.zig
+++ b/src/type.zig
@@ -3596,12 +3596,12 @@ pub const Type = extern union {
 
     fn intAbiSize(bits: u16, target: Target) u64 {
         const alignment = intAbiAlignment(bits, target);
-        return std.mem.alignForwardGeneric(u64, (bits + 7) / 8, alignment);
+        return std.mem.alignForwardGeneric(u64, @intCast(u16, (@as(u17, bits) + 7) / 8), alignment);
     }
 
     fn intAbiAlignment(bits: u16, target: Target) u32 {
         return @min(
-            std.math.ceilPowerOfTwoPromote(u16, (bits + 7) / 8),
+            std.math.ceilPowerOfTwoPromote(u16, @intCast(u16, (@as(u17, bits) + 7) / 8)),
             target.maxIntAlignment(),
         );
     }
diff --git a/test/behavior/basic.zig b/test/behavior/basic.zig
index 8727d0dd41..073be26288 100644
--- a/test/behavior/basic.zig
+++ b/test/behavior/basic.zig
@@ -1124,3 +1124,24 @@ test "runtime-known globals initialized with undefined" {
     try expect(S.s[0] == 1);
     try expect(S.s[4] == 5);
 }
+
+test "arrays and vectors with big integers" {
+    if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
+
+    // TODO: only aarch64-windows didn't pass in the PR that added this code.
+    //       figure out why if you can run this target.
+    if (builtin.os.tag == .windows and builtin.cpu.arch == .aarch64) return error.SkipZigTest;
+
+    inline for (.{ u65528, u65529, u65535 }) |Int| {
+        var a: [1]Int = undefined;
+        a[0] = std.math.maxInt(Int);
+        try expect(a[0] == comptime std.math.maxInt(Int));
+        var b: @Vector(1, Int) = undefined;
+        b[0] = std.math.maxInt(Int);
+        try expect(b[0] == comptime std.math.maxInt(Int));
+    }
+}

From 3dd0afe787e1cab4db41e685e67ce5594e92f7ef Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Sun, 30 Apr 2023 14:32:12 +0100
Subject: [PATCH 144/725] std.c: adding freebsd's kinfo_proc type.

---
 lib/std/c/freebsd.zig | 186 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 186 insertions(+)

diff --git a/lib/std/c/freebsd.zig b/lib/std/c/freebsd.zig
index 352c12c695..ddcbe3b8da 100644
--- a/lib/std/c/freebsd.zig
+++ b/lib/std/c/freebsd.zig
@@ -64,6 +64,7 @@ pub extern "c" fn getpid() pid_t;
 
 pub extern "c" fn kinfo_getfile(pid: pid_t, cntp: *c_int) ?[*]kinfo_file;
 pub extern "c" fn kinfo_getvmmap(pid: pid_t, cntp: *c_int) ?[*]kinfo_vmentry;
+pub extern "c" fn kinfo_getproc(pid: pid_t) ?[*]kinfo_proc;
 
 pub extern "c" fn cpuset_getaffinity(level: cpulevel_t, which: cpuwhich_t, id: id_t, setsize: usize, mask: *cpuset_t) c_int;
 pub extern "c" fn cpuset_setaffinity(level: cpulevel_t, which: cpuwhich_t, id: id_t, setsize: usize, mask: *const cpuset_t) c_int;
@@ -676,6 +677,191 @@ comptime {
     std.debug.assert(@sizeOf(kinfo_vmentry) == KINFO_VMENTRY_SIZE);
 }
 
+pub const WMESGLEN = 8;
+pub const LOCKNAMELEN = 8;
+pub const TDNAMLEN = 16;
+pub const COMMLEN = 19;
+pub const MAXCOMLEN = 19;
+pub const KI_EMULNAMELEN = 16;
+pub const KI_NGROUPS = 16;
+pub const LOGNAMELEN = 17;
+pub const LOGINCLASSLEN = 17;
+
+pub const KI_NSPARE_INT = 2;
+pub const KI_NSPARE_LONG = 12;
+pub const KI_NSPARE_PTR = 5;
+
+pub const RUSAGE_SELF = 0;
+pub const RUSAGE_CHILDREN = -1;
+pub const RUSAGE_THREAD = 1;
+
+pub const proc = opaque {};
+pub const thread = opaque {};
+pub const vnode = opaque {};
+pub const filedesc = opaque {};
+pub const pwddesc = opaque {};
+pub const vmspace = opaque {};
+pub const pcb = opaque {};
+pub const lwpid_t = i32;
+pub const fixpt_t = u32;
+pub const vm_size_t = usize;
+pub const segsz_t = isize;
+
+pub const itimerval = extern struct {
+    interval: timeval,
+    value: timeval,
+};
+
+pub const pstats = extern struct {
+    cru: rusage,
+    timer: [3]itimerval,
+    prof: extern struct {
+        base: u8,
+        size: c_ulong,
+        off: c_ulong,
+        scale: c_ulong,
+    },
+    start: timeval,
+};
+
+pub const user = extern struct {
+    stats: pstats,
+    kproc: kinfo_proc,
+};
+
+pub const pargs = extern struct {
+    ref: c_uint,
+    length: c_uint,
+    args: [1]u8,
+};
+
+pub const priority = extern struct {
+    class: u8,
+    level: u8,
+    native: u8,
+    user: u8,
+};
+
+pub const rusage = extern struct {
+    utime: timeval,
+    stime: timeval,
+    maxrss: c_long,
+    ixrss: c_long,
+    idrss: c_long,
+    isrss: c_long,
+    minflt: c_long,
+    majflt: c_long,
+    nswap: c_long,
+    inblock: c_long,
+    oublock: c_long,
+    msgsnd: c_long,
+    msgrcv: c_long,
+    nsignals: c_long,
+    nvcsw: c_long,
+    nivcsw: c_long,
+};
+
+pub const kinfo_proc = extern struct {
+    structsize: c_int,
+    layout: c_int,
+    args: *pargs,
+    paddr: *proc,
+    addr: *user,
+    tracep: *vnode,
+    textvp: *vnode,
+    fd: *filedesc,
+    vmspace: *vmspace,
+    wchan: ?*const anyopaque,
+    pid: pid_t,
+    ppid: pid_t,
+    pgid: pid_t,
+    tpgid: pid_t,
+    sid: pid_t,
+    tsid: pid_t,
+    jobc: c_short,
+    spare_short1: c_short,
+    tdev_freebsd11: u32,
+    siglist: sigset_t,
+    sigmask: sigset_t,
+    sigignore: sigset_t,
+    sigcatch: sigset_t,
+    uid: uid_t,
+    ruid: uid_t,
+    svuid: uid_t,
+    rgid: gid_t,
+    svgid: gid_t,
+    ngroups: c_short,
+    spare_short2: c_short,
+    groups: [KI_NGROUPS]gid_t,
+    size: vm_size_t,
+    rssize: segsz_t,
+    swrss: segsz_t,
+    tsize: segsz_t,
+    dsize: segsz_t,
+    ssize: segsz_t,
+    xstat: c_ushort,
+    acflag: c_ushort,
+    pctcpu: fixpt_t,
+    estcpu: c_uint,
+    slptime: c_uint,
+    swtime: c_uint,
+    cow: c_uint,
+    runtime: u64,
+    start: timeval,
+    childtime: timeval,
+    flag: c_long,
+    kiflag: c_long,
+    traceflag: c_int,
+    stat: u8,
+    nice: i8,
+    lock: u8,
+    rqindex: u8,
+    oncpu_old: u8,
+    lastcpu_old: u8,
+    tdname: [TDNAMLEN + 1]u8,
+    wmesg: [WMESGLEN + 1]u8,
+    login: [LOGNAMELEN + 1]u8,
+    lockname: [LOCKNAMELEN + 1]u8,
+    comm: [COMMLEN + 1]u8,
+    emul: [KI_EMULNAMELEN + 1]u8,
+    loginclass: [LOGINCLASSLEN + 1]u8,
+    moretdname: [MAXCOMLEN - TDNAMLEN + 1]u8,
+    sparestrings: [46]u8,
+    spareints: [KI_NSPARE_INT]c_int,
+    tdev: u64,
+    oncpu: c_int,
+    lastcpu: c_int,
+    tracer: c_int,
+    flag2: c_int,
+    fibnum: c_int,
+    cr_flags: c_uint,
+    jid: c_int,
+    numthreads: c_int,
+    tid: lwpid_t,
+    pri: priority,
+    rusage: rusage,
+    rusage_ch: rusage,
+    pcb: *pcb,
+    stack: ?*anyopaque,
+    udata: ?*anyopaque,
+    tdaddr: *thread,
+    pd: *pwddesc,
+    spareptrs: [KI_NSPARE_PTR]?*anyopaque,
+    sparelongs: [KI_NSPARE_LONG]c_long,
+    sflag: c_long,
+    tdflag: c_long,
+};
+
+pub const KINFO_PROC_SIZE = switch (builtin.cpu.arch) {
+    .x86 => 768,
+    .arm => 816,
+    else => 1088,
+};
+
+comptime {
+    assert(@sizeOf(kinfo_proc) == KINFO_PROC_SIZE);
+}
+
 pub const CTL = struct {
     pub const KERN = 1;
     pub const DEBUG = 5;

From 1585ed637d101ed16adb6b9ebdfa465299bfdb13 Mon Sep 17 00:00:00 2001
From: David Carlier 
Date: Wed, 3 May 2023 22:56:37 +0100
Subject: [PATCH 145/725] std.c: openbsd sigcontext/ucontext for arm64.

---
 lib/std/c/openbsd.zig | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lib/std/c/openbsd.zig b/lib/std/c/openbsd.zig
index 51c4bcb6dd..3551f50020 100644
--- a/lib/std/c/openbsd.zig
+++ b/lib/std/c/openbsd.zig
@@ -1127,6 +1127,18 @@ pub usingnamespace switch (builtin.cpu.arch) {
             sc_cookie: c_long,
         };
     },
+    .arm64 => struct {
+        pub const ucontext_t = extern struct {
+            __sc_unused: c_int,
+            sc_mask: c_int,
+            sc_sp: c_ulong,
+            sc_lr: c_ulong,
+            sc_elr: c_ulong,
+            sc_spsr: c_ulong,
+            sc_x: [30]c_ulong,
+            sc_cookie: c_long,
+        };
+    },
     else => struct {},
 };
 

From e8f76b452bf86481069c2eeeb02d46161ea79a08 Mon Sep 17 00:00:00 2001
From: David CARLIER 
Date: Mon, 1 May 2023 15:29:42 +0100
Subject: [PATCH 146/725] std.c: adding freebsd's domainset api

---
 lib/std/c/freebsd.zig | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/lib/std/c/freebsd.zig b/lib/std/c/freebsd.zig
index ddcbe3b8da..e035a408d0 100644
--- a/lib/std/c/freebsd.zig
+++ b/lib/std/c/freebsd.zig
@@ -2548,3 +2548,28 @@ pub extern "c" fn mincore(
     length: usize,
     vec: [*]u8,
 ) c_int;
+
+pub const MAXMEMDOM = 8;
+pub const domainid_t = u8;
+
+pub const LIST_ENTRY = opaque {};
+
+pub const DOMAINSET = struct {
+    pub const POLICY_INVALID = 0;
+    pub const POLICY_ROUNDROBIN = 1;
+    pub const POLICY_FIRSTOUCH = 2;
+    pub const POLICY_PREFER = 3;
+    pub const POLICY_INTERLEAVE = 4;
+    pub const POLICY_MAX = DOMAINSET.POLICY_INTERLEAVE;
+};
+
+pub const domainset_t = extern struct {
+    link: LIST_ENTRY,
+    policy: u16,
+    prefer: domainid_t,
+    cnt: domainid_t,
+    order: [MAXMEMDOM]domainid_t,
+};
+
+pub extern "c" fn cpuset_getdomain(level: cpulevel_t, which: cpuwhich_t, id: id_t, len: usize, domain: *domainset_t, r: *c_int) c_int;
+pub extern "c" fn cpuset_setdomain(level: cpulevel_t, which: cpuwhich_t, id: id_t, len: usize, domain: *const domainset_t, r: c_int) c_int;

From e21739dd8caf7d6a9aefe68c37eddb6406bcb810 Mon Sep 17 00:00:00 2001
From: David Carlier 
Date: Tue, 25 Apr 2023 20:16:39 +0100
Subject: [PATCH 147/725] std.fs: selfExePath haiku using constants instead

---
 lib/std/c/haiku.zig | 11 +++++++++--
 lib/std/fs.zig      |  3 +--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/std/c/haiku.zig b/lib/std/c/haiku.zig
index 9b693a59c2..b0fcb710af 100644
--- a/lib/std/c/haiku.zig
+++ b/lib/std/c/haiku.zig
@@ -11,9 +11,9 @@ extern "c" fn _errnop() *c_int;
 
 pub const _errno = _errnop;
 
-pub extern "c" fn find_directory(which: c_int, volume: i32, createIt: bool, path_ptr: [*]u8, length: i32) status_t;
+pub extern "c" fn find_directory(which: directory_which, volume: i32, createIt: bool, path_ptr: [*]u8, length: i32) status_t;
 
-pub extern "c" fn find_path(codePointer: *const u8, baseDirectory: c_int, subPath: [*:0]const u8, pathBuffer: [*:0]u8, bufferSize: usize) status_t;
+pub extern "c" fn find_path(codePointer: *const u8, baseDirectory: path_base_directory, subPath: [*:0]const u8, pathBuffer: [*:0]u8, bufferSize: usize) status_t;
 
 pub extern "c" fn find_thread(thread_name: ?*anyopaque) i32;
 
@@ -1024,6 +1024,13 @@ pub const directory_which = enum(c_int) {
     _,
 };
 
+// TODO fill out if needed
+pub const path_base_directory = enum(c_int) {
+    B_FIND_PATH_IMAGE_PATH = 1000,
+};
+
+pub const B_APP_IMAGE_SYMBOL = null;
+
 pub const cc_t = u8;
 pub const speed_t = u8;
 pub const tcflag_t = u32;
diff --git a/lib/std/fs.zig b/lib/std/fs.zig
index 9f64387bd8..7327a3a913 100644
--- a/lib/std/fs.zig
+++ b/lib/std/fs.zig
@@ -2999,8 +2999,7 @@ pub fn selfExePath(out_buffer: []u8) SelfExePathError![]u8 {
         .haiku => {
             // The only possible issue when looking for the self image path is
             // when the buffer is too short.
-            // TODO replace with proper constants
-            if (os.find_path(null, 1000, null, out_buffer.ptr, out_buffer.len) != 0)
+            if (os.find_path(os.B_APP_IMAGE_SYMBOL, os.path_base_directory.B_FIND_IMAGE_PATH, null, out_buffer.ptr, out_buffer.len) != 0)
                 return error.Overflow;
             return mem.sliceTo(out_buffer, 0);
         },

From e02a5924374c7b8d804b90220d0224ab73fbccb1 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 9 May 2023 14:24:21 -0700
Subject: [PATCH 148/725] Autodoc: update favicon to latest logo mark

ziglang/logo commit 6446ba8e37a0651da720d8869e1ce9264fa0c0b9
---
 src/autodoc/render_source.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/autodoc/render_source.zig b/src/autodoc/render_source.zig
index 4777ba399b..439b43dce3 100644
--- a/src/autodoc/render_source.zig
+++ b/src/autodoc/render_source.zig
@@ -25,7 +25,7 @@ pub fn genHtml(
     );
     try out.print("    {s} - source view\n", .{src.sub_file_path});
     try out.writeAll(
-        \\    
+        \\    
         \\