Tokenizer bug fixes and improvements

Fixes many error messages corresponding to invalid bytes displaying the
wrong byte. Additionaly improves handling of UTF-8 in some places.

test/compile_errors.zig

@@ -42,8 +42,8 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
         const case = ctx.obj("isolated carriage return in multiline string literal", b.graph.host);
 
         case.addError("const foo = \\\\\test\r\r rogue carriage return\n;", &[_][]const u8{
-            ":1:19: error: expected ';' after declaration",
-            ":1:20: note: invalid byte: '\\r'",
+            ":1:13: error: expected expression, found 'invalid bytes'",
+            ":1:19: note: invalid byte: '\\r'",
         });
     }
 
@@ -217,4 +217,40 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
             \\pub fn anytypeFunction(_: anytype) void {}
         );
     }
+
+    {
+        const case = ctx.obj("invalid byte in string", b.graph.host);
+
+        case.addError("_ = \"\x01Q\";", &[_][]const u8{
+            ":1:5: error: expected expression, found 'invalid bytes'",
+            ":1:6: note: invalid byte: '\\x01'",
+        });
+    }
+
+    {
+        const case = ctx.obj("invalid byte in comment", b.graph.host);
+
+        case.addError("//\x01Q", &[_][]const u8{
+            ":1:1: error: expected type expression, found 'invalid bytes'",
+            ":1:3: note: invalid byte: '\\x01'",
+        });
+    }
+
+    {
+        const case = ctx.obj("control character in character literal", b.graph.host);
+
+        case.addError("const c = '\x01';", &[_][]const u8{
+            ":1:11: error: expected expression, found 'invalid bytes'",
+            ":1:12: note: invalid byte: '\\x01'",
+        });
+    }
+
+    {
+        const case = ctx.obj("invalid byte at start of token", b.graph.host);
+
+        case.addError("x = \x00Q", &[_][]const u8{
+            ":1:5: error: expected expression, found 'invalid bytes'",
+            ":1:5: note: invalid byte: '\\x00'",
+        });
+    }
 }