From d28c5069b8421628fcb333d8e558a41d2a2d93d5 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Tue, 24 Mar 2026 11:23:28 +0000 Subject: [PATCH] llvm: rework handling of globals The main goal here is to make incremental compilation work a bit better. I also slightly expanded some `std.zig.llvm.Builder` APIs so that we don't need to pointlessly create new `Global`s whenever e.g. a function turns into a variable or vice versa. Also, lean into aliases for exports! If we just use aliases for every export, everything becomes simpler. Besides, we can't just go around renaming the globals of `Nav`s: the export could disappear on a future update, in which case we'd have to somehow revert that change, which is easier said than done. --- lib/std/zig/llvm/Builder.zig | 155 +++-- src/Sema.zig | 1 - src/Zcu.zig | 4 +- src/Zcu/PerThread.zig | 9 +- src/codegen/llvm.zig | 1204 ++++++++++++++++------------------ src/codegen/llvm/FuncGen.zig | 60 +- 6 files changed, 704 insertions(+), 729 deletions(-) diff --git a/lib/std/zig/llvm/Builder.zig b/lib/std/zig/llvm/Builder.zig index 5001a0c375..276012b09a 100644 --- a/lib/std/zig/llvm/Builder.zig +++ b/lib/std/zig/llvm/Builder.zig @@ -2343,12 +2343,13 @@ pub const Global = struct { none = maxInt(u32), _, - pub fn unwrap(self: Index, builder: *const Builder) Index { - var cur = self; + pub fn unwrap(orig_index: Index, builder: *const Builder) Index { + var cur = orig_index; while (true) { - const replacement = cur.getReplacement(builder); - if (replacement == .none) return cur; - cur = replacement; + switch (builder.globals.values()[@intFromEnum(cur)].kind) { + .replaced => |replacement| cur = replacement, + else => return cur, + } } } @@ -2388,8 +2389,12 @@ pub const Global = struct { return self.ptrConst(builder).type; } - pub fn toConst(self: Index) Constant { - return @enumFromInt(@intFromEnum(Constant.first_global) + @intFromEnum(self)); + pub fn toConst(global: Index) Constant { + return @enumFromInt(@intFromEnum(Constant.first_global) + @intFromEnum(global)); + } + + pub fn toValue(global: Index) Value { + return global.toConst().toValue(); } pub fn setLinkage(self: Index, linkage: Linkage, builder: *Builder) void { @@ -2450,6 +2455,42 @@ pub const Global = struct { self.ptr(builder).kind = .{ .replaced = .none }; } + /// Replaces whatever this `Global` currently contains with a new `Function`. Similar to + /// `Builder.addFunction`, but the same `Global` is reused. + pub fn toNewFunction(global: Index, builder: *Builder) Allocator.Error!Function.Index { + try builder.functions.ensureUnusedCapacity(builder.gpa, 1); + errdefer comptime unreachable; + const function: Function.Index = @enumFromInt(builder.functions.items.len); + builder.functions.appendAssumeCapacity(.{ + .global = global, + .strip = undefined, + }); + global.ptr(builder).kind = .{ .function = function }; + return function; + } + + /// Replaces whatever this `Global` currently contains with a new `Variable`. Similar to + /// `Builder.addVariable`, but the same `Global` is reused. + pub fn toNewVariable(global: Index, builder: *Builder) Allocator.Error!Variable.Index { + try builder.variables.ensureUnusedCapacity(builder.gpa, 1); + errdefer comptime unreachable; + const variable: Variable.Index = @enumFromInt(builder.variables.items.len); + builder.variables.appendAssumeCapacity(.{ .global = global }); + global.ptr(builder).kind = .{ .variable = variable }; + return variable; + } + + /// Replaces whatever this `Global` currently contains with a new `Alias`. Similar to + /// `Builder.addAlias`, but the same `Global` is reused. + pub fn toNewAlias(global: Index, builder: *Builder) Allocator.Error!Alias.Index { + try builder.aliases.ensureUnusedCapacity(builder.gpa, 1); + errdefer comptime unreachable; + const alias: Alias.Index = @enumFromInt(builder.aliases.items.len); + builder.aliass.appendAssumeCapacity(.{ .global = global, .aliasee = .none }); + global.ptr(builder).kind = .{ .alias = alias }; + return alias; + } + fn updateDsoLocal(self: Index, builder: *Builder) void { const self_ptr = self.ptr(builder); switch (self_ptr.linkage) { @@ -2494,13 +2535,6 @@ pub const Global = struct { self.renameAssumeCapacity(builder.next_replaced_global, builder); self.ptr(builder).kind = .{ .replaced = other.unwrap(builder) }; } - - fn getReplacement(self: Index, builder: *const Builder) Index { - return switch (builder.globals.values()[@intFromEnum(self)].kind) { - .replaced => |replacement| replacement, - else => .none, - }; - } }; }; @@ -2593,22 +2627,6 @@ pub const Variable = struct { return self.toConst(builder).toValue(); } - pub fn setLinkage(self: Index, linkage: Linkage, builder: *Builder) void { - return self.ptrConst(builder).global.setLinkage(linkage, builder); - } - - pub fn setVisibility(self: Index, visibility: Visibility, builder: *Builder) void { - return self.ptrConst(builder).global.setVisibility(visibility, builder); - } - - pub fn setDllStorageClass(self: Index, class: DllStorageClass, builder: *Builder) void { - return self.ptrConst(builder).global.setDllStorageClass(class, builder); - } - - pub fn setUnnamedAddr(self: Index, unnamed_addr: UnnamedAddr, builder: *Builder) void { - return self.ptrConst(builder).global.setUnnamedAddr(unnamed_addr, builder); - } - pub fn setThreadLocal(self: Index, thread_local: ThreadLocal, builder: *Builder) void { self.ptr(builder).thread_local = thread_local; } @@ -9692,8 +9710,12 @@ pub fn print(self: *Builder, w: *Writer) (Writer.Error || Allocator.Error)!void if (self.variables.items.len > 0) { if (need_newline) try w.writeByte('\n') else need_newline = true; - for (self.variables.items) |variable| { - if (variable.global.getReplacement(self) != .none) continue; + for (self.variables.items, 0..) |variable, variable_i| { + // Skip the variable if its global has been repurposed for something else. + switch (variable.global.ptrConst(self).kind) { + .variable => |v| if (@intFromEnum(v) != variable_i) continue, + else => continue, + } const global = variable.global.ptrConst(self); metadata_formatter.need_comma = true; defer metadata_formatter.need_comma = undefined; @@ -9723,8 +9745,12 @@ pub fn print(self: *Builder, w: *Writer) (Writer.Error || Allocator.Error)!void if (self.aliases.items.len > 0) { if (need_newline) try w.writeByte('\n') else need_newline = true; - for (self.aliases.items) |alias| { - if (alias.global.getReplacement(self) != .none) continue; + for (self.aliases.items, 0..) |alias, alias_i| { + // Skip the alias if its global has been repurposed for something else. + switch (alias.global.ptrConst(self).kind) { + .alias => |a| if (@intFromEnum(a) != alias_i) continue, + else => continue, + } const global = alias.global.ptrConst(self); metadata_formatter.need_comma = true; defer metadata_formatter.need_comma = undefined; @@ -9750,7 +9776,11 @@ pub fn print(self: *Builder, w: *Writer) (Writer.Error || Allocator.Error)!void defer attribute_groups.deinit(self.gpa); for (0.., self.functions.items) |function_i, function| { - if (function.global.getReplacement(self) != .none) continue; + // Skip the function if its global has been repurposed for something else. + switch (function.global.ptrConst(self).kind) { + .function => |f| if (@intFromEnum(f) != function_i) continue, + else => continue, + } if (need_newline) try w.writeByte('\n') else need_newline = true; const function_index: Function.Index = @enumFromInt(function_i); const global = function.global.ptrConst(self); @@ -13687,20 +13717,32 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco self.aliases.items.len, ); - for (self.variables.items) |variable| { - if (variable.global.getReplacement(self) != .none) continue; + for (self.variables.items, 0..) |variable, variable_i| { + // Skip the variable if its global has been repurposed for something else. + switch (variable.global.ptrConst(self).kind) { + .variable => |v| if (@intFromEnum(v) != variable_i) continue, + else => continue, + } globals.putAssumeCapacity(variable.global, {}); } - for (self.functions.items) |function| { - if (function.global.getReplacement(self) != .none) continue; + for (self.functions.items, 0..) |function, function_i| { + // Skip the function if its global has been repurposed for something else. + switch (function.global.ptrConst(self).kind) { + .function => |f| if (@intFromEnum(f) != function_i) continue, + else => continue, + } globals.putAssumeCapacity(function.global, {}); } - for (self.aliases.items) |alias| { - if (alias.global.getReplacement(self) != .none) continue; + for (self.aliases.items, 0..) |alias, alias_i| { + // Skip the alias if its global has been repurposed for something else. + switch (alias.global.ptrConst(self).kind) { + .alias => |a| if (@intFromEnum(a) != alias_i) continue, + else => continue, + } globals.putAssumeCapacity(alias.global, {}); } @@ -13742,8 +13784,12 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco defer section_map.deinit(self.gpa); try section_map.ensureUnusedCapacity(self.gpa, globals.count()); - for (self.variables.items) |variable| { - if (variable.global.getReplacement(self) != .none) continue; + for (self.variables.items, 0..) |variable, variable_i| { + // Skip the variable if its global has been repurposed for something else. + switch (variable.global.ptrConst(self).kind) { + .variable => |v| if (@intFromEnum(v) != variable_i) continue, + else => continue, + } const section = blk: { if (variable.section == .none) break :blk 0; @@ -13789,8 +13835,12 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco }); } - for (self.functions.items) |func| { - if (func.global.getReplacement(self) != .none) continue; + for (self.functions.items, 0..) |func, func_i| { + // Skip the function if its global has been repurposed for something else. + switch (func.global.ptrConst(self).kind) { + .function => |f| if (@intFromEnum(f) != func_i) continue, + else => continue, + } const section = blk: { if (func.section == .none) break :blk 0; @@ -13830,8 +13880,12 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco }); } - for (self.aliases.items) |alias| { - if (alias.global.getReplacement(self) != .none) continue; + for (self.aliases.items, 0..) |alias, alias_i| { + // Skip the alias if its global has been repurposed for something else. + switch (alias.global.ptrConst(self).kind) { + .alias => |a| if (@intFromEnum(a) != alias_i) continue, + else => continue, + } const strtab = alias.global.strtab(self); @@ -14635,8 +14689,13 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco }; for (self.functions.items, 0..) |func, func_index| { + // Skip the function if its global has been repurposed for something else. + switch (func.global.ptrConst(self).kind) { + .function => |f| if (@intFromEnum(f) != func_index) continue, + else => continue, + } + const FunctionBlock = ir.ModuleBlock.FunctionBlock; - if (func.global.getReplacement(self) != .none) continue; if (func.instructions.len == 0) continue; diff --git a/src/Sema.zig b/src/Sema.zig index ab7e8b4c15..bda2659d8b 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -5751,7 +5751,6 @@ fn zirExport(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void if (ptr_info.byte_offset != 0) { return sema.fail(block, ptr_src, "TODO: export pointer in middle of value", .{}); } - if (zcu.llvm_object != null and options.linkage == .internal) return; try sema.exports.append(zcu.gpa, .{ .opts = options, .src = src, diff --git a/src/Zcu.zig b/src/Zcu.zig index 820524d718..88258b236a 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -3731,7 +3731,9 @@ pub fn resetUnit(zcu: *Zcu, unit: AnalUnit) void { }; for (zcu.all_exports.items[base..][0..len], base..) |exp, exp_index_usize| { const exp_index: Export.Index = @enumFromInt(exp_index_usize); - if (zcu.comp.bin_file) |lf| { + if (zcu.llvm_object) |llvm_object| { + _ = llvm_object; // TODO: delete exports from LLVM + } else if (zcu.comp.bin_file) |lf| { lf.deleteExport(exp.exported, exp.opts.name); } if (zcu.failed_exports.fetchSwapRemove(exp_index)) |failed_kv| { diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index c9972a300a..7d7e029041 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -1910,14 +1910,7 @@ fn analyzeNavVal( try sema.flushExports(); - queue_codegen: { - if (!queue_linker_work) break :queue_codegen; - - if (!nav_ty.hasRuntimeBits(zcu)) { - if (comp.config.use_llvm) break :queue_codegen; - if (file.mod.?.strip) break :queue_codegen; - } - + if (queue_linker_work) { comp.link_prog_node.increaseEstimatedTotalItems(1); try comp.link_queue.enqueueZcu(comp, pt.tid, .{ .link_nav = nav_id }); } diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 6716b1caea..bd3be4f96f 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -550,17 +550,13 @@ pub const Object = struct { debug_anyerror_fwd_ref: Builder.Metadata.Optional, zcu: *Zcu, - /// Ideally we would use `llvm_module.getNamedFunction` to go from *Decl to LLVM function, - /// but that has some downsides: - /// * we have to compute the fully qualified name every time we want to do the lookup - /// * for externally linked functions, the name is not fully qualified, but when - /// a Decl goes from exported to not exported and vice-versa, we would use the wrong - /// version of the name and incorrectly get function not found in the llvm module. - /// * it works for functions not all globals. - /// Therefore, this table keeps track of the mapping. + /// Maps a `Nav` to the corresponding LLVM global. nav_map: std.AutoHashMapUnmanaged(InternPool.Nav.Index, Builder.Global.Index), - /// Same deal as `decl_map` but for anonymous declarations, which are always global constants. - uav_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Global.Index), + /// Same as `nav_map` but for UAVs (which are always global constants). + uav_map: std.AutoHashMapUnmanaged(struct { + val: InternPool.Index, + @"addrspace": std.builtin.AddressSpace, + }, Builder.Variable.Index), /// Maps enum types to their corresponding LLVM functions for implementing the `tag_name` instruction. enum_tag_name_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Function.Index), /// Serves the same purpose as `enum_tag_name_map` but for the `is_named_enum_value` instruction. @@ -717,13 +713,13 @@ pub const Object = struct { for (llvm_errors[1..], error_name_list) |*llvm_error, name| { const name_string = try o.builder.stringNull(name.toSlice(ip)); const name_init = try o.builder.stringConst(name_string); - const name_variable_index = - try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); + const name_variable_index = try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); try name_variable_index.setInitializer(name_init, &o.builder); - name_variable_index.setLinkage(.private, &o.builder); name_variable_index.setMutability(.constant, &o.builder); - name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); name_variable_index.setAlignment(comptime .fromByteUnits(1), &o.builder); + const global_index = name_variable_index.ptrConst(&o.builder).global; + global_index.setLinkage(.private, &o.builder); + global_index.setUnnamedAddr(.unnamed_addr, &o.builder); llvm_error.* = try o.builder.structConst(llvm_slice_ty, &.{ name_variable_index.toConst(&o.builder), @@ -790,9 +786,9 @@ pub const Object = struct { array_llvm_ty, .default, ); - compiler_used_variable.setLinkage(.appending, &o.builder); - compiler_used_variable.setSection(try o.builder.string("llvm.metadata"), &o.builder); try compiler_used_variable.setInitializer(init_val, &o.builder); + compiler_used_variable.setSection(try o.builder.string("llvm.metadata"), &o.builder); + compiler_used_variable.ptrConst(&o.builder).global.setLinkage(.appending, &o.builder); } if (!o.builder.strip) { @@ -1140,6 +1136,7 @@ pub const Object = struct { ) Zcu.CodegenFailError!void { const zcu = o.zcu; const comp = zcu.comp; + const gpa = comp.gpa; const ip = &zcu.intern_pool; const func = zcu.funcInfo(func_index); const nav = ip.getNav(func.owner_nav); @@ -1149,9 +1146,42 @@ pub const Object = struct { const fn_info = zcu.typeToFunc(fn_ty).?; const target = &owner_mod.resolved_target.result; - const function_index = try o.resolveLlvmFunction(func.owner_nav); + const gop = try o.nav_map.getOrPut(gpa, func.owner_nav); + if (!gop.found_existing) { + errdefer assert(o.nav_map.remove(func.owner_nav)); + // First time lowering this NAV! Create a fresh global. + const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip)); + gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{ + .type = .void, // placeholder; populated below + .kind = .{ .alias = .none }, // placeholder; populated below + }); + } + const llvm_global = gop.value_ptr.*; - var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder); + const llvm_function: Builder.Function.Index = switch (llvm_global.ptrConst(&o.builder).kind) { + .function => |function| function, // re-use existing `Builder.Function` + .replaced, .alias, .variable => try llvm_global.toNewFunction(&o.builder), + }; + { + const global = llvm_function.ptrConst(&o.builder).global.ptr(&o.builder); + global.type = try o.lowerType(fn_ty); + global.addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", target); + global.linkage = if (o.builder.strip) .private else .internal; + global.visibility = .default; + global.dll_storage_class = .default; + global.unnamed_addr = .unnamed_addr; + } + llvm_function.setAlignment(switch (nav.resolved.?.@"align") { + .none => fn_ty.abiAlignment(zcu).toLlvm(), + else => |a| a.toLlvm(), + }, &o.builder); + llvm_function.setSection(s: { + const section = nav.resolved.?.@"linksection".toSlice(ip) orelse break :s .none; + break :s try o.builder.string(section); + }, &o.builder); + try o.addLlvmFunctionAttributes(pt, func.owner_nav, llvm_function); + + var attributes = try llvm_function.ptrConst(&o.builder).attributes.toWip(&o.builder); defer attributes.deinit(&o.builder); const func_analysis = func.analysisUnordered(ip); @@ -1221,47 +1251,41 @@ pub const Object = struct { } }, &o.builder); } - if (nav.resolved.?.@"linksection".toSlice(ip)) |section| - function_index.setSection(try o.builder.string(section), &o.builder); - var deinit_wip = true; var wip = try Builder.WipFunction.init(&o.builder, .{ - .function = function_index, + .function = llvm_function, .strip = owner_mod.strip, }); defer if (deinit_wip) wip.deinit(); wip.cursor = .{ .block = try wip.block(0, "Entry") }; - var llvm_arg_i: u32 = 0; - - const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: { - const param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; - break :param param; - } else .none; - if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) { .signed => try attributes.addRetAttr(.signext, &o.builder), .unsigned => try attributes.addRetAttr(.zeroext, &o.builder), }; - const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing; - - const err_ret_trace: Builder.Value = if (err_return_tracing) param: { - const param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; - break :param param; - } else .none; - // This is the list of args we will use that correspond directly to the AIR arg // instructions. Depending on the calling convention, this list is not necessarily // a bijection with the actual LLVM parameters of the function. - const gpa = o.gpa; var args: std.ArrayList(Builder.Value) = .empty; defer args.deinit(gpa); - { + const ret_ptr: Builder.Value, const err_ret_trace: Builder.Value = implicit_args: { var it = iterateParamTypes(o, fn_info); + + const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: { + const param = wip.arg(it.llvm_index); + it.llvm_index += 1; + break :param param; + } else .none; + + const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing; + const err_ret_trace: Builder.Value = if (err_return_tracing) param: { + const param = wip.arg(it.llvm_index); + it.llvm_index += 1; + break :param param; + } else .none; + while (try it.next()) |lowering| { try args.ensureUnusedCapacity(gpa, 1); @@ -1271,7 +1295,7 @@ pub const Object = struct { assert(!it.byval_attr); const param_index = it.zig_index - 1; const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); - const param = wip.arg(llvm_arg_i); + const param = wip.arg(it.llvm_index - 1); if (isByRef(param_ty, zcu)) { const alignment = param_ty.abiAlignment(zcu).toLlvm(); @@ -1281,146 +1305,116 @@ pub const Object = struct { args.appendAssumeCapacity(arg_ptr); } else { args.appendAssumeCapacity(param); - - try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, llvm_arg_i); } - llvm_arg_i += 1; }, .byref => { - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(param_ty); - const param = wip.arg(llvm_arg_i); - const alignment = param_ty.abiAlignment(zcu).toLlvm(); - - try o.addByRefParamAttrs(&attributes, llvm_arg_i, alignment, it.byval_attr, param_llvm_ty); - llvm_arg_i += 1; + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param = wip.arg(it.llvm_index - 1); if (isByRef(param_ty, zcu)) { args.appendAssumeCapacity(param); } else { + const param_llvm_ty = try o.lowerType(param_ty); + const alignment = param_ty.abiAlignment(zcu).toLlvm(); args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, "")); } }, .byref_mut => { - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(param_ty); - const param = wip.arg(llvm_arg_i); - const alignment = param_ty.abiAlignment(zcu).toLlvm(); - - try attributes.addParamAttr(llvm_arg_i, .noundef, &o.builder); - llvm_arg_i += 1; + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param = wip.arg(it.llvm_index - 1); if (isByRef(param_ty, zcu)) { args.appendAssumeCapacity(param); } else { + const param_llvm_ty = try o.lowerType(param_ty); + const alignment = param_ty.abiAlignment(zcu).toLlvm(); args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, "")); } }, .abi_sized_int => { assert(!it.byval_attr); - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param = wip.arg(it.llvm_index - 1); const param_llvm_ty = try o.lowerType(param_ty); const alignment = param_ty.abiAlignment(zcu).toLlvm(); const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target); _ = try wip.store(.normal, param, arg_ptr, alignment); - args.appendAssumeCapacity(if (isByRef(param_ty, zcu)) - arg_ptr - else - try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); + if (isByRef(param_ty, zcu)) { + args.appendAssumeCapacity(arg_ptr); + } else { + args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); + } }, .slice => { assert(!it.byval_attr); - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const ptr_info = param_ty.ptrInfo(zcu); - - if (std.math.cast(u5, it.zig_index - 1)) |i| { - if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { - try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); - } - } - if (param_ty.zigTypeTag(zcu) != .optional and - !ptr_info.flags.is_allowzero and - ptr_info.flags.address_space == .generic) - { - try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); - } - if (ptr_info.flags.is_const) { - try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder); - } - const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) { - else => |a| .wrap(a.toLlvm()), - .none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)), - }; - try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder); - const ptr_param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; - const len_param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; - - const slice_llvm_ty = try o.lowerType(param_ty); - args.appendAssumeCapacity( - try wip.buildAggregate(slice_llvm_ty, &.{ ptr_param, len_param }, ""), + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + assert(!isByRef(param_ty, zcu)); + const slice_val = try wip.buildAggregate( + try o.lowerType(param_ty), + &.{ wip.arg(it.llvm_index - 2), wip.arg(it.llvm_index - 1) }, + "", ); + args.appendAssumeCapacity(slice_val); }, .multiple_llvm_types => { assert(!it.byval_attr); const field_types = it.types_buffer[0..it.types_len]; - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); const param_llvm_ty = try o.lowerType(param_ty); const param_alignment = param_ty.abiAlignment(zcu).toLlvm(); const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, param_alignment, target); const llvm_ty = try o.builder.structType(.normal, field_types); - for (0..field_types.len) |field_i| { - const param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; + const llvm_args_start = it.llvm_index - field_types.len; + for (0..field_types.len, llvm_args_start..) |field_i, llvm_arg_index| { + const param = wip.arg(@intCast(llvm_arg_index)); const field_ptr = try wip.gepStruct(llvm_ty, arg_ptr, field_i, ""); - const alignment = Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8)); + const alignment: Builder.Alignment = .fromByteUnits(@divExact(target.ptrBitWidth(), 8)); _ = try wip.store(.normal, param, field_ptr, alignment); } - const is_by_ref = isByRef(param_ty, zcu); - args.appendAssumeCapacity(if (is_by_ref) - arg_ptr - else - try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment, "")); + if (isByRef(param_ty, zcu)) { + args.appendAssumeCapacity(arg_ptr); + } else { + args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment, "")); + } }, .float_array => { - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); const param_llvm_ty = try o.lowerType(param_ty); - const param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; + const param = wip.arg(it.llvm_index - 1); const alignment = param_ty.abiAlignment(zcu).toLlvm(); const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target); _ = try wip.store(.normal, param, arg_ptr, alignment); - args.appendAssumeCapacity(if (isByRef(param_ty, zcu)) - arg_ptr - else - try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); + if (isByRef(param_ty, zcu)) { + args.appendAssumeCapacity(arg_ptr); + } else { + args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); + } }, .i32_array, .i64_array => { - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); const param_llvm_ty = try o.lowerType(param_ty); - const param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; + const param = wip.arg(it.llvm_index - 1); const alignment = param_ty.abiAlignment(zcu).toLlvm(); const arg_ptr = try buildAllocaInner(&wip, param.typeOfWip(&wip), alignment, target); _ = try wip.store(.normal, param, arg_ptr, alignment); - args.appendAssumeCapacity(if (isByRef(param_ty, zcu)) - arg_ptr - else - try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); + if (isByRef(param_ty, zcu)) { + args.appendAssumeCapacity(arg_ptr); + } else { + args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); + } }, } } - } + + break :implicit_args .{ ret_ptr, err_ret_trace }; + }; const file, const subprogram = if (!wip.strip) debug_info: { const file = try o.getDebugFile(file_scope); @@ -1432,7 +1426,7 @@ pub const Object = struct { const subprogram = try o.builder.debugSubprogram( file, try o.builder.metadataString(nav.name.toSlice(ip)), - try o.builder.metadataStringFromStrtabString(function_index.name(&o.builder)), + try o.builder.metadataString(nav.fqn.toSlice(ip)), line_number, line_number + func.lbrace_line, debug_decl_type, @@ -1449,7 +1443,7 @@ pub const Object = struct { }, o.debug_compile_unit.unwrap().?, ); - function_index.setSubprogram(subprogram, &o.builder); + llvm_function.setSubprogram(subprogram, &o.builder); break :debug_info .{ file, subprogram }; } else .{undefined} ** 2; @@ -1466,7 +1460,7 @@ pub const Object = struct { const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len}); const counters_variable = try o.builder.addVariable(anon_name, .void, .default); try o.used.append(gpa, counters_variable.toConst(&o.builder)); - counters_variable.setLinkage(.private, &o.builder); + counters_variable.ptrConst(&o.builder).global.setLinkage(.private, &o.builder); counters_variable.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); if (target.ofmt == .macho) { @@ -1524,7 +1518,7 @@ pub const Object = struct { _ = try attributes.removeFnAttr(.null_pointer_is_valid); } - function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); + llvm_function.setAttributes(try attributes.finish(&o.builder), &o.builder); if (fg.fuzz) |*f| { { @@ -1539,158 +1533,162 @@ pub const Object = struct { // Due to error "members of llvm.compiler.used must be named", this global needs a name. const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len}); const pcs_variable = try o.builder.addVariable(anon_name, array_llvm_ty, .default); - try o.used.append(gpa, pcs_variable.toConst(&o.builder)); - pcs_variable.setLinkage(.private, &o.builder); - pcs_variable.setMutability(.constant, &o.builder); - pcs_variable.setAlignment(Type.usize.abiAlignment(zcu).toLlvm(), &o.builder); - if (target.ofmt == .macho) { - pcs_variable.setSection(try o.builder.string("__DATA,__sancov_pcs1"), &o.builder); - } else { - pcs_variable.setSection(try o.builder.string("__sancov_pcs1"), &o.builder); - } try pcs_variable.setInitializer(init_val, &o.builder); + pcs_variable.setMutability(.constant, &o.builder); + pcs_variable.setSection(switch (target.ofmt) { + .macho => try o.builder.string("__DATA,__sancov_pcs1"), + else => try o.builder.string("__sancov_pcs1"), + }, &o.builder); + pcs_variable.setAlignment(Type.usize.abiAlignment(zcu).toLlvm(), &o.builder); + const pcs_global = pcs_variable.ptrConst(&o.builder).global; + pcs_global.setLinkage(.private, &o.builder); + try o.used.append(gpa, pcs_global.toConst()); } try fg.wip.finish(); try o.flushTypePool(pt); } - pub fn updateNav(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) !void { + pub fn updateNav(o: *Object, pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) !void { const zcu = o.zcu; const ip = &zcu.intern_pool; + const comp = zcu.comp; + const gpa = comp.gpa; - const nav = ip.getNav(nav_index); + const nav = ip.getNav(nav_id); const resolved = nav.resolved.?; - const lib_name, const linkage, const visibility: Builder.Visibility, const is_dll_import, const init_val, const owner_nav = switch (ip.indexToKey(resolved.value)) { - else => .{ .none, .internal, .default, false, resolved.value, nav_index }, - .@"extern" => |e| .{ e.lib_name, e.linkage, .fromSymbolVisibility(e.visibility), e.is_dll_import, .none, e.owner_nav }, + const opt_extern: ?InternPool.Key.Extern = switch (ip.indexToKey(resolved.value)) { + .@"extern" => |@"extern"| @"extern", + else => null, }; - const ty: Type = .fromInterned(nav.resolved.?.type); - - if (linkage != .internal and ip.isFunctionType(ty.toIntern())) { - const function_index = try o.resolveLlvmFunction(owner_nav); - // Add parameter attributes which weren't set by `resolveLlvmFunction` - const fn_info = zcu.typeToFunc(ty).?; - var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder); - defer attributes.deinit(&o.builder); - var it = iterateParamTypes(o, fn_info); - if (firstParamSRet(fn_info, zcu, zcu.getTarget())) it.llvm_index += 1; - if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) it.llvm_index += 1; - while (try it.next()) |lowering| switch (lowering) { - .byval => { - const param_index = it.zig_index - 1; - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); - if (!isByRef(param_ty, zcu)) { - try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1); - } - }, - .byref => { - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(param_ty); - const alignment = param_ty.abiAlignment(zcu); - try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty); - }, - .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder), - // No attributes needed for these. - .no_bits, - .abi_sized_int, - .multiple_llvm_types, - .float_array, - .i32_array, - .i64_array, - => continue, - - .slice => unreachable, // extern functions do not support slice types. - }; - function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); + const nav_ty: Type = .fromInterned(resolved.type); + const llvm_ty: Builder.Type = if (opt_extern != null) ty: { + // We *must* lower this declaration no matter what. If it has a type we can't actually + // represent (because it doesn't have runtime bits), we instead lower as the zero-size + // type `[0 x i8]`. I don't think the type on an extern declaration actually does much + // anyway. + if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty); + break :ty try o.builder.arrayType(0, .i8); + } else if (nav_ty.hasRuntimeBits(zcu)) ty: { + break :ty try o.lowerType(nav_ty); } else { - const variable_index = try o.resolveGlobalNav(nav_index); - variable_index.setAlignment(zcu.navAlignment(nav_index).toLlvm(), &o.builder); - if (resolved.@"linksection".toSlice(ip)) |section| - variable_index.setSection(try o.builder.string(section), &o.builder); - if (resolved.@"const") variable_index.setMutability(.constant, &o.builder); - try variable_index.setInitializer(switch (init_val) { - .none => .no_init, - else => try o.lowerValue(init_val), - }, &o.builder); - variable_index.setVisibility(visibility, &o.builder); + // This is a non-extern zero-bit `Nav`---we're not interested in it. + // TODO: we might need to rethink this a little under incremental compilation. If a + // declaration becomes zero-bit, we can't just leave its old value there, because it + // might now be ill-formed. + return; + }; - const file_scope = zcu.navFileScopeIndex(nav_index); + const gop = try o.nav_map.getOrPut(gpa, nav_id); + if (!gop.found_existing) { + errdefer assert(o.nav_map.remove(nav_id)); + // First time lowering this NAV! Create a fresh global. + const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip)); + gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{ + .type = .void, // placeholder; populated below + .kind = .{ .alias = .none }, // placeholder; populated below + }); + } + const llvm_global = gop.value_ptr.*; + + llvm_global.ptr(&o.builder).type = llvm_ty; + llvm_global.ptr(&o.builder).addr_space = toLlvmAddressSpace(resolved.@"addrspace", zcu.getTarget()); + + if (opt_extern) |@"extern"| { + const name = name: { + const name_slice = nav.name.toSlice(ip); + if (zcu.getTarget().cpu.arch.isWasm() and nav_ty.zigTypeTag(zcu) == .@"fn") { + if (@"extern".lib_name.toSlice(ip)) |lib_name_slice| { + if (!std.mem.eql(u8, lib_name_slice, "c")) { + break :name try o.builder.strtabStringFmt("{s}|{s}", .{ name_slice, lib_name_slice }); + } + } + } + break :name try o.builder.strtabString(name_slice); + }; + if (o.builder.getGlobal(name)) |other_global| { + if (other_global != llvm_global) { + // Another global already has this name; just use it in place of this global. + try llvm_global.replace(other_global, &o.builder); + return; + } + } + try llvm_global.rename(name, &o.builder); + llvm_global.ptr(&o.builder).unnamed_addr = .default; + llvm_global.ptr(&o.builder).dll_storage_class = switch (@"extern".is_dll_import) { + true => .dllimport, + false => .default, + }; + llvm_global.ptr(&o.builder).linkage = switch (@"extern".linkage) { + .internal => if (o.builder.strip) .private else .internal, + .strong => .external, + .weak => .extern_weak, + .link_once => unreachable, + }; + llvm_global.ptr(&o.builder).visibility = .fromSymbolVisibility(@"extern".visibility); + } else { + llvm_global.ptr(&o.builder).linkage = if (o.builder.strip) .private else .internal; + llvm_global.ptr(&o.builder).visibility = .default; + llvm_global.ptr(&o.builder).dll_storage_class = .default; + llvm_global.ptr(&o.builder).unnamed_addr = .unnamed_addr; + } + + const llvm_align = switch (resolved.@"align") { + .none => nav_ty.abiAlignment(zcu).toLlvm(), + else => |a| a.toLlvm(), + }; + const llvm_section: Builder.String = if (resolved.@"linksection".toSlice(ip)) |section| s: { + break :s try o.builder.string(section); + } else .none; + + // Actual function bodies with AIR go through `updateFunc` instead, so the only functions we + // can see are extern functions or other comptime function body values (e.g. undefined). Of + // these, only extern functions need to be lowered to LLVM functions. + if (opt_extern != null and nav_ty.zigTypeTag(zcu) == .@"fn" and nav_ty.fnHasRuntimeBits(zcu)) { + const llvm_function: Builder.Function.Index = switch (llvm_global.ptrConst(&o.builder).kind) { + .function => |function| function, // re-use existing `Builder.Function` + .replaced, .alias, .variable => try llvm_global.toNewFunction(&o.builder), + }; + llvm_function.setAlignment(llvm_align, &o.builder); + llvm_function.setSection(llvm_section, &o.builder); + try o.addLlvmFunctionAttributes(pt, nav_id, llvm_function); + } else { + const file_scope = nav.srcInst(ip).resolveFile(ip); const mod = zcu.fileByIndex(file_scope).mod.?; - if (resolved.@"threadlocal" and !mod.single_threaded) - variable_index.setThreadLocal(.generaldynamic, &o.builder); - const line_number = zcu.navSrcLine(nav_index) + 1; + const llvm_variable: Builder.Variable.Index = switch (llvm_global.ptrConst(&o.builder).kind) { + .variable => |variable| variable, // re-use existing `Builder.Variable` + .replaced, .alias, .function => try llvm_global.toNewVariable(&o.builder), + }; + llvm_variable.setAlignment(llvm_align, &o.builder); + llvm_variable.setSection(llvm_section, &o.builder); + llvm_variable.setMutability(if (resolved.@"const") .constant else .global, &o.builder); + try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value), &o.builder); + llvm_variable.setThreadLocal(tl: { + if (resolved.@"threadlocal" and !mod.single_threaded) break :tl .generaldynamic; + break :tl .default; + }, &o.builder); if (!mod.strip) { const debug_file = try o.getDebugFile(file_scope); - - const debug_global_var = try o.builder.debugGlobalVar( - try o.builder.metadataString(nav.name.toSlice(ip)), // Name - try o.builder.metadataStringFromStrtabString(variable_index.name(&o.builder)), // Linkage name - debug_file, // File - debug_file, // Scope - line_number, - try o.getDebugType(pt, ty), - variable_index, - .{ .local = linkage == .internal }, + const debug_global_var_expr = try o.builder.debugGlobalVarExpression( + try o.builder.debugGlobalVar( + try o.builder.metadataString(nav.name.toSlice(ip)), // Name + try o.builder.metadataString(nav.fqn.toSlice(ip)), // Linkage name + debug_file, // File + debug_file, // Scope + zcu.navSrcLine(nav_id) + 1, + try o.getDebugType(pt, nav_ty), + llvm_variable, + .{ .local = llvm_global.ptrConst(&o.builder).linkage == .internal }, + ), + try o.builder.debugExpression(&.{}), ); - - const debug_expression = try o.builder.debugExpression(&.{}); - - const debug_global_var_expression = try o.builder.debugGlobalVarExpression( - debug_global_var, - debug_expression, - ); - - variable_index.setGlobalVariableExpression(debug_global_var_expression, &o.builder); - try o.debug_globals.append(o.gpa, debug_global_var_expression); + llvm_variable.setGlobalVariableExpression(debug_global_var_expr, &o.builder); + try o.debug_globals.append(o.gpa, debug_global_var_expr); } } - - switch (linkage) { - .internal => {}, - .strong, .weak => { - const global_index = o.nav_map.get(nav_index).?; - - const decl_name = decl_name: { - if (zcu.getTarget().cpu.arch.isWasm() and ty.zigTypeTag(zcu) == .@"fn") { - if (lib_name.toSlice(ip)) |lib_name_slice| { - if (!std.mem.eql(u8, lib_name_slice, "c")) { - break :decl_name try o.builder.strtabStringFmt("{f}|{s}", .{ nav.name.fmt(ip), lib_name_slice }); - } - } - } - break :decl_name try o.builder.strtabString(nav.name.toSlice(ip)); - }; - - if (o.builder.getGlobal(decl_name)) |other_global| { - if (other_global != global_index) { - // Another global already has this name; just use it in place of this global. - try global_index.replace(other_global, &o.builder); - return; - } - } - - try global_index.rename(decl_name, &o.builder); - global_index.setUnnamedAddr(.default, &o.builder); - if (is_dll_import) { - global_index.setDllStorageClass(.dllimport, &o.builder); - } else if (zcu.comp.config.dll_export_fns) { - global_index.setDllStorageClass(.default, &o.builder); - } - - global_index.setLinkage(switch (linkage) { - .internal => unreachable, - .strong => .external, - .weak => .extern_weak, - .link_once => unreachable, - }, &o.builder); - global_index.setVisibility(visibility, &o.builder); - }, - .link_once => unreachable, - } } fn flushTypePool(o: *Object, pt: Zcu.PerThread) Allocator.Error!void { @@ -1698,18 +1696,43 @@ pub const Object = struct { } pub fn updateExports( - self: *Object, + o: *Object, exported: Zcu.Exported, export_indices: []const Zcu.Export.Index, ) link.File.UpdateExportsError!void { - const zcu = self.zcu; - const nav_index = switch (exported) { - .nav => |nav| nav, - .uav => |uav| return updateExportedValue(self, uav, export_indices), - }; + const zcu = o.zcu; const ip = &zcu.intern_pool; - const global_index = self.nav_map.get(nav_index).?; + const ty: Type, const llvm_ptr: Builder.Constant = switch (exported) { + .nav => |nav| exp: { + const nav_ty: Type = .fromInterned(ip.getNav(nav).resolved.?.type); + const nav_ref = try o.lowerNavRef(nav); + break :exp .{ nav_ty, nav_ref }; + }, + .uav => |uav| exp: { + const uav_ty = Value.fromInterned(uav).typeOf(zcu); + const uav_ref = try o.lowerUavRef( + uav, + uav_ty.abiAlignment(zcu), + target_util.defaultAddressSpace(zcu.getTarget(), .global_constant), + ); + break :exp .{ uav_ty, uav_ref }; + }, + }; + switch (llvm_ptr.unwrap()) { + .global => |global| return o.updateExportedGlobal(global, ty, export_indices), + .constant => @panic("LLVM TODO: export zero-bit value"), + } + } + + fn updateExportedGlobal( + o: *Object, + global_index: Builder.Global.Index, + ty: Type, + export_indices: []const Zcu.Export.Index, + ) link.File.UpdateExportsError!void { + const zcu = o.zcu; const comp = zcu.comp; + const ip = &zcu.intern_pool; // If we're on COFF and linking with LLD, the linker cares about our exports to determine the subsystem in use. coff_export_flags: { @@ -1719,7 +1742,7 @@ pub const Object = struct { .elf, .wasm => break :coff_export_flags, .coff => |*coff| coff, }; - if (!ip.isFunctionType(ip.getNav(nav_index).resolved.?.type)) break :coff_export_flags; + if (ty.zigTypeTag(zcu) != .@"fn") break :coff_export_flags; const flags = &coff.lld_export_flags; for (export_indices) |export_index| { const name = export_index.ptr(zcu).opts.name; @@ -1732,152 +1755,88 @@ pub const Object = struct { } } - if (export_indices.len != 0) { - return updateExportedGlobal(self, zcu, global_index, export_indices); - } else { - const fqn = try self.builder.strtabString(ip.getNav(nav_index).fqn.toSlice(ip)); - try global_index.rename(fqn, &self.builder); - global_index.setLinkage(if (self.builder.strip) .private else .internal, &self.builder); - if (comp.config.dll_export_fns) - global_index.setDllStorageClass(.default, &self.builder); - global_index.setUnnamedAddr(.unnamed_addr, &self.builder); - } - } - - fn updateExportedValue( - o: *Object, - exported_value: InternPool.Index, - export_indices: []const Zcu.Export.Index, - ) link.File.UpdateExportsError!void { - const zcu = o.zcu; - const gpa = zcu.gpa; - const ip = &zcu.intern_pool; - const main_exp_name = try o.builder.strtabString(export_indices[0].ptr(zcu).opts.name.toSlice(ip)); - const global_index = i: { - const gop = try o.uav_map.getOrPut(gpa, exported_value); - if (gop.found_existing) { - const global_index = gop.value_ptr.*; - try global_index.rename(main_exp_name, &o.builder); - break :i global_index; - } - const llvm_addr_space = toLlvmAddressSpace(.generic, zcu.getTarget()); - const variable_index = try o.builder.addVariable( - main_exp_name, - try o.lowerType(.fromInterned(ip.typeOf(exported_value))), - llvm_addr_space, - ); - const global_index = variable_index.ptrConst(&o.builder).global; - gop.value_ptr.* = global_index; - // This line invalidates `gop`. - const init_val = try o.lowerValue(exported_value); - try variable_index.setInitializer(init_val, &o.builder); - break :i global_index; - }; - return updateExportedGlobal(o, zcu, global_index, export_indices); - } - - fn updateExportedGlobal( - o: *Object, - zcu: *Zcu, - global_index: Builder.Global.Index, - export_indices: []const Zcu.Export.Index, - ) link.File.UpdateExportsError!void { - const comp = zcu.comp; - const ip = &zcu.intern_pool; - const first_export = export_indices[0].ptr(zcu); - - // We will rename this global to have a name matching `first_export`. - // Successive exports become aliases. - // If the first export name already exists, then there is a corresponding - // extern global - we replace it with this global. - const first_exp_name = try o.builder.strtabString(first_export.opts.name.toSlice(ip)); - if (o.builder.getGlobal(first_exp_name)) |other_global| replace: { - if (other_global.toConst().getBase(&o.builder) == global_index.toConst().getBase(&o.builder)) { - break :replace; // this global already has the name we want - } - try global_index.takeName(other_global, &o.builder); - try other_global.replace(global_index, &o.builder); - // Problem: now we need to replace in the decl_map that - // the extern decl index points to this new global. However we don't - // know the decl index. - // Even if we did, a future incremental update to the extern would then - // treat the LLVM global as an extern rather than an export, so it would - // need a way to check that. - // This is a TODO that needs to be solved when making - // the LLVM backend support incremental compilation. - } else { - try global_index.rename(first_exp_name, &o.builder); + // If the first export specifies a linksection, set the exported variable's section to that + // one. This is kind of a hack because `std.builtin.ExportOptions.section` doesn't actually + // make much sense: the linksection should be associated with the declaration itself rather + // than some particular symbol it is exported as! + if (export_indices[0].ptr(zcu).opts.section.toSlice(ip)) |section_slice| { + const variable = &global_index.ptrConst(&o.builder).kind.variable; + variable.setSection(try o.builder.string(section_slice), &o.builder); } - global_index.setUnnamedAddr(.default, &o.builder); - if (comp.config.dll_export_fns and first_export.opts.visibility != .hidden) - global_index.setDllStorageClass(.dllexport, &o.builder); - global_index.setLinkage(switch (first_export.opts.linkage) { - .internal => unreachable, - .strong => .external, - .weak => .weak_odr, - .link_once => .linkonce_odr, - }, &o.builder); - global_index.setVisibility(switch (first_export.opts.visibility) { - .default => .default, - .hidden => .hidden, - .protected => .protected, - }, &o.builder); - if (first_export.opts.section.toSlice(ip)) |section| - switch (global_index.ptrConst(&o.builder).kind) { - .variable => |impl_index| impl_index.setSection( - try o.builder.string(section), - &o.builder, - ), - .function => unreachable, - .alias => unreachable, - .replaced => unreachable, - }; + const llvm_global_ty = global_index.typeOf(&o.builder); - // If a Decl is exported more than one time (which is rare), - // we add aliases for all but the first export. - // TODO LLVM C API does not support deleting aliases. - // The planned solution to this is https://github.com/ziglang/zig/issues/13265 - // Until then we iterate over existing aliases and make them point - // to the correct decl, or otherwise add a new alias. Old aliases are leaked. - for (export_indices[1..]) |export_idx| { + // All exports are represented as aliases to the original global. + + // TODO: we currently do not delete old exports. To do that we'll need to track which + // globals actually *are* exports. + + for (export_indices) |export_idx| { const exp = export_idx.ptr(zcu); const exp_name = try o.builder.strtabString(exp.opts.name.toSlice(ip)); - if (o.builder.getGlobal(exp_name)) |global| { - switch (global.ptrConst(&o.builder).kind) { + + // Our goal is to make an alias with the name `exp_name`, but if that name is already + // taken by some existing global, we need to figure out what to do with that existing + // global. + // + // The name, aliasee, and type will be set within this block. Other properties of the + // alias will be set below. + const alias_global: Builder.Global.Index = global: { + const existing_global = o.builder.getGlobal(exp_name) orelse { + // There is no existing global with this name, so make a new alias. + const alias = try o.builder.addAlias( + exp_name, + llvm_global_ty, + .default, + global_index.toConst(), + ); + break :global alias.ptrConst(&o.builder).global; + }; + // There is an existing global with this name, so we can't just create an alias. We + // need to figure out what to do with the existing global instead. + switch (existing_global.ptrConst(&o.builder).kind) { .alias => |alias| { + // We can just repurpose the existing alias. alias.setAliasee(global_index.toConst(), &o.builder); - continue; + alias.ptrConst(&o.builder).global.ptr(&o.builder).type = global_index.typeOf(&o.builder); + break :global existing_global; }, .variable, .function => { - // This existing global is an `extern` corresponding to this export. - // Replace it with the global being exported. - // This existing global must be replaced with the alias. - try global.rename(.empty, &o.builder); - try global.replace(global_index, &o.builder); + // This must be an extern, which is no good to us---we need an alias. The + // extern should refer to the value we're exporting, so replace it with the + // exported value. That will free up the name for us to create a new alias. + // We need to make a new global which is an alias. Replace this existing one + // with the target global, making the name available and fixing references + // to this global to point to the target. + try existing_global.replace(global_index, &o.builder); + // The name is now free, so create an alias. + const alias = try o.builder.addAlias( + exp_name, + llvm_global_ty, + .default, + global_index.toConst(), + ); + break :global alias.ptrConst(&o.builder).global; }, - .replaced => unreachable, + .replaced => unreachable, // a replaced global would have lost the name `exp_name` } - } - const alias_index = try o.builder.addAlias( - .empty, - global_index.typeOf(&o.builder), - .default, - global_index.toConst(), - ); - try alias_index.rename(exp_name, &o.builder); + }; - const alias_global_index = alias_index.ptrConst(&o.builder).global; - alias_global_index.setUnnamedAddr(.default, &o.builder); - if (comp.config.dll_export_fns and first_export.opts.visibility != .hidden) - alias_global_index.setDllStorageClass(.dllexport, &o.builder); - alias_global_index.setLinkage(switch (first_export.opts.linkage) { - .internal => unreachable, + // Now for a bit of setup which + + // We need the alias to *not* be `unnamed_addr` to ensure that the alias address equals + // the address of the original global. + alias_global.setUnnamedAddr(.default, &o.builder); + + if (comp.config.dll_export_fns and exp.opts.visibility != .hidden) + alias_global.setDllStorageClass(.dllexport, &o.builder); + alias_global.setLinkage(switch (exp.opts.linkage) { + .internal => if (o.builder.strip) .private else .internal, // we still did useful work in replacing an existing symbol if there was one .strong => .external, .weak => .weak_odr, .link_once => .linkonce_odr, }, &o.builder); - alias_global_index.setVisibility(switch (first_export.opts.visibility) { + alias_global.setVisibility(switch (exp.opts.visibility) { .default => .default, .hidden => .hidden, .protected => .protected, @@ -1940,7 +1899,12 @@ pub const Object = struct { assert(val != .anyerror_type); const fwd_ref = o.debug_types.items[@intFromEnum(index)]; const name_str = try o.builder.metadataStringFmt("{f}", .{ty.fmt(pt)}); - const debug_incomplete_type = try o.builder.debugSignedType(name_str, 0); + // If `ty` is a function, use a dummy *function* type to prevent existing debug + // subprograms from becoming ill-formed. + const debug_incomplete_type = switch (ty.zigTypeTag(zcu)) { + .@"fn" => try o.builder.debugSubroutineType(null), + else => try o.builder.debugSignedType(name_str, 0), + }; o.builder.resolveDebugForwardReference(fwd_ref, debug_incomplete_type); } } @@ -2269,7 +2233,9 @@ pub const Object = struct { }, .@"fn" => { if (!ty.fnHasRuntimeBits(zcu)) { - return o.builder.debugSignedType(name, 0); + // Use a dummy *function* type to prevent existing debug subprograms from + // becoming ill-formed. + return o.builder.debugSubroutineType(null); } const fn_info = zcu.typeToFunc(ty).?; @@ -2716,75 +2682,38 @@ pub const Object = struct { return o.getDebugType(pt, .fromInterned(namespace.owner_type)); } - /// If the llvm function does not exist, create it. - /// Note that this can be called before the function's semantic analysis has - /// completed, so if any attributes rely on that, they must be done in updateFunc, not here. - pub fn resolveLlvmFunction( + /// Sets the attributes and callconv of the given `Builder.Function`, which corresponds to the + /// given `Nav` (which is a function). + fn addLlvmFunctionAttributes( o: *Object, - nav_index: InternPool.Nav.Index, - ) Allocator.Error!Builder.Function.Index { + pt: Zcu.PerThread, + nav_id: InternPool.Nav.Index, + function_index: Builder.Function.Index, + ) Allocator.Error!void { const zcu = o.zcu; const ip = &zcu.intern_pool; - const gpa = o.gpa; - const nav = ip.getNav(nav_index); - const owner_mod = zcu.navFileScope(nav_index).mod.?; + const nav = ip.getNav(nav_id); + const owner_mod = zcu.navFileScope(nav_id).mod.?; const ty: Type = .fromInterned(nav.resolved.?.type); - const gop = try o.nav_map.getOrPut(gpa, nav_index); - if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.function; const fn_info = zcu.typeToFunc(ty).?; const target = &owner_mod.resolved_target.result; - const is_extern, const lib_name = if (nav.getExtern(ip)) |@"extern"| - .{ true, @"extern".lib_name } - else - .{ false, .none }; - const function_index = try o.builder.addFunction( - try o.lowerType(ty), - try o.builder.strtabString((if (is_extern) nav.name else nav.fqn).toSlice(ip)), - toLlvmAddressSpace(nav.resolved.?.@"addrspace", target), - ); - gop.value_ptr.* = function_index.ptrConst(&o.builder).global; - var attributes: Builder.FunctionAttributes.Wip = .{}; defer attributes.deinit(&o.builder); - if (!is_extern) { - function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); - function_index.setUnnamedAddr(.unnamed_addr, &o.builder); - } else { - if (target.cpu.arch.isWasm()) { - try attributes.addFnAttr(.{ .string = .{ - .kind = try o.builder.string("wasm-import-name"), - .value = try o.builder.string(nav.name.toSlice(ip)), + if (target.cpu.arch.isWasm()) if (nav.getExtern(ip)) |@"extern"| { + try attributes.addFnAttr(.{ .string = .{ + .kind = try o.builder.string("wasm-import-name"), + .value = try o.builder.string(nav.name.toSlice(ip)), + } }, &o.builder); + if (@"extern".lib_name.toSlice(ip)) |lib_name_slice| { + if (!std.mem.eql(u8, lib_name_slice, "c")) try attributes.addFnAttr(.{ .string = .{ + .kind = try o.builder.string("wasm-import-module"), + .value = try o.builder.string(lib_name_slice), } }, &o.builder); - if (lib_name.toSlice(ip)) |lib_name_slice| { - if (!std.mem.eql(u8, lib_name_slice, "c")) try attributes.addFnAttr(.{ .string = .{ - .kind = try o.builder.string("wasm-import-module"), - .value = try o.builder.string(lib_name_slice), - } }, &o.builder); - } } - } - - var llvm_arg_i: u32 = 0; - if (firstParamSRet(fn_info, zcu, target)) { - // Sret pointers must not be address 0 - try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); - try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); - - const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type)); - try attributes.addParamAttr(llvm_arg_i, .{ .sret = raw_llvm_ret_ty }, &o.builder); - - llvm_arg_i += 1; - } - - const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing; - - if (err_return_tracing) { - try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); - llvm_arg_i += 1; - } + }; if (fn_info.cc == .async) { @panic("TODO: LLVM backend lower async function"); @@ -2859,9 +2788,6 @@ pub const Object = struct { } } - if (nav.resolved.?.@"align" != .none) - function_index.setAlignment(nav.resolved.?.@"align".toLlvm(), &o.builder); - // Function attributes that are independent of analysis results of the function body. try o.addCommonFnAttributes( &attributes, @@ -2877,8 +2803,71 @@ pub const Object = struct { if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder); + var it = iterateParamTypes(o, fn_info); + if (firstParamSRet(fn_info, zcu, target)) { + // Sret pointers must not be address 0 + try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder); + try attributes.addParamAttr(it.llvm_index, .@"noalias", &o.builder); + + const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type)); + try attributes.addParamAttr(it.llvm_index, .{ .sret = raw_llvm_ret_ty }, &o.builder); + it.llvm_index += 1; + } + const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing; + if (err_return_tracing) { + try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder); + it.llvm_index += 1; + } + while (try it.next()) |lowering| switch (lowering) { + .byval => { + const param_index = it.zig_index - 1; + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[param_index]); + if (!isByRef(param_ty, zcu)) { + try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1); + } + }, + .byref => { + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param_llvm_ty = try o.lowerType(param_ty); + const alignment = param_ty.abiAlignment(zcu); + try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty); + }, + .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder), + .slice => { + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const ptr_info = param_ty.ptrInfo(zcu); + const llvm_ptr_index = it.llvm_index - 2; + if (std.math.cast(u5, it.zig_index - 1)) |i| { + if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { + try attributes.addParamAttr(llvm_ptr_index, .@"noalias", &o.builder); + } + } + if (param_ty.zigTypeTag(zcu) != .optional and + !ptr_info.flags.is_allowzero and + ptr_info.flags.address_space == .generic) + { + try attributes.addParamAttr(llvm_ptr_index, .nonnull, &o.builder); + } + if (ptr_info.flags.is_const) { + try attributes.addParamAttr(llvm_ptr_index, .readonly, &o.builder); + } + const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) { + else => |a| .wrap(a.toLlvm()), + .none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)), + }; + try attributes.addParamAttr(llvm_ptr_index, .{ .@"align" = elem_align }, &o.builder); + }, + // No attributes needed for these. + .no_bits, + .abi_sized_int, + .multiple_llvm_types, + .float_array, + .i32_array, + .i64_array, + => continue, + }; + function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); - return function_index; } fn addCommonFnAttributes( @@ -2951,97 +2940,6 @@ pub const Object = struct { } } - fn resolveGlobalUav( - o: *Object, - uav: InternPool.Index, - llvm_addr_space: Builder.AddrSpace, - alignment: InternPool.Alignment, - ) Allocator.Error!Builder.Variable.Index { - assert(alignment != .none); - // TODO: Add address space to the anon_decl_map - const gop = try o.uav_map.getOrPut(o.gpa, uav); - if (gop.found_existing) { - // Keep the greater of the two alignments. - const variable_index = gop.value_ptr.ptr(&o.builder).kind.variable; - const old_alignment = InternPool.Alignment.fromLlvm(variable_index.getAlignment(&o.builder)); - const max_alignment = old_alignment.maxStrict(alignment); - variable_index.setAlignment(max_alignment.toLlvm(), &o.builder); - return variable_index; - } - errdefer assert(o.uav_map.remove(uav)); - - const zcu = o.zcu; - const decl_ty = zcu.intern_pool.typeOf(uav); - - const variable_index = try o.builder.addVariable( - try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav)}), - try o.lowerType(.fromInterned(decl_ty)), - llvm_addr_space, - ); - gop.value_ptr.* = variable_index.ptrConst(&o.builder).global; - - try variable_index.setInitializer(try o.lowerValue(uav), &o.builder); - variable_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); - variable_index.setMutability(.constant, &o.builder); - variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); - variable_index.setAlignment(alignment.toLlvm(), &o.builder); - return variable_index; - } - - fn resolveGlobalNav( - o: *Object, - nav_index: InternPool.Nav.Index, - ) Allocator.Error!Builder.Variable.Index { - const gop = try o.nav_map.getOrPut(o.gpa, nav_index); - if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.variable; - errdefer assert(o.nav_map.remove(nav_index)); - - const zcu = o.zcu; - const ip = &zcu.intern_pool; - const nav = ip.getNav(nav_index); - const linkage: std.builtin.GlobalLinkage, const visibility: Builder.Visibility, const is_dll_import: bool = switch (nav.resolved.?.value) { - .none => .{ .internal, .default, false }, // this is a source declaration which is *not* marked `extern` - else => |val| switch (ip.indexToKey(val)) { - else => .{ .internal, .default, false }, - .@"extern" => |e| .{ e.linkage, .fromSymbolVisibility(e.visibility), e.is_dll_import }, - }, - }; - - const variable_index = try o.builder.addVariable( - try o.builder.strtabString(switch (linkage) { - .internal => nav.fqn, - .strong, .weak => nav.name, - .link_once => unreachable, - }.toSlice(ip)), - try o.lowerType(.fromInterned(nav.resolved.?.type)), - toLlvmGlobalAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget()), - ); - gop.value_ptr.* = variable_index.ptrConst(&o.builder).global; - - // This is needed for declarations created by `@extern`. - switch (linkage) { - .internal => { - variable_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); - variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); - }, - .strong, .weak => { - variable_index.setLinkage(switch (linkage) { - .internal => unreachable, - .strong => .external, - .weak => .extern_weak, - .link_once => unreachable, - }, &o.builder); - variable_index.setUnnamedAddr(.default, &o.builder); - if (nav.resolved.?.@"threadlocal" and !zcu.navFileScope(nav_index).mod.?.single_threaded) - variable_index.setThreadLocal(.generaldynamic, &o.builder); - if (is_dll_import) variable_index.setDllStorageClass(.dllimport, &o.builder); - }, - .link_once => unreachable, - } - variable_index.setVisibility(visibility, &o.builder); - return variable_index; - } - pub fn errorIntType(o: *Object) Allocator.Error!Builder.Type { return o.builder.intType(o.zcu.errorSetBits()); } @@ -3051,7 +2949,7 @@ pub const Object = struct { const target = zcu.getTarget(); const ip = &zcu.intern_pool; return switch (t.toIntern()) { - .u0_type, .i0_type => unreachable, + .u0_type, .i0_type => unreachable, // no runtime bits inline .u1_type, .u8_type, .i8_type, @@ -3100,18 +2998,18 @@ pub const Object = struct { return .i8; }, .bool_type => .i1, - .void_type => .void, - .type_type => unreachable, .anyerror_type => try o.errorIntType(), - .comptime_int_type, - .comptime_float_type, - .noreturn_type, - => unreachable, + .void_type => unreachable, // no runtime bits + .type_type => unreachable, // no runtime bits + .comptime_int_type => unreachable, // no runtime bits + .comptime_float_type => unreachable, // no runtime bits + .noreturn_type => unreachable, // no runtime bits + .null_type => unreachable, // no runtime bits + .undefined_type => unreachable, // no runtime bits + .enum_literal_type => unreachable, // no runtime bits + .optional_noreturn_type => unreachable, // no runtime bits + .empty_tuple_type => unreachable, // no runtime bits .anyframe_type => @panic("TODO implement lowerType for AnyFrame types"), - .null_type, - .undefined_type, - .enum_literal_type, - => unreachable, .ptr_usize_type, .ptr_const_comptime_int_type, .manyptr_u8_type, @@ -3121,13 +3019,10 @@ pub const Object = struct { .slice_const_u8_type, .slice_const_u8_sentinel_0_type, => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize) }), - .optional_noreturn_type => unreachable, .anyerror_void_error_union_type, .adhoc_inferred_error_set_type, => try o.errorIntType(), - .generic_poison_type, - .empty_tuple_type, - => unreachable, + .generic_poison_type => unreachable, // values, not types .undef, .undef_bool, @@ -3176,7 +3071,11 @@ pub const Object = struct { ), .opt_type => |child_ty| { // Must stay in sync with `opt_payload` logic in `lowerPtr`. - if (!Type.fromInterned(child_ty).hasRuntimeBits(zcu)) return .i8; + switch (Type.fromInterned(child_ty).classify(zcu)) { + .no_possible_value, .fully_comptime => unreachable, + .one_possible_value => return .i8, + .runtime, .partially_comptime => {}, + } const payload_ty = try o.lowerType(.fromInterned(child_ty)); if (t.optionalReprIsPayload(zcu)) return payload_ty; @@ -3198,8 +3097,13 @@ pub const Object = struct { // Must stay in sync with `codegen.errUnionPayloadOffset`. // See logic in `lowerPtr`. const error_type = try o.errorIntType(); - if (!Type.fromInterned(error_union_type.payload_type).hasRuntimeBits(zcu)) - return error_type; + + switch (Type.fromInterned(error_union_type.payload_type).classify(zcu)) { + .fully_comptime => unreachable, + .no_possible_value, .one_possible_value => return error_type, + .runtime, .partially_comptime => {}, + } + const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type)); const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu); @@ -3245,6 +3149,8 @@ pub const Object = struct { return int_ty; } + assert(struct_type.size > 0); + var llvm_field_types: std.ArrayList(Builder.Type) = .empty; defer llvm_field_types.deinit(o.gpa); // Although we can estimate how much capacity to add, these cannot be @@ -3311,7 +3217,7 @@ pub const Object = struct { comptime assert(struct_layout_version == 2); var offset: u64 = 0; - var big_align: InternPool.Alignment = .none; + var big_align: InternPool.Alignment = .@"1"; for ( tuple_type.types.get(ip), @@ -3345,6 +3251,7 @@ pub const Object = struct { try o.builder.arrayType(padding_len, .i8), ); } + assert(offset > 0); return o.builder.structType(.normal, llvm_field_types.items); }, .union_type => { @@ -3358,6 +3265,8 @@ pub const Object = struct { return int_ty; } + assert(union_obj.size > 0); + const layout = Type.getUnionLayout(union_obj, zcu); if (layout.payload_size == 0) { @@ -3421,15 +3330,9 @@ pub const Object = struct { ); return ty; }, - .opaque_type => { - const gop = try o.type_map.getOrPut(o.gpa, t.toIntern()); - if (!gop.found_existing) { - gop.value_ptr.* = try o.builder.opaqueType(try o.builder.string(t.containerTypeName(ip).toSlice(ip))); - } - return gop.value_ptr.*; - }, + .opaque_type => unreachable, // no runtime bits .enum_type => try o.lowerType(t.intTagType(zcu)), - .func_type => |func_type| try o.lowerFnType(func_type), + .func_type => |func_type| try o.lowerFnType(t, func_type), .error_set_type, .inferred_error_set_type => try o.errorIntType(), // values, not types .undef, @@ -3455,10 +3358,13 @@ pub const Object = struct { }; } - fn lowerFnType(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { + fn lowerFnType(o: *Object, fn_ty: Type, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { const zcu = o.zcu; const ip = &zcu.intern_pool; const target = zcu.getTarget(); + + assert(fn_ty.fnHasRuntimeBits(zcu)); + const ret_ty = try lowerFnRetTy(o, fn_info); var llvm_params: std.ArrayList(Builder.Type) = .empty; @@ -3526,15 +3432,12 @@ pub const Object = struct { const ip = &zcu.intern_pool; const target = zcu.getTarget(); - const val = Value.fromInterned(arg_val); + const val: Value = .fromInterned(arg_val); const val_key = ip.indexToKey(val.toIntern()); - if (val.isUndef(zcu)) { - return o.builder.undefConst(try o.lowerType(.fromInterned(val_key.typeOf()))); - } - const ty: Type = .fromInterned(val_key.typeOf()); ty.assertHasLayout(zcu); + assert(ty.hasRuntimeBits(zcu)); return switch (val_key) { .int_type, @@ -3555,7 +3458,7 @@ pub const Object = struct { .inferred_error_set_type, => unreachable, // types, not values - .undef => unreachable, // handled above + .undef => return o.builder.undefConst(try o.lowerType(ty)), .simple_value => |simple_value| switch (simple_value) { .void => unreachable, // non-runtime value .null => unreachable, // non-runtime value @@ -3565,14 +3468,8 @@ pub const Object = struct { .true => .true, }, .enum_literal => unreachable, // non-runtime value - .@"extern" => |@"extern"| { - const function_index = try o.resolveLlvmFunction(@"extern".owner_nav); - return function_index.ptrConst(&o.builder).global.toConst(); - }, - .func => |func| { - const function_index = try o.resolveLlvmFunction(func.owner_nav); - return function_index.ptrConst(&o.builder).global.toConst(); - }, + .@"extern" => unreachable, // non-runtime value + .func => unreachable, // non-runtime value .int => { var bigint_space: Value.BigIntSpace = undefined; const bigint = val.toBigInt(&bigint_space, zcu); @@ -3815,7 +3712,7 @@ pub const Object = struct { comptime assert(struct_layout_version == 2); var llvm_index: usize = 0; var offset: u64 = 0; - var big_align: InternPool.Alignment = .none; + var big_align: InternPool.Alignment = .@"1"; var need_unnamed = false; for ( tuple.types.get(ip), @@ -4033,7 +3930,7 @@ pub const Object = struct { const offset: u64 = prev_offset + ptr.byte_offset; return switch (ptr.base_addr) { .nav => |nav| { - const base_ptr = try o.lowerNavRefValue(nav); + const base_ptr = try o.lowerNavRef(nav); return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{ try o.builder.intConst(.i64, offset), }); @@ -4092,8 +3989,19 @@ pub const Object = struct { }; } - /// This logic is very similar to `lowerNavRefValue` but for anonymous declarations. - /// Maybe the logic could be unified. + pub fn lowerPtrToVoid( + o: *Object, + /// Must not be `.none`. + @"align": InternPool.Alignment, + @"addrspace": std.builtin.AddressSpace, + ) Allocator.Error!Builder.Constant { + const addr: u64 = @"align".toByteUnits().?; + const llvm_usize = try o.lowerType(.usize); + const llvm_addr = try o.builder.intConst(llvm_usize, addr); + const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", o.zcu.getTarget())); + return o.builder.castConst(.inttoptr, llvm_addr, llvm_ptr_ty); + } + pub fn lowerUavRef( o: *Object, uav_val: InternPool.Index, @@ -4105,6 +4013,8 @@ pub const Object = struct { const zcu = o.zcu; const ip = &zcu.intern_pool; + const gpa = zcu.comp.gpa; + const uav_ty: Type = .fromInterned(ip.typeOf(uav_val)); switch (ip.indexToKey(uav_val)) { @@ -4118,63 +4028,63 @@ pub const Object = struct { } const llvm_addrspace = toLlvmAddressSpace(@"addrspace", zcu.getTarget()); - const llvm_global = (try o.resolveGlobalUav(uav_val, llvm_addrspace, @"align")).ptrConst(&o.builder).global; - return o.builder.convConst( - llvm_global.toConst(), - try o.builder.ptrType(llvm_addrspace), - ); + const gop = try o.uav_map.getOrPut(gpa, .{ .val = uav_val, .@"addrspace" = @"addrspace" }); + if (gop.found_existing) { + // Keep the greater of the two alignments. + const llvm_variable = gop.value_ptr.*; + const old_align: InternPool.Alignment = .fromLlvm(llvm_variable.getAlignment(&o.builder)); + llvm_variable.setAlignment(old_align.maxStrict(@"align").toLlvm(), &o.builder); + return llvm_variable.ptrConst(&o.builder).global.toConst(); + } + errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" })); + + const llvm_ty = try o.lowerType(uav_ty); + const llvm_name = try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav_val)}); + const llvm_variable = try o.builder.addVariable(llvm_name, llvm_ty, llvm_addrspace); + gop.value_ptr.* = llvm_variable; + try llvm_variable.setInitializer(try o.lowerValue(uav_val), &o.builder); + llvm_variable.setMutability(.constant, &o.builder); + llvm_variable.setAlignment(@"align".toLlvm(), &o.builder); + const llvm_global = llvm_variable.ptrConst(&o.builder).global; + llvm_global.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); + llvm_global.setUnnamedAddr(.unnamed_addr, &o.builder); + return llvm_global.toConst(); } - pub fn lowerNavRefValue(o: *Object, nav_index: InternPool.Nav.Index) Allocator.Error!Builder.Constant { + pub fn lowerNavRef(o: *Object, nav_id: InternPool.Nav.Index) Allocator.Error!Builder.Constant { const zcu = o.zcu; const ip = &zcu.intern_pool; + const gpa = zcu.comp.gpa; - const nav = ip.getNav(nav_index); - + const nav = ip.getNav(nav_id); const nav_ty: Type = .fromInterned(nav.resolved.?.type); - - if (nav.getExtern(ip) == null and !nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) { - return o.lowerPtrToVoid(nav.resolved.?.@"align", nav.resolved.?.@"addrspace"); + if (!nav_ty.isRuntimeFnOrHasRuntimeBits(zcu) and nav.getExtern(ip) == null) { + const nav_align = switch (nav.resolved.?.@"align") { + .none => nav_ty.abiAlignment(zcu), + else => |a| a, + }; + return o.lowerPtrToVoid(nav_align, nav.resolved.?.@"addrspace"); } - const llvm_global = if (nav_ty.zigTypeTag(zcu) == .@"fn") - (try o.resolveLlvmFunction(nav_index)).ptrConst(&o.builder).global - else - (try o.resolveGlobalNav(nav_index)).ptrConst(&o.builder).global; + const gop = try o.nav_map.getOrPut(gpa, nav_id); + if (!gop.found_existing) { + errdefer assert(o.nav_map.remove(nav_id)); + // The NAV hasn't been lowered yet, so generate a placeholder global whose details will + // be filled in later. + const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip)); + gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{ + .type = .void, // placeholder; populated by `updateNav`/`updateFunc` + .kind = .{ .alias = .none }, // placeholder; populated by `updateNav`/`updateFunc` + }); + } + const llvm_global = gop.value_ptr.*; - return try o.builder.convConst( - llvm_global.toConst(), - try o.builder.ptrType(toLlvmAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget())), - ); - } - - pub fn lowerPtrToVoid( - o: *Object, - @"align": InternPool.Alignment, - @"addrspace": std.builtin.AddressSpace, - ) Allocator.Error!Builder.Constant { - const target = o.zcu.getTarget(); - // Even though we are pointing at something which has zero bits (e.g. `void`), - // Pointers are defined to have bits. So we must return something here. - // The value cannot be undefined, because we use the `nonnull` annotation - // for non-optional pointers. We also need to respect the alignment, even though - // the address will never be dereferenced. - const int: u64 = @"align".toByteUnits() orelse - // Note that these 0xaa values are appropriate even in release-optimized builds - // because we need a well-defined value that is not null, and LLVM does not - // have an "undef_but_not_null" attribute. As an example, if this `alloc` AIR - // instruction is followed by a `wrap_optional`, it will return this value - // verbatim, and the result should test as non-null. - switch (target.ptrBitWidth()) { - 16 => 0xaaaa, - 32 => 0xaaaaaaaa, - 64 => 0xaaaaaaaa_aaaaaaaa, - else => unreachable, - }; - const llvm_usize = try o.lowerType(.usize); - const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", target)); - return o.builder.castConst(.inttoptr, try o.builder.intConst(llvm_usize, int), llvm_ptr_ty); + // We need to make sure the global's address space is up to date, because that affects the + // type of a pointer to this global. But everything else about the global will be populated + // by `updateNav` or `updateFunc`. + llvm_global.ptr(&o.builder).addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget()); + return llvm_global.toConst(); } pub fn addByValParamAttrs( @@ -4243,13 +4153,14 @@ pub const Object = struct { const name = try o.builder.strtabString("__zig_error_name_table"); // TODO: Address space const variable_index = try o.builder.addVariable(name, .ptr, .default); - variable_index.setLinkage(.private, &o.builder); variable_index.setMutability(.constant, &o.builder); - variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); variable_index.setAlignment( Type.slice_const_u8_sentinel_0.abiAlignment(o.zcu).toLlvm(), &o.builder, ); + const global_index = variable_index.ptrConst(&o.builder).global; + global_index.setLinkage(.private, &o.builder); + global_index.setUnnamedAddr(.unnamed_addr, &o.builder); o.error_name_table = variable_index; return variable_index; @@ -4261,10 +4172,11 @@ pub const Object = struct { const llvm_err_int_ty = try o.errorIntType(); const name = try builder.strtabString("__zig_errors_len"); const variable_index = try builder.addVariable(name, llvm_err_int_ty, .default); - variable_index.setLinkage(.private, builder); variable_index.setMutability(.constant, builder); - variable_index.setUnnamedAddr(.unnamed_addr, builder); variable_index.setAlignment(Type.errorAbiAlignment(o.zcu).toLlvm(), builder); + const global_index = variable_index.ptrConst(&o.builder).global; + global_index.setLinkage(.private, builder); + global_index.setUnnamedAddr(.unnamed_addr, builder); o.errors_len_variable = variable_index; } return o.errors_len_variable; @@ -4332,16 +4244,16 @@ pub const Object = struct { for (0..loaded_enum.field_names.len) |field_index| { const name = try o.builder.stringNull(loaded_enum.field_names.get(ip)[field_index].toSlice(ip)); const name_init = try o.builder.stringConst(name); - const name_variable_index = - try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); + const name_variable_index = try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); try name_variable_index.setInitializer(name_init, &o.builder); - name_variable_index.setLinkage(.private, &o.builder); name_variable_index.setMutability(.constant, &o.builder); - name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); + const name_global_index = name_variable_index.ptrConst(&o.builder).global; + name_global_index.setLinkage(.private, &o.builder); + name_global_index.setUnnamedAddr(.unnamed_addr, &o.builder); const name_val = try o.builder.structValue(llvm_ret_ty, &.{ - name_variable_index.toConst(&o.builder), + name_global_index.toConst(), try o.builder.intConst(llvm_usize_ty, name.slice(&o.builder).?.len - 1), }); diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig index e70091ceba..0c7e794c78 100644 --- a/src/codegen/llvm/FuncGen.zig +++ b/src/codegen/llvm/FuncGen.zig @@ -581,8 +581,17 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif else => unreachable, }; const fn_info = zcu.typeToFunc(zig_fn_ty).?; - const return_type = Type.fromInterned(fn_info.return_type); - const llvm_fn = try self.resolveInst(air_call.callee); + const return_type: Type = .fromInterned(fn_info.return_type); + const llvm_fn = llvm_fn: { + // If the callee is a function *body*, we need to use a pointer to the global. + if (air_call.callee.toInterned()) |ip_index| switch (ip.indexToKey(ip_index)) { + .@"extern" => |e| break :llvm_fn (try o.lowerNavRef(e.owner_nav)).toValue(), + .func => |f| break :llvm_fn (try o.lowerNavRef(f.owner_nav)).toValue(), + else => {}, + }; + // Otherwise, the operand is already a function pointer (possibly runtime-known). + break :llvm_fn try self.resolveInst(air_call.callee); + }; const target = zcu.getTarget(); const sret = firstParamSRet(fn_info, zcu, target); @@ -875,7 +884,9 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!v const target = zcu.getTarget(); const panic_func = zcu.funcInfo(zcu.builtin_decl_values.get(panic_id.toBuiltin())); const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?; - const panic_global = try o.resolveLlvmFunction(panic_func.owner_nav); + const llvm_panic_fn_ty = try o.lowerType(.fromInterned(panic_func.ty)); + + const llvm_panic_fn_ref = try o.lowerNavRef(panic_func.owner_nav); const has_err_trace = zcu.comp.config.any_error_tracing and fn_info.cc == .auto; if (has_err_trace) assert(fg.err_ret_trace != .none); @@ -884,8 +895,8 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!v .normal, llvm.toLlvmCallConvTag(fn_info.cc, target).?, .none, - panic_global.typeOf(&o.builder), - panic_global.toValue(&o.builder), + llvm_panic_fn_ty, + llvm_panic_fn_ref.toValue(), if (has_err_trace) &.{fg.err_ret_trace} else &.{}, "", ); @@ -1745,8 +1756,9 @@ fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Tod .default, ); try table_variable.setInitializer(table_val, &o.builder); - table_variable.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); - table_variable.setUnnamedAddr(.unnamed_addr, &o.builder); + const table_global = table_variable.ptrConst(&o.builder).global; + table_global.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); + table_global.setUnnamedAddr(.unnamed_addr, &o.builder); const table_includes_else = item_count != table_len; @@ -1759,7 +1771,7 @@ fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Tod .likely => .likely, .unlikely => .unlikely, }, - .table = table_variable.toConst(&o.builder), + .table = table_global.toConst(), .table_includes_else = table_includes_else, }; }; @@ -3255,8 +3267,8 @@ fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = fg.object; const ty_nav = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; - const llvm_ptr_const = try o.lowerNavRefValue(ty_nav.nav); - return llvm_ptr_const.toValue(); + const llvm_ptr = try o.lowerNavRef(ty_nav.nav); + return llvm_ptr.toValue(); } fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { @@ -4636,29 +4648,27 @@ fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value const o = self.object; const zcu = o.zcu; const ptr_ty = self.typeOfIndex(inst); - const pointee_type = ptr_ty.childType(zcu); - if (!pointee_type.hasRuntimeBits(zcu)) { - const ptr_info = ptr_ty.ptrInfo(zcu); - return (try o.lowerPtrToVoid(ptr_info.flags.alignment, ptr_info.flags.address_space)).toValue(); + const ptr_align = ptr_ty.ptrAlignment(zcu); + const elem_ty = ptr_ty.childType(zcu); + if (!elem_ty.hasRuntimeBits(zcu)) { + return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue(); } - const pointee_llvm_ty = try o.lowerType(pointee_type); - const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); - return self.buildAlloca(pointee_llvm_ty, alignment); + const llvm_elem_ty = try o.lowerType(elem_ty); + return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm()); } fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { + if (self.ret_ptr != .none) return self.ret_ptr; const o = self.object; const zcu = o.zcu; const ptr_ty = self.typeOfIndex(inst); - const ret_ty = ptr_ty.childType(zcu); - if (!ret_ty.hasRuntimeBits(zcu)) { - const ptr_info = ptr_ty.ptrInfo(zcu); - return (try o.lowerPtrToVoid(ptr_info.flags.alignment, ptr_info.flags.address_space)).toValue(); + const ptr_align = ptr_ty.ptrAlignment(zcu); + const elem_ty = ptr_ty.childType(zcu); + if (!elem_ty.hasRuntimeBits(zcu)) { + return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue(); } - if (self.ret_ptr != .none) return self.ret_ptr; - const ret_llvm_ty = try o.lowerType(ret_ty); - const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); - return self.buildAlloca(ret_llvm_ty, alignment); + const llvm_elem_ty = try o.lowerType(elem_ty); + return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm()); } /// Use this instead of builder.buildAlloca, because this function makes sure to