Rollup merge of #155721 - cezarbbb:fix-archive-ice-148217, r=bjorn3

When archive format is wrong produce an error instead of ICE

Fix rust-lang/rust#145624. Fix rust-lang/rust#147094. Fix rust-lang/rust#148217.
There are now two-step solutions to replace the original ICE：
Step 1: BSD format archive on a GNU/Linux target should emit a format mismatch warning.
Step 2: Corrupt archive with member offset exceeding file boundary should produce an error, not an ICE.

r? @bjorn3

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -9,7 +9,7 @@
     ArchiveKind, COFFShortExport, MachineTypes, NewArchiveMember, write_archive_to_stream,
 };
 pub use ar_archive_writer::{DEFAULT_OBJECT_READER, ObjectReader};
-use object::read::archive::ArchiveFile;
+use object::read::archive::{ArchiveFile, ArchiveKind as ObjectArchiveKind};
 use object::read::macho::FatArch;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_data_structures::memmap::Mmap;
@@ -320,6 +320,50 @@ fn add_archive(
     fn build(self: Box<Self>, output: &Path) -> bool;
 }
 
+fn target_archive_format_to_object_kind(format: &str) -> Option<ObjectArchiveKind> {
+    match format {
+        "gnu" => Some(ObjectArchiveKind::Gnu),
+        "bsd" => Some(ObjectArchiveKind::Bsd),
+        "darwin" => Some(ObjectArchiveKind::Bsd64),
+        "coff" => Some(ObjectArchiveKind::Coff),
+        "aix_big" => Some(ObjectArchiveKind::AixBig),
+        _ => None,
+    }
+}
+
+fn archive_kinds_compatible(actual: ObjectArchiveKind, expected: ObjectArchiveKind) -> bool {
+    if actual == expected {
+        return true;
+    }
+    matches!(
+        (actual, expected),
+        // An archive without long filenames or symbol table is detected as Unknown;
+        // this is compatible with any target format.
+        (ObjectArchiveKind::Unknown, _)
+        // 64-bit symbol table variants are compatible with their 32-bit counterparts
+        | (ObjectArchiveKind::Gnu64, ObjectArchiveKind::Gnu)
+        | (ObjectArchiveKind::Gnu, ObjectArchiveKind::Gnu64)
+        | (ObjectArchiveKind::Bsd64, ObjectArchiveKind::Bsd)
+        | (ObjectArchiveKind::Bsd, ObjectArchiveKind::Bsd64)
+        // GNU and COFF archives share the same magic and member header format;
+        // only the symbol table layout differs.
+        | (ObjectArchiveKind::Gnu, ObjectArchiveKind::Coff)
+        | (ObjectArchiveKind::Coff, ObjectArchiveKind::Gnu)
+        | (ObjectArchiveKind::Gnu64, ObjectArchiveKind::Coff)
+    )
+}
+
+fn archive_kind_display_name(kind: ObjectArchiveKind) -> String {
+    match kind {
+        ObjectArchiveKind::Gnu | ObjectArchiveKind::Gnu64 => "GNU".to_string(),
+        ObjectArchiveKind::Bsd => "BSD".to_string(),
+        ObjectArchiveKind::Bsd64 => "Darwin".to_string(),
+        ObjectArchiveKind::Coff => "COFF".to_string(),
+        ObjectArchiveKind::AixBig => "AIX big".to_string(),
+        _ => format!("{kind:?}"),
+    }
+}
+
 pub struct ArArchiveBuilderBuilder;
 
 impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
@@ -420,6 +464,19 @@ fn add_archive(
             .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
         let archive_index = self.src_archives.len();
 
+        if let Some(expected_kind) =
+            target_archive_format_to_object_kind(&self.sess.target.archive_format)
+        {
+            let actual_kind = archive.kind();
+            if !archive_kinds_compatible(actual_kind, expected_kind) {
+                self.sess.dcx().emit_warn(crate::errors::IncompatibleArchiveFormat {
+                    path: archive_path.clone(),
+                    actual: archive_kind_display_name(actual_kind),
+                    expected: archive_kind_display_name(expected_kind),
+                });
+            }
+        }
+
         for entry in archive.members() {
             let entry = entry.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
             let file_name = String::from_utf8(entry.name().to_vec())
@@ -482,9 +539,24 @@ fn build_inner(self, output: &Path) -> io::Result<bool> {
                 match entry {
                     ArchiveEntry::FromArchive { archive_index, file_range } => {
                         let src_archive = &self.src_archives[archive_index];
-
-                        let data = &src_archive.1
-                            [file_range.0 as usize..file_range.0 as usize + file_range.1 as usize];
+                        let archive_data = &src_archive.1;
+                        let start = file_range.0 as usize;
+                        let end = start + file_range.1 as usize;
+                        let Some(data) = archive_data.get(start..end) else {
+                            return Err(io_error_context(
+                                "invalid archive member",
+                                io::Error::new(
+                                    io::ErrorKind::InvalidData,
+                                    format!(
+                                        "archive member at offset {start} with size {} \
+                                         exceeds archive size {} in `{}`",
+                                        file_range.1,
+                                        archive_data.len(),
+                                        src_archive.0.display(),
+                                    ),
+                                ),
+                            ));
+                        };
 
                         Box::new(data) as Box<dyn AsRef<[u8]>>
                     }

compiler/rustc_codegen_ssa/src/errors.rs

@@ -677,6 +677,18 @@ pub(crate) struct UnknownArchiveKind<'a> {
     pub kind: &'a str,
 }
 
+#[derive(Diagnostic)]
+#[diag("archive `{$path}` was built as {$actual} format, but the target expects {$expected}")]
+#[help(
+    "this often occurs when using BSD-format archive tools on a Linux target; \
+    rebuild the archive with the correct format for the target platform"
+)]
+pub(crate) struct IncompatibleArchiveFormat {
+    pub path: PathBuf,
+    pub actual: String,
+    pub expected: String,
+}
+
 #[derive(Diagnostic)]
 #[diag("linking static libraries is not supported for BPF")]
 pub(crate) struct BpfStaticlibNotSupported;

tests/run-make/archive-corrupt-error/corrupt.a

@@ -0,0 +1,3 @@
+!<arch>
+corrupt.o/      0           0     0     100644  10000     `
+small_data

tests/run-make/archive-corrupt-error/lib.rs

@@ -0,0 +1,3 @@
+extern "C" {
+    fn foo() -> i32;
+}

tests/run-make/archive-corrupt-error/rmake.rs

@@ -0,0 +1,21 @@
+// Regression test for https://github.com/rust-lang/rust/issues/148217
+// A corrupt archive with member offset exceeding file boundary should produce
+// an error, not an ICE.
+
+//@ ignore-cross-compile
+
+use run_make_support::{path, rfs, rustc, static_lib_name};
+
+fn main() {
+    rfs::create_dir("archive");
+    rfs::copy("corrupt.a", path("archive").join(static_lib_name("corrupt")));
+    rustc()
+        .input("lib.rs")
+        .crate_type("rlib")
+        .library_search_path("archive")
+        .arg("-lstatic=corrupt")
+        .run_fail()
+        .assert_stderr_not_contains("panicked")
+        .assert_stderr_not_contains("unexpectedly panicked")
+        .assert_stderr_contains("invalid archive member");
+}

tests/run-make/archive-format-error/lib.rs

@@ -0,0 +1,3 @@
+extern "C" {
+    fn foo() -> i32;
+}

tests/run-make/archive-format-error/native.c

@@ -0,0 +1 @@
+int foo() { return 42; }

tests/run-make/archive-format-error/rmake.rs

@@ -0,0 +1,22 @@
+// Regression test for https://github.com/rust-lang/rust/issues/148217
+// BSD format archive on a Linux target should emit a format mismatch warning.
+
+//@ ignore-cross-compile
+//@ only-linux
+
+use run_make_support::{cc, llvm_ar, path, rfs, rustc, static_lib_name};
+
+fn main() {
+    rfs::create_dir("archive");
+
+    cc().arg("-c").input("native.c").output("archive/native.o").run();
+    let bsd_archive = path("archive").join(static_lib_name("native_bsd"));
+    llvm_ar().arg("rcus").arg("--format=bsd").output_input(&bsd_archive, "archive/native.o").run();
+    rustc()
+        .input("lib.rs")
+        .crate_type("rlib")
+        .library_search_path("archive")
+        .arg("-lstatic=native_bsd")
+        .run()
+        .assert_stderr_contains("was built as BSD format, but the target expects GNU");
+}