Merge pull request #118 from overdrivenpotato/lto

Add support for LTO and remove link-dead-code

Author: sajattack

README.md

@@ -56,7 +56,7 @@ implementations of graphics functions, and the addition of missing libraries.
 ## Dependencies
 
 To compile for the PSP, you will need a Rust **nightly** version equal to or
-later than `2022-05-22` and the `rust-src` component. Please install Rust using
+later than `2022-06-11` and the `rust-src` component. Please install Rust using
 https://rustup.rs/
 
 Use the following if you are new to Rust. (Feel free to set an override manually

cargo-psp/Cargo.toml

@@ -29,4 +29,5 @@ rustc_version = "0.2.3"
 
 serde = "1.0.111"
 serde_derive = "1.0.111"
+bincode = "1.3.1"
 toml = "0.5.6"

cargo-psp/src/fix_imports.rs

@@ -0,0 +1,118 @@
+use std::{mem, path::Path, collections::HashMap};
+use goblin::elf::Elf;
+
+/// A stub library entry like the one found in the `psp` crate, but with types
+/// changed to be cross-platform.
+///
+/// In particular, pointers are swapped to `u32`.
+#[derive(serde_derive::Deserialize, serde_derive::Serialize, Debug)]
+struct SceStubLibraryEntry {
+    name: u32,
+    version: [u8; 2],
+    flags: u16,
+    len: u8,
+    v_stub_count: u8,
+    stub_count: u16,
+    nid_table: u32,
+    stub_table: u32,
+}
+
+/// Fixes the stub count for all imported modules.
+///
+/// When linking, sometimes it is possible for imported functions to be
+/// completely stripped. This can happen with LTO, for example. Because of this,
+/// we need a way to write the stub count post-linking. This function does
+/// exactly this.
+pub fn fix<T: AsRef<Path>>(path: T) {
+    let mut bytes = std::fs::read(&path).unwrap();
+    let elf = Elf::parse(&bytes).unwrap();
+
+    let shstrtab = {
+        let sh = &elf.section_headers[elf.header.e_shstrndx as usize];
+        let start = sh.sh_offset as usize;
+        let end = (sh.sh_offset + sh.sh_size) as usize;
+
+        &bytes[start..end]
+    };
+
+    // Map of name -> section header
+    let sections = elf.section_headers
+        .iter()
+        .map(|sh| {
+            let name = shstrtab[sh.sh_name..]
+                .iter()
+                .take_while(|b| **b != 0)
+                .map(|b| *b as char)
+                .collect::<String>();
+
+            (name, sh)
+        })
+        .collect::<HashMap<_, _>>();
+
+    let lib_stub = match sections.get(".lib.stub") {
+        Some(s) => s,
+
+        // The binary might not import any functions, in some rare cases.
+        None => return,
+    };
+
+    // TODO: Use module info instead of these sections, as they can technically
+    // be stripped.
+
+    // If we have .lib.stub, then .lib.stub.btm must exist.
+    let lib_stub_btm = sections.get(".lib.stub.btm")
+        .expect("could not find .lib.stub.btm section");
+
+    let rodata_sce_nid = sections.get(".rodata.sceNid")
+        .expect("Could not find .rodata.sceNid section");
+
+    let start = lib_stub.sh_offset as usize;
+    let end = lib_stub_btm.sh_offset as usize;
+
+    // Rough check for the length.
+    assert_eq!((end - start) % mem::size_of::<SceStubLibraryEntry>(), 0);
+
+    // List of (stub index in .lib.stub, stub entry). Ordered by appearance in
+    // .rodata.sceNid section.
+    let stubs_nid_sorted = {
+        let mut entries = bytes[start..end]
+            .chunks(mem::size_of::<SceStubLibraryEntry>())
+            .map(bincode::deserialize)
+            .map(Result::unwrap)
+            .enumerate()
+            .collect::<Vec<(_, SceStubLibraryEntry)>>();
+
+        // Sort by order of appearance in .rodata.sceNid section.
+        entries.sort_unstable_by_key(|(_, s)| s.nid_table);
+
+        entries
+    };
+
+    // Iterator of mutable byte slices (of stub lib entries) in raw ELF binary.
+    let stub_entry_bufs = bytes[start..end]
+        .chunks_mut(mem::size_of::<SceStubLibraryEntry>());
+
+    for (i, stub_entry_buf) in stub_entry_bufs.enumerate() {
+        // A NID is a 32-bit value.
+        const NID_SIZE: u32 = 4;
+
+        let mut stub: SceStubLibraryEntry = bincode::deserialize(stub_entry_buf)
+            .unwrap();
+
+        let nid_end = stubs_nid_sorted.get(
+                1 + stubs_nid_sorted.iter()
+                    .position(|&(j, _)| i == j)
+                    .unwrap()
+            )
+            .map(|(_, s)| s.nid_table)
+            .unwrap_or(rodata_sce_nid.sh_addr as u32 + rodata_sce_nid.sh_size as u32);
+
+        stub.stub_count = ((nid_end - stub.nid_table) / NID_SIZE) as u16;
+
+        // Re-serialize the stub and save.
+        let serialized = bincode::serialize(&stub).unwrap();
+        stub_entry_buf.copy_from_slice(&serialized);
+    }
+
+    std::fs::write(path, bytes).unwrap();
+}

cargo-psp/src/main.rs

@@ -6,6 +6,8 @@ use std::{
     process::{self, Command, Stdio},
 };
 
+mod fix_imports;
+
 const CONFIG_NAME: &str = "Psp.toml";
 
 #[derive(serde_derive::Deserialize, Default)]
@@ -118,11 +120,13 @@ impl fmt::Display for CommitDate {
     }
 }
 
-// Minimum 2022-05-22, remember to update both commit date and version too, below.
-const MINIMUM_COMMIT_DATE: CommitDate = CommitDate { year: 2022, month: 05, day: 22 };
+// Minimum 2022-06-11, remember to update both commit date and version too,
+// below. Note that the `day` field lags by one day, as the toolchain always
+// contains the previous days' nightly rustc.
+const MINIMUM_COMMIT_DATE: CommitDate = CommitDate { year: 2022, month: 06, day: 10 };
 const MINIMUM_RUSTC_VERSION: Version = Version {
     major: 1,
-    minor: 62,
+    minor: 63,
     patch: 0,
     pre: Vec::new(),
     build: Vec::new(),
@@ -191,18 +195,13 @@ fn main() {
         },
     };
 
-    // FIXME: This is a workaround. This should eventually be removed.
-    let rustflags = env::var("RUSTFLAGS").unwrap_or("".into())
-        + " -C link-dead-code";
-
     let mut process = Command::new("cargo")
         .arg("build")
         .arg("-Z")
         .arg(build_std_flag)
         .arg("--target")
         .arg("mipsel-sony-psp")
         .args(args)
-        .env("RUSTFLAGS", rustflags)
         .stdin(Stdio::inherit())
         .stdout(Stdio::inherit())
         .stderr(Stdio::inherit())
@@ -239,6 +238,7 @@ fn main() {
     for id in metadata.clone().workspace_members {
         let package = metadata[&id].clone();
 
+        // TODO: Error if no bin is ever found.
         for target in package.targets {
             if target.kind.iter().any(|k| k == "bin") {
                 let elf_path = bin_dir.join(&target.name);
@@ -247,6 +247,8 @@ fn main() {
                 let sfo_path = bin_dir.join("PARAM.SFO");
                 let pbp_path = bin_dir.join("EBOOT.PBP");
 
+                fix_imports::fix(&elf_path);
+
                 Command::new("prxgen")
                     .arg(&elf_path)
                     .arg(&prx_path)

ci/concourse/build-rust.yml

@@ -11,8 +11,7 @@ image_resource:
     tag: 1.44-slim
 
 params:
-  RUSTFLAGS: "-C link-dead-code"
-  RUSTUP_TOOLCHAIN: nightly-2022-05-22
+  RUSTUP_TOOLCHAIN: nightly-2022-06-11
 
 inputs:
   - name: repo

libunwind/.gitignore

@@ -1,4 +1,4 @@
 src
 include
-libunwind.a
+libunwind*.a
 *.o

libunwind/Makefile

@@ -1,3 +1,8 @@
+SUFFIX ?=
+CPPFLAGS ?=
+
+OUTLIB := libunwind$(SUFFIX).a
+
 CC      := clang
 CXX     := clang++
 AR      := llvm-ar
@@ -9,20 +14,25 @@ S_OBJLIST   := UnwindRegistersRestore.o UnwindRegistersSave.o
 
 CFLAGS   := -std=c99
 CXXFLAGS := -std=c++11 -nostdinc++ -fno-exceptions -fno-rtti
-CPPFLAGS := -target mipsel-unknown-unknown -mcpu=mips2 -msingle-float \
-	-fstrict-aliasing -funwind-tables -O3 \
+CPPFLAGS := $(CPPFLAGS) -target mipsel-unknown-unknown -mcpu=mips2 \
+	-msingle-float -fstrict-aliasing -funwind-tables -O3 \
 	-D __LITTLE_ENDIAN__ -D __ELF__ -D _LIBUNWIND_IS_BAREMETAL \
 	-D _LIBUNWIND_HAS_NO_THREADS -D _LIBUNWIND_IS_NATIVE_ONLY \
 	-DNDEBUG \
 	-I /usr/local/pspdev/psp/include/ \
 	-I include
 
-../psp/libunwind.a: libunwind.a
-	cp libunwind.a ../psp/
+all:
+	make ../psp/libunwind.a
+	SUFFIX=_lto CPPFLAGS=-flto make ../psp/libunwind_lto.a
+
+../psp/$(OUTLIB): $(OUTLIB)
+	cp $^ ../psp/
 	touch ../psp/build.rs
+	make clean
 
-libunwind.a: $(CPP_OBJLIST) $(C_OBJLIST) $(S_OBJLIST)
-	$(AR) $(ARFLAGS) libunwind.a $^
+$(OUTLIB): $(CPP_OBJLIST) $(C_OBJLIST) $(S_OBJLIST)
+	$(AR) $(ARFLAGS) $(OUTLIB) $^
 
 $(CPP_OBJLIST): %.o: src/%.cpp
 	$(COMPILE.cc) $^
@@ -32,7 +42,7 @@ $(S_OBJLIST): %.o: src/%.S
 $(C_OBJLIST) $(S_OBJLIST):
 	$(COMPILE.c) $^
 
-.PHONY: clean patch
+.PHONY: all clean patch
 
 patch:
 	git submodule update --init --depth 1 -- ./rustc
@@ -41,4 +51,4 @@ patch:
 	patch -p0 < ./no-sdc1.patch
 
 clean:
-	rm -r *.o libunwind.a
+	rm -r *.o

psp/Cargo.toml

@@ -23,7 +23,7 @@ std = []
 stub-only = []
 
 [dependencies]
-paste = "0.1.12"
+paste = "1.0.1"
 bitflags = "1.2.1"
 embedded-graphics = { version = "0.7.1", optional = true, features = ["fixed_point"] }
 unstringify = "0.1.4"

psp/build.rs

@@ -1,18 +1,31 @@
-use std::{env, path::Path};
+use std::{env, path::Path, os::unix::prelude::OsStrExt};
 
 fn main() {
     println!("cargo:rerun-if-changed=build.rs");
     println!("cargo:rerun-if-changed=libunwind.a");
+    println!("cargo:rerun-if-env-changed=RUSTFLAGS");
 
     if env::var("CARGO_FEATURE_STUB_ONLY").is_ok() {
         return;
     }
 
+    // Figure out whether to use the LTO libunwind, or the regular one.
+    let libunwind = if env::var_os("CARGO_ENCODED_RUSTFLAGS")
+        .expect("could not get `CARGO_ENCODED_RUSTFLAGS` variable")
+        .as_bytes()
+        .split(|b| *b == 0x1f)
+        .any(|flags| flags.starts_with(b"-Clinker-plugin-lto"))
+    {
+        "./libunwind_lto.a"
+    } else {
+        "./libunwind.a"
+    };
+
     // TODO: Do we even need to copy the library over? Maybe we can just link
     // directly from the current directory.
     let out_dir = env::var("OUT_DIR").unwrap();
     let out_file = Path::new(&out_dir).join("libunwind.a");
-    std::fs::copy("./libunwind.a", out_file).unwrap();
+    std::fs::copy(libunwind, out_file).unwrap();
 
     println!("cargo:rustc-link-lib=static=unwind");
     println!("cargo:rustc-link-search=native={}", out_dir);