Overhauled common BSS support & more

- With a map, attempts to detect and handle common BSS automatically
- With a map, attempts to detect and correct inflated common BSS bug (< GC 2.7 linker)
- Support for "stripped" symbols, sometimes required to match inflated common BSS sizes
- Warns on duplicated TUs in a map (other than common BSS)
- Automatically adds `comment:0` to `.s` TUs from a map (avoids linker crash)

Author: encounter

Cargo.lock

@@ -3,7 +3,7 @@ name = "decomp-toolkit"
 description = "Yet another GameCube/Wii decompilation toolkit."
 authors = ["Luke Street <[email protected]>"]
 license = "MIT OR Apache-2.0"
-version = "0.6.3"
+version = "0.6.4"
 edition = "2021"
 publish = false
 repository = "https://github.com/encounter/decomp-toolkit"

Cargo.toml

@@ -739,7 +739,7 @@ fn load_analyze_dol(config: &ProjectConfig) -> Result<AnalyzeResult> {
     }
 
     if let Some(map_path) = &config.base.map {
-        apply_map_file(map_path, &mut obj)?;
+        apply_map_file(map_path, &mut obj, config.common_start, config.mw_comment_version)?;
         dep.push(map_path.clone());
     }
 
@@ -963,7 +963,7 @@ fn load_analyze_rel(config: &ProjectConfig, module_config: &ModuleConfig) -> Res
 
     let mut dep = vec![module_config.object.clone()];
     if let Some(map_path) = &module_config.map {
-        apply_map_file(map_path, &mut module_obj)?;
+        apply_map_file(map_path, &mut module_obj, None, None)?;
         dep.push(map_path.clone());
     }
 
@@ -1451,11 +1451,10 @@ fn diff(args: DiffArgs) -> Result<()> {
     log::info!("Loading {}", args.elf_file.display());
     let linked_obj = process_elf(&args.elf_file)?;
 
-    for orig_sym in obj
-        .symbols
-        .iter()
-        .filter(|s| !matches!(s.kind, ObjSymbolKind::Unknown | ObjSymbolKind::Section))
-    {
+    let common_bss = obj.sections.common_bss_start();
+    for orig_sym in obj.symbols.iter().filter(|s| {
+        !matches!(s.kind, ObjSymbolKind::Unknown | ObjSymbolKind::Section) && !s.flags.is_stripped()
+    }) {
         let Some(orig_section_index) = orig_sym.section else { continue };
         let orig_section = &obj.sections[orig_section_index];
         let (linked_section_index, linked_section) =
@@ -1474,7 +1473,12 @@ fn diff(args: DiffArgs) -> Result<()> {
         let mut found = false;
         if let Some((_, linked_sym)) = linked_sym {
             if linked_sym.name.starts_with(&orig_sym.name) {
-                if linked_sym.size != orig_sym.size {
+                if linked_sym.size != orig_sym.size &&
+                    // TODO validate common symbol sizes
+                    // (need to account for inflation bug)
+                    matches!(common_bss, Some((idx, addr)) if
+                        orig_section_index == idx && orig_sym.address as u32 >= addr)
+                {
                     log::error!(
                         "Expected {} (type {:?}) to have size {:#X}, but found {:#X}",
                         orig_sym.name,

src/cmd/dol.rs

@@ -57,7 +57,7 @@ pub fn run(args: Args) -> Result<()> {
 
 fn entries(args: EntriesArgs) -> Result<()> {
     let file = map_file(&args.map_file)?;
-    let entries = process_map(&mut file.as_reader())?;
+    let entries = process_map(&mut file.as_reader(), None, None)?;
     match entries.unit_entries.get_vec(&args.unit) {
         Some(vec) => {
             println!("Entries for {}:", args.unit);
@@ -89,7 +89,7 @@ fn entries(args: EntriesArgs) -> Result<()> {
 fn symbol(args: SymbolArgs) -> Result<()> {
     let file = map_file(&args.map_file)?;
     log::info!("Processing map...");
-    let entries = process_map(&mut file.as_reader())?;
+    let entries = process_map(&mut file.as_reader(), None, None)?;
     log::info!("Done!");
     let mut opt_ref: Option<(String, SymbolEntry)> = None;
 

src/cmd/map.rs

@@ -125,6 +125,13 @@ impl ObjSections {
         self.iter()
             .flat_map(|(idx, s)| s.splits.iter().map(move |(addr, split)| (idx, s, addr, split)))
     }
+
+    pub fn common_bss_start(&self) -> Option<(usize, u32)> {
+        let Ok(Some((section_index, section))) = self.by_name(".bss") else {
+            return None;
+        };
+        section.splits.iter().find(|(_, split)| split.common).map(|(addr, _)| (section_index, addr))
+    }
 }
 
 impl Index<usize> for ObjSections {

src/obj/sections.rs

@@ -26,9 +26,9 @@ pub enum ObjSymbolScope {
 }
 
 flags! {
-    #[repr(u8)]
+    #[repr(u32)]
     #[derive(Deserialize_repr, Serialize_repr)]
-    pub enum ObjSymbolFlags: u8 {
+    pub enum ObjSymbolFlags: u32 {
         Global,
         Local,
         Weak,
@@ -39,6 +39,9 @@ flags! {
         RelocationIgnore,
         /// Symbol won't be written to symbols file
         NoWrite,
+        /// Symbol was stripped from the original object,
+        /// but is still useful for common BSS matching.
+        Stripped,
     }
 }
 
@@ -83,6 +86,9 @@ impl ObjSymbolFlagSet {
     #[inline]
     pub fn is_no_write(&self) -> bool { self.0.contains(ObjSymbolFlags::NoWrite) }
 
+    #[inline]
+    pub fn is_stripped(&self) -> bool { self.0.contains(ObjSymbolFlags::Stripped) }
+
     #[inline]
     pub fn set_scope(&mut self, scope: ObjSymbolScope) {
         match scope {
@@ -119,7 +125,8 @@ impl ObjSymbolFlagSet {
         self.0
             & (ObjSymbolFlags::ForceActive
                 | ObjSymbolFlags::NoWrite
-                | ObjSymbolFlags::RelocationIgnore)
+                | ObjSymbolFlags::RelocationIgnore
+                | ObjSymbolFlags::Stripped)
     }
 }
 
@@ -212,7 +219,10 @@ impl ObjSymbols {
     }
 
     pub fn add(&mut self, in_symbol: ObjSymbol, replace: bool) -> Result<SymbolIndex> {
-        let opt = if let Some(section_index) = in_symbol.section {
+        let opt = if in_symbol.flags.is_stripped() {
+            // Stripped symbols don't overwrite existing symbols
+            None
+        } else if let Some(section_index) = in_symbol.section {
             self.at_section_address(section_index, in_symbol.address as u32).find(|(_, symbol)| {
                 symbol.kind == in_symbol.kind ||
                     // Replace auto symbols with real symbols
@@ -228,7 +238,8 @@ impl ObjSymbols {
             let replace = replace || (is_auto_symbol(existing) && !is_auto_symbol(&in_symbol));
             let size =
                 if existing.size_known && in_symbol.size_known && existing.size != in_symbol.size {
-                    log::warn!(
+                    // TODO fix this and restore to warning
+                    log::debug!(
                         "Conflicting size for {}: was {:#X}, now {:#X}",
                         existing.name,
                         existing.size,
@@ -336,6 +347,8 @@ impl ObjSymbols {
             .into_iter()
             .flatten()
             .map(move |&idx| (idx, &self.symbols[idx]))
+            // "Stripped" symbols don't actually exist at the address
+            .filter(|(_, sym)| !sym.flags.is_stripped())
     }
 
     pub fn kind_at_section_address(
@@ -513,7 +526,7 @@ impl Index<SymbolIndex> for ObjSymbols {
 impl ObjSymbol {
     /// Whether this symbol can be referenced by the given relocation kind.
     pub fn referenced_by(&self, reloc_kind: ObjRelocKind) -> bool {
-        if self.flags.is_relocation_ignore() {
+        if self.flags.is_relocation_ignore() || self.flags.is_stripped() {
             return false;
         }
 

src/obj/symbols.rs

@@ -281,9 +281,10 @@ impl CommentSym {
             vis_flags |= 0xD;
         }
         let mut active_flags = 0;
-        if symbol.flags.is_force_active()
-            || (force_active
-                && matches!(symbol.kind, ObjSymbolKind::Function | ObjSymbolKind::Object))
+        if !symbol.flags.is_stripped()
+            && (symbol.flags.is_force_active()
+                || (force_active
+                    && matches!(symbol.kind, ObjSymbolKind::Function | ObjSymbolKind::Object)))
         {
             active_flags |= 0x8;
         }

src/util/comment.rs

@@ -130,6 +130,9 @@ pub fn parse_symbol_line(line: &str, obj: &mut ObjInfo) -> Result<Option<ObjSymb
                     "force_active" => {
                         symbol.flags.0 |= ObjSymbolFlags::ForceActive;
                     }
+                    "stripped" => {
+                        symbol.flags.0 |= ObjSymbolFlags::Stripped;
+                    }
                     "noreloc" => {
                         ensure!(
                             symbol.size != 0,
@@ -270,6 +273,9 @@ where W: Write + ?Sized {
     // if symbol.flags.is_force_active() {
     //     write!(w, " force_active")?;
     // }
+    if symbol.flags.is_stripped() {
+        write!(w, " stripped")?;
+    }
     if let Some(section) = symbol.section {
         if obj.blocked_ranges.contains_key(&SectionAddress::new(section, symbol.address as u32)) {
             write!(w, " noreloc")?;