The first commit

tools/bk_crc/bk_crc.cpp

@@ -0,0 +1,73 @@
+#include <cstdint>
+#include <fstream>
+#include <iostream>
+#include <stdio.h>
+#include <string>
+#include <utility>
+#include <vector>
+#include <algorithm>
+#include <iomanip>
+
+typedef enum{
+    BINARY,
+    DEFINES,
+} OutType;
+
+typedef std::pair<uint32_t, uint32_t> CRC;
+
+CRC bk_crc(const std::vector<uint8_t> &buffer){
+    CRC crc = {0, 0xFFFFFFFF};
+    for (auto byte : buffer){
+        crc.first += byte;
+        crc.second ^= (byte << (crc.first & 0x17));
+    }
+    return crc;
+}
+
+/* calculate Banjo-Kazooie CRC and write value to bin OR txt file with gcc defines */
+/// -D output as defines
+int main(int argc, char *argv[]){
+    struct {
+        OutType type;
+        std::string name;
+    } config = {BINARY, ""};
+
+    //parse arguments
+    for(int i = 1; i < argc - 1; i++){
+        std::string argi = argv[i];
+        if(argi == "-D" || argi == "-d") //no loop
+            config.type = DEFINES;
+            config.name = argv[++i];
+            std::transform(config.name.begin(), config.name.end(), config.name.begin(), ::toupper);
+    }
+
+    std::string in_bin = argv[argc -1];
+    std::ifstream in_f(in_bin, std::ios::in | std::ios::binary | std::ios::ate);
+    size_t in_size = in_f.tellg();
+    in_f.seekg(0);
+    std::vector<uint8_t> in_data;
+    in_data.resize(in_size);
+    in_f.read(reinterpret_cast<char*>(in_data.data()), in_size);
+    
+    auto result = bk_crc(in_data);
+
+    switch(config.type){
+        case OutType::BINARY:
+            std::cout << (char)((result.first >> 24) & 0xff) << (char)((result.first >> 16) & 0xff) << (char)((result.first >> 8) & 0xff) << (char)((result.first >> 0) & 0xff);
+            std::cout << (char)((result.second >> 24) & 0xff) << (char)((result.second >> 16) & 0xff) << (char)((result.second >> 8) & 0xff) << (char)((result.second >> 0) & 0xff);
+            std::cout << std::flush;
+            break;
+
+        case OutType::DEFINES:
+            std::cout << "-D'" << config.name << "_CRC1=0x";
+            std::cout << std::hex <<std::setfill('0') << std::setw(8) << result.first;
+            std::cout << "' ";
+            std::cout << "-D'" << config.name << "_CRC2=0x";
+            std::cout << std::hex <<std::setfill('0') << std::setw(8) << result.second;
+            std::cout << "'";
+            std::cout << std::endl;
+            break;
+    }
+
+    return 0; 
+}

tools/decomp_me_ctx.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+./tools/m2ctx.py $1
+
+sed -i 's/sizeof(long)/8/g' ctx.c

tools/m2ctx.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import sys
+import subprocess
+from pathlib import Path
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.abspath(os.path.join(script_dir, ".."))
+src_dir = root_dir + "src/"
+
+# Project-specific
+CPP_FLAGS = [
+    "-Iinclude",
+    "-Iinclude/2.0L",
+    "-Iinclude/2.0L/PR",
+    "-Isrc",
+    "-Iver/current/build/include",
+    "-D_LANGUAGE_C",
+    "-D_FINALROM",
+    "-DF3DEX_GBI",
+    "-D_MIPS_SZLONG=32",
+    "-ffreestanding",
+]
+
+def import_c_file(in_file) -> str:
+    in_file = os.path.relpath(in_file, root_dir)
+    cpp_command = ["gcc", "-E", "-P", "-dM", *CPP_FLAGS, in_file]
+    cpp_command2 = ["gcc", "-E", "-P", *CPP_FLAGS, in_file]
+
+    out_text = ""
+    try:
+        out_text += subprocess.check_output(cpp_command, cwd=root_dir, encoding="utf-8")
+        out_text += subprocess.check_output(cpp_command2, cwd=root_dir, encoding="utf-8")
+    except subprocess.CalledProcessError:
+        print(
+            "Failed to preprocess input file, when running command:\n"
+            + cpp_command,
+            file=sys.stderr,
+            )
+        sys.exit(1)
+
+    if not out_text:
+        print("Output is empty - aborting")
+        sys.exit(1)
+    return out_text
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="""Create a context file which can be used for mips_to_c"""
+    )
+    parser.add_argument(
+        "c_file",
+        help="""File from which to create context""",
+    )
+    args = parser.parse_args()
+
+    output = import_c_file(args.c_file)
+
+    with open(os.path.join(root_dir, "ctx.c"), "w", encoding="UTF-8") as f:
+        f.write(output)
+
+
+if __name__ == "__main__":
+    main()

tools/mips_to_c_ctx.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+./tools/m2ctx.py $1
+
+sed -i 's/\[\]/\[0\]/g' ctx.c
+sed -i 's/sizeof(long)/8/g' ctx.c

tools/progress.py

@@ -0,0 +1,87 @@
+import argparse
+import os
+import re
+import sys
+import subprocess
+import glob
+
+def get_functions(elffile, section, ending=None):
+    try:
+        result = subprocess.run(['objdump', '-x', elffile], stdout=subprocess.PIPE)
+        nm_lines = result.stdout.decode().split("\n")
+    except:
+        sys.stderr.write(f"Error: Could not run objdump on {elffile} - make sure that the project is built")
+        sys.exit(1)
+
+    functions = {}
+
+    for line in nm_lines:
+        if f"g     F {section}" in line or "g     F *ABS*	" in line:
+            components = line.split()
+            size = int(components[4], 16)
+            name = components[5]
+            functions[name] = {"function": name, "length": size}
+
+    return functions
+
+def generate_csv(functions, nonmatching_funcs, version, section):
+    ret = []
+    ret.append("version,section,function,length,matching")
+    for func in functions:
+        length = functions[func]["length"]
+        if length > 0:
+            matching = "no" if func in nonmatching_funcs else "yes"
+            ret.append(f"{version},{section},{func},{length},{matching}")
+    return "\n".join(ret)
+
+def get_nonmatching_funcs(basedir, subcode):
+    grepstr = r'#pragma GLOBAL_ASM\(.*/\K.*(?=\.s)'
+    if subcode:
+        try:
+            funcs = set(subprocess.check_output(['grep', '-ohPR', grepstr, basedir + '/src/' + subcode]).decode('ascii').split())
+        except subprocess.CalledProcessError as grepexc:
+            if grepexc.returncode != 1:
+                raise grepexc
+            funcs = set()
+    else:
+        args = ['grep', '-ohPs', '-d', 'skip', grepstr]
+        args.extend(glob.glob(basedir + '/src/*'))
+        try:
+            funcs = set(subprocess.check_output(args).decode('ascii').split())
+        except subprocess.CalledProcessError as grepexc:
+            if grepexc.returncode != 1:
+                raise grepexc
+            funcs = set()
+        try:
+            funcs = funcs.union(set(subprocess.check_output(['grep', '-ohPR', grepstr, basedir + '/src/done']).decode('ascii').split()))
+        except subprocess.CalledProcessError as grepexc:
+            if grepexc.returncode != 1:
+                raise grepexc
+    return funcs
+
+
+def main(basedir, elffile, section, ending, version, subcode):
+    functions = get_functions(elffile, section, ending)
+    section_name = section.split("_")[-1] # .code_game -> game
+    nonmatching_funcs = get_nonmatching_funcs(basedir, subcode)
+    csv = generate_csv(functions, nonmatching_funcs, version, section_name)
+    print(csv)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Create progress csv based on map file',
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('basedir', type=str,
+                        help="base directory (containing src/)")
+    parser.add_argument('elffile', type=str,
+                        help=".elf file to be read")
+    parser.add_argument('section', type=str,
+                        help=".text section of the map")
+    parser.add_argument('--ending', type=str,
+                        help="section name that marks the end of 'section'")
+    parser.add_argument('--version', type=str, default='us',
+                        help="ROM version, us, eu, debug, ects")
+    parser.add_argument('--subcode', type=str, default=None,
+                        help="Subcode for section to get progress of")
+    args = parser.parse_args()
+
+    main(args.basedir, args.elffile, args.section, args.ending, args.version, args.subcode)

tools/progress_read.py

@@ -0,0 +1,77 @@
+import argparse
+import os
+import re
+import sys
+import csv
+import anybadge
+
+# Read using `mips-linux-gnu-readelf -S`
+overlay_sizes = {
+    'bk_boot'   : (0x5BE0 - 0x1000),
+    'core1'     : 0x034b70 + 0x003080,
+    'core2'     : 0x0dc600,
+    'CC'        : 0x0036b0,
+    'MMM'       : 0x0055f0,
+    'GV'        : 0x00a7e0,
+    'TTC'       : 0x005fc0,
+    'MM'        : 0x0034a0,
+    'BGS'       : 0x00a2a0,
+    'RBB'       : 0x009c60,
+    'FP'        : 0x00b600,
+    'SM'        : 0x0046d0,
+    'cutscenes' : 0x006f60,
+    'lair'      : 0x00c8c0,
+    'fight'     : 0x00af90,
+    'CCW'       : 0x008760,
+}
+
+def RGB_to_hex(RGB):
+    ''' [255,255,255] -> "#FFFFFF" '''
+    # Components need to be integers for hex to make sense
+    RGB = [int(x) for x in RGB]
+    return "#"+"".join(["0{0:x}".format(v) if v < 16 else
+                        "{0:x}".format(v) for v in RGB])
+
+def main(csv_name, version, overlay):
+    with open(csv_name, mode='r') as csv_file:
+        csv_reader = csv.DictReader(csv_file)
+        line_count = 0
+        total_func = 0
+        incomplete_func = 0
+        if overlay == 'total':
+            total_byte = sum(overlay_sizes.values())
+        else:
+            total_byte = overlay_sizes[overlay]
+        incomplete_byte = 0
+        for row in csv_reader:
+            if(row["version"] == version):
+                total_func += 1
+                if row['matching'] != 'yes':
+                    incomplete_func += 1
+                    incomplete_byte += int(row['length'])
+        done_byte = total_byte - incomplete_byte
+        done_func = total_func - incomplete_func
+        percent = ((done_byte/total_byte) * 100)
+        print("%s: bytes: %3.4f%% (%d/%d), nonstatic funcs: %3.4f%% (%d/%d)" % (overlay, percent, done_byte, total_byte,((done_func/total_func) *100), done_func, total_func ))
+        green = min(255, round(min(1, (percent / 100) * 2) * 224))
+        red = min(255, round(min(1, ((100 - percent) / 100) * 2) * 224))
+        color = RGB_to_hex([red, green, 0])
+        if overlay == 'total':
+            badge = anybadge.Badge("Banjo-Kazooie (us.v10)", "%3.4f%%" % (percent), default_color=color)
+        else:
+            badge = anybadge.Badge(overlay, "%3.4f%%" % (percent), default_color=color)
+        badge.write_badge('progress/progress_' + overlay + '.svg',overwrite=True)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Create progress csv based on map file',
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('csv_name', type=str,
+                        help="csv to read")
+    parser.add_argument('ver_str', type=str,
+                        help="version")
+    parser.add_argument('overlay', type=str,
+                        help="overlay name")
+    args = parser.parse_args()
+
+    main(args.csv_name, args.ver_str, args.overlay)

tools/rareunzip.py

@@ -0,0 +1,28 @@
+import sys
+import zlib
+
+
+def runzip_with_leftovers(data):
+    d = zlib.decompressobj(wbits=-15) # raw deflate bytestream
+    res = d.decompress(data[4:])      # drop 4 byte length header
+    return (res, d.unused_data)
+
+def runzip(data):
+    res, leftovers = runzip_with_leftovers(data)
+    return res
+
+def main():
+    with open(sys.argv[1], "rb") as f:
+        with open(sys.argv[2], "wb") as o:
+            data = f.read()
+            # banjo header
+            if data[:2] == b'\x11\x72':
+                data = data[2:]
+            o.write(runzip(data))
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        print("usage %s infile outfile" % sys.argv[0])
+    else:
+        main()

tools/set_o32abi_bit.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import argparse, struct, sys
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('file', help='input file')
+    args = parser.parse_args()
+
+    with open(args.file, 'r+b') as f:
+        magic = struct.unpack('>I', f.read(4))[0]
+        if magic != 0x7F454C46:
+            print('Error: Not an ELF file')
+            sys.exit(1)
+
+        f.seek(36)
+        flags = struct.unpack('>I', f.read(4))[0]
+        if flags & 0xF0000000 != 0x20000000: # test for mips3
+            print('Error: Architecture not mips3')
+            sys.exit(1)
+
+        flags |= 0x00001000 # set EF_MIPS_ABI_O32
+        f.seek(36)
+        f.write(struct.pack('>I', flags))
+

tools/sound_func_val_unwrap

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+SFX_ID=$(( $1 & 0x7ff))
+VOLUME=$(echo "$((($1 >> 21) & 0x7ff)).0 / 1023.0" | bc -l)
+SAMPLE_RATE=$(((($1 >> 11) & 0x3ff)<<5))
+printf "SFX_%X, %f, %d\n" $SFX_ID $VOLUME $SAMPLE_RATE

tools/splat_inputs.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# Script to get a list of input files that are referenced by a splat file
+import argparse
+import sys
+sys.path.append("./tools/n64splat")
+
+from split import *
+
+def main(config_path):
+    # Load config
+    with open(config_path) as f:
+        config = yaml.load(f.read(), Loader=yaml.SafeLoader)
+
+    options.initialize(config, config_path, None, None)
+    options.set("modes", [])
+    options.set("verbose", False)
+
+    all_segments = initialize_segments(config["segments"])
+
+    objs = ""
+
+    for segment in all_segments:
+        linker_entries = segment.get_linker_entries()
+        for entry in linker_entries:
+            src_paths = entry.src_paths
+            for path in src_paths:
+                objs += str(path) + " "
+    
+    return objs
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Get objects for splat file',
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('yamls', nargs='+', help="Splat files")
+    args = parser.parse_args()
+
+    obj_lists = map(main, args.yamls)
+    # map(print, obj_lists)
+    for obj_list in obj_lists:
+        print(obj_list)
+