diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..589cfcd0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.dockerignore +.gitlab-ci.yml +Dockerfile +README.md +LICENSE + +asm/ +assets/ +include/ +ido/ +build/ +bin/ +src/ + +baserom* +*.z64 + +Makefile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..3c539533 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,72 @@ +# This CI will run: +workflow: + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # ... on merge requests + - if: $CI_COMMIT_TAG # ... on tags + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # ... on default (master/main) branch + +stages: + - build-image + - test + + +variables: + # Fetch submodules + GIT_SUBMODULE_STRATEGY: recursive + # Only fetch the latest commit (shallow clone, faster) + GIT_SUBMODULE_DEPTH: 1 + GIT_DEPTH: 1 + +# Builds the image and pushes it to the registry +# This image contains all the tooling necessary to run the compilation tools +build-image: + stage: build-image + # Run image build only if packages changed + only: + changes: + - Dockerfile + - packages.txt + - requirements.txt + - .gitlab-ci.yml + - tools/* + # Set up the docker daemon for building the image + image: docker:latest + services: + - docker:dind + variables: + DOCKER_DRIVER: overlay2 + script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY # Login to the registry + - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME || true # Pull the image if it exists + # Build the image and tag it with the branch name and latest + - | + docker build \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --cache-from $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME \ + -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME \ + -t $CI_REGISTRY_IMAGE:latest \ + . + - docker push --all-tags $CI_REGISTRY_IMAGE # Push the image to the registry + +# Try to compile the code inside the image to make sure it works (run docker container) +build-test-us10: + stage: test + image: + name: $CI_REGISTRY_IMAGE:latest + entrypoint: [""] + before_script: + # Download the baserom from $BASEROM__URL, decrypt with $BASEROM__KEY and save as baserom.us.v10.z64, and check the sha1sum against $BASEROM__SHA1 + - curl -L "$BASEROM_US10_URL" -o baserom.us.v10.enc.z64 + - openssl enc -d -aes-256-cbc -in baserom.us.v10.enc.z64 -out baserom.us.v10.z64 -k "$BASEROM_US10_KEY" + - FILE_SHA1=$(sha1sum baserom.us.v10.z64 | awk '{ print $1 }') + - echo "Calculated SHA1 - $FILE_SHA1" + - echo "Expected SHA1 - $BASEROM_US10_SHA1" + - if [ "${FILE_SHA1}" != "${BASEROM_US10_SHA1}" ]; then echo "Checksum verification failed"; exit 1; else echo "Checksum verification passed"; fi + script: + # Compile the code + - make + # Check if the resulting ROM is the same as the expected one + - FILE_SHA1=$(sha1sum build/us.v10/banjo.us.v10.z64 | awk '{ print $1 }') + - echo "Calculated SHA1 - $FILE_SHA1" + - echo "Expected SHA1 - $BASEROM_US10_SHA1" + - if [ "${FILE_SHA1}" != "${BASEROM_US10_SHA1}" ]; then echo "Checksum verification failed"; exit 1; else echo "Checksum verification passed"; fi diff --git a/Dockerfile b/Dockerfile index 797a089c..469dc11f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,22 @@ -FROM ubuntu:20.04 as build - +FROM ubuntu:20.04 AS build ENV DEBIAN_FRONTEND=noninteractive -COPY packages.txt / -RUN apt-get update && apt-get install -y $(cat packages.txt) +# (for debug purposes) +RUN echo "System arch: $(uname -a)\nDPKG arch: $(dpkg --print-architecture)" -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +# Install package dependencies +COPY packages.txt ./ +RUN apt-get update && apt-get install -y $(cat packages.txt) && rm packages.txt -COPY requirements.txt / +# Install Rust/Cargo +# NOTE: We replace /proc/self/exe with /bin/sh in the script to avoid issues with Docker +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sed 's#/proc/self/exe#\/bin\/sh#g' | CARGO_HOME=/opt/cargo sh -s -- -y +ENV PATH=/opt/cargo/bin:$PATH + +# Install Python dependencies +COPY requirements.txt ./ +COPY tools/n64splat/requirements.txt ./tools/n64splat/requirements.txt RUN python3 -m pip install -r requirements.txt -RUN mkdir /banjo WORKDIR /banjo +ENTRYPOINT ["/bin/bash", "-c"] diff --git a/README.md b/README.md index 49bc063e..2bde9578 100644 --- a/README.md +++ b/README.md @@ -2,23 +2,68 @@ +### Baserom checksums + +- `baserom.us.v10.z64`: `1fe1632098865f639e22c11b9a81ee8f29c75d7a` +- `baserom.us.v11.z64`: `ded6ee166e740ad1bc810fd678a84b48e245ab80` +- `baserom.jp.z64`: `90726d7e7cd5bf6cdfd38f45c9acbf4d45bd9fd8` +- `baserom.pal.z64`: `bb359a75941df74bf7290212c89fbc6e2c5601fe` + ## Building -Grab tools +The following instructions should work on the following platforms: +- Ubuntu 18.04 or higher (x86_64) +- Docker only + - Linux (x86_64, ARM) + - macOS (x86_64, ARM) + +### Local (Linux) + +Works with Ubuntu 18.04 or higher. + +#### 1. Install dependencies + +```sh +sudo apt-get update && sudo apt-get install -y $(cat packages.txt) +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +python3 -m pip install -r requirements.txt +``` + +#### 2. Grab tools ```sh git submodule update --init --recursive +cd tools/n64splat +python3 -m pip install -r requirements.txt ``` -Drop in `US v1.0` as `baserom.us.v10.z64` (sha1sum: `1fe1632098865f639e22c11b9a81ee8f29c75d7a`) +#### 3. Add baserom -To extract and build everything +Add the file for `US v1.0` as `baserom.us.v10.z64` in the project folder. + +(optional): Check the baserom checksum + +```sh +sha1sum baserom.us.v10.z64 +``` + +The output should match the checksum specified above. + +#### 4. Build + +To extract and build everything simply run: ```sh make ``` -where the following are supported values of `` +If you want to build a specific module, instead do: + +```sh +make +``` + +...where the following are supported values of `` - `core1` - `core2` - `MM` @@ -35,19 +80,87 @@ where the following are supported values of `` - `fight` - `cutscenes` -### Prerequisites +### Local (Docker - Linux/macOS) -Ubuntu 18.04 or higher. +#### 1. Get the Docker image + +(if available) you can pull it from GitLab (but you need to be logged in): ```sh -sudo apt-get update && sudo apt-get install -y $(cat packages.txt) - -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - -python3 -m pip install -r requirements.txt +docker login registry.gitlab.com +docker pull registry.gitlab.com/banjo.decomp/banjo-kazooie:latest ``` -### Other versions +(otherwise) you can build it yourself: + +```sh +docker build -t banjo-kazooie . +``` + +**NOTE for ARM users** (Windows ARM, Raspberry Pi and similar, or Apple Silicon): Use this command instead: + +```sh +docker build --platform linux/amd64 -t banjo-kazooie . +``` + +#### 2. Add baserom + +Follow the same instructions as Step 3 above in "Local (Linux)". + +#### 3. Run the Docker container + +```sh +docker run -it --rm -v $(pwd):/banjo banjo-kazooie +``` + +**NOTE for ARM users**: Use this command instead: + +```sh +docker run --platform linux/amd64 -it --rm -v $(pwd):/banjo banjo-kazooie +``` + +#### 4. Build + +Follow the same instructions as Step 4 above in "Local (Linux)". + +To exit Docker, simply type `exit`. + +### Cloud (GitLab CI) + +These are the instructions for building on GitLab CI. +This applies to the main repo - **if you have a fork**, you will need to follow these steps too! + +#### 1. Upload the baserom + +Upload the file for `US v1.0` as `baserom.us.v10.enc.z64` to a remote server where it can be downloaded from with `wget` or `curl`. The file has to be encrypted with `AES-256-CBC`, as follows: + +```sh +openssl enc -aes-256-cbc -salt -in baserom.us.v10.z64 -out baserom.us.v10.enc.z64 +``` + +Then, upload the encrypted file to a server and get a direct download link. + +Sharing services like Google Drive, Dropbox, or OneDrive might not work, as they require manual interaction to download the file. + +#### 2. Set up environment variables + +In your GitLab project, go to `Settings > CI/CD > Variables` and add the following variables (for each version): + +- `BASEROM__URL`: a direct download URL for the baserom.us.v10.z64 file (see above); this file has to be encrypted with `AES-256-CBC` +- `BASEROM__KEY`: the AES key used to encrypt the baserom file above +- `BASEROM__SHA1`: the SHA1 checksum of the baserom file; simply use the one mentioned above + +Replace `` with the version you are using: +- `USA10` +- `USA11` +- `JP` +- `PAL` + +#### 3. Trigger the pipeline + +Push a commit to your repository and you should see a new pipeline starting in the `CI/CD > Pipelines` section! + +## Other versions Drop in `us.v11`, `jp`, or `pal` as `baserom..z64` e.g. `baserom.us.v11.z64` diff --git a/packages.txt b/packages.txt index cd64f4fb..d96c3d07 100644 --- a/packages.txt +++ b/packages.txt @@ -10,3 +10,4 @@ python3 python3-pip unzip wget +openssl