Run tests against BYOND when changed/added

.github/assets/CIScripts/dm.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+
+dmepath=""
+retval=1
+
+for var
+do
+	if [[ $var != -* && $var == *.dme ]]
+	then
+		dmepath=`echo $var | sed -r 's/.{4}$//'`
+		break
+	fi
+done
+
+if [[ $dmepath == "" ]]
+then
+	echo "No .dme file specified, aborting."
+	exit 1
+fi
+
+if [[ -a $dmepath.m.dme ]]
+then
+	rm $dmepath.m.dme
+fi
+
+cp $dmepath.dme $dmepath.m.dme
+if [[ $? != 0 ]]
+then
+	echo "Failed to make modified dme, aborting."
+	exit 2
+fi
+
+for var
+do
+	arg=`echo $var | sed -r 's/^.{2}//'`
+	if [[ $var == -I* ]]
+	then
+		sed -i "3i#include $arg" $dmepath.m.dme
+		continue
+	fi
+done
+
+#windows
+if [[ `uname` == MINGW* ]]
+then
+	dm=""
+
+	if hash dm.exe 2>/dev/null
+	then
+		dm='dm.exe'
+	elif [[ -a '/c/Program Files (x86)/BYOND/bin/dm.exe' ]]
+	then
+		dm='/c/Program Files (x86)/BYOND/bin/dm.exe'
+	elif [[ -a '/c/Program Files/BYOND/bin/dm.exe' ]]
+	then
+		dm='/c/Program Files/BYOND/bin/dm.exe'
+	fi
+
+	if [[ $dm == "" ]]
+	then
+		echo "Couldn't find the DreamMaker executable, aborting."
+		exit 3
+	fi
+
+	"$dm" $dmepath.m.dme 2>&1 | tee result.log
+	retval=$?
+	if ! grep '\- 0 errors, ' result.log
+	then
+		retval=1 #hard fail, due to errors
+	fi
+else
+	if hash DreamMaker 2>/dev/null
+	then
+		DreamMaker -max_errors 0 $dmepath.m.dme 2>&1 | tee result.log
+		retval=$?
+		if ! grep '\- 0 errors, ' result.log
+		then
+			retval=1 #hard fail, due to errors
+		fi
+	else
+		echo "Couldn't find the DreamMaker executable, aborting."
+		exit 3
+	fi
+fi
+
+mv $dmepath.m.dmb $dmepath.dmb
+mv $dmepath.m.rsc $dmepath.rsc
+
+rm $dmepath.m.dme
+
+exit $retval

.github/assets/CIScripts/install_byond.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -euo pipefail
+
+source buildByond.conf
+
+if [ -d "$HOME/BYOND/byond/bin" ] && grep -Fxq "${BYOND_MAJOR_VERSION}.${BYOND_MINOR_VERSION}" $HOME/BYOND/version.txt;
+then
+  echo "Using cached directory."
+else
+  echo "Setting up BYOND."
+  rm -rf "$HOME/BYOND"
+  mkdir -p "$HOME/BYOND"
+  cd "$HOME/BYOND"
+  if ! curl --fail --connect-timeout 2 --max-time 10 "https://spacestation13.github.io/byond-builds/${BYOND_MAJOR_VERSION}/${BYOND_MAJOR_VERSION}.${BYOND_MINOR_VERSION}_byond_linux.zip" -o byond.zip -A "GithubCI/2.0"; then
+      echo "Mirror download failed, falling back to byond.com"
+      if ! curl --fail --connect-timeout 2 --max-time 10 "http://www.byond.com/download/build/${BYOND_MAJOR_VERSION}/${BYOND_MAJOR_VERSION}.${BYOND_MINOR_VERSION}_byond_linux.zip" -o byond.zip -A "GithubCI/2.0"; then
+          echo "BYOND download failed too :("
+          exit 1
+      fi
+  fi
+  unzip byond.zip
+  rm byond.zip
+  cd byond
+  make here
+  echo "$BYOND_MAJOR_VERSION.$BYOND_MINOR_VERSION" > "$HOME/BYOND/version.txt"
+  cd ~/
+fi

.github/assets/CIScripts/run_tests.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+set -euo pipefail
+
+rm -f summary.log
+touch summary.log
+base="Content.Tests/DMProject/environment.dme"
+basedir="$(dirname "$base")"
+testsfailed=0
+byondcrashes=0
+testspassed=0
+
+while read -r file; do
+	first_line=$(head -n 1 "$file" || echo "")
+	second_line=$((head -n 2 "$file" | tail -n 1) || echo "")
+	relative=$(realpath --relative-to="$(dirname "$base")" "$file")
+	if [[ $first_line == "// NOBYOND"* || $second_line == "// NOBYOND"* ]]; then
+		#skip this one, it won't work in byond
+		echo "Skipping $relative due to NOBYOND mark"
+		continue
+	fi
+	if [[ $first_line == "// IGNORE"* ]]; then
+		#skip this one, it won't work in byond
+		echo "Skipping $relative due to IGNORE mark"
+		continue
+	fi
+
+
+	echo "Compiling $relative"
+	if ! .github/assets/CIScripts/dm.sh -I\"$relative\" -I\"crashwrapper.dm\" $base; then
+		if [[ $first_line == "// COMPILE ERROR"* || $first_line == "//COMPILE ERROR"* ]] then	#expected compile error, should fail to compile
+			echo "Expected compile failure, test passed"
+			testspassed=$((testspassed + 1))
+			continue
+		else
+			echo "TEST FAILED: $relative"
+			echo "TEST FAILED: $relative" >> summary.log
+			testsfailed=$((testsfailed + 1))
+			continue
+		fi
+	else
+		if [[ $first_line == "// COMPILE ERROR"* || $first_line == "//COMPILE ERROR"* ]] then	#expected compile error, should fail to compile
+			echo "TEST FAILED: Expected compile failure"
+			echo "TEST FAILED: $relative" >> summary.log
+			testsfailed=$((testsfailed + 1))
+			continue
+		fi
+	fi
+
+	echo "Running $relative"
+	touch $basedir/errors.log
+	if ! DreamDaemon $basedir/environment.dmb -once -close -trusted -verbose -invisible; then
+		echo "TEST FAILED: BYOND CRASHED!"
+		echo "TEST FAILED: $relative" >> summary.log
+		byondcrashes=$((byondcrashes+1))
+		sed -i '/^[[:space:]]*$/d' $basedir/errors.log
+		cat $basedir/errors.log
+		rm $basedir/errors.log
+	fi
+	if [ -s "$basedir/errors.log" ]; then
+		if [[ $first_line == "// RUNTIME ERROR"* || $first_line == "//RUNTIME ERROR"* ]]	then #expected runtime error, should compile but then fail to run
+			echo "Expected runtime error, test passed"
+			rm $basedir/errors.log
+			testspassed=$((testspassed + 1))
+			continue
+		else
+			echo "Errors detected!"
+			sed -i '/^[[:space:]]*$/d' $basedir/errors.log
+			cat $basedir/errors.log
+			echo "TEST FAILED: $relative"
+			rm $basedir/errors.log
+			echo "TEST FAILED: $relative" >> summary.log
+			testsfailed=$((testsfailed + 1))
+			continue
+		fi
+	else
+		echo "Test passed: $relative"
+		testspassed=$((testspassed + 1))
+	fi
+done < test_file_diffs
+
+
+echo "--------------------------------------------------------------------------------"
+echo "Test Summary"
+echo "--------------------------------------------------------------------------------"
+echo "passed: $testspassed, failed: $testsfailed, BYOND crashes: $byondcrashes"
+echo "--------------------------------------------------------------------------------"
+echo "failed tests:"
+cat summary.log
+
+echo "passed=$testspassed" >> "$GITHUB_OUTPUT"
+echo "failed=$testsfailed" >> "$GITHUB_OUTPUT"
+echo "crashes=$byondcrashes" >> "$GITHUB_OUTPUT"
+
+exit $testsfailed

.github/workflows/check-test-parity.yml

@@ -0,0 +1,69 @@
+name: Run BYOND Against Modified Tests
+
+on:
+  pull_request:
+    branches: [master]
+  push:
+    branches:
+      - "master"
+
+jobs:
+  unit_test:
+    name: Checking parity
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+            fetch-depth: 2
+
+      - name: Check for test changes
+        id: changed-files
+        run: |
+            git diff --diff-filter=ACMR --name-only HEAD^1 HEAD Content.Tests/DMProject/Tests Content.Tests/DMProject/BrokenTests >> test_file_diffs
+            echo "changed_files_count=$(wc -l < test_file_diffs)" >> $GITHUB_OUTPUT
+
+      - name: Add Architecture
+        if: ${{ steps.changed-files.outputs.changed_files_count > 0 }}
+        run: |
+          sudo dpkg --add-architecture i386
+          sudo sed -i'' -E 's/^(deb|deb-src) http:\/\/(azure.archive|security).ubuntu.com/\1 [arch=amd64,i386] http:\/\/\2.ubuntu.com/' /etc/apt/sources.list
+          sudo apt-get update
+
+      - name: Install Dependencies
+        if: ${{ steps.changed-files.outputs.changed_files_count > 0 }}
+        uses: Eeems-Org/apt-cache-action@v1
+        with:
+          packages: libssl-dev:i386 libgcc-s1:i386 libcurl4:i386
+
+      - name: Get latest version number
+        if: ${{ steps.changed-files.outputs.changed_files_count > 0 }}
+        run: |
+          set -e
+          set -o pipefail
+          curl --fail -s https://www.byond.com/download/version.txt | awk -F. '{print "BYOND_MAJOR_VERSION="$1"\nBYOND_MINOR_VERSION="$2}' > buildByond.conf          
+
+      - name: Cache BYOND
+        if: ${{ steps.changed-files.outputs.changed_files_count > 0 }}
+        uses: actions/cache@v4
+        with:
+          path: ~/BYOND
+          key: ${{ runner.os }}-byond-${{ hashFiles('buildByond.conf')}}
+          restore-keys: ${{ runner.os }}-byond
+
+      - name: Setup BYOND
+        if: ${{ steps.changed-files.outputs.changed_files_count > 0 }}
+        run: |
+          .github/assets/CIScripts/install_byond.sh
+          cd $GITHUB_WORKSPACE
+          printenv
+          echo "BYOND_SYSTEM=/home/runner/BYOND/byond" >> $GITHUB_ENV
+          echo "/home/runner/BYOND/byond/bin" >> $GITHUB_PATH
+          echo "LD_LIBRARY_PATH=/home/runner/BYOND/byond/bin:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+          echo "MANPATH=/home/runner/BYOND/byond/man:$MANPATH" >> $GITHUB_ENV
+
+      - name: Run Full Unit Tests
+        if: ${{ steps.changed-files.outputs.changed_files_count > 0 }}
+        id: run_tests
+        run: |
+          .github/assets/CIScripts/run_tests.sh

Content.Tests/DMProject/crashwrapper.dm

@@ -0,0 +1,5 @@
+//this is so we can use CRASH(X) in list index lookups like L[CRASH("accessed index")]
+/proc/_crash_wrapper(X)
+    CRASH(X)
+    return 0
+#define CRASH(X) _crash_wrapper(X)