WIP test for new build system design

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 resolver = "3"
 members = [
-    "Modules/_base64",
+    "Modules/_base64", "Modules/cpython-build-helper", "Modules/cpython-rust-staticlib",
     "Modules/cpython-sys"
 ]

Makefile.pre.in

@@ -178,7 +178,8 @@ CONFINCLUDEPY=	$(CONFINCLUDEDIR)/python$(LDVERSION)
 SHLIB_SUFFIX=	@SHLIB_SUFFIX@
 EXT_SUFFIX=	@EXT_SUFFIX@
 LDSHARED=	@LDSHARED@ $(PY_LDFLAGS)
-BLDSHARED=	@BLDSHARED@ $(PY_CORE_LDFLAGS)
+BLDSHARED_EXE= @BLDSHARED_EXE@
+BLDSHARED_ARGS=	@BLDSHARED_ARGS@ $(PY_CORE_LDFLAGS)
 LDCXXSHARED=	@LDCXXSHARED@ $(PY_LDFLAGS)
 DESTSHARED=	$(BINLIBDEST)/lib-dynload
 
@@ -3374,7 +3375,7 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h
 # Module dependencies and platform-specific files
 
 cpython-sys: Modules/cpython-sys/Cargo.toml Modules/cpython-sys/build.rs Modules/cpython-sys/wrapper.h Modules/cpython-sys/parser.h
-	CARGO_TARGET_DIR=$(abs_builddir)/target PYTHON_BUILD_DIR=$(abs_builddir) \$(CARGO_HOME)/bin/cargo build --lib --locked --package cpython-sys --profile $(CARGO_PROFILE) $(if $(CARGO_TARGET),--target=$(CARGO_TARGET)) --manifest-path $(srcdir)/Cargo.toml
+	CARGO_TARGET_DIR=$(abs_builddir)/target PYTHON_BUILD_DIR=$(abs_builddir) RUST_SHARED_BUILD=\$(PY_ENABLE_SHARED) \$(CARGO_HOME)/bin/cargo build --lib --locked --package cpython-sys --profile $(CARGO_PROFILE) $(if $(CARGO_TARGET),--target=$(CARGO_TARGET)) --manifest-path $(srcdir)/Cargo.toml
 
 # force rebuild when header file or module build flavor (static/shared) is changed
 MODULE_DEPS_STATIC=Modules/config.c

Modules/_base64/Cargo.toml

@@ -6,6 +6,10 @@ edition = "2024"
 [dependencies]
 cpython-sys ={ path = "../cpython-sys" }
 
+[build-dependencies]
+cpython-build-helper = { path = "../cpython-build-helper" }
+
 [lib]
 name = "_base64"
-crate-type = ["staticlib"]
+# For shared builds, we generate a c dynamic library. Static builds use the rlib
+crate-type = ["cdylib", "rlib"]

Modules/_base64/build.rs

@@ -0,0 +1,5 @@
+use cpython_build_helper::print_linker_args;
+
+fn main() {
+    print_linker_args();
+}

Modules/cpython-build-helper/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "cpython-build-helper"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+shlex = "1.3"

Modules/cpython-build-helper/README.md

@@ -0,0 +1,4 @@
+# cpython-build-helper
+
+This crate is used in Rust modules `build.rs` files to help do common tasks such
+as passing necessary link arguments.

Modules/cpython-build-helper/src/lib.rs

@@ -0,0 +1,17 @@
+use std::env;
+
+/// Print necessary link arguments for the library depending on the build
+/// configuration (static or shared)
+pub fn print_linker_args() {
+    let shared_build = env::var("RUST_SHARED_BUILD").expect("RUST_SHARED_BUILD not set in Makefile?");
+    if shared_build == "1" {
+        let build_shared_args =
+            env::var("BLDSHARED_ARGS").expect("BLDSHARED_ARGS not set in Makefile?");
+        // TODO(emmatyping): Ideally, we would not need to split the args here and take shlex
+        // as a dependency.
+        for arg in shlex::split(&build_shared_args).expect("Invalid BUILDSHARED_ARGS") {
+            println!("cargo:rustc-link-arg={}", arg);
+        }
+    }
+    // Static linker configuration is in cpython-rust-staticlib
+}

Modules/cpython-rust-staticlib/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "cpython-rust-staticlib"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+_base64 ={ path = "../_base64" }
+
+[lib]
+name = "cpython_rust_staticlib"
+crate-type = ["staticlib"]

Modules/cpython-rust-staticlib/src/lib.rs

@@ -0,0 +1 @@
+pub use _base64::PyInit__base64;

Modules/cpython-sys/Cargo.toml

@@ -6,4 +6,5 @@ edition = "2024"
 [dependencies]
 
 [build-dependencies]
-bindgen = "0.72.1"
+bindgen = "0.72.1"
+cpython-build-helper = { path = "../cpython-build-helper" }

Modules/cpython-sys/build.rs

@@ -1,7 +1,10 @@
 use std::env;
 use std::path::{Path, PathBuf};
 
+use cpython_build_helper::print_linker_args;
+
 fn main() {
+    print_linker_args();
     let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
     let srcdir = manifest_dir
         .parent()

Modules/makesetup

@@ -232,7 +232,6 @@ sed -e 's/[ 	]*#.*//' -e '/^[ 	]*$/d' |
 		yes)	continue;;
 		esac
 		objs=''
-		custom_ldflags=''
 		if test "x$rust" = "x"; then
 			for src in $srcs
 			do
@@ -272,51 +271,48 @@ sed -e 's/[ 	]*#.*//' -e '/^[ 	]*$/d' |
 				esac
 				echo "$rule" >>$rulesf
 			done
+			case $doconfig in
+			yes)	OBJS="$OBJS $objs";;
+			esac
+			for mod in $mods
+			do
+				file="$srcdir/$mod\$(EXT_SUFFIX)"
+				case $doconfig in
+				no)
+					SHAREDMODS="$SHAREDMODS $file"
+					BUILT_SHARED="$BUILT_SHARED $mod"
+					;;
+				esac
+				rule="$file: $objs \$(MODULE_${mods_upper}_LDEPS)"
+				rule="$rule; \$(BLDSHARED_EXE) \$(BLDSHARED_ARGS) $objs $libs \$(LIBPYTHON) -o $file"
+				echo "$rule" >>$rulesf
+			done
 		else
 			prefixed_srcs=
 			for src in $srcs
 			do
 				prefixed_srcs="$prefixed_srcs $srcdir/$src"
 			done
 			objs=
-			# there's actually only one obj, so just set it to the lib
-			for lib in $libs
-			do
-				objs="target/\$(if \$(CARGO_TARGET),\$(CARGO_TARGET)/\$(CARGO_TARGET_DIR),\$(CARGO_TARGET_DIR))/$lib"
-			done
 			libs=
-			# depends on the headers through cpython-sys
-			rule="$objs: cpython-sys \$(srcdir)/Cargo.toml \$(srcdir)/Cargo.lock \$(srcdir)/$srcdir/$manifest $prefixed_srcs \$(PYTHON_HEADERS)"
-			rule="$rule; CARGO_TARGET_DIR=\$(abs_builddir)/target PYTHON_BUILD_DIR=\$(abs_builddir) \$(CARGO_HOME)/bin/cargo build --lib --locked --package ${mods} --profile \$(CARGO_PROFILE) \$(if \$(CARGO_TARGET),--target=\$(CARGO_TARGET)) --manifest-path \$(srcdir)/Cargo.toml"
-			echo "$rule" >>$rulesf
+
 			for mod in $mods
 			do
-				case $UNAME_SYSTEM in
-				Darwin*)
-					custom_ldflags="$custom_ldflags -Wl,-u,_PyInit_$mod"
-					;;
-				*)
-					custom_ldflags="$custom_ldflags -Wl,--defsym=PyInit_$mod=PyInit_$mod"
+				rust_shared="target/\$(if \$(CARGO_TARGET),\$(CARGO_TARGET)/\$(CARGO_TARGET_DIR),\$(CARGO_TARGET_DIR))/lib$mod\$(SHLIB_SUFFIX)"
+				file="$srcdir/$mod\$(EXT_SUFFIX)"
+				case $doconfig in
+				no)
+					SHAREDMODS="$SHAREDMODS $file"
+					BUILT_SHARED="$BUILT_SHARED $mod"
 					;;
 				esac
+				# depends on the headers through cpython-sys
+				rule="$rust_shared: cpython-sys \$(srcdir)/Cargo.toml \$(srcdir)/Cargo.lock \$(srcdir)/$srcdir/$manifest $prefixed_srcs \$(PYTHON_HEADERS) \$(MODULE_${mods_upper}_LDEPS) \$(LIBRARY)"
+				rule="$rule; CARGO_TARGET_DIR=\$(abs_builddir)/target PYTHON_BUILD_DIR=\$(abs_builddir) RUST_SHARED_BUILD=\$(PY_ENABLE_SHARED) \$(CARGO_HOME)/bin/cargo build -vvv --lib --locked --package ${mod} --profile \$(CARGO_PROFILE) \$(if \$(CARGO_TARGET),--target=\$(CARGO_TARGET)) --manifest-path \$(srcdir)/Cargo.toml"
+				echo "$rule" >>$rulesf
+				echo "$file: $rust_shared; mv $rust_shared $file" >>$rulesf
 			done
 		fi
-		case $doconfig in
-		yes)	OBJS="$OBJS $objs";;
-		esac
-		for mod in $mods
-		do
-			file="$srcdir/$mod\$(EXT_SUFFIX)"
-			case $doconfig in
-			no)
-				SHAREDMODS="$SHAREDMODS $file"
-				BUILT_SHARED="$BUILT_SHARED $mod"
-				;;
-			esac
-			rule="$file: $objs \$(MODULE_${mods_upper}_LDEPS)"
-			rule="$rule; \$(BLDSHARED) $custom_ldflags $objs $libs \$(LIBPYTHON) -o $file"
-			echo "$rule" >>$rulesf
-		done
 	done
 
 	case $SHAREDMODS in