Makefile

##########################################################################
# Configuration:

# Git version & date which are injected into built binaries.
VERSION = $(shell git describe --dirty --tags)
DATE    = $(shell date +%F-%T-%Z)
# Number of available processors for parallel builds.
NPROC := $(if ${NPROC},${NPROC},$(shell nproc))
# Repository root (the directory of the invoked Makefile).
ROOTDIR  = $(abspath $(dir $(firstword $(MAKEFILE_LIST))))
# Configured Go installation path of built targets.
GOBIN = $(shell go env GOPATH)/bin
# Configured Rust installation path of built release targets.
# Caller may override with a CARGO_TARGET_DIR environment variable.
# See: https://doc.rust-lang.org/cargo/reference/environment-variables.html
CARGO_TARGET_DIR ?= ${ROOTDIR}/target
RUSTBIN = ${CARGO_TARGET_DIR}/release
# Location to place intermediate files and output artifacts
# during the build process. Note the go tool ignores directories
# with leading '.' or '_'.
WORKDIR  = ${ROOTDIR}/.build
# Packaged build outputs.
PKGDIR = ${WORKDIR}/package
# All invocations can reference installed tools, Rust, and Go binaries.
# Each takes precedence over the configured $PATH
PATH := ${RUSTBIN}:${GOBIN}:${PATH}

# Etcd release we pin within Flow distributions.
ETCD_VERSION = v3.4.13
ETCD_SHA256 = 2ac029e47bab752dacdb7b30032f230f49e2f457cbc32e8f555c2210bb5ff107

PACKAGE_TARGETS = \
	${PKGDIR}/bin/etcd \
	${PKGDIR}/bin/flowctl \
	${PKGDIR}/bin/gazette \
	${PKGDIR}/bin/sops

##########################################################################
# Configure Go build & test behaviors.

# Enable the sqlite3 JSON extension.
GO_BUILD_TAGS += json1

# Targets which Go targets rely on in order to build.
GO_BUILD_DEPS = \
	${RUSTBIN}/libbindings.a \
	${RUSTBIN}/librocks-exp/librocksdb.a \
	crates/bindings/flow_bindings.h

##########################################################################
# Build rules:

.PHONY: default
default: rust-build package

# `etcd` is used for testing, and packaged as a release artifact.
${GOBIN}/etcd:
	curl -L -o /tmp/etcd.tgz \
			https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz \
		&& echo "${ETCD_SHA256} /tmp/etcd.tgz" | sha256sum -c - \
		&& tar --extract \
			--file /tmp/etcd.tgz \
			--directory /tmp/ \
		&& mkdir -p ${GOBIN}/ \
		&& mv /tmp/etcd-${ETCD_VERSION}-linux-amd64/etcd /tmp/etcd-${ETCD_VERSION}-linux-amd64/etcdctl ${GOBIN}/ \
		&& chown ${UID}:${UID} ${GOBIN}/etcd ${GOBIN}/etcdctl \
		&& rm -r /tmp/etcd-${ETCD_VERSION}-linux-amd64/ \
		&& rm /tmp/etcd.tgz \
		&& $@ --version

# Rule for building Go targets.
# go-install rules never correspond to actual files, and are always re-run each invocation.
go-install/%: ${RUSTBIN}/libbindings.a crates/bindings/flow_bindings.h
	MBP=go.gazette.dev/core/mainboilerplate ;\
	./go.sh install -v --tags "${GO_BUILD_TAGS}" \
		-ldflags "-X $${MBP}.Version=${VERSION} -X $${MBP}.BuildDate=${DATE}" $*

${GOBIN}/gazette: go-install/go.gazette.dev/core/cmd/gazette
${GOBIN}/gazctl:  go-install/go.gazette.dev/core/cmd/gazctl
${GOBIN}/flowctl: $(GO_BUILD_DEPS) go-install/github.com/estuary/flow/go/flowctl

# `sops` is used for encrypt/decrypt of connector configurations.
${GOBIN}/sops:
	go install go.mozilla.org/sops/v3/cmd/sops@v3.7.1

# The & here declares that this single invocation will produce all of the files on the left hand
# side. flow_bindings.h is generated by the bindings build.rs.
${RUSTBIN}/libbindings.a crates/bindings/flow_bindings.h &:
	FLOW_VERSION=${VERSION} cargo build --release --locked -p bindings

${RUSTBIN}/librocks-exp/librocksdb.a:
	cargo build --release --locked -p librocks-exp

${PKGDIR}:
	mkdir -p ${PKGDIR}/bin
	mkdir ${PKGDIR}/lib
${PKGDIR}/bin/etcd: ${PKGDIR} ${GOBIN}/etcd
	cp ${GOBIN}/etcd $@
${PKGDIR}/bin/sops: ${PKGDIR} ${GOBIN}/sops
	cp ${GOBIN}/sops $@
${PKGDIR}/bin/flowctl:     ${PKGDIR} ${GOBIN}/flowctl
	cp ${GOBIN}/flowctl $@
${PKGDIR}/bin/gazctl: ${PKGDIR} ${GOBIN}/gazctl
	cp ${GOBIN}/gazctl $@
${PKGDIR}/bin/gazette: ${PKGDIR} ${GOBIN}/gazette
	cp ${GOBIN}/gazette $@

##########################################################################
# Make targets used by CI:

# We use LLVM for faster linking. See .cargo/config.
.PHONY: extra-ci-setup
extra-ci-runner-setup:
	sudo ln --force --symbolic /usr/bin/ld.lld-12 /usr/bin/ld.lld

.PHONY: print-versions
print-versions:
	echo "Resolved repository version: ${VERSION}" \
		&& /usr/bin/ld.lld --version \
		&& cargo version --verbose \
		&& docker --version \
		&& gcloud info \
		&& go version \
		&& jq --version \
		&& node --version \
		&& npm --version \
		&& rustc --version \

.PHONY: install-tools
install-tools: ${GOBIN}/etcd ${GOBIN}/sops

.PHONY: rust-build
rust-build:
	FLOW_VERSION=${VERSION} cargo build --release --locked

.PHONY: rust-test
rust-test:
	FLOW_VERSION=${VERSION} cargo test --release --locked

.PHONY: go-test-fast
go-test-fast: $(GO_BUILD_DEPS) ${GOBIN}/etcd ${GOBIN}/sops
	./go.sh test -p ${NPROC} --tags "${GO_BUILD_TAGS}" ./go/...

.PHONY: go-test-ci
go-test-ci:   $(GO_BUILD_DEPS) ${GOBIN}/etcd ${GOBIN}/sops
	GORACE="halt_on_error=1" \
	./go.sh test -p ${NPROC} --tags "${GO_BUILD_TAGS}" --race --count=15 --failfast ./go/...

.PHONY: catalog-test
catalog-test: ${GOBIN}/flowctl ${GOBIN}/gazette ${GOBIN}/etcd ${GOBIN}/sops flow.schema.json
	${GOBIN}/flowctl test --source ${ROOTDIR}/examples/local-sqlite.flow.yaml $(ARGS)

.PHONY: end-to-end-test
end-to-end-test: ${GOBIN}/flowctl ${GOBIN}/gazette ${GOBIN}/etcd ${GOBIN}/sops
	PATH="${PATH}:${GOBIN}" ./tests/run-end-to-end.sh

flow.schema.json: ${GOBIN}/flowctl
	${GOBIN}/flowctl json-schema > $@

.PHONY: package
package: $(PACKAGE_TARGETS)

# These docker targets intentionally don't depend on any upstream targets. This is because the
# upstream targes are all PHONY as well, so there would be no way to prevent them from running twice if you
# invoke e.g. `make package` followed by `make docker-image`. If the `docker-image` target depended
# on the `package` target, it would not skip the package step when you invoke `docker-image`.
# For now, the github action workflow manually invokes make to perform each of these tasks.
.PHONY: docker-image
docker-image:
	docker build \
		--file ${ROOTDIR}/.devcontainer/release.Dockerfile \
		--tag ghcr.io/estuary/flow:${VERSION} \
		--tag ghcr.io/estuary/flow:dev \
		${PKGDIR}/

.PHONY: docker-push
docker-push:
	docker push ghcr.io/estuary/flow:${VERSION}

# This is used by the GH Action workflow to push the 'dev' tag.
# It is invoked only for builds on the master branch.
.PHONY: docker-push-dev
docker-push-dev:
	docker push ghcr.io/estuary/flow:dev