diff --git a/.gitlab/ci/container-build.gitlab-ci.yml b/.gitlab/ci/container-build.gitlab-ci.yml index a23dc23..5a0078b 100644 --- a/.gitlab/ci/container-build.gitlab-ci.yml +++ b/.gitlab/ci/container-build.gitlab-ci.yml @@ -196,6 +196,20 @@ manifest:ruby: - generate-environment - container:ruby +container:config-generator: + extends: + - .single-image-build-base + needs: + - generate-environment + - manifest:ruby + +manifest:config-generator: + extends: + - .manifest-create-base + needs: + - generate-environment + - container:config-generator + container:postgresql: extends: - .single-image-build-base diff --git a/container/config-generator/Dockerfile b/container/config-generator/Dockerfile new file mode 100644 index 0000000..02844af --- /dev/null +++ b/container/config-generator/Dockerfile @@ -0,0 +1,12 @@ +ARG RETICULUM_IMAGE_TAG=local + +FROM ghcr.io/code0-tech/reticulum/ci-builds/ruby:$RETICULUM_IMAGE_TAG + +ARG RETICULUM_IMAGE_TAG=local +LABEL org.opencontainers.image.version=$RETICULUM_IMAGE_TAG + +WORKDIR /config-generator +COPY container/config-generator/templates templates +COPY container/config-generator/generate.rb generate.rb + +CMD ["ruby", "generate.rb"] diff --git a/container/config-generator/generate.rb b/container/config-generator/generate.rb new file mode 100644 index 0000000..a22824b --- /dev/null +++ b/container/config-generator/generate.rb @@ -0,0 +1,85 @@ +# config-generator/generate.rb +require 'erb' +require 'fileutils' + +class ConfigGenerator + REQUIRED_VARS = %w[ + SAGITTARIUS_RAILS_HOST + SAGITTARIUS_RAILS_PORT + SAGITTARIUS_GRPC_HOST + SAGITTARIUS_GRPC_PORT + SCULPTOR_HOST + SCULPTOR_PORT + HOSTNAME + ] + TEMPLATES_DIR = '/config-generator/templates' + OUTPUT_DIR = '/generated-configs' + + def initialize + @env = ENV.to_h + validate_env! + end + + def generate_all + puts "Generating configuration files..." + + template_files = Dir.glob(File.join(TEMPLATES_DIR, '**', '*.erb')) + + if template_files.empty? + puts "WARNING: No template files found in #{TEMPLATES_DIR}" + return + end + + template_files.each do |template_path| + generate_from_template(template_path) + end + + puts "Configuration generation complete! Generated #{template_files.size} file(s)." + end + + private + + def validate_env! + missing = REQUIRED_VARS.reject { |var| @env[var] } + + if missing.any? + abort "ERROR: Missing required environment variables: #{missing.join(', ')}" + end + end + + def generate_from_template(template_path) + # Calculate relative path from templates directory + relative_path = template_path.sub("#{TEMPLATES_DIR}/", '') + + # Remove .erb extension for output file + output_filename = relative_path.sub(/\.erb$/, '') + output_path = File.join(OUTPUT_DIR, output_filename) + + # Read and render template + template = ERB.new(File.read(template_path), trim_mode: '-') + result = template.result(binding) + + # Ensure output directory exists + FileUtils.mkdir_p(File.dirname(output_path)) + + # Write rendered config + File.write(output_path, result) + + puts "Generated: #{output_path}" + rescue StandardError => e + puts "ERROR generating #{template_path}: #{e.message}" + raise + end + + # Helper methods for templates + def env(key, default = nil) + @env.fetch(key, default) + end + + def env?(key) + @env[key] == 'true' + end +end + +# Run the generator +ConfigGenerator.new.generate_all diff --git a/container/config-generator/templates/nginx.default.conf.erb b/container/config-generator/templates/nginx.default.conf.erb new file mode 100644 index 0000000..406bf72 --- /dev/null +++ b/container/config-generator/templates/nginx.default.conf.erb @@ -0,0 +1,103 @@ +upstream sagittarius_rails_web { + server <%= env('SAGITTARIUS_RAILS_HOST') %>:<%= env('SAGITTARIUS_RAILS_PORT') %>; +} + +upstream sagittarius_grpc { + server <%= env('SAGITTARIUS_GRPC_HOST') %>:<%= env('SAGITTARIUS_GRPC_PORT') %>; +} + +upstream sculptor { + server <%= env('SCULPTOR_HOST') %>:<%= env('SCULPTOR_PORT') %>; +} + +<% if env?('SSL_ENABLED') %> +server { + listen 80; + server_name <%= env('HOSTNAME') %>; + + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl http2; + server_name <%= env('HOSTNAME') %>; + + ssl_certificate /etc/nginx/certs/<%= env('SSL_CERT_FILE', "#{env('HOSTNAME')}.pem") %>; + ssl_certificate_key /etc/nginx/certs/<%= env('SSL_KEY_FILE', "#{env('HOSTNAME')}.key") %>; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 10m; +<% else %> +server { + listen 80 http2; + server_name <%= env('HOSTNAME') %>; +<% end %> + + location = /graphql { + proxy_pass http://sagittarius_rails_web; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /files/upload { + proxy_pass http://sagittarius_rails_web; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Increase limits for file uploads + client_max_body_size 100M; + } + + location = /health/liveness { + proxy_pass http://sagittarius_rails_web; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /good_job { + proxy_pass http://sagittarius_rails_web; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location = /error502grpc { + internal; + default_type application/grpc; + add_header grpc-status 14; + add_header grpc-message "unavailable"; + return 204; + } + + location / { + if ($content_type = "application/grpc") { + grpc_pass grpc://sagittarius_grpc; + error_page 502 = /error502grpc; + } + grpc_set_header X-Real-IP $remote_addr; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Disable buffering for streaming + grpc_read_timeout 1h; + grpc_send_timeout 1h; + + # Keep connection alive for long-running streams + grpc_socket_keepalive on; + + proxy_pass http://sculptor; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} diff --git a/container/config-generator/templates/sagittarius.sagittarius.yml.erb b/container/config-generator/templates/sagittarius.sagittarius.yml.erb new file mode 100644 index 0000000..c6a4ab1 --- /dev/null +++ b/container/config-generator/templates/sagittarius.sagittarius.yml.erb @@ -0,0 +1,8 @@ +rails: + web: + force_ssl: false + log_level: <%= env('SAGITTARIUS_LOG_LEVEL', 'info') %> + threads: 3 + db: + host: <%= env('POSTGRES_HOST') %> + port: <%= env('POSTGRES_PORT') %> diff --git a/container/sculptor/Dockerfile.erb b/container/sculptor/Dockerfile.erb index eba3a4f..53514fb 100644 --- a/container/sculptor/Dockerfile.erb +++ b/container/sculptor/Dockerfile.erb @@ -4,6 +4,7 @@ FROM ghcr.io/code0-tech/reticulum/ci-builds/node:$RETICULUM_IMAGE_TAG WORKDIR /sculptor +COPY projects/sculptor/public public COPY projects/sculptor/edition.mjs \ projects/sculptor/graphql-imports.d.ts \ projects/sculptor/next.config.ts \ diff --git a/docker-compose/.env b/docker-compose/.env new file mode 100644 index 0000000..b4e9e94 --- /dev/null +++ b/docker-compose/.env @@ -0,0 +1,37 @@ +# IDE config +HOSTNAME=localhost +HTTP_PORT=80 +HTTPS_PORT=443 +SSL_ENABLED=false +SSL_CERT_FILE= # must be located in ./certs, defaults to ".pem" +SSL_KEY_FILE= # must be located in ./certs, defaults to ".key" + +INITIAL_ROOT_PASSWORD=root +INITIAL_ROOT_MAIL=root@code0.tech + +# Runtime config +AQUILA_SAGITTARIUS_URL=http://nginx:80 +AQUILA_SAGITTARIUS_TOKEN= +DRACO_REST_PORT=8084 + +# Active services +COMPOSE_PROFILES=ide,runtime + +# Image config +IMAGE_REGISTRY=registry.gitlab.com/code0-tech/packages +IMAGE_TAG= +IMAGE_EDITION= # ce or ee + +# Internal config options +SAGITTARIUS_RAILS_HOST=sagittarius-rails-web +SAGITTARIUS_RAILS_PORT=3000 +SAGITTARIUS_GRPC_HOST=sagittarius-grpc +SAGITTARIUS_GRPC_PORT=50051 +SAGITTARIUS_LOG_LEVEL=info +SCULPTOR_HOST=sculptor +SCULPTOR_PORT=3000 +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +POSTGRES_DB=sagittarius_production +POSTGRES_USER=sagittarius +POSTGRES_PASSWORD=sagittarius diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml new file mode 100644 index 0000000..0eaadf4 --- /dev/null +++ b/docker-compose/docker-compose.yml @@ -0,0 +1,186 @@ +services: + config-generator: + image: ${IMAGE_REGISTRY}/config-generator:${IMAGE_TAG} + env_file: + - .env + volumes: + - generated-configs:/generated-configs + restart: "no" + profiles: + - ide + + postgres: + image: postgres:16.1 + environment: + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}" ] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + profiles: + - ide + + sagittarius-rails-web: + image: ${IMAGE_REGISTRY}/sagittarius:${IMAGE_TAG}-${IMAGE_EDITION} + depends_on: + config-generator: + condition: service_completed_successfully + postgres: + condition: service_healthy + environment: + INITIAL_ROOT_PASSWORD: ${INITIAL_ROOT_PASSWORD} + INITIAL_ROOT_MAIL: ${INITIAL_ROOT_MAIL} + volumes: + - generated-configs:/tmp/generated-configs:ro + entrypoint: | + sh -c " + cp /tmp/generated-configs/sagittarius.sagittarius.yml config/sagittarius.yml + exec bin/docker-entrypoint ./bin/rails server + " + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "curl --fail http://localhost:3000/health/liveness"] + interval: 10s + timeout: 5s + retries: 5 + profiles: + - ide + + sagittarius-rails-background: + image: ${IMAGE_REGISTRY}/sagittarius:${IMAGE_TAG}-${IMAGE_EDITION} + depends_on: + sagittarius-rails-web: + condition: service_healthy + volumes: + - generated-configs:/tmp/generated-configs:ro + entrypoint: | + sh -c " + cp /tmp/generated-configs/sagittarius.sagittarius.yml config/sagittarius.yml + exec bin/docker-entrypoint bundle exec good_job + " + restart: unless-stopped + profiles: + - ide + + sagittarius-grpc: + image: ${IMAGE_REGISTRY}/sagittarius:${IMAGE_TAG}-${IMAGE_EDITION} + depends_on: + sagittarius-rails-web: + condition: service_healthy + volumes: + - generated-configs:/tmp/generated-configs:ro + entrypoint: | + sh -c " + cp /tmp/generated-configs/sagittarius.sagittarius.yml config/sagittarius.yml + exec bin/docker-entrypoint ./bin/grpc_server + " + restart: unless-stopped + profiles: + - ide + + sculptor: + image: ${IMAGE_REGISTRY}/sculptor:${IMAGE_TAG}-${IMAGE_EDITION} + restart: unless-stopped + profiles: + - ide + + nginx: + image: nginx:1.29.5-alpine-slim + depends_on: + config-generator: + condition: service_completed_successfully + sagittarius-rails-web: + condition: service_started + sagittarius-grpc: + condition: service_started + sculptor: + condition: service_started + volumes: + - generated-configs:/tmp/generated-configs:ro + - ./certs:/etc/nginx/certs:ro + entrypoint: | + sh -c " + cp /tmp/generated-configs/nginx.default.conf /etc/nginx/conf.d/default.conf + nginx -t + exec nginx -g 'daemon off;' + " + ports: + - "${HTTP_PORT}:80" + - "${HTTPS_PORT}:443" + restart: unless-stopped + profiles: + - ide + + nats: + image: nats:2.11.9 + command: + - -js + profiles: + - runtime + + aquila: + depends_on: + - nats + image: ${IMAGE_REGISTRY}/aquila:${IMAGE_TAG} + environment: + MODE: dynamic + NATS_URL: nats://nats:4222 + NATS_BUCKET: 'flow_store' + GRPC_HOST: 0.0.0.0 + SAGITTARIUS_URL: "${AQUILA_SAGITTARIUS_URL}" + RUNTIME_TOKEN: "${AQUILA_SAGITTARIUS_TOKEN}" + profiles: + - runtime + + taurus: + depends_on: + - nats + - aquila + image: ${IMAGE_REGISTRY}/taurus:${IMAGE_TAG} + environment: + MODE: dynamic + AQUILA_URL: 'http://aquila:8081' + NATS_URL: nats://nats:4222 + DEFINITION_PATH: '/definitions' + profiles: + - runtime + + draco-rest: + depends_on: + - nats + image: ${IMAGE_REGISTRY}/draco:${IMAGE_TAG}-rest + environment: + MODE: dynamic + AQUILA_URL: 'http://aquila:8081' + NATS_URL: nats://nats:4222 + NATS_BUCKET: 'flow_store' + DEFINITION_PATH: '/definitions' + HTTP_SERVER_PORT: 8084 + HTTP_SERVER_HOST: "0.0.0.0" + ports: + - "${DRACO_REST_PORT}:8084" + profiles: + - runtime + + draco-cron: + depends_on: + - nats + image: ${IMAGE_REGISTRY}/draco:${IMAGE_TAG}-cron + environment: + MODE: dynamic + AQUILA_URL: 'http://aquila:8081' + NATS_URL: nats://nats:4222 + NATS_BUCKET: 'flow_store' + DEFINITION_PATH: '/definitions' + profiles: + - runtime + +volumes: + generated-configs: + postgres-data: \ No newline at end of file diff --git a/support/helpers.sh b/support/helpers.sh index 906b0c8..227b9f1 100644 --- a/support/helpers.sh +++ b/support/helpers.sh @@ -98,7 +98,7 @@ function get_image_tag() { function get_component_version() { component=$1 - override_variable="OVERRIDE_${component}_VERSION" + override_variable="OVERRIDE_${component//-/_}_VERSION" if [[ -n "${!override_variable:+x}" ]]; then echo ${!override_variable}