docker-compose-dev.yml

services:

# Docker compose without dataset generator services
  ruuter-public:
    container_name: ruuter-public
    image: ruuter
    environment:
      - application.cors.allowedOrigins=http://localhost:8086,http://localhost:3001,http://localhost:3003,http://localhost:3004,http://localhost:8080,http://localhost:8000,http://localhost:8090
      - application.httpCodesAllowList=200,201,202,204,400,401,403,500
      - application.internalRequests.allowedIPs=127.0.0.1
      - application.logging.displayRequestContent=true
      - application.logging.displayResponseContent=true
      - application.logging.printStackTrace=true
      - application.internalRequests.disabled=true
      - server.port=8086
    volumes:
      - ./DSL/Ruuter.public:/DSL
      - ./constants.ini:/app/constants.ini
    ports:
      - 8086:8086
    networks:
      - bykstack
    cpus: "0.5"
    mem_limit: "512M"
  
  ruuter-private:
    container_name: ruuter-private
    image: ruuter
    environment:
      - application.cors.allowedOrigins=http://localhost:3001,http://localhost:3003,http://localhost:8088,http://localhost:3002,http://localhost:3004,http://localhost:8000
      - application.httpCodesAllowList=200,201,202,400,401,403,500
      - application.internalRequests.allowedIPs=127.0.0.1
      - application.logging.displayRequestContent=true
      - application.logging.displayResponseContent=true
      - application.logging.printStackTrace=true
      - application.internalRequests.disabled=true
      - server.port=8088
    volumes:
      - ./DSL/Ruuter.private:/DSL
      - ./constants.ini:/app/constants.ini
    ports:
      - 8088:8088
    networks:
      - bykstack
    cpus: "0.5"
    mem_limit: "512M"

  data-mapper:
    container_name: data-mapper
    image: data-mapper
    environment:
      - PORT=3000
      - CONTENT_FOLDER=/data
    volumes:
      - ./DSL:/data
      - ./DSL/DMapper/global-classifier/hbs:/workspace/app/views/global-classifier
      - ./DSL/DMapper/global-classifier/lib:/workspace/app/lib
    ports:
      - 3000:3000
    networks:
      - bykstack

  tim:
    container_name: tim
    image: tim
    depends_on:
      tim-postgresql:
        condition: service_started
    environment:
      - SECURITY_ALLOWLIST_JWT=ruuter-private,ruuter-public,data-mapper,resql,tim,tim-postgresql,chat-widget,authentication-layer,127.0.0.1,::1
      - KEY_PASS=ppjjpp
    ports:
      - 8085:8085
    networks:
      - bykstack
    extra_hosts:
      - "host.docker.internal:host-gateway"
    cpus: "0.5"
    mem_limit: "512M"

  tim-postgresql:
    container_name: tim-postgresql
    image: postgres:14.1
    environment:
      - POSTGRES_USER=tim
      - POSTGRES_PASSWORD=123
      - POSTGRES_DB=tim
      # - POSTGRES_HOST_AUTH_METHOD=trust
    volumes:
      - ./tim-db:/var/lib/postgresql/data
    ports:
      - 9876:5432
    networks:
      - bykstack

  authentication-layer:
    container_name: authentication-layer
    image: authentication-layer
    ports:
      - 3004:3004
    networks:
      - bykstack

  resql:
    container_name: resql
    image: resql
    depends_on:
      users_db:
        condition: service_started
    environment:
      - sqlms.datasources.[0].name=byk
      - sqlms.datasources.[0].jdbcUrl=jdbc:postgresql://users_db:5432/global-classifier #For LocalDb Use
      # sqlms.datasources.[0].jdbcUrl=jdbc:postgresql://171.22.247.13:5435/byk?sslmode=require
      - sqlms.datasources.[0].username=postgres
      - sqlms.datasources.[0].password=dbadmin
      - logging.level.org.springframework.boot=INFO
    ports:
      - 8082:8082
    volumes:
      - ./DSL/Resql:/DSL
      - ./shared:/shared
      - ./DSL/DatasetGenerator/output_datasets:/app/output_datasets
    networks:
      - bykstack

  users_db:
    container_name: users_db
    image: postgres:14.1
    environment:
      - POSTGRES_USER=postgres
      - POSTGRES_PASSWORD=dbadmin
      - POSTGRES_DB=global-classifier
    ports:
      - 5435:5432
    volumes:
      - ./global-classifier-db/db_files:/var/lib/postgresql/data
      - ./shared:/shared
      - ./DSL/DatasetGenerator/output_datasets:/app/output_datasets

    networks:
      - bykstack
    restart: always

  init:
    image: busybox
    command: ["sh", "-c", "chmod -R 777 /shared && chmod -R 777 /app/model_trainer && chmod -R 777 /app/data"]
    volumes:
      - shared-volume:/shared
      - ./model_trainer:/app/model_trainer
      - cron_data:/app/data
    networks:
      - bykstack

# It's important to build the cron-manager container from the Dockerfile.python containing the Python enviro 
  cron-manager:
    container_name: cron-manager
    image: cron-manager-python:latest
    user: "root"
    volumes:
      - ./DSL/CronManager/DSL:/DSL
      - ./DSL/CronManager/script:/app/scripts
      - ./DSL/DatasetGenerator/output_datasets:/app/output_datasets
      - ./src/s3_dataset_processor:/app/src/s3_dataset_processor
      - ./DSL/DatasetGenerator/config:/app/config
      - ./src/model-training:/app/src/training
      - ./mlflow/mlflow_artifacts:/mlflow/mlflow_artifacts
      - ./data/processed:/app/data/processed
      - shared-volume:/app/shared
      - ./models:/app/models
      - ./src/inference/inference_scripts:/app/inference_scripts
      - ./grafana-configs/loki_logger.py:/app/inference_scripts/loki_logger.py
      - ./grafana-configs/loki_logger.py:/app/src/training/loki_logger.py
      - ./constants.ini:/app/inference_scripts/constants.ini
      - cron_data:/app/data
      - ./src/scripts:/app/src/scripts
    runtime: nvidia 
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - server.port=9010
      - MLFLOW_TRACKING_URI=http://mlflow:5000
      - PYTHONPATH=/app:/app/src/model_training:/app/src/s3_dataset_processor:/app/src
    ports:
      - 9010:8080
    networks:
      - bykstack
    depends_on:
      init:
        condition: service_completed_successfully
  
  mlflow:
    build:
      context: ./mlflow
      dockerfile: Dockerfile
    container_name: mlflow
    ports:
      - "5001:5000"
    env_file:
      - sidecar.env
    environment:
      - MLFLOW_TRACKING_USERNAME=${MLFLOW_TRACKING_USERNAME}
      - MLFLOW_TRACKING_PASSWORD=${MLFLOW_TRACKING_PASSWORD}
      - MLFLOW_HOST=${MLFLOW_HOST}
      - MLFLOW_PORT=${MLFLOW_PORT}
      - MLFLOW_BACKEND_STORE_URI=${MLFLOW_BACKEND_STORE_URI}
      - MLFLOW_DEFAULT_ARTIFACT_ROOT=${MLFLOW_DEFAULT_ARTIFACT_ROOT}
      - MLFLOW_FLASK_SERVER_SECRET_KEY=${MLFLOW_FLASK_SERVER_SECRET_KEY}
    volumes:
      - ./mlflow/mlflow_data:/mlflow/mlflow_data
      - ./mlflow/mlflow_artifacts:/mlflow/mlflow_artifacts
    networks:
      - bykstack
  
  minio:
    image: minio/minio:latest
    container_name: minio
    env_file:
      - config.env

    # command: server /data --console-address ":9001"
    entrypoint:  >
          sh -c "
            export MINIO_ROOT_USER=$${S3_ACCESS_KEY_ID} && \
            export MINIO_ROOT_PASSWORD=$${S3_SECRET_ACCESS_KEY} && \
            export MINIO_BROWSER_REDIRECT_URL=$${MINIO_BROWSER_REDIRECT_URL} && \

            minio server /app/minio_data --console-address ":9001"
          "
        
    volumes:
      - minio_data:/app/minio_data
    ports:
      - "9000:9000"  # API port
      - "9001:9001"  # Console port
    networks:
      - bykstack

  gc-s3-ferry:
    image: s3-ferry:latest
    container_name: gc-s3-ferry
    volumes:
      - ./DSL/DatasetGenerator/output_datasets:/app/output_datasets
      - shared-volume:/app/shared
      - ./models:/app/models
      - ./src/inference/dummy_model:/app/dummy_model
      - cron_data:/app/data
    env_file:
      - config.env
    ports:
      - "3006:3000"
    user: "root"
    networks:
      - bykstack
    depends_on:
      minio:
        condition: service_started 

  opensearch-node:
    image: opensearchproject/opensearch:2.11.1
    container_name: opensearch-node
    environment:
      - node.name=opensearch-node
      - discovery.seed_hosts=opensearch
      - discovery.type=single-node
      - bootstrap.memory_lock=true
      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
      - plugins.security.disabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    volumes:
      - opensearch-data:/usr/share/opensearch/data
    ports:
      - 9200:9200
      - 9600:9600
    networks:
      - bykstack 

  notifications-node:
    container_name: notifications-node
    build:
      context: ./notification-server
      dockerfile: Dockerfile
    ports:
      - 4040:4040
    depends_on:
      - opensearch-node
    environment:
      OPENSEARCH_PROTOCOL: http
      OPENSEARCH_HOST: opensearch-node
      OPENSEARCH_PORT: 9200
      OPENSEARCH_USERNAME: admin
      OPENSEARCH_PASSWORD: admin
      PORT: 4040
      REFRESH_INTERVAL: 1000
      CORS_WHITELIST_ORIGINS: http://localhost:3001,http://localhost:3002,http://localhost:3003,http://localhost:3004,http://localhost:8080,http://localhost:8088
      RUUTER_URL: http://ruuter-public:8086
    volumes:
      - /app/node_modules
      - ./notification-server:/app
    networks:
      - bykstack 

  opensearch-dashboards:
      image: opensearchproject/opensearch-dashboards:2.11.1
      container_name: opensearch-dashboards
      environment:
        - OPENSEARCH_HOSTS=http://opensearch-node:9200
        - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true
      ports:
        - 5601:5601
      networks:
        - bykstack                                                                                                                                                                                                                    
        
  gui:
    container_name: gui
    environment:
      - NODE_ENV=local
      - REACT_APP_RUUTER_API_URL=http://localhost:8086
      - REACT_APP_RUUTER_PRIVATE_API_URL=http://localhost:8088
      - REACT_APP_EXTERNAL_API_URL=http://localhost:8000
      - REACT_APP_CUSTOMER_SERVICE_LOGIN=http://localhost:3004/et/dev-auth
      - REACT_APP_NOTIFICATION_NODE_URL=http://localhost:4040
      - REACT_APP_CSP=upgrade-insecure-requests; default-src 'self'; font-src 'self' data:; img-src 'self' data:; script-src 'self' 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; object-src 'none'; connect-src 'self' http://localhost:8086 http://localhost:8088 http://localhost:8085 http://localhost:4040 http://localhost:3001 http://localhost:8000;
      - DEBUG_ENABLED=true
      - CHOKIDAR_USEPOLLING=true
      - PORT=3001
      - REACT_APP_SERVICE_ID=conversations,settings,monitoring
      - REACT_APP_ENABLE_HIDDEN_FEATURES=TRUE
      - VITE_HOST=0.0.0.0
      - VITE_ALLOWED_HOSTS=global-classifier-dev.rootcode.software,localhost,127.0.0.1

    build:
      context: ./GUI
      dockerfile: Dockerfile.dev
    ports:
      - 3003:3001
    volumes:
      - /app/node_modules
      - ./GUI:/app
    networks:
      - bykstack
    cpus: "0.5"
    mem_limit: "1G"

  dataset-gen-service:
    image: synthesisai/dataset-generator:latest
    container_name: dataset-gen-service
    ports:
      - "8000:8000"
    env_file:
      - .env
    volumes:
      - ./DSL/DatasetGenerator/config:/app/config
      - ./DSL/DatasetGenerator/templates:/app/templates
      - ./DSL/DatasetGenerator/user_configs:/app/user_configs
      - cron_data:/app/data
      - ./DSL/DatasetGenerator/output_datasets:/app/output_datasets
      - ./DSL/DatasetGenerator/logs:/app/logs
    networks:
      - bykstack

  redis:
    image: redis:7-alpine
    container_name: redis-embeddings
    ports:
      - "6378:6379"
    volumes:
      - redis_data:/data
      - ./DSL/DatasetGenerator/config/redis.conf:/usr/local/etc/redis/redis.conf
    command: redis-server /usr/local/etc/redis/redis.conf
    environment:
      - REDIS_REPLICATION_MODE=master
    restart: unless-stopped
    networks:
      - bykstack
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
      timeout: 3s
      retries: 3

  #temporarary container to initialize S3 storage with necessary buckets and models  
  init-storage:

    container_name: init-storage
    image: amazon/aws-cli:latest
    volumes:
      - cron_data:/data_models
      - ./src/inference/model-repository:/app/model-repository
    entrypoint: >
        sh -c "
          echo 'Setting AWS KEYS' && 
          aws configure set aws_access_key_id $${S3_ACCESS_KEY_ID} && 
          aws configure set aws_secret_access_key $${S3_SECRET_ACCESS_KEY} && 

          echo \"WAITING for S3 storage to be ready...\" &&
          until curl -s $${S3_HEALTH_ENDPOINT}; do sleep 2; done &&

          echo \"Checking for S3 bucket: $${S3_DATA_BUCKET_NAME} ...\" && 
          if ! aws --endpoint-url $${S3_ENDPOINT_URL} s3api head-bucket --bucket $${S3_DATA_BUCKET_NAME} 2>/dev/null; then
            echo \"Bucket $${S3_DATA_BUCKET_NAME} does not exist. Creating...\" && 
            aws --endpoint-url $${S3_ENDPOINT_URL} s3 mb s3://$${S3_DATA_BUCKET_NAME} && 
            echo \"Bucket $${S3_DATA_BUCKET_NAME} created successfully.\"
          else
            echo \"Bucket $${S3_DATA_BUCKET_NAME} already exists. Skipping creation.\"
          fi && 


          echo 'S3 storage is ready.' && 
          echo 'Uploading models...' && 

          echo 'Uploading initial dummy model to testing registry...' && 
          aws --endpoint-url $${S3_ENDPOINT_URL} s3 cp --recursive /app/model-repository/testing s3://$${S3_DATA_BUCKET_NAME}/$${S3_DATA_BUCKET_PATH}/models/testing && 

          echo 'Uploading initial dummy model to production registry...' && 
          aws --endpoint-url $${S3_ENDPOINT_URL} s3 cp --recursive /app/model-repository/production s3://$${S3_DATA_BUCKET_NAME}/$${S3_DATA_BUCKET_PATH}/models/production && 

          echo 'Uploading initial dummy model to undeployed registry...' && 
          aws --endpoint-url $${S3_ENDPOINT_URL} s3 cp --recursive /app/model-repository/undeployed s3://$${S3_DATA_BUCKET_NAME}/$${S3_DATA_BUCKET_PATH}/models/undeployed && 

          echo 'Upload complete.'
        "

    env_file:
      - config.env
    networks:
      - bykstack
    depends_on:
      minio:
        condition: service_started
      cron-manager:
        condition: service_started
      ruuter-private:
        condition: service_started
      ruuter-public:
        condition: service_started
      gc-s3-ferry:
        condition: service_started

  triton-production-server:
    container_name: triton-production-server
    runtime: nvidia
    build:
      context: ./src/inference
      dockerfile: Dockerfile
    command: >
      sh -c "
        echo 'Setting environment variables for S3 access...' && 
        export AWS_ACCESS_KEY_ID=$${S3_ACCESS_KEY_ID} && 
        export AWS_SECRET_ACCESS_KEY=$${S3_SECRET_ACCESS_KEY} && 
        export AWS_REGION=$${S3_REGION} && 
        export AWS_ENDPOINT_URL=$${S3_ENDPOINT_URL} && 

        echo 'Starting Triton server for production models...' && 
        tritonserver --model-repository=s3://$${S3_ENDPOINT_NAME}/$${S3_DATA_BUCKET_NAME}/$${S3_DATA_BUCKET_PATH}/models/production  --model-control-mode=explicit --log-verbose=1
      "
    ports:
      - "6000:8000"
      - "6001:8001"
      - "6002:8002"
    volumes:
      - cron_data:/data_models

    env_file:
      - config.env
    networks:
      - bykstack
    depends_on:
      init-storage:
        condition: service_completed_successfully
      cron-manager:
        condition: service_started
      ruuter-private:
        condition: service_started
      minio:
        condition: service_started

  triton-test-server:
    container_name: triton-test-server
    runtime: nvidia
    build:
      context: ./src/inference
      dockerfile: Dockerfile
    command: >
      sh -c "

        echo 'Setting environment variables for S3 access...' && 

        export AWS_ACCESS_KEY_ID=$${S3_ACCESS_KEY_ID} && 
        export AWS_SECRET_ACCESS_KEY=$${S3_SECRET_ACCESS_KEY} && 
        export AWS_REGION=$${S3_REGION} && 
        export AWS_ENDPOINT_URL=$${S3_ENDPOINT_URL} && 

        echo 'Starting Triton server for testing models...' && \ 

        tritonserver  --model-repository=s3://$${S3_ENDPOINT_NAME}/$${S3_DATA_BUCKET_NAME}/$${S3_DATA_BUCKET_PATH}/models/testing --model-control-mode=explicit --log-verbose=1
      "
    ports:
      - "4000:8000"
      - "4001:8001"
      - "4002:8002"
    volumes:
      - cron_data:/data_models

    env_file:
      - config.env
    networks:
      - bykstack
    depends_on:
      init-storage:
        condition: service_completed_successfully
      cron-manager:
        condition: service_started
      ruuter-private:
        condition: service_started
      minio:
        condition: service_started

  # Logging Stack - Loki and Grafana
  loki:
    image: grafana/loki:2.9.0
    container_name: loki
    ports:
      - "3100:3100"
    command: -config.file=/etc/loki/local-config.yaml
    volumes:
      - ./grafana-configs/loki-config.yaml:/etc/loki/local-config.yaml
      - loki-data:/loki
    networks:
      - bykstack
    restart: unless-stopped

  grafana:
    image: grafana/grafana:10.0.0
    container_name: grafana
    ports:
      - "4005:3000"
    env_file:
      - config.env
    volumes:
      - grafana-data:/var/lib/grafana
      - ./grafana-configs/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
      - ./grafana-configs/grafana-dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml
      - ./grafana-configs/grafana-dashboard-deployment.json:/etc/grafana/dashboards/deployment.json
    networks:
      - bykstack
    depends_on:
      - loki
    restart: unless-stopped


volumes:
  shared-volume:
    name: shared-volume
  opensearch-data:
    name: opensearch-data
  dataset_gen_ollama_models:
    name: dataset_gen_ollama_models
  cron_data:
    name: cron_data
  minio_data:
    name: minio_data
  loki-data:
    name: loki-data
  grafana-data:
    name: grafana-data
  redis_data:
    name: redis_data


networks:
  bykstack:
    name: bykstack
    driver: bridge