Chatbot Creation Framework - Deployment Guide

Overview

This guide provides comprehensive instructions for deploying the Chatbot Creation Framework using Docker containerization with PostgreSQL, MinIO, and Redis.

Prerequisites

• Docker Desktop or Docker Engine (v20.10+)
• Docker Compose (v2.0+)
• Node.js 18+ (for local development)
• Git

Quick Start

1. Clone and Setup

git clone https://github.com/your-org/chatbot-creation-framework.git
cd chatbot-creation-framework
cp .env.example .env.local

2. Configure Environment Variables

Edit .env.local with your configuration:

# Application
NODE_ENV=production
PORT=3000
APP_URL=http://localhost:3000

# Database
DATABASE_URL=postgresql://chatbot_user:secure_password@postgres:5432/chatbot_framework
POSTGRES_USER=chatbot_user
POSTGRES_PASSWORD=secure_password
POSTGRES_DB=chatbot_framework

# Redis
REDIS_URL=redis://redis:6379

# MinIO S3-Compatible Storage
MINIO_ENDPOINT=minio
MINIO_PORT=9000
MINIO_ACCESS_KEY=chatbot_access
MINIO_SECRET_KEY=chatbot_secret_key_min_32_chars
MINIO_USE_SSL=false
MINIO_BUCKET_PREFIX=chatbot-

# NextAuth.js
NEXTAUTH_URL=http://localhost:3000
NEXTAUTH_SECRET=your-super-secret-key-min-32-chars

# n8n Integration
N8N_BASE_URL=http://your-n8n-instance.com
N8N_API_KEY=your-n8n-api-key

# File Upload Configuration
MAX_FILE_SIZE=52428800  # 50MB
ALLOWED_FILE_TYPES=pdf,md,docx,txt
UPLOAD_PATH=/app/uploads

# Security
JWT_SECRET=your-jwt-secret-key-min-32-chars
API_RATE_LIMIT=100
API_RATE_WINDOW=900000  # 15 minutes

# Monitoring
LOG_LEVEL=info
ENABLE_METRICS=true

3. Deploy with Docker Compose

docker compose up -d

Docker Configuration Files

docker-compose.yml

version: '3.8'

services:
  # Next.js Application
  app:
    build:
      context: .
      dockerfile: Dockerfile
      target: production
    ports:
      - "3000:3000"
    environment:
      - NODE_ENV=production
      - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
      - REDIS_URL=redis://redis:6379
      - MINIO_ENDPOINT=minio
      - NEXTAUTH_URL=${NEXTAUTH_URL}
      - NEXTAUTH_SECRET=${NEXTAUTH_SECRET}
    env_file:
      - .env.local
    depends_on:
      postgres:
        condition: service_healthy
      redis:
        condition: service_healthy
      minio:
        condition: service_healthy
    restart: unless-stopped
    volumes:
      - app_uploads:/app/uploads
    networks:
      - chatbot_network

  # PostgreSQL Main Database
  postgres:
    image: pgvector/pgvector:pg15
    environment:
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C"
    ports:
      - "5432:5432"
    volumes:
      - postgres_data:/var/lib/postgresql/data
      - ./docker/postgres/init:/docker-entrypoint-initdb.d
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s
    restart: unless-stopped
    networks:
      - chatbot_network

  # Redis for Caching
  redis:
    image: redis:7-alpine
    ports:
      - "6379:6379"
    volumes:
      - redis_data:/data
      - ./docker/redis/redis.conf:/usr/local/etc/redis/redis.conf
    command: redis-server /usr/local/etc/redis/redis.conf
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s
    restart: unless-stopped
    networks:
      - chatbot_network

  # MinIO S3-Compatible Storage
  minio:
    image: minio/minio:latest
    ports:
      - "9000:9000"
      - "9001:9001"
    environment:
      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
      MINIO_BROWSER_REDIRECT_URL: http://localhost:9001
    volumes:
      - minio_data:/data
    command: server /data --console-address ":9001"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3
      start_period: 60s
    restart: unless-stopped
    networks:
      - chatbot_network

  # Nginx Reverse Proxy
  nginx:
    image: nginx:alpine
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - ./docker/nginx/nginx.conf:/etc/nginx/nginx.conf
      - ./docker/nginx/conf.d:/etc/nginx/conf.d
      - nginx_logs:/var/log/nginx
      - ./certs:/etc/nginx/certs:ro
    depends_on:
      - app
    restart: unless-stopped
    networks:
      - chatbot_network

volumes:
  postgres_data:
    driver: local
  redis_data:
    driver: local
  minio_data:
    driver: local
  app_uploads:
    driver: local
  nginx_logs:
    driver: local

networks:
  chatbot_network:
    driver: bridge

docker-compose.override.yml (Development)

version: '3.8'

services:
  app:
    build:
      target: development
    environment:
      - NODE_ENV=development
    volumes:
      - .:/app
      - /app/node_modules
      - /app/.next
    command: npm run dev
    ports:
      - "3000:3000"

  postgres:
    ports:
      - "5432:5432"
    environment:
      - POSTGRES_DB=chatbot_framework_dev

  redis:
    ports:
      - "6379:6379"

  minio:
    ports:
      - "9000:9000"
      - "9001:9001"

Dockerfile

# Multi-stage build for optimized production image
FROM node:18-alpine AS base

# Install dependencies only when needed
FROM base AS deps
RUN apk add --no-cache libc6-compat
WORKDIR /app

# Copy package files
COPY package.json package-lock.json* ./
RUN npm ci

# Rebuild the source code only when needed
FROM base AS builder
WORKDIR /app
COPY --from=deps /app/node_modules ./node_modules
COPY . .

# Build application
RUN npm run build

# Production image, copy all the files and run next
FROM base AS production
WORKDIR /app

ENV NODE_ENV production
RUN addgroup --system --gid 1001 nodejs
RUN adduser --system --uid 1001 nextjs

# Copy built application
COPY --from=builder --chown=nextjs:nodejs /app/.next ./.next
COPY --from=builder --chown=nextjs:nodejs /app/node_modules ./node_modules
COPY --from=builder --chown=nextjs:nodejs /app/package.json ./package.json
COPY --from=builder --chown=nextjs:nodejs /app/public ./public

# Create upload directory
RUN mkdir -p /app/uploads && chown nextjs:nodejs /app/uploads

USER nextjs

EXPOSE 3000

ENV PORT 3000
ENV HOSTNAME "0.0.0.0"

CMD ["npm", "start"]

# Development target
FROM base AS development
WORKDIR /app

RUN npm install -g nodemon
COPY package*.json ./
RUN npm install

COPY . .
EXPOSE 3000

CMD ["npm", "run", "dev"]

Configuration Files

docker/postgres/init/01-init-extensions.sql

-- Enable required PostgreSQL extensions
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE EXTENSION IF NOT EXISTS "vector";

-- Create additional databases for development
CREATE DATABASE chatbot_framework_test;
CREATE DATABASE chatbot_framework_dev;

docker/redis/redis.conf

# Redis Configuration
bind 0.0.0.0
port 6379
protected-mode no

# Memory management
maxmemory 256mb
maxmemory-policy allkeys-lru

# Persistence
save 900 1
save 300 10
save 60 10000

# Logging
loglevel notice
logfile /var/log/redis/redis-server.log

# Security
requirepass your_redis_password

# Performance
tcp-keepalive 300
timeout 0

docker/nginx/nginx.conf

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log notice;
pid /var/run/nginx.pid;

events {
    worker_connections 1024;
    use epoll;
    multi_accept on;
}

http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;
    
    # Logging
    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for"';
    access_log /var/log/nginx/access.log main;
    
    # Performance
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    client_max_body_size 50M;
    
    # Gzip compression
    gzip on;
    gzip_vary on;
    gzip_min_length 1024;
    gzip_proxied expired no-cache no-store private auth;
    gzip_types
        text/plain
        text/css
        text/xml
        text/javascript
        application/json
        application/javascript
        application/xml+rss
        application/atom+xml
        image/svg+xml;
    
    # Rate limiting
    limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s;
    limit_req_zone $binary_remote_addr zone=login:10m rate=1r/s;
    
    include /etc/nginx/conf.d/*.conf;
}

docker/nginx/conf.d/default.conf

upstream nextjs {
    server app:3000;
}

upstream minio {
    server minio:9000;
}

upstream minio_console {
    server minio:9001;
}

# Main application
server {
    listen 80;
    server_name localhost _;
    
    # Security headers
    add_header X-Frame-Options DENY;
    add_header X-Content-Type-Options nosniff;
    add_header X-XSS-Protection "1; mode=block";
    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
    
    # API rate limiting
    location /api/ {
        limit_req zone=api burst=20 nodelay;
        proxy_pass http://nextjs;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_cache_bypass $http_upgrade;
    }
    
    # Login rate limiting
    location /api/auth/ {
        limit_req zone=login burst=5 nodelay;
        proxy_pass http://nextjs;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
    
    # Main app
    location / {
        proxy_pass http://nextjs;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_cache_bypass $http_upgrade;
    }
}

# MinIO API
server {
    listen 80;
    server_name minio.localhost;
    
    location / {
        proxy_pass http://minio;
        proxy_set_header Host $http_host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        
        # Handle large file uploads
        client_max_body_size 50M;
        proxy_request_buffering off;
    }
}

# MinIO Console
server {
    listen 80;
    server_name minio-console.localhost;
    
    location / {
        proxy_pass http://minio_console;
        proxy_set_header Host $http_host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}

Environment Setup

Development Environment

# Clone repository
git clone https://github.com/your-org/chatbot-creation-framework.git
cd chatbot-creation-framework

# Install dependencies
npm install

# Copy environment file
cp .env.example .env.local

# Start development environment
docker compose -f docker-compose.yml -f docker-compose.override.yml up -d

# Run database migrations
npm run db:migrate

# Start development server
npm run dev

Production Environment

# Clone repository
git clone https://github.com/your-org/chatbot-creation-framework.git
cd chatbot-creation-framework

# Copy environment file and configure
cp .env.example .env.local
# Edit .env.local with production values

# Build and start production environment
docker compose up -d

# Run database migrations
docker compose exec app npm run db:migrate:prod

Health Checks and Monitoring

Health Check Endpoints

• Application: GET /api/health
• Database: GET /api/health/database
• Redis: GET /api/health/redis
• MinIO: GET /api/health/storage

Monitoring Commands

# View logs
docker compose logs -f app
docker compose logs -f postgres
docker compose logs -f redis
docker compose logs -f minio

# Check container status
docker compose ps

# Monitor resource usage
docker stats

# Check disk usage
docker system df

Backup and Recovery

Database Backup

# Create backup
docker compose exec postgres pg_dump -U chatbot_user chatbot_framework > backup_$(date +%Y%m%d_%H%M%S).sql

# Restore backup
docker compose exec -T postgres psql -U chatbot_user -d chatbot_framework < backup_file.sql

MinIO Backup

# Install MinIO client
curl https://dl.min.io/client/mc/release/linux-amd64/mc -o mc
chmod +x mc

# Configure client
./mc alias set local http://localhost:9000 chatbot_access chatbot_secret_key_min_32_chars

# Backup buckets
./mc mirror local/chatbot-bucket-name ./backup/minio/

# Restore buckets
./mc mirror ./backup/minio/ local/chatbot-bucket-name/

Full System Backup Script

#!/bin/bash
# backup.sh

BACKUP_DIR="/backups/$(date +%Y%m%d_%H%M%S)"
mkdir -p $BACKUP_DIR

echo "Creating system backup..."

# Database backup
docker compose exec postgres pg_dump -U chatbot_user chatbot_framework > $BACKUP_DIR/database.sql

# MinIO data backup
docker compose exec minio tar -czf - /data > $BACKUP_DIR/minio_data.tar.gz

# Configuration backup
cp -r docker/ $BACKUP_DIR/
cp .env.local $BACKUP_DIR/
cp docker-compose.yml $BACKUP_DIR/

echo "Backup completed: $BACKUP_DIR"

Scaling and Performance

Horizontal Scaling

# docker-compose.scale.yml
services:
  app:
    deploy:
      replicas: 3
      resources:
        limits:
          cpus: '1'
          memory: 1G
        reservations:
          cpus: '0.5'
          memory: 512M

  nginx:
    volumes:
      - ./docker/nginx/nginx.conf:/etc/nginx/nginx.conf
      - ./docker/nginx/upstream.conf:/etc/nginx/conf.d/upstream.conf

Performance Optimization

services:
  postgres:
    environment:
      - POSTGRES_SHARED_BUFFERS=256MB
      - POSTGRES_EFFECTIVE_CACHE_SIZE=1GB
      - POSTGRES_WORK_MEM=4MB
    deploy:
      resources:
        limits:
          memory: 2G
        reservations:
          memory: 1G

  redis:
    deploy:
      resources:
        limits:
          memory: 512M
        reservations:
          memory: 256M

SSL Configuration

Let's Encrypt with Certbot

# Install certbot
sudo apt update
sudo apt install certbot python3-certbot-nginx

# Obtain certificates
sudo certbot --nginx -d yourdomain.com -d api.yourdomain.com

# Auto-renewal
sudo crontab -e
# Add: 0 12 * * * /usr/bin/certbot renew --quiet

Manual SSL Configuration

# Add to nginx configuration
server {
    listen 443 ssl http2;
    server_name yourdomain.com;
    
    ssl_certificate /etc/nginx/certs/cert.pem;
    ssl_certificate_key /etc/nginx/certs/key.pem;
    
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512;
    ssl_prefer_server_ciphers off;
    ssl_session_cache shared:SSL:10m;
    ssl_session_timeout 10m;
}

Troubleshooting

Common Issues

1. Database Connection Issues

   # Check database status
   docker compose logs postgres
   
   # Test connection
   docker compose exec postgres psql -U chatbot_user -d chatbot_framework -c "SELECT 1;"
   

2. MinIO Access Issues

   # Check MinIO logs
   docker compose logs minio
   
   # Verify credentials
   curl http://localhost:9000/minio/health/live
   

3. Memory Issues

   # Check container memory usage
   docker stats
   
   # Increase Docker memory limits
   # Docker Desktop: Settings > Resources > Memory
   

4. File Upload Issues

   # Check upload directory permissions
   docker compose exec app ls -la /app/uploads
   
   # Fix permissions
   docker compose exec app chown -R nextjs:nodejs /app/uploads
   

Performance Tuning

1. Database Optimization

   -- Add indexes for better performance
   CREATE INDEX CONCURRENTLY idx_documents_project_status ON documents(chatbot_project_id, processing_status);
   CREATE INDEX CONCURRENTLY idx_messages_conversation_created ON messages(conversation_id, created_at);
   
   -- Analyze table statistics
   ANALYZE documents;
   ANALYZE messages;
   

2. Redis Configuration

   # Increase memory limit
   maxmemory 512mb
   
   # Enable lazy freeing
   lazyfree-lazy-eviction yes
   lazyfree-lazy-expire yes
   

This comprehensive deployment guide provides everything needed to successfully deploy and maintain the chatbot creation framework in various environments.