Ollama GPU Scheduling: Running Inference and ComfyUI on One RTX Without OOM

# Example: Daily schedule for balanced usage
# 6:00 AM - 10:00 AM: Ollama priority (code assistant during work hours)
# 10:00 AM - 12:00 PM: ComfyUI priority (creative work)
# 12:00 PM - 2:00 PM: Shared/lunch break (light usage both)
# 2:00 PM - 6:00 PM: Ollama priority (continued work)
# 6:00 PM - 10:00 PM: ComfyUI priority (evening creative sessions)
# 10:00 PM - 6:00 AM: Ollama priority (background processing, embeddings)

# Implementation via systemd timers or cron
# Ollama gets GPU during work hours
0 6 * * * /usr/local/bin/gpu-grant ollama
0 10 * * * /usr/local/bin/gpu-revoke ollama
0 12 * * * /usr/local/bin/gpu-grant comfyui
0 14 * * * /usr/local/bin/gpu-revoke comfyui
# ... and so on

# Example: Priority queue implementation
# Priority levels (higher number = higher priority)
# 10: Interactive user requests (chat, immediate image generation)
# 5: Background processing (embeddings, batch jobs)
# 1: Maintenance/cleanup tasks

# Pseudocode for GPU arbiter
class GpuArbiter {
  constructor() {
    this.queue = new PriorityQueue();
    this.currentHolder = null;
  }

  async requestAccess(application, priority, estimatedDuration) {
    // Add request to queue
    await this.queue.enqueue({
      application,
      priority,
      estimatedDuration,
      timestamp: Date.now()
    });

    // If we're not holding GPU or new request has higher priority
    if (!this.currentHolder ||
        priority > this.currentHolder.priority) {
      await this.preemptAndGrant(application, priority);
    }
  }

  async releaseAccess(application) {
    if (this.currentHolder?.application === application) {
      this.currentHolder = null;
      await this.grantNextInQueue();
    }
  }
}

// Usage in Ollama wrapper
class OllamaManager {
  constructor(gpuArbiter) {
    this.gpuArbiter = gpuArbiter;
  }

  async generate(prompt) {
    // Request GPU with medium priority
    await this.gpuArbiter.requestAccess('ollama', 5, 30000); // 30s estimate

    try {
      return await this.ollamaInstance.generate(prompt);
    } finally {
      await this.gpuArbiter.releaseAccess('ollama');
    }
  }
}

// Usage in ComfyUI wrapper
class ComfyUIManager {
  constructor(gpuArbiter) {
    this.gpuArbiter = gpuArbiter;
  }

  async generateImage(params) {
    // Request GPU with high priority for interactive use
    await this.gpuArbiter.requestAccess('comfyui', 10, 15000); // 15s estimate

    try {
      return await this.comfyuiInstance.generateImage(params);
    } finally {
      await this.gpuArbiter.releaseAccess('comfyui');
    }
  }
}

# Implementation using CUDA MPS (Multi-Process Service) or similar
# Actually, for most users, a simpler approach works better:

# 1. Monitor VRAM usage in real-time
# 2. Set soft limits that applications should stay under
# 3. Allow temporary exceeding with graceful degradation
# 4. Send warnings when approaching hard limits

# Example monitoring script
#!/usr/bin/env python3
import subprocess
import time
import requests

def get_vram_usage():
    """Get current VRAM usage in MB"""
    try:
        result = subprocess.run([
            'nvidia-smi',
            '--query-gpu=memory.used,memory.total',
            '--format=csv,noheader,nounits'
        ], capture_output=True, text=True, check=True)

        used, total = map(int, result.stdout.strip().split(', '))
        return used, total
    except Exception as e:
        print(f"Error getting VRAM usage: {e}")
        return 0, 16384  # 16GB in MB

def check_and_warn():
    used, total = get_vram_usage()
    percent = (used / total) * 100

    if percent > 90:
        print(f"⚠️  HIGH VRAM USAGE: {used}MB / {total}MB ({percent:.1f}%)")
        # Could trigger notifications or automatic cleanup
    elif percent > 80:
        print(f"⚠️  Elevated VRAM usage: {used}MB / {total}MB ({percent:.1f}%)")

    return percent > 95  # Return True if approaching critical

# Main monitoring loop
if __name__ == "__main__":
    print("Starting VRAM monitor...")
    while True:
        if check_and_warn():
            # In a real implementation, you might:
            # 1. Send a desktop notification
            # 2. Trigger graceful cleanup in applications
            # 3. Log the event for analysis
            pass
        time.sleep(5)  # Check every 5 seconds

# Example: Ollama model quantization choices
# Pull different quantizations for different use cases

# High-quality coding assistant (when you have the VRAM to spare)
ollama pull qwen2.5-coder:14b-q5_K_M

# General purpose assistant (balanced)
ollama pull qwen2.5-coder:14b-q4_K_M

# Lightweight assistant (when sharing GPU heavily)
ollama pull qwen2.5-coder:14b-q3_K_M

# Embedding model (tiny footprint)
ollama pull nomic-embed-text:latest

# Switch between them based on current needs
# You can even run multiple quantizations of same model!

# Example: ComfyUI launch with VRAM optimizations
# Add these to your ComfyUI startup script or command line

#!/bin/bash
# Launch ComfyUI with memory optimizations

export COMMANDLINE_ARGS="--medvram --opencv --disable-numpy-random-seed"
# --medvram: Medium VRAM optimization (enables attention slicing, etc.)
# --lowvram: More aggressive (slower but uses less VRAM)
# --disable-numpy-random-seed: Fixes nondeterminism in some cases

# Alternative: Use environment variables
export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128
# Helps with memory fragmentation

# Start ComfyUI
python main.py --listen --port 8188

# For extreme VRAM constraints, consider:
# --ckpt sd-v1-5.ckpt  # Use smaller model
# --no-half-vae        # Disable VAE FP16 (saves VRAM, costs quality)

# Simple request queue for Ollama
import asyncio
import time
from collections import deque
from dataclasses import dataclass
from typing import Callable, Any

@dataclass
class OllamaRequest:
    prompt: str
    model: str
    future: asyncio.Future
    timestamp: float
    priority: int = 0  # Higher = more urgent

class OllamaRequestQueue:
    def __init__(self, max_concurrent=1):
        self.max_concurrent = max_concurrent
        self.queue = deque()
        self.processing = 0
        self.results = {}

    async def add_request(self, prompt, model="qwen2.5-coder:14b", priority=0):
        """Add a request to the queue and return a future for the result"""
        future = asyncio.Future()
        request = OllamaRequest(prompt, model, future, time.time(), priority)

        # Insert in priority order
        inserted = False
        for i, req in enumerate(self.queue):
            if priority > req.priority:
                self.queue.insert(i, request)
                inserted = True
                break

        if not inserted:
            self.queue.append(request)

        await self._process_queue()
        return future

    async def _process_queue(self):
        """Process requests from the queue"""
        while (self.queue and
               self.processing < self.max_concurrent):
            request = self.queue.popleft()
            self.processing += 1

        # Process the request (this would call your Ollama client)
        asyncio.create_task(self._handle_request(request))

    async def _handle_request(self, request):
        """Actually process a single request"""
        try:
            # Simulate calling Ollama - replace with actual implementation
            start = time.time()
            # result = await ollama_client.generate(
            #     model=request.model,
            #     prompt=request.prompt
            # )
            await asyncio.sleep(2)  # Simulate processing time
            result = f"Response to: {request.prompt[:50]}..."

            # Calculate and store result
            request.future.set_result({
                'response': result,
                'processing_time': time.time() - start
            })
        except Exception as e:
            request.future.set_exception(e)
        finally:
            self.processing -= 1
            await self._process_queue()  # Process next in queue

# Usage
queue = OllamaRequestQueue(max_concurrent=1)  # Sequential processing

# Add requests - they'll be processed sequentially
future1 = queue.add_request("Explain quantum computing")
future2 = queue.add_request("Write a Python function to sort a list")

# Wait for results
result1 = await future1
result2 = await future2

# Simple ComfyUI job queue
import asyncio
import uuid
from dataclasses import dataclass
from enum import Enum

class JobStatus(Enum):
    QUEUED = "queued"
    PROCESSING = "processing"
    COMPLETED = "completed"
    FAILED = "failed"

@dataclass
class ComfyUIJob:
    id: str
    prompt: str
    params: dict
    status: JobStatus
    result: any = None
    error: str = None
    created_at: float = 0
    started_at: float = 0
    completed_at: float = 0

class ComfyUIJobQueue:
    def __init__(self, max_concurrent=1):
        self.max_concurrent = max_concurrent
        self.queue = []  # List of ComfyUIJob
        self.processing = {}  # id -> ComfyUIJob
        self.results = {}  # id -> result

    async def add_job(self, prompt, params=None, priority=0):
        """Add a job to the queue"""
        if params is None:
            params = {}

        job = ComfyUIJob(
            id=str(uuid.uuid4()),
            prompt=prompt,
            params=params,
            status=JobStatus.QUEUED,
            created_at=time.time()
        )

        # Add to queue (simple FIFO for now - could add priority)
        self.queue.append(job)

        await self._process_queue()
        return job.id

    async def _process_queue(self):
        """Start processing queued jobs"""
        while (len(self.queue) > 0 and
               len(self.processing) < self.max_concurrent):
            job = self.queue.pop(0)
            job.status = JobStatus.PROCESSING
            job.started_at = time.time()
            self.processing[job.id] = job

            # Process the job
            asyncio.create_task(self._handle_job(job))

    async def _handle_job(self, job):
        """Actually process a ComfyUI job"""
        try:
            # Simulate calling ComfyUI - replace with actual API call
            await asyncio.sleep(5)  # Simulate image generation time

            # In reality, you'd call ComfyUI's API here
            result = {
                'image_path': f"/tmp/comfyui_{job.id}.png",
                'prompt': job.prompt,
                'params': job.params,
                'generation_time': time.time() - job.started_at
            }

            job.status = JobStatus.COMPLETED
            job.result = result
            job.completed_at = time.time()
            self.results[job.id] = result

        except Exception as e:
            job.status = JobStatus.FAILED
            job.error = str(e)
            job.completed_at = time.time()
        finally:
            # Clean up processing tracking
            if job.id in self.processing:
                del self.processing[job.id]

            # Process next job
            await self._process_queue()

    def get_job_status(self, job_id):
        """Get the status of a job"""
        if job_id in self.results:
            return self.results[job_id]
        elif job_id in self.processing:
            return self.processing[job_id]
        else:
            # Find in queue
            for job in self.queue:
                if job.id == job_id:
                    return job
            return None

# Usage
job_queue = ComfyUIJobQueue(max_concurrent=1)

# Add jobs
job1_id = await job_queue.add_job("A beautiful sunset over mountains", {"steps": 20})
job2_id = await job_queue.add_job("Portrait of a cyberpunk hacker", {"steps": 25})

# Check status later
status1 = job_queue.get_job_status(job1_id)
status2 = job_queue.get_job_status(job2_id)

# Comprehensive monitoring script
#!/usr/bin/env python3
import subprocess
import time
import json
import requests
from datetime import datetime

def get_gpu_stats():
    """Get comprehensive GPU statistics"""
    try:
        # Get basic memory usage
        mem_result = subprocess.run([
            'nvidia-smi',
            '--query-gpu=memory.used,memory.total,utilization.gpu,temperature.gpu,power.draw',
            '--format=csv,noheader,nounits'
        ], capture_output=True, text=True, check=True)

        used_mem, total_mem, gpu_util, gpu_temp, power_draw = map(
            int, mem_result.strip().split(', ')
        )

        # Get process-specific info
        proc_result = subprocess.run([
            'nvidia-smi',
            '--query-compute-apps=pid,process_name,used_memory',
            '--format=csv,noheader,nounits'
        ], capture_output=True, text=True)

        processes = []
        if proc_result.stdout.strip():
            for line in proc_result.stdout.strip().split('\n'):
                if line:
                    pid, name, mem_used = line.split(', ')
                    processes.append({
                        'pid': int(pid),
                        'name': name,
                        'vram_mb': int(mem_used)
                    })

        return {
            'timestamp': datetime.now().isoformat(),
            'vram_used_mb': used_mem,
            'vram_total_mb': total_mem,
            'vram_free_mb': total_mem - used_mem,
            'vram_utilization_percent': round((used_mem / total_mem) * 100, 1),
            'gpu_utilization_percent': gpu_util,
            'gpu_temperature_c': gpu_temp,
            'power_draw_w': power_draw,
            'processes': processes
        }
    except Exception as e:
        return {'error': str(e), 'timestamp': datetime.now().isoformat()}

def check_ollama_health():
    """Check if Ollama is responsive"""
    try:
        response = requests.get('http://localhost:11434/api/tags', timeout=5)
        return response.status_code == 200
    except:
        return False

def check_comfyui_health():
    """Check if ComfyUI is responsive"""
    try:
        response = requests.get('http://localhost:8188', timeout=5)
        return response.status_code == 200
    except:
        return False

def log_stats(stats):
    """Log stats to file and check for alerts"""
    # Log to file
    with open('/tmp/gpu_monitor.log', 'a') as f:
        f.write(json.dumps(stats) + '\n')

    # Check for alert conditions
    alerts = []

    if stats.get('vram_utilization_percent', 0) > 90:
        alerts.append(f"HIGH VRAM USAGE: {stats['vram_utilization_percent']}%")

    if stats.get('gpu_temperature_c', 0) > 80:
        alerts.append(f"HIGH TEMPERATURE: {stats['gpu_temperature_c']}°C")

    if not check_ollama_health():
        alerts.append("OLLAMA UNRESPONSIVE")

    if not check_comfyui_health():
        alerts.append("COMFYUI UNRESPONSIVE")

    # Print alerts if any
    if alerts:
        print(f"[{stats['timestamp']}] ALERT: {' | '.join(alerts)}")

    return alerts

# Main monitoring loop
if __name__ == "__main__":
    print("Starting GPU monitoring...")
    while True:
        stats = get_gpu_stats()
        log_stats(stats)
        time.sleep(10)  # Update every 10 seconds