Distributed Training Setup

Overview

Multi-GPU and multi-node training with PyTorch DDP. This guide covers practical implementation for production ML systems.

Why This Matters in MLOps

Modern ML systems require rigorous operations practices:

Reliability: Models must perform consistently in production

Reproducibility: Experiments must be tracked and reproducible

Scalability: Systems must handle growing data and traffic

Observability: You need visibility into what models are doing

Setup

bash
Install required tools
pip install pytorch mlflow pandas numpy scikit-learn
Or with Docker
docker pull python:3.11-slim

Core Implementation

python
import os
import json
import logging
from datetime import datetime
from pathlib import Path
logger = logging.getLogger(__name__)
class DistributedTrainingSetup:
    """
    Distributed Training Setup implementation.
    
    Handles: training
    Tool: pytorch
    """
    
    def __init__(self, config: dict = None):
        self.config = config or self._default_config()
        self._setup()
    
    def _default_config(self) -> dict:
        return {
            "tool": "pytorch",
            "environment": os.getenv("ENVIRONMENT", "development"),
            "log_level": "INFO",
        }
    
    def _setup(self):
        """Initialize pytorch connection and resources."""
        logging.basicConfig(level=self.config.get("log_level", "INFO"))
        logger.info(f"Initialized Distributed Training Setup with config: {self.config}")
    
    def run(self, **kwargs) -> dict:
        """Execute training."""
        start = datetime.utcnow()
        
        try:
            result = self._execute(**kwargs)
            
            elapsed = (datetime.utcnow() - start).total_seconds()
            logger.info(f"Distributed Training Setup completed in {elapsed:.2f}s")
            
            return {
                "status": "success",
                "result": result,
                "elapsed_seconds": elapsed
            }
        
        except Exception as e:
            logger.error(f"Distributed Training Setup failed: {e}")
            return {
                "status": "failed", 
                "error": str(e)
            }
    
    def _execute(self, **kwargs) -> dict:
        """Core training logic. Override to customize."""
        return {"completed": True, "tool": "pytorch"}
Configuration
config = {
    "tool": "pytorch",
    "tracking_uri": os.getenv("MLFLOW_TRACKING_URI", "http://localhost:5000"),
    "artifact_root": "./artifacts",
}
Initialize
processor = DistributedTrainingSetup(config)
result = processor.run()
print(json.dumps(result, indent=2))

PYTORCH Integration

python
Specific pytorch integration for training
import subprocess
def setup_pytorch():
    """Configure pytorch for training."""
    
    # Initialize project
    print(f"Setting up pytorch for training...")
    
    # Example configuration
    config = {
        "project": "my-ml-project",
        "tool": "pytorch",
        "specialty": "training",
        "version": "1.0.0"
    }
    
    # Save configuration
    Path(".pytorch").mkdir(exist_ok=True)
    with open(f".pytorch/config.json", "w") as f:
        json.dump(config, f, indent=2)
    
    print(f"pytorch configured for training")
    return config
config = setup_pytorch()

Monitoring and Alerting

python
from dataclasses import dataclass
import time
@dataclass
class MetricSnapshot:
    timestamp: float
    metric_name: str
    value: float
    labels: dict
class MLOpsMonitor:
    """Monitor training metrics."""
    
    def __init__(self):
        self.metrics: list[MetricSnapshot] = []
        self.thresholds = {
            "error_rate": 0.05,
            "latency_p99_ms": 1000,
            "data_drift_score": 0.3
        }
    
    def record(self, metric: str, value: float, labels: dict = None):
        snapshot = MetricSnapshot(
            timestamp=time.time(),
            metric_name=metric,
            value=value,
            labels=labels or {}
        )
        self.metrics.append(snapshot)
        self._check_threshold(metric, value)
    
    def _check_threshold(self, metric: str, value: float):
        threshold = self.thresholds.get(metric)
        if threshold and value > threshold:
            logger.warning(f"ALERT: {metric}={value:.3f} exceeds threshold {threshold}")
monitor = MLOpsMonitor()

CI/CD Integration

yaml
.github/workflows/ml-pipeline.yml
name: ML Pipeline
on:
  push:
    paths: ['src/', 'data/']
jobs:
  train-and-evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      
      - name: Install dependencies
        run: pip install -r requirements.txt
      
      - name: Run training
        run: python -m src.distributed_training_setup
        env:
          MLFLOW_TRACKING_URI: ${{ secrets.MLFLOW_URI }}
      
      - name: Check model quality
        run: python -m src.validate_model

Best Practices

Version everything — models, data, configs, and code

Automate testing — catch regressions before production

Monitor continuously — don't wait for users to report issues

Document experiments — future you will thank present you

Use feature flags — control rollouts without code changes

Resources

MLflow documentation: https://mlflow.org/docs

DVC documentation: https://dvc.org/doc

Kubeflow documentation: https://www.kubeflow.org/docs

Made With ML MLOps course: https://madewithml.com