Video-Sys: Distributed Video Generation That Actually Scales

# Install Video-Sys with distributed support
git clone https://github.com/mu-sz/Video-Sys.git
cd Video-Sys
pip install -e ".[distributed]"

# Install Flash Attention 2 for faster attention
pip install flash-attn --no-build-isolation

# Install DeepSpeed for ZeRO optimization
pip install deepspeed

#!/usr/bin/env python
"""
Distributed video generation training with Video-Sys.
Run with: torchrun --nproc_per_node=4 train_distributed.py
"""
import torch
from video_sys import VideoSysConfig, Trainer, VideoDataset
from video_sys.optimizers import AdamW8bit

def main():
    # Configuration
    config = VideoSysConfig(
        # Model
        model_name="stable-video-diffusion-xt",
        resolution=(720, 1280),
        num_frames=120,

        # Distributed
        distributed=True,
        backend="nccl",
        gradient_reduction="reduce_scatter",
        find_unused_parameters=False,

        # Memory optimization
        gradient_checkpointing=True,
        mixed_precision="bf16",
        loss_scale="dynamic",

        # Training
        batch_size_per_gpu=2,
        gradient_accumulation_steps=8,
        num_workers=4,
        pin_memory=True,

        # Optimizer
        optimizer="adamw_8bit",
        learning_rate=1e-5,
        weight_decay=0.01,
        lr_scheduler="cosine",
        warmup_steps=1000,

        # Logging
        log_every_n_steps=10,
        save_every_n_steps=500,
        checkpointing=True
    )

    # Create trainer
    trainer = Trainer(config)

    # Load dataset
    train_dataset = VideoDataset(
        video_dir="/path/to/videos",
        resolution=(720, 1280),
        num_frames=120,
        augmentation=True
    )

    # Train
    trainer.fit(
        train_dataset=train_dataset,
        val_dataset=None,  # Or provide validation dataset
        num_epochs=100,
        resume_from_checkpoint=None  # Or path to checkpoint
    )

if __name__ == "__main__":
    main()

#!/usr/bin/env python
"""
Multi-GPU inference with Video-Sys.
Run with: torchrun --nproc_per_node=8 infer.py
"""
import torch
from video_sys import VideoGenerator
from video_sys.parallel import TensorParallelInference

def generate_video_batch(
    prompts: list[str],
    output_dir: str,
    num_gpus: int = 8
):
    """
    Generate multiple videos in parallel across GPUs.
    """
    # Load model on each GPU
    local_rank = int(os.environ["LOCAL_RANK"])
    torch.cuda.set_device(local_rank)

    generator = VideoGenerator.from_pretrained(
        "your-finetuned-model",
        torch_dtype=torch.float16,
        device=f"cuda:{local_rank}"
    )

    # Convert to tensor parallel
    tp_generator = TensorParallelInference(
        model=generator,
        num_gpus=num_gpus,
        use_flash_attention_2=True
    )

    # Generate videos
    for i, prompt in enumerate(prompts):
        if i % num_gpus == local_rank:
            output = tp_generator.generate(
                prompt=prompt,
                num_frames=120,
                height=1080,
                width=1920,
                chunk_size=24,
                overlap=8,
                guidance_scale=7.5,
                num_inference_steps=50
            )

            # Save
            output_path = f"{output_dir}/video_{i}.mp4"
            tp_generator.save_video(output, output_path)
            print(f"GPU {local_rank}: Generated {output_path}")

if __name__ == "__main__":
    prompts = [
        "A person walking through a rainy city",
        "Drone shot of mountains at sunset",
        "Close-up of a flower blooming",
        # ... more prompts
    ]

    generate_video_batch(prompts, output_dir="./output")

# Monitor GPU utilization
watch -n 1 nvidia-smi

# Check NCCL communication
NCCL_DEBUG=INFO python train.py 2>&1 | grep NCCL

# Profile training
torchrun --nproc_per_node=4 \
    pytorch_profiler.py \
    --output_dir ./profiler_logs

# Check distributed setup
python -m torch.distributed.run \
    --nproc_per_node=4 \
    test_distributed.py