Dropout та комбіновані стратегії регуляризації

import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error

# Генерація даних
np.random.seed(42)
n_samples = 20
X_train = np.sort(np.random.uniform(0, 1, n_samples))
y_train = np.sin(2 * np.pi * X_train) + np.random.normal(0, 0.1, n_samples)

# Тестові дані (справжня функція)
X_test = np.linspace(0, 1, 100)
y_test_true = np.sin(2 * np.pi * X_test)

# Апроксимація поліномами різних ступенів
degrees = [1, 4, 15]
models = {}

for degree in degrees:
    poly_features = PolynomialFeatures(degree=degree)
    X_train_poly = poly_features.fit_transform(X_train.reshape(-1, 1))
    X_test_poly = poly_features.transform(X_test.reshape(-1, 1))
    
    model = LinearRegression()
    model.fit(X_train_poly, y_train)
    
    y_train_pred = model.predict(X_train_poly)
    y_test_pred = model.predict(X_test_poly)
    
    train_mse = mean_squared_error(y_train, y_train_pred)
    test_mse = mean_squared_error(y_test_true, y_test_pred)
    
    models[degree] = {
        'model': model,
        'y_test_pred': y_test_pred,
        'train_mse': train_mse,
        'test_mse': test_mse
    }

# Візуалізація
fig, axes = plt.subplots(1, 3, figsize=(18, 5))

for idx, degree in enumerate(degrees):
    ax = axes[idx]
    
    # Справжня функція
    ax.plot(X_test, y_test_true, 'g--', linewidth=2, alpha=0.7, label='Справжня функція')
    
    # Тренувальні точки
    ax.scatter(X_train, y_train, color='blue', s=80, alpha=0.7, 
               edgecolor='black', linewidth=1.5, label='Тренувальні дані', zorder=5)
    
    # Апроксимація
    ax.plot(X_test, models[degree]['y_test_pred'], 'r-', linewidth=2.5, 
            label=f'Поліном степеня {degree}')
    
    ax.set_xlabel('x', fontsize=12)
    ax.set_ylabel('y', fontsize=12)
    ax.set_ylim([-1.5, 1.5])
    ax.grid(alpha=0.3)
    ax.legend(fontsize=10)
    
    # Статистика
    train_mse = models[degree]['train_mse']
    test_mse = models[degree]['test_mse']
    
    if degree == 1:
        title = f'Underfitting (ступінь {degree})'
        status = '❌ Занадто проста'
    elif degree == 4:
        title = f'Хороша генералізація (ступінь {degree})'
        status = '✅ Баланс'
    else:
        title = f'Overfitting (ступінь {degree})'
        status = '❌ Надто складна'
    
    ax.set_title(f'{title}\nTrain MSE: {train_mse:.4f}, Test MSE: {test_mse:.4f}',
                 fontsize=11, fontweight='bold')
    ax.text(0.5, -1.3, status, ha='center', fontsize=12, 
            bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.5))

plt.tight_layout()
plt.show()

import torch
import torch.nn as nn

# Модель з Dropout
class MLPWithDropout(nn.Module):
    def __init__(self):
        super(MLPWithDropout, self).__init__()
        self.fc1 = nn.Linear(100, 256)
        self.dropout1 = nn.Dropout(p=0.5)
        
        self.fc2 = nn.Linear(256, 128)
        self.dropout2 = nn.Dropout(p=0.5)
        
        self.fc3 = nn.Linear(128, 10)
        self.relu = nn.ReLU()
    
    def forward(self, x):
        x = self.relu(self.fc1(x))
        x = self.dropout1(x)  # Dropout після активації
        
        x = self.relu(self.fc2(x))
        x = self.dropout2(x)
        
        x = self.fc3(x)  # Останній шар БЕЗ Dropout
        return x

model = MLPWithDropout()
x = torch.randn(4, 100)  # Батч з 4 прикладів

print("🔬 Експеримент: Dropout у train vs eval режимах\n")

# Train mode
model.train()
out1_train = model(x)
out2_train = model(x)

print("📊 Training Mode (Dropout активний):")
print(f"   1-й forward: {out1_train[0, :3].detach().numpy()}")
print(f"   2-й forward: {out2_train[0, :3].detach().numpy()}")
print(f"   Різниця: {torch.norm(out1_train - out2_train).item():.6f}")
print(f"   Виходи РІЗНІ через випадкову маску! ✅\n")

# Eval mode
model.eval()
out1_eval = model(x)
out2_eval = model(x)

print("📊 Evaluation Mode (Dropout вимкнений):")
print(f"   1-й forward: {out1_eval[0, :3].detach().numpy()}")
print(f"   2-й forward: {out2_eval[0, :3].detach().numpy()}")
print(f"   Різниця: {torch.norm(out1_eval - out2_eval).item():.6f}")
print(f"   Виходи ІДЕНТИЧНІ (детермінований inference)! ✅")

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
import time

# 5 різних моделей
class CNN_Baseline(nn.Module):
    """Без регуляризації"""
    def __init__(self):
        super().__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 64, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
            nn.Conv2d(64, 128, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
            nn.Conv2d(128, 256, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
        )
        self.classifier = nn.Sequential(
            nn.Linear(256*4*4, 512), nn.ReLU(),
            nn.Linear(512, 10)
        )
    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), -1)
        return self.classifier(x)

class CNN_BatchNormOnly(nn.Module):
    """Лише BatchNorm"""
    def __init__(self):
        super().__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 64, 3, padding=1), nn.BatchNorm2d(64), nn.ReLU(), nn.MaxPool2d(2),
            nn.Conv2d(64, 128, 3, padding=1), nn.BatchNorm2d(128), nn.ReLU(), nn.MaxPool2d(2),
            nn.Conv2d(128, 256, 3, padding=1), nn.BatchNorm2d(256), nn.ReLU(), nn.MaxPool2d(2),
        )
        self.classifier = nn.Sequential(
            nn.Linear(256*4*4, 512), nn.BatchNorm1d(512), nn.ReLU(),
            nn.Linear(512, 10)
        )
    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), -1)
        return self.classifier(x)

class CNN_DropoutOnly(nn.Module):
    """Лише Dropout"""
    def __init__(self):
        super().__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 64, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
            nn.Conv2d(64, 128, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
            nn.Conv2d(128, 256, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
        )
        self.classifier = nn.Sequential(
            nn.Linear(256*4*4, 512), nn.ReLU(), nn.Dropout(0.5),
            nn.Linear(512, 10)
        )
    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), -1)
        return self.classifier(x)

class CNN_BN_Dropout(nn.Module):
    """BatchNorm + Dropout (зменшений)"""
    def __init__(self):
        super().__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 64, 3, padding=1), nn.BatchNorm2d(64), nn.ReLU(), nn.MaxPool2d(2),
            nn.Conv2d(64, 128, 3, padding=1), nn.BatchNorm2d(128), nn.ReLU(), nn.MaxPool2d(2),
            nn.Conv2d(128, 256, 3, padding=1), nn.BatchNorm2d(256), nn.ReLU(), nn.MaxPool2d(2),
        )
        self.classifier = nn.Sequential(
            nn.Linear(256*4*4, 512), nn.BatchNorm1d(512), nn.ReLU(), nn.Dropout(0.3),
            nn.Linear(512, 10)
        )
    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), -1)
        return self.classifier(x)

# Трансформації
transform_simple = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))
])

transform_augmented = transforms.Compose([
    transforms.RandomHorizontalFlip(),
    transforms.RandomCrop(32, padding=4),
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))
])

# Датасети
train_simple = datasets.CIFAR10('./data', train=True, download=True, transform=transform_simple)
train_augmented = datasets.CIFAR10('./data', train=True, download=True, transform=transform_augmented)
test_dataset = datasets.CIFAR10('./data', train=False, download=True, transform=transform_simple)

def train_model(model, train_loader, test_loader, epochs=20, lr=0.01, weight_decay=0):
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    model = model.to(device)
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=weight_decay)
    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs)
    
    history = {'train_acc': [], 'test_acc': []}
    
    for epoch in range(epochs):
        # Train
        model.train()
        train_correct, train_total = 0, 0
        for images, labels in train_loader:
            images, labels = images.to(device), labels.to(device)
            outputs = model(images)
            loss = criterion(outputs, labels)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            _, predicted = outputs.max(1)
            train_total += labels.size(0)
            train_correct += predicted.eq(labels).sum().item()
        
        # Test
        model.eval()
        test_correct, test_total = 0, 0
        with torch.no_grad():
            for images, labels in test_loader:
                images, labels = images.to(device), labels.to(device)
                outputs = model(images)
                _, predicted = outputs.max(1)
                test_total += labels.size(0)
                test_correct += predicted.eq(labels).sum().item()
        
        train_acc = 100. * train_correct / train_total
        test_acc = 100. * test_correct / test_total
        history['train_acc'].append(train_acc)
        history['test_acc'].append(test_acc)
        scheduler.step()
    
    return history

# Експеримент
configs = [
    ("Baseline (без регул.)", CNN_Baseline(), train_simple, 0.01, 0),
    ("BatchNorm only", CNN_BatchNormOnly(), train_simple, 0.1, 0),
    ("Dropout only", CNN_DropoutOnly(), train_simple, 0.01, 0),
    ("BN + Dropout", CNN_BN_Dropout(), train_simple, 0.1, 0),
    ("Повний стек", CNN_BN_Dropout(), train_augmented, 0.1, 5e-4),
]

results = {}
test_loader = DataLoader(test_dataset, batch_size=128, shuffle=False, num_workers=2)

print("🚀 Експеримент: порівняння регуляризаційних стратегій\n")

for name, model, train_data, lr, wd in configs:
    print(f"📊 Навчання: {name}")
    torch.manual_seed(42)
    train_loader = DataLoader(train_data, batch_size=128, shuffle=True, num_workers=2)
    history = train_model(model, train_loader, test_loader, epochs=20, lr=lr, weight_decay=wd)
    results[name] = history
    
    final_train = history['train_acc'][-1]
    final_test = history['test_acc'][-1]
    gap = final_train - final_test
    print(f"   Фінал: Train={final_train:.2f}%, Test={final_test:.2f}%, Gap={gap:.2f}%\n")

print("🎉 Експеримент завершено!")

import matplotlib.pyplot as plt

fig, axes = plt.subplots(1, 2, figsize=(16, 6))

colors = {
    "Baseline (без регул.)": '#ef4444',
    "BatchNorm only": '#f59e0b',
    "Dropout only": '#3b82f6',
    "BN + Dropout": '#8b5cf6',
    "Повний стек": '#22c55e'
}

# Графік 1: Test Accuracy
ax1 = axes[0]
for name, history in results.items():
    ax1.plot(range(1, 21), history['test_acc'], 
             linewidth=2.5, label=name, color=colors[name], marker='o', markersize=4)
ax1.set_xlabel('Епоха', fontsize=13)
ax1.set_ylabel('Test Accuracy (%)', fontsize=13)
ax1.set_title('Точність на тестовому датасеті', fontsize=15, fontweight='bold')
ax1.legend(fontsize=10, loc='lower right')
ax1.grid(alpha=0.3)
ax1.set_ylim([60, 90])

# Графік 2: Overfitting Gap
ax2 = axes[1]
final_gaps = {}
for name, history in results.items():
    gap = history['train_acc'][-1] - history['test_acc'][-1]
    final_gaps[name] = gap

names = list(final_gaps.keys())
gaps = list(final_gaps.values())
bars = ax2.barh(names, gaps, color=[colors[n] for n in names], alpha=0.7)

for i, (bar, gap) in enumerate(zip(bars, gaps)):
    ax2.text(gap + 0.5, i, f'{gap:.1f}%', va='center', fontsize=11, fontweight='bold')

ax2.set_xlabel('Train-Test Accuracy Gap (%)', fontsize=13)
ax2.set_title('Міра перенавчання (менше = краще)', fontsize=15, fontweight='bold')
ax2.grid(axis='x', alpha=0.3)
ax2.invert_yaxis()

plt.tight_layout()
plt.show()

# Підсумкова таблиця
print("\n📊 Порівняльна таблиця:\n")
print(f"{'Стратегія':<25} | {'Train Acc':<10} | {'Test Acc':<10} | {'Gap':<8} | {'Overfit?':<10}")
print("-" * 80)
for name, history in results.items():
    train = history['train_acc'][-1]
    test = history['test_acc'][-1]
    gap = train - test
    overfit = "❌ Так" if gap > 15 else "⚠️ Помірно" if gap > 10 else "✅ Ні"
    print(f"{name:<25} | {train:<10.2f} | {test:<10.2f} | {gap:<8.2f} | {overfit:<10}")