inputs, labels = Variable(inputs).cuda(), Variable(labels).cuda()

C:\\Program Files\\NVIDIA Corporation\\NVSMI>nvidia-smi

from torch.nn import Linear
myLayer = Linear(in_features = 10, out_features = 5, bias = True)

import torch
from torch.nn import Linear
m = Linear(20, 30)
inp = torch.randn(128, 20)
out = m(inp)
print(out.size())
# output:torch.Size([128, 30])

# 查询weight
w = m.weight
# 查询bias
b = m.bias

torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)

torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[30,80], gamma=0.1)

torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1)

import os
import numpy as np

'''
数据集的分配
'''
path = 'train'
# 读取文件夹内的所有文件
files= os.listdir(path) #得到文件夹下的所有文件名称
print(f'Total no of images {len(files)}')
no_of_images = len(files)
# 创建验证集的随机文件索引
shuffle = np.random.permutation(no_of_images)
# 将训练数据集进行划分，2000个样本归入验证集，剩下的样本归入训练集
validation_index = shuffle[:2000]
train_index = shuffle[2000:]
# 创建验证集和训练集文件夹
os.mkdir(os.path.join(path, 'validation'))
os.mkdir(os.path.join(path, 'training'))
for t in ['training', 'validation']:
    for folder in ['dog', 'cat']:
        os.mkdir(os.path.join(path, t, folder))
# 将图片的一小部分复制到validation文件夹      
for i in validation_index:
    folder = files[i].split('/')[-1].split('.')[0]
    image = files[i].split('/')[-1]
    os.rename(os.path.join(path, files[i]), os.path.join(path, 'validation', folder, image))
# 将剩下的图片复制到training文件夹
for i in train_index:
    folder = files[i].split('/')[-1].split('.')[0]
    image = files[i].split('/')[-1]
    os.rename(os.path.join(path, files[i]), os.path.join(path, 'training', folder, image))

print(train.class_to_idx)
# out: {'cat': 0, 'dog': 1}
print(train.classes)
# out: ['cat', 'dog']

# 对张量进行可视化
def imshow(inp):
    inp = inp.numpy().transpose((1, 2, 0))
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    inp = std * inp + mean
    inp = np.clip(inp, 0, 1)
    plt.imshow(inp)

imshow(train[50][0])

import torch 
# 按批加载Pytorch张量
train_data_gen = torch.utils.data.DataLoader(train, batch_size = 64, num_workers = 3)
valid_data_gen = torch.utils.data.DataLoader(valid, batch_size = 64, num_workers = 3)
dataset_sizes = {'train':len(train_data_gen.dataset),'valid':len(valid_data_gen.dataset)}
dataloaders = {'train':train_data_gen,'valid':valid_data_gen}

[Errno 32] Broken pipe

train_data_gen = torch.utils.data.DataLoader(train, batch_size = 64)
valid_data_gen = torch.utils.data.DataLoader(valid, batch_size = 64)

import torchvision.models as models
resnet18 = models.resnet18()
alexnet = models.alexnet()
vgg16 = models.vgg16()
squeezenet = models.squeezenet1_0()
densenet = models.densenet161()
inception = models.inception_v3()
googlenet = models.googlenet()
shufflenet = models.shufflenet_v2_x1_0()
mobilenet = models.mobilenet_v2()
resnext50_32x4d = models.resnext50_32x4d()
wide_resnet50_2 = models.wide_resnet50_2()
mnasnet = models.mnasnet1_0()

# 构建网络架构
import torchvision.models as models
import torch.nn as nn
import torch.optim as op
model_ft = models.resnet18(pretrained = True)
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, 2)

# 检查是否可以在GPU上运行
if torch.cuda.is_available():
    model_ft = model_ft.cuda()

# 损失函数和优化器
learning_rate = 0.001
criterion = nn.CrossEntropyLoss()
optimizer_ft = op.SGD(model_ft.parameters(), lr = learning_rate, momentum = 0.9)
exp_lr_scheduler = op.lr_scheduler.StepLR(optimizer_ft, step_size = 7, gamma = 0.1)

import time
from torch.autograd import Variable
# 训练模型
def train_model(model, criterion, optimizer, scheduler, num_epochs = 25):
    since = time.time()
    
    best_model_wts = model.state_dict()
    best_acc = 0.0
    
    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs-1))
        print('-'*10)
        
        # 每轮都有训练和验证的阶段
        for phase in ['train', 'valid']:
            if phase == 'train':
                scheduler.step()
                model.train(True) # 模型设置为训练模式
            else:
                model.train(False) # 模型设置为评估模式
            
            running_loss = 0.0
            running_correct = 0
            
            # 在数据上迭代
            for data in dataloaders[phase]:
                # 获取输入
                inputs, labels = data
                # 封装成变量
                inputs, labels = Variable(inputs), Variable(labels)
                # 梯度参数清零
                optimizer.zero_grad()
                # 前向
                outputs = model(inputs)
                _, preds = torch.max(outputs.data, 1)
                loss = criterion(outputs, labels)
                # 只在训练阶段反向优化
                if phase == 'train':
                    loss.backward()
                    optimizer.step()
                # 统计
                running_loss += loss.item()
                running_correct += torch.sum(preds == labels.data)
            
            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_correct / dataset_sizes[phase]
            
            print('{} Loss: (:.4f) Acc: (:.4f)'.format(phase, epoch_loss, epoch_acc))
            
            # 深度复刻模型
            if phase == 'valid' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = model.state_dict()
                
        print()
    
    time_elapsed = time.time() - since
    print('Training Complete in {:.0f}m {:.0f}s'.format(time_elapsed//60, time_elapsed%60))
    
    #加载最优权重
    model.load_state_dict(best_model_wts)
    return model

model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler, num_epochs=2)

a = torch.Tensor(([2,3], [4,8], [7,9]))
print('a is {}'.format(a))
print('a size is {}'.format(a.size()))
'''
out:
a is tensor([[2., 3.],
        [4., 8.],
        [7., 9.]])
a size is torch.Size([3, 2])
'''

a[0,1] = 100
print('a is changed to {}'.format(a))
'''
out:
a is changed to tensor([[  2., 100.],
        [  4.,   8.],
        [  7.,   9.]])
'''

numpy_a = a.numpy()
print('conver to numpy is \\n {}'.format(numpy_a))
'''
out:
conver to numpy is 
 [[  2. 100.]
 [  4.   8.]
 [  7.   9.]]
'''

import numpy as np
b = np.array([[2,3], [4,5]])
torch_b = torch.from_numpy(b)
print('from numpy to torch.Tensor is {}'.format(torch_b))
'''
out:
from numpy to torch.Tensor is tensor([[2, 3],
        [4, 5]], dtype=torch.int32)
'''

a = torch.LongTensor(([2,3], [4,8], [7,9]))
print('a is {}'.format(a))
'''
out:
a is tensor([[2, 3],
        [4, 8],
        [7, 9]])
'''

a = torch.zeros((3,2))
print('a is {}'.format(a))
'''
out:
a is tensor([[0., 0.],
        [0., 0.],
        [0., 0.]])
'''
a = torch.randn((3,2))
print('a is {}'.format(a))
'''
out:
a is tensor([[ 0.9284,  0.4900],
        [ 0.3578, -1.0652],
        [ 0.5255, -1.2100]])
'''

X = Variable(torch.Tensor ( [1) ) , requìres_grad=True) 

import torch.utils.data.dataset as dataset
import pandas as pd

class myDataset(dataset):
    def __init__(self, csv_file, txt_file, root_dir, other_file):
        self.csv_data = pd.read_csv(csv_file)
        with open(txt_file, 'r') as f:
            data_list = f.readlines()
        self.txt_data = data_list
        self.root_dir = root_dir
    
    def __len__(self):
        return len(self.csv_data)
    
    def __getitem__(self, idx):
        data = (self.csv_data(idx), self.txt_data[idx])
        return data

optimizer = torch.optim.SGD(model.parameters() , lr=0.01 , momentum=0.9 ) 
# 将模型的参数作为需要更新的参数传入优化器，设定学习率是 0.01 ，动量是 0.9 的随机梯度下降
optimizer.zeros() # 在优化之前需要先将梯度归零
loss.backward() # 反向传播，自动求导得到每个参数的梯度
optimizer.step() # 以通过梯度做一步参数更新

# 保存
torch.save(model, './model/pth')
# 加载
load_model = torch.load('model.pth')

# 保存
torch.save(model.state_dict(), './model_state.pth')
# 加载
model.load_state_dic(torch.load('model_state.pth'))

import torch.nn as nn

class simpleNet(nn.Module):
    def __init__(self, in_dim, n_hidden_1, n_hidden_2, out_dim):
        super(simpleNet, self).__init__()
        self.layer1 = nn.Linear(in_dim, n_hidden_1)
        self.layer2 = nn.Linear(n_hidden_1, n_hidden_2)
        self.layer3 = nn.Linear(n_hidden_2, out_dim)
        
    def forward(self, x):
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        return x

class Activation_Net(nn.Module):
    def __init__(self, in_dim, n_hidden_1, n_hidden_2, out_dim):
        super(Activation_Net, self).__init__()
        self.layer1 = nn.Sequential(nn.Linear(in_dim, n_hidden_1),
                                    nn.ReLU(True))
        self.layer2 = nn.Sequential(nn.Linear(n_hidden_1, n_hidden_2),
                                    nn.ReLU(True))
        self.layer3 = nn.Sequential(nn.Linear(n_hidden_2, out_dim))
    
    def forward(self, x):
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        return x

class Batch_Net(nn.Module):
    def __init__(self, in_dim, n_hidden_1, n_hidden_2, out_dim):
        super(Batch_Net, self).__init__()
        self.layer1 = nn.Sequential(nn.Linear(in_dim, n_hidden_1),
                                    nn.BatchNorm1d(n_hidden_1),
                                    nn.ReLU(True))
        self.layer2 = nn.Sequential(nn.Linear(n_hidden_1, n_hidden_2),
                                    nn.BatchNorm1d(n_hidden_2),
                                    nn.ReLU(True))
        self.layer3 = nn.Sequential(nn.Linear(n_hidden_2, out_dim))
    
    def forward(self, x):
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        return x

import torch
from torch import nn, optim
from torch.autograd import Variable
from torch.utils.data import DataLoader
from torchvision import datasets, transforms

import net

# 设置超参数
batch_size = 64
learning_rate = 1e-2
num_epoches = 20

#数据预处理
data_tf = transforms.Compose([transforms.ToTensor(),
                              transforms.Normalize([0.5], [0.5])])

#下载训练集-MNIST手写数字训练集
train_dataset = datasets.MNIST(root="./data", train=True, transform=data_tf, download=True)
test_dataset = datasets.MNIST(root="./data", train=False, transform=data_tf)
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)

model = net.simpleNet(28*28, 300, 100, 10)
if torch.cuda.is_available():
    model = model.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr = learning_rate)

# 训练模型
def train_model(model, criterion, optimizer, num_epoches):
    for epoch in range(num_epoches):
        print('epoch {}/{}'.format(epoch, num_epoches-1))
        print('-'*10)
        ## training------------------
#        model.train()
        train_loss = 0.0
        train_acc = 0.0
        # 获取数据输入和标签，封装成变量
        for data in train_loader: #获得一个batch样本
            img, label = data # 获得图片和标签
            img = img.view(img.size(0), -1) #将图片进行img的转换
            if torch.cuda.is_available():
                img = Variable(img).cuda()
                label = Variable(label).cuda()
            else:
                img, label = Variable(img), Variable(label)
            # 梯度参数清零
            optimizer.zero_grad()
            # 前向
            out = model(img) # 等价于 out = model.forward(img)
            loss = criterion(out, label)
            _, preds = torch.max(out.data, 1)
            # 反向传播
            loss.backward()
            optimizer.step()
            # 统计
            train_loss += loss.item()
            train_correct = torch.sum(preds == label.data)
            train_acc += train_correct
        print('Train Loss: {:.6f}, Acc: {:.6f}'.format(train_loss/(len(train_loader)), train_acc/(len(train_loader))))
        
        ## evaluation-------------
        model.eval()
        eval_loss = 0.0
        eval_acc = 0.0
        for data in test_loader:
            img, label = data
            img = img.view(img.size(0), -1)
            if torch.cuda.is_available():
                with torch.no_grad():
                    img = Variable(img).cuda()
                    label = Variable(label).cuda()
            else:
                img = Variable(img, volatile = True)
                label = Variable(label, volatile = True)
            out = model(img)
            loss = criterion(out, label)
            eval_loss += loss.item()
            _, preds = torch.max(out.data, 1)
            num_correct = torch.sum(preds == label.data)
            eval_acc += num_correct
        print('Test Loss:{:.6f}, Acc: {:.6f}'.format(eval_loss/(len(test_loader)), eval_acc/(len(test_loader))))

import torch
 
a = torch.Tensor(2,3)
print(a)
# tensor([[0.0000, 0.0000, 0.0000],
#        [0.0000, 0.0000, 0.0000]])
 
print(a.view(1,-1))
# tensor([[0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000]])

train_model(model, criterion, optimizer, num_epoches) 

import torch.nn as nn
import torch.nn.functional as F

class SimpleCNN(nn.Module):
    def __init__(self):
        super(SimpleCNN, self).__init__()
        layer1 = nn.Sequential()
        layer1.add_module('conv1', nn.Conv2d(3, 32, 3, 1, padding = 1))
        layer1.add_module('relu1', nn.ReLU(True))
        layer1.add_module('pool1', nn.MaxPool2d(2,2))
        self.conv1 = layer1
        
        layer2 = nn.Sequential()
        layer2.add_module('conv2', nn.Conv2d(32, 64, 3, 1, padding = 1))
        layer2.add_module('relu2', nn.ReLU(True))
        layer2.add_module('pool2', nn.MaxPool2d(2,2))
        self.conv2 = layer2
        
        layer3 = nn.Sequential()
        layer3.add_module('conv3', nn.Conv2d(64, 128, 3, 1, padding = 1))
        layer3.add_module('relu3', nn.ReLU(True))
        layer3.add_module('pool3', nn.MaxPool2d(2,2))
        self.conv3 = layer3
        
        layer4 = nn.Sequential()
        layer4.add_module('fc1', nn.Linear(2048, 512))
        layer4.add_module('fc_relu1', nn.ReLU(True))
        layer4.add_module('fc2', nn.Linear(512, 64))
        layer4.add_module('fc_relu2', nn.ReLU(True))
        layer4.add_module('fc3', nn.Linear(64, 10))
        self.fc = layer4
    
    def forward(self, x):
        conv1 = self.conv1(x)
        conv2 = self.conv2(conv1)
        conv3 = self.conv3(conv2)
        fc_input = conv3.view(conv3.size(0), -1)
        fc_out = self.fc(fc_input)
        return fc_out

torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True, padding_mode='zeros')

torch.nn.MaxPool2d(kernel_size, stride=None, padding=0, dilation=1, return_indices=False, ceil_mode=False)

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets,transforms
from torch.autograd import Variable
import matplotlib.pyplot as plt

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Sequential(nn.Conv2d(1, 10, kernel_size = 5),
                                   nn.ReLU(True),
                                   nn.MaxPool2d(2, 2))
        self.conv2 = nn.Sequential(nn.Conv2d(10, 20, kernel_size = 5),
                                   nn.Dropout2d(),
                                   nn.ReLU(True),
                                   nn.MaxPool2d(2, 2))
        self.fc1 = nn.Sequential(nn.Linear(320, 50),
                                 nn.ReLU(True),
                                 nn.Dropout2d())
        self.fc2 = nn.Sequential(nn.Linear(50, 10),
                                 nn.LogSoftmax(dim = 1))
    
    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = x.view(-1, 320)
        x = self.fc1(x)
        x = self.fc2(x)
        return x

#数据预处理  
transformation = transforms.Compose([transforms.ToTensor(),
                                     transforms.Normalize((0.1307,), (0.3081,))])

train_dataset = datasets.MNIST('data/',train=True, transform=transformation, download=True)
test_dataset = datasets.MNIST('data/',train=False, transform=transformation, download=True)

train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=32, shuffle=True)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=True)

is_cuda=False
if torch.cuda.is_available():
    is_cuda = True
model = Net()
if is_cuda:
    model.cuda()

optimizer = optim.SGD(model.parameters(),lr=0.01)

def fit(epoch, model, data_loader, phase='training', volatile=False):
    if phase == 'training':
        model.train()
    if phase == 'validation':
        model.eval()
        volatile=True
    running_loss = 0.0
    running_correct = 0
    for batch_idx , (data,target) in enumerate(data_loader):
        if is_cuda:
            data,target = data.cuda(),target.cuda()
        data , target = Variable(data,volatile),Variable(target)
        if phase == 'training':
            optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output,target)
        
        running_loss += F.nll_loss(output,target,reduction = 'sum').item()
        preds = output.data.max(dim=1,keepdim=True)[1]
        running_correct += preds.eq(target.data.view_as(preds)).cpu().sum()
        if phase == 'training':
            loss.backward()
            optimizer.step()
    
    loss = running_loss/len(data_loader.dataset)
    accuracy = 100. * running_correct/len(data_loader.dataset)
    
    print(f'{phase} loss is {loss:{5}.{2}} and {phase} accuracy is {running_correct}/{len(data_loader.dataset)}{accuracy:{10}.{4}}')
    return loss,accuracy

train_losses , train_accuracy = [],[]
val_losses , val_accuracy = [],[]
num_epoches = 20
for epoch in range(num_epoches):
    print('-'*10)
    print('epoch {}/{}'.format(epoch+1, num_epoches))
    epoch_loss, epoch_accuracy = fit(epoch, model, train_loader, phase='training')
    val_epoch_loss , val_epoch_accuracy = fit(epoch, model, test_loader, phase='validation')
    train_losses.append(epoch_loss)
    train_accuracy.append(epoch_accuracy)
    val_losses.append(val_epoch_loss)
    val_accuracy.append(val_epoch_accuracy)

import matplotlib.pyplot as plt
from torchvision import transforms
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
from torch import optim
from torchvision.datasets import ImageFolder

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Sequential(nn.Conv2d(3, 10, kernel_size = 5),
                                   nn.ReLU(),
                                   nn.MaxPool2d(2, 2))
        self.conv2 = nn.Sequential(nn.Conv2d(10, 20, kernel_size = 5),
                                   nn.Dropout2d(),
                                   nn.ReLU(),
                                   nn.MaxPool2d(2, 2))
        self.fc1 = nn.Sequential(nn.Linear(56180, 500),
                                 nn.ReLU(),
                                 nn.Dropout2d())
        self.fc2 = nn.Sequential(nn.Linear(500, 50),
                                 nn.ReLU(),
                                 nn.Dropout2d())
        self.fc3 = nn.Sequential(nn.Linear(50, 2),
                                 nn.LogSoftmax(dim = 1))
    
    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = x.view(x.size(0), -1)
        x = self.fc1(x)
        x = self.fc2(x)
        x = self.fc3(x)
        return x

#数据预处理  
simple_transform = transforms.Compose([transforms.Resize((224,224)),
                                       transforms.ToTensor(),
                                       transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
                                      ])
train = ImageFolder('cat_and_dog/train/training/',simple_transform)
valid = ImageFolder('cat_and_dog/train/validation/',simple_transform)
batch_size = 32
train_data_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, num_workers=0, shuffle=True)
valid_data_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, num_workers=0, shuffle=True)

is_cuda=False
if torch.cuda.is_available():
    is_cuda = True

model = Net()

if is_cuda:
    model.cuda()
optimizer = optim.SGD(model.parameters(),lr=0.01)

model = SimpleCNN()
new_model = nn.Sequential(*list(model.children())[:2])
print(new_model)
'''
out:
Sequential(
  (0): Sequential(
    (conv1): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (relu1): ReLU(inplace=True)
    (pool1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (1): Sequential(
    (conv2): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (relu2): ReLU(inplace=True)
    (pool2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
)
'''

model = SimpleCNN()
for param in model.named_parameters():
    print(param[0])
'''
out:
conv1.conv1.weight
conv1.conv1.bias
conv2.conv2.weight
conv2.conv2.bias
conv3.conv3.weight
conv3.conv3.bias
fc.fc1.weight
fc.fc1.bias
fc.fc2.weight
fc.fc2.bias
fc.fc3.weight
fc.fc3.bias
'''

from torchvision import models

vgg = models.vgg16(pretrained = True)

VGG(
  (features): Sequential(
    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): ReLU(inplace=True)
    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (3): ReLU(inplace=True)
    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (6): ReLU(inplace=True)
    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (8): ReLU(inplace=True)
    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (11): ReLU(inplace=True)
    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (13): ReLU(inplace=True)
    (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (15): ReLU(inplace=True)
    (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (18): ReLU(inplace=True)
    (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (20): ReLU(inplace=True)
    (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (22): ReLU(inplace=True)
    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (25): ReLU(inplace=True)
    (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (27): ReLU(inplace=True)
    (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (29): ReLU(inplace=True)
    (30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (avgpool): AdaptiveAvgPool2d(output_size=(7, 7))
  (classifier): Sequential(
    (0): Linear(in_features=25088, out_features=4096, bias=True)
    (1): ReLU(inplace=True)
    (2): Dropout(p=0.5, inplace=False)
    (3): Linear(in_features=4096, out_features=4096, bias=True)
    (4): ReLU(inplace=True)
    (5): Dropout(p=0.5, inplace=False)
    (6): Linear(in_features=4096, out_features=1000, bias=True)
  )
)

# 冻结features 模型的所有参数
for param in vgg.features.parameters():
    param.requires_grad = False

# 修改最后一层的输出
vgg.classifier[-1] = nn.Linear(in_features=4096, out_features=2, bias=True)

# 构建优化器和损失函数
optimizer = optim.SGD(vgg.classifier.parameters(), lr = 0.0001, momentum = 0.5)
cost = torch.nn.CrossEntropyLoss()

def fit(epoch, model, data_loader, phase='training', volatile=False):
    if phase == 'training':
        model.train()
    if phase == 'validation':
        model.eval()
        volatile=True
    running_loss = 0.0
    running_correct = 0
    for batch_idx , (data,target) in enumerate(data_loader):
        if is_cuda:
            data,target = data.cuda(),target.cuda()
        data, target = Variable(data,volatile),Variable(target)
        if phase == 'training':
            optimizer.zero_grad()
        output = model(data)
        preds = output.data.max(dim=1,keepdim=True)[1]
        loss = cost(output,target)
        
        running_loss += loss.item()
        running_correct += preds.eq(target.data.view_as(preds)).cpu().sum()
        if phase == 'training':
            loss.backward()
            optimizer.step()
    
    loss = running_loss/len(data_loader.dataset)
    accuracy = 100. * running_correct/len(data_loader.dataset)
    
    print(f'{phase} loss is {loss:{5}.{2}} and {phase} accuracy is {running_correct}/{len(data_loader.dataset)}{accuracy:{10}.{4}}')
    return loss,accuracy

train_losses , train_accuracy = [],[]
val_losses , val_accuracy = [],[]
num_epoches = 1
for epoch in range(num_epoches):
    print('-'*10)
    print('epoch {}/{}'.format(epoch+1, num_epoches))
    epoch_loss, epoch_accuracy = fit(epoch, vgg, train_data_loader, phase='training')
    val_epoch_loss , val_epoch_accuracy = fit(epoch, vgg, valid_data_loader, phase='validation')
    train_losses.append(epoch_loss)
    train_accuracy.append(epoch_accuracy)
    val_losses.append(val_epoch_loss)
    val_accuracy.append(val_epoch_accuracy)

vgg = models.vgg16(pretrained = True)
vgg = vgg.cuda()
features = vgg.features

def preconvfeat(dataset,model):
    conv_features = []
    labels_list = []
    for data in dataset:
        inputs,labels = data
        if is_cuda:
            inputs , labels = inputs.cuda(),labels.cuda()       
        inputs , labels = Variable(inputs),Variable(labels)
        output = model(inputs)
        conv_features.extend(output.data.cpu().numpy())
        labels_list.extend(labels.data.cpu().numpy())
    conv_features = np.concatenate([[feat] for feat in conv_features])
    
    return (conv_features,labels_list)

conv_feat_train,labels_train = preconvfeat(train_data_loader,features)
conv_feat_val,labels_val = preconvfeat(valid_data_loader,features)

class My_dataset(Dataset):
    def __init__(self,feat,labels):
        self.conv_feat = feat
        self.labels = labels
    
    def __len__(self):
        return len(self.conv_feat)
    
    def __getitem__(self,idx):
        return self.conv_feat[idx],self.labels[idx]

train_feat_dataset = My_dataset(conv_feat_train,labels_train)
val_feat_dataset = My_dataset(conv_feat_val,labels_val)

batch_size = 5
train_feat_loader = DataLoader(train_feat_dataset,batch_size=batch_size,shuffle=True)
val_feat_loader = DataLoader(val_feat_dataset,batch_size=batch_size,shuffle=True)

def data_gen(conv_feat,labels,batch_size=64,shuffle=True):
    labels = np.array(labels)
    if shuffle:
        index = np.random.permutation(len(conv_feat))
        conv_feat = conv_feat[index]
        labels = labels[index]
    for idx in range(0,len(conv_feat),batch_size):
        yield(conv_feat[idx:idx+batch_size],labels[idx:idx+batch_size])

train_batches = data_gen(conv_feat_train,labels_train)
val_batches = data_gen(conv_feat_val,labels_val)

class autoencoder(nn.Module):
    def __init__(self):
        super(autoencoder, self).__init__()
        
        self.encoder = nn.Sequential(
            nn.Linear(28*28, 128),
            nn.ReLU(True),
            nn.Linear(128, 64),
            nn.ReLU(True),
            nn.Linear(64, 12),
            nn.ReLU(True),
            nn.Linear(12, 3) # 输出的 code 是 3 维，便于可视化
        )
        
        self.decoder = nn.Sequential(
            nn.Linear(3, 12),
            nn.ReLU(True),
            nn.Linear(12, 64),
            nn.ReLU(True),
            nn.Linear(64, 128),
            nn.ReLU(True),
            nn.Linear(128, 28*28),
            nn.Tanh()
        )

    def forward(self, x):
        encode = self.encoder(x)
        decode = self.decoder(encode)
        return encode, decode

net = autoencoder()
    
criterion = nn.MSELoss(size_average=False)
optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)

def to_img(x):
    '''
    定义一个函数将最后的结果转换回图片
    '''
    x = 0.5 * (x + 1.)
    x = x.clamp(0, 1)
    x = x.view(x.shape[0], 1, 28, 28)
    return x

# 开始训练自动编码器
for e in range(100):
    for im, _ in train_data:
        im = im.view(im.shape[0], -1)
        im = Variable(im)
        # 前向传播
        _, output = net(im)
        loss = criterion(output, im) / im.shape[0] # 平均
        # 反向传播
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    
    if (e+1) % 20 == 0: # 每 20 次，将生成的图片保存一下
        print('epoch: {}, Loss: {:.4f}'.format(e + 1, loss.data[0]))
        pic = to_img(output.cpu().data)
        if not os.path.exists('./simple_autoencoder'):
            os.mkdir('./simple_autoencoder')
        save_image(pic, './simple_autoencoder/image_{}.png'.format(e + 1))

class discriminator(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(nn.Linear(784, 256),
                                 nn.LeakyReLU(0.2),
                                 nn.Linear(256, 256),
                                 nn.LeakyReLU(0.2),
                                 nn.Linear(256, 1)
                                 )
    
    def forward(self, x):
        x = self.net(x)
        return x

class generator(nn.Module):
    def __init__(self, input_size):
        super(generator, self).__init__()
        self.gen =nn.Sequential(
                nn.Linear(input_size, 1024),
                nn.ReLU(True),
                nn.Linear(1024, 1024),
                nn.ReLU(True),
                nn.Linear(1024, 784),
                nn.Tanh()
                )
    
    def forward(self, x):
        x = self.gen(x)
        return x

bce_loss = nn.BCEWithLogitsLoss()

def discriminator_loss(logits_real, logits_fake): # 判别器的 loss
    size = logits_real.shape[0]
    true_labels = Variable(torch.ones(size, 1)).float().cuda()
    false_labels = Variable(torch.zeros(size, 1)).float().cuda()
    loss = bce_loss(logits_real, true_labels) + bce_loss(logits_fake, false_labels)
    return loss

def generator_loss(logits_fake): # 生成器的 loss  
    size = logits_fake.shape[0]
    true_labels = Variable(torch.ones(size, 1)).float().cuda()
    loss = bce_loss(logits_fake, true_labels)
    return loss

# 使用 adam 来进行训练，学习率是 3e-4, beta1 是 0.5, beta2 是 0.999
def get_optimizer(net):
    optimizer = torch.optim.Adam(net.parameters(), lr=3e-4, betas=(0.5, 0.999))
    return optimizer

def train_a_gan(D_net, G_net, D_optimizer, G_optimizer, discriminator_loss, generator_loss, show_every=250, 
                noise_size=NOISE_DIM, num_epochs=10):
    iter_count = 0
    for epoch in range(num_epochs):
        for x, _ in train_data:
            bs = x.shape[0]
            # 判别网络
            real_data = Variable(x).view(bs, -1).cuda() # 真实数据
            logits_real = D_net(real_data) # 判别网络得分
            
            sample_noise = (torch.rand(bs, noise_size) - 0.5) / 0.5 # -1 ~ 1 的均匀分布
            g_fake_seed = Variable(sample_noise).cuda()
            fake_images = G_net(g_fake_seed) # 生成的假的数据
            logits_fake = D_net(fake_images) # 判别网络得分

            d_total_error = discriminator_loss(logits_real, logits_fake) # 判别器的 loss
            D_optimizer.zero_grad()
            d_total_error.backward()
            D_optimizer.step() # 优化判别网络
            
            # 生成网络
            g_fake_seed = Variable(sample_noise).cuda()
            fake_images = G_net(g_fake_seed) # 生成的假的数据

            gen_logits_fake = D_net(fake_images)
            g_error = generator_loss(gen_logits_fake) # 生成网络的 loss
            G_optimizer.zero_grad()
            g_error.backward()
            G_optimizer.step() # 优化生成网络

            if (iter_count % show_every == 0):
                print('Iter: {}, D: {:.4}, G:{:.4}'.format(iter_count, d_total_error.item(), g_error.item()))
                imgs_numpy = deprocess_img(fake_images.data.cpu().numpy())
                show_images(imgs_numpy[0:16])
                plt.show()
                print()
            iter_count += 1

def ls_discriminator_loss(scores_real, scores_fake):
    loss = 0.5 * ((scores_real - 1) ** 2).mean() + 0.5 * (scores_fake ** 2).mean()
    return loss

def ls_generator_loss(scores_fake):
    loss = 0.5 * ((scores_fake - 1) ** 2).mean()
    return loss

class build_dc_classifier(nn.Module):
    def __init__(self):
        super(build_dc_classifier, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(1, 32, 5, 1),
            nn.LeakyReLU(0.01),
            nn.MaxPool2d(2, 2),
            nn.Conv2d(32, 64, 5, 1),
            nn.LeakyReLU(0.01),
            nn.MaxPool2d(2, 2)
        )
        self.fc = nn.Sequential(
            nn.Linear(1024, 1024),
            nn.LeakyReLU(0.01),
            nn.Linear(1024, 1)
        )
        
    def forward(self, x):
        x = self.conv(x)
        x = x.view(x.shape[0], -1)
        x = self.fc(x)
        return x

class build_dc_generator(nn.Module): 
    def __init__(self, noise_dim=NOISE_DIM):
        super(build_dc_generator, self).__init__()
        self.fc = nn.Sequential(
            nn.Linear(noise_dim, 1024),
            nn.ReLU(True),
            nn.BatchNorm1d(1024),
            nn.Linear(1024, 7 * 7 * 128),
            nn.ReLU(True),
            nn.BatchNorm1d(7 * 7 * 128)
        )
        
        self.conv = nn.Sequential(
            nn.ConvTranspose2d(128, 64, 4, 2, padding=1),
            nn.ReLU(True),
            nn.BatchNorm2d(64),
            nn.ConvTranspose2d(64, 1, 4, 2, padding=1),
            nn.Tanh()
        )
        
    def forward(self, x):
        x = self.fc(x)
        x = x.view(x.shape[0], 128, 7, 7) # reshape 通道是 128，大小是 7x7
        x = self.conv(x)
        return x