【PyTorch学习笔记】6.循环神经网络

痛定思痛。 2022-09-04 09:49 390阅读 0赞

文章目录

  • 40.时间序列表示
  • 41.循环神经网络
  • 42.RNN Layer使用
    • 42.1 nn.RNN
    • 42.2 nn.RNNCell
    1. 时间序列预测
  • 44.RNN训练难题
  • 45.LSTM Layer使用
    • 45.1 nn.LSTM
    • 45.2 nn.LSTMCell
  • 46.情感分类实战

根据龙良曲Pytorch学习视频整理，视频链接：
【计算机-AI】PyTorch学这个就够了！
(好课推荐)深度学习与PyTorch入门实战——主讲人龙良曲

40.时间序列表示

Sequence representation

  • [seq_len, feature_len]
  • [word, word_vec]
    one-hot
  • [words, word vec]
    sparse
    high-dim
    semantic similarity

word2vec vs Glove

  1. import torch
  2. import torch.nn as nn
  3. from torchnlp.word_to_vector import GloVe
  5. word_to_idx = { 'hello': 0, 'world': 1}
  6. lookup_tensor = torch.tensor([word_to_idx['hello']], dtype=torch.long)
  7. embeds = nn.Embedding(2, 5) # 2 words in vocab, 5 dimensional embeddings
  8. hello_embed = embeds(lookup_tensor)
  9. print(hello_embed)
  10. """ tensor([[ 0.2565, -0.2827, -0.0259, -1.9533, 0.8330]], grad_fn=<EmbeddingBackward>) """
  12. vectors = GloVe()
  13. print(vectors['hello']) # 2GB文件

torchnlp包安装 pip install pytorch-nlp

41.循环神经网络

Weight sharing
Consistent memory

42.RNN Layer使用

input dim, hidden dim

  1. rnn = nn.RNN(100, 10)   # word_dim, memory/h
  2. print(rnn._parameters.keys())   # odict_keys(['weight_ih_l0', 'weight_hh_l0', 'bias_ih_l0', 'bias_hh_l0'])
  3. print(rnn.weight_hh_l0.shape, rnn.weight_ih_l0.shape)   # torch.Size([10, 10]) torch.Size([10, 100])
  4. print(rnn.bias_hh_l0.shape, rnn.bias_ih_l0.shape)   # torch.Size([10]) torch.Size([10])

42.1 nn.RNN

  • .__init__
    (input_size, hidden_size, num_layers)
  • out, ht = forward(x, h0)
    x: [seq_len, b, word_vec]
    ho/ht: [num_layers, b, h_dim]
    out: [seq_len, b, h_dim]

Single layer RNN

  1. rnn = nn.RNN(input_size=100, hidden_size=20, num_layers=1)
  2. print(rnn)  # RNN(100, 20)
  3. x = torch.randn(10, 3, 100)
  4. out, h = rnn(x, torch.zeros(1, 3, 20))
  5. print(out.shape, h.shape)   # torch.Size([10, 3, 20]) torch.Size([1, 3, 20])

h是最后一个时间序列所有的memory状态；out是所有时间序列的最后一个memory状态

2 layer RNN

  1. rnn = nn.RNN(100, 10, num_layers=2)   # word_dim, memory/h
  2. print(rnn._parameters.keys())   # odict_keys(['weight_ih_l0', 'weight_hh_l0', 'bias_ih_l0', 'bias_hh_l0', 'weight_ih_l1', 'weight_hh_l1', 'bias_ih_l1', 'bias_hh_l1'])
  3. print(rnn.weight_hh_l0.shape, rnn.weight_ih_l0.shape)   # torch.Size([10, 10]) torch.Size([10, 100])
  4. print(rnn.bias_hh_l0.shape, rnn.bias_ih_l0.shape)   # torch.Size([10]) torch.Size([10])

[T, b, h_dim], [layers, b, h_dim]

  1. rnn = nn.RNN(input_size=100, hidden_size=20, num_layers=4)
  2. print(rnn)  # RNN(100, 20, num_layers=4)
  3. x = torch.randn(10, 3, 100)
  4. out, h = rnn(x)
  5. print(out.shape, h.shape)   # torch.Size([10, 3, 20]) torch.Size([4, 3, 20])

42.2 nn.RNNCell

  • __init__
    (input_size, hidden_size, num_layers)
  • ht = rnncell(xt, ht_1)
    x: [b, word_vec]
    ht_1/ht: [num_layers, b, h_dim]
    out = torch.stack([h1, h2,…ht])

Functional

  1. cell1 = nn.RNNCell(100, 30)
  2. cell2 = nn.RNNCell(30, 20)
  3. h1 = torch.zeros(3, 30)
  4. h2 = torch.zeros(3, 20)
  5. for xt in x:
  6.     h1 = cell1(xt, h1)
  7.     h2 = cell2(h1, h2)
  8. print(h1.shape)     # torch.Size([3, 30])
  9. print(h2.shape)     # torch.Size([3, 20])

43. 时间序列预测

  1. import numpy as np
  2. import matplotlib.pyplot as plt
  3. import torch
  4. import torch.nn as nn
  5. import torch.optim as optim
  7. num_time_steps = 50
  8. input_size = 1
  9. hidden_size = 16
  10. output_size = 1
  11. lr = 0.01
  13. class Net(nn.Module):
  14.     def __init__(self):
  15.         super(Net, self).__init__()
  17.         self.rnn = nn.RNN(
  18.             input_size=input_size,
  19.             hidden_size=hidden_size,
  20.             num_layers=1,
  21.             batch_first=True
  22.         )
  23.         for p in self.rnn.parameters():
  24.             nn.init.normal_(p, mean=0.0, std=0.001)
  26.         self.linear = nn.Linear(hidden_size, output_size)
  28.     def forward(self, x, hidden_prev):
  29.         out, hidden_prev = self.rnn(x, hidden_prev)
  30.         # [b, seq, h] => [seq, h] b=1
  31.         out = out.view(-1, hidden_size)
  32.         # [seq, h] => [seq, 1]
  33.         out = self.linear(out)
  34.         # [seq, 1] => [1, seq, 1]
  35.         out = out.unsqueeze(dim=0)
  36.         return out, hidden_prev
  38. model = Net()
  39. criterion = nn.MSELoss()
  40. optimizer = optim.Adam(model.parameters(), lr)
  42. hidden_prev = torch.zeros(1, 1, hidden_size)
  44. for iter in range(6000):
  45.     start = np.random.randint(3, size=1)[0]
  46.     time_steps = np.linspace(start, start + 10, num_time_steps)
  47.     data = np.sin(time_steps)
  48.     data = data.reshape(num_time_steps, 1)
  49.     x = torch.tensor(data[:-1]).float().view(1, num_time_steps - 1, 1)
  50.     y = torch.tensor(data[1:]).float().view(1, num_time_steps - 1, 1)
  52.     output, hidden_prev = model(x, hidden_prev)
  53.     hidden_prev = hidden_prev.detach()  # 改requirse_grad为false，切断反向传播
  55.     loss = criterion(output, y)
  56.     model.zero_grad()
  57.     loss.backward()
  58.     # 检查梯度信息
  59.     # for p in model.parameters():
  60.     # print(p.grad.norm())
  61.     # torch.nn.utils.clip_grad_norm_(p, 10)
  62.     optimizer.step()
  64.     if iter % 100 == 0:
  65.         print('Iteration: {} loss {}'.format(iter, loss.item()))
  67. start = np.random.randint(3, size=1)[0]
  68. time_steps = np.linspace(start, start + 10, num_time_steps)
  69. data = np.sin(time_steps)
  70. data = data.reshape(num_time_steps, 1)
  71. x = torch.tensor(data[:-1]).float().view(1, num_time_steps - 1, 1)
  72. y = torch.tensor(data[1:]).float().view(1, num_time_steps - 1, 1)
  74. predictions = []
  75. input = x[:, 0, :]  # [b, seq, feature] => [b, feature]
  76. for _ in range(x.shape[1]):
  77.     input = input.view(1, 1, 1)
  78.     (pred, hidden_prev) = model(input, hidden_prev)
  79.     input = pred    # tensor([[[0.8720]]], grad_fn=<UnsqueezeBackward0>)
  80.     predictions.append(pred.detach().numpy().ravel()[0])
  82. x = x.data.numpy().ravel()
  83. plt.scatter(time_steps[:-1], x.ravel(), s=90)
  84. plt.plot(time_steps[:-1], x.ravel())
  86. plt.scatter(time_steps[1:], predictions)
  87. plt.show()

在这里插入图片描述

44.RNN训练难题

  • Gradient Exploding
    解决办法：Gradient Clipping
  • Gradient Vanishing
    解决办法：LSTM

45.LSTM Layer使用

45.1 nn.LSTM

  • __init__
    (input_size, hidden_size, num_layers)

LSTM.forward()

  • out, (ht, ct) = lstm(x, [ht_0, ct_0])
    x: [seq_len, b, word_vec]
    h/c: [num_layers, b, h_dim]
    out: [seq_len, b, h_dim]

    lstm = nn.LSTM(input_size=100, hidden_size=20, num_layers=4)
    print(lstm) # LSTM(100, 20, num_layers=4)
    x = torch.rand(10, 3, 100)
    out, (h, c) = lstm(x)
    print(out.shape, h.shape, c.shape) # torch.Size([10, 3, 20]) torch.Size([4, 3, 20]) torch.Size([4, 3, 20])

45.2 nn.LSTMCell

  • __init__
    (input_size, hidden_size, num_layers)

LSTMCell.forward()

  • ht, ct = lstmcell(xt, [ht_0, ct_0])
    xt: [b, word_vec]
    h/c: [ b, h_dim]

Single layer

  1. cell = nn.LSTMCell(input_size=100, hidden_size=20)
  2. print(cell)  # LSTMCell(100, 20)
  3. h = torch.zeros(3, 20)
  4. c = torch.zeros(3, 20)
  5. for xt in x:
  6.     h, c = cell(xt, [h, c])
  7. print(h.shape, c.shape)     # torch.Size([3, 20]) torch.Size([3, 20])

Two layers

  1. cell1 = nn.LSTMCell(input_size=100, hidden_size=30)
  2. cell2 = nn.LSTMCell(input_size=30, hidden_size=20)
  3. h1 = torch.zeros(3, 30)
  4. c1 = torch.zeros(3, 30)
  5. h2 = torch.zeros(3, 20)
  6. c2 = torch.zeros(3, 20)
  7. for xt in x:
  8.     h1, c1 = cell1(xt, [h1, c1])
  9.     h2, c2 = cell2(h1, [h2, c2])
  10. print(h2.shape, c2.shape)   # torch.Size([3, 20]) torch.Size([3, 20])

46.情感分类实战

Google CoLab

  • Continuous 12 hours
  • free K80 for GPU
  • no need to cross GFW

没有谷歌账号不能使用Colaboratory，码一下

  1. import torch
  2. import torch.nn as nn
  3. import numpy as np
  4. from torchtext import data, datasets
  6. # load dataset
  7. TEXT = data.Field(tokenize='spacy')
  8. LABEL = data.LabelField(dtype=torch.float)
  9. train_data, test_data = datasets.IMDB.split(TEXT, LABEL)
  11. print('len of train data:', len(train_data))
  12. print('len of test data:', len(test_data))
  13. print(train_data.examples[15].text)
  14. print(train_data.examples[15].label)
  16. class RNN(nn.Module):
  17.     def __init__(self, vocab_size, embedding_dim, hidden_dim):
  18.         super(RNN, self).__init__()
  20.         # [0-10001] => [100]
  21.         self.embedding = nn.Embedding(vocab_size, embedding_dim)
  22.         # [100] => [256]
  23.         self.rnn = nn.LSTM(embedding_dim, hidden_dim, num_layers=2, bidirectional=True, dropout=0.5)
  24.         # [25*2] => [1]
  25.         self.fc = nn.Linear(hidden_dim*2, 1)
  26.         self.dropout = nn.Dropout(0.5)
  28.     def forward(self, x):
  29.         # [seq, b, 1] => [seq, b, 100]
  30.         embedding = self.dropout(self.embedding(x))
  31.         # output: [seq, b, hid_dim*2]
  32.         # hidden/h, [num_layer*2, b, hid_dim]
  33.         # cell/c: [num_layer*2, b, hid_dim]
  34.         output, (hidden, cell) = self.rnn(embedding)
  35.         # [num_layer*2, b, hid_dim] => 2 of [b, hid_dim] => [b, hid_dim*2]
  36.         hidden = torch.cat([hidden[-2], hidden[-1]], dim=1)
  37.         # [b, hid_dim*2] => [b, 1]
  38.         hidden = self.dropout(hidden)
  39.         out = self.fc(hidden)
  40.         return out
  42. # load word embedding
  43. rnn = RNN(len(TEXT.vocab), 100, 256)
  44. pretrained_embedding = TEXT.vocab.vectors
  45. print('pretrained_embedding:', pretrained_embedding.shape)
  46. rnn.embedding.weight.data.copy_(pretrained_embedding)
  47. print('embedding layer inited.')
  50. def binary_acc(preds, y):
  51.     preds = torch.round((torch.sigmoid(preds)))
  52.     correct = torch.eq(preds, y).float()
  53.     acc = correct.sum() / len(correct)
  54.     return acc
  56. def eval(rnn, iterator, criteon):
  57.     avg_acc = []
  58.     rnn.eval()
  59.     with torch.no_grad():
  60.         for batch in iterator:
  61.             # [b, 1] => [b]
  62.             pred = rnn(batch.text).squeeze(1)
  63.             loss = criteon(pred, batch.label)
  64.             acc = binary_acc(pred, batch.label).item()
  65.             avg_acc.append(acc)
  66.     avg_acc = np.array(avg_acc).mean()
  67.     print('>>>test:', avg_acc)
  69. def train(rnn, iterator, optimizer, criteon):
  70.     avg_acc = []
  71.     rnn.train()
  73.     for i, batch in enumerate(iterator):
  74.         # [seq, b] => [b, 1] => [b]
  75.         pred = rnn(batch.text).squeeze(1)
  76.         loss = criteon(pred, batch.label)
  77.         acc = binary_acc(pred, batch.label).item()
  78.         avg_acc.append(acc)
  80.         optimizer.zero_grad()
  81.         loss.backward()
  82.         optimizer.step()
  84. def main():
  85.     pass
  87. if __name__ == '__main__':
  88.     main()

只是抄了一下代码，并没有动脑子

发表评论

表情：
评论列表 （有 0 条评论，390人围观）

还没有评论，来说两句吧...

相关阅读

    相关 pytorch-循环神经网络

    循环神经网络 本节介绍循环神经网络，下图展示了如何基于循环神经网络实现语言模型。我们的目的是基于当前的输入与过去的输入序列，预测序列的下一个字符。循环神经网络引入一个隐藏

    清疚清疚/ 0 赞/ 224 阅读