" class="reference-link">

日萌社

人工智能AI：Keras PyTorch MXNet TensorFlow PaddlePaddle 深度学习实战（不定时更新）

tensorflow 2.0 深度学习（第一部分 part1）

tensorflow 2.0 深度学习（第一部分 part2）

tensorflow 2.0 深度学习（第一部分 part3）

tensorflow 2.0 深度学习（第二部分 part1）

tensorflow 2.0 深度学习（第二部分 part2）

tensorflow 2.0 深度学习（第二部分 part3）

tensorflow 2.0 深度学习 (第三部分 卷积神经网络 part1)

tensorflow 2.0 深度学习 (第三部分 卷积神经网络 part2)

tensorflow 2.0 深度学习（第四部分 循环神经网络）

tensorflow 2.0 深度学习（第五部分 GAN生成神经网络 part1）

tensorflow 2.0 深度学习（第五部分 GAN生成神经网络 part2）

tensorflow 2.0 深度学习（第六部分 强化学习）

基础知识

梯度传播相关原理

" class="reference-link">梯度传播原理

" class="reference-link">梯度弥散、梯度爆炸

>>> import tensorflow as tf
>>> W = tf.ones([2,2]) # 任意创建某矩阵
>>> W
<tf.Tensor: id=2, shape=(2, 2), dtype=float32, numpy=
array([[1., 1.],
       [1., 1.]], dtype=float32)>
>>> tf.linalg.eigh(W) 
(<tf.Tensor: id=3, shape=(2,), dtype=float32, numpy=array([0., 2.], dtype=float32)>, 
 <tf.Tensor: id=4, shape=(2, 2), dtype=float32, numpy=
array([[-0.70710677,  0.70710677],
       [ 0.70710677,  0.70710677]], dtype=float32)>)
>>> tf.linalg.eigh(W)[0] # 计算特征值：此时的W矩阵的最大特征值为2，那么下面演示的是最大特征值大于1时的矩阵相乘
<tf.Tensor: id=5, shape=(2,), dtype=float32, numpy=array([0., 2.], dtype=float32)>
>>> tf.linalg.eigh(W)[1]
<tf.Tensor: id=8, shape=(2, 2), dtype=float32, numpy=
array([[-0.70710677,  0.70710677],
       [ 0.70710677,  0.70710677]], dtype=float32)>
>>> val = [W]
>>> val
[<tf.Tensor: id=2, shape=(2, 2), dtype=float32, numpy=
array([[1., 1.],
       [1., 1.]], dtype=float32)>]
>>> for i in range(10): # 矩阵相乘 n 次方
...  val.append([val[-1]@W])
...
>>> val
[<tf.Tensor: id=2, shape=(2, 2), dtype=float32, numpy=
array([[1., 1.],
       [1., 1.]], dtype=float32)>, 
[<tf.Tensor: id=9, shape=(2, 2), dtype=float32, numpy=
array([[2., 2.],
       [2., 2.]], dtype=float32)>], 
[<tf.Tensor: id=11, shape=(1, 2, 2), dtype=float32, numpy=
array([[[4., 4.],
        [4., 4.]]], dtype=float32)>], 
[<tf.Tensor: id=13, shape=(1, 1, 2, 2), dtype=float32, numpy=
array([[[[8., 8.],
         [8., 8.]]]], dtype=float32)>], 
[<tf.Tensor: id=15, shape=(1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[16., 16.],
          [16., 16.]]]]], dtype=float32)>], 
[<tf.Tensor: id=17, shape=(1, 1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[[32., 32.],
           [32., 32.]]]]]], dtype=float32)>], 
[<tf.Tensor: id=19, shape=(1, 1, 1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[[[64., 64.],
            [64., 64.]]]]]]], dtype=float32)>], 
[<tf.Tensor: id=21, shape=(1, 1, 1, 1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[[[[128., 128.],
             [128., 128.]]]]]]]], dtype=float32)>], 
[<tf.Tensor: id=23, shape=(1, 1, 1, 1, 1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[[[[[256., 256.],
              [256., 256.]]]]]]]]], dtype=float32)>], 
[<tf.Tensor: id=25, shape=(1, 1, 1, 1, 1, 1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[[[[[[512., 512.],
               [512., 512.]]]]]]]]]], dtype=float32)>], 
[<tf.Tensor: id=27, shape=(1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[[[[[[[1024., 1024.],
                [1024., 1024.]]]]]]]]]]], dtype=float32)>]]
>>> l = list(map(lambda x:x,val)) #map遍历val中的每个array到list中作为list的每个元素
>>> len(l)
11
>>> norm = list(map(lambda x:tf.norm(x).numpy(),val)) #tf.norm(x)默认执行L2的tf.norm(x,ord=2)等同于tf.sqrt(tf.reduce_sum(tf.square(a)))
>>> len(norm)
11 
>>> norm
[2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0, 2048.0]
>>> m = list(map(lambda x:tf.sqrt(tf.reduce_sum(tf.square(x))),val)) #tf.norm(x)默认执行L2的tf.norm(x,ord=2)等同于tf.sqrt(tf.reduce_sum(tf.square(a)))
>>> m
[<tf.Tensor: id=130, shape=(), dtype=float32, numpy=2.0>, 
<tf.Tensor: id=135, shape=(), dtype=float32, numpy=4.0>, 
<tf.Tensor: id=140, shape=(), dtype=float32, numpy=8.0>, 
<tf.Tensor: id=145, shape=(), dtype=float32, numpy=16.0>, 
<tf.Tensor: id=150, shape=(), dtype=float32, numpy=32.0>, 
<tf.Tensor: id=155, shape=(), dtype=float32, numpy=64.0>, 
<tf.Tensor: id=160, shape=(), dtype=float32, numpy=128.0>, 
<tf.Tensor: id=165, shape=(), dtype=float32, numpy=256.0>, 
<tf.Tensor: id=170, shape=(), dtype=float32, numpy=512.0>, 
<tf.Tensor: id=175, shape=(), dtype=float32, numpy=1024.0>, 
<tf.Tensor: id=180, shape=(), dtype=float32, numpy=2048.0>]
>>> from matplotlib import pyplot as plt
>>> plt.plot(range(1,12),norm)
[<matplotlib.lines.Line2D object at 0x0000025773D525C8>]
>>> plt.show()
>>> eigenvalues = tf.linalg.eigh(W)
>>> eigenvalues #此时的 W 矩阵最大特征值是 0.8
(<tf.Tensor: id=14, shape=(2,), dtype=float32, numpy=array([0. , 0.8], dtype=float32)>, 
 <tf.Tensor: id=15, shape=(2, 2), dtype=float32, 
  numpy=array([[-0.70710677,  0.70710677],
               [ 0.70710677,  0.70710677]], dtype=float32)>)

>>> import tensorflow as tf
>>> from matplotlib import pyplot as plt
>>> W = tf.ones([2,2])*0.4 # 任意创建某矩阵
>>> W
<tf.Tensor: id=185, shape=(2, 2), dtype=float32, numpy=
array([[0.4, 0.4],
       [0.4, 0.4]], dtype=float32)>
>>> tf.linalg.eigh(W)
(<tf.Tensor: id=186, shape=(2,), dtype=float32, numpy=array([0. , 0.8], dtype=float32)>, 
 <tf.Tensor: id=187, shape=(2, 2), dtype=float32, numpy=
array([[-0.70710677,  0.70710677],
       [ 0.70710677,  0.70710677]], dtype=float32)>)
>>> eigenvalues = tf.linalg.eigh(W)[0] # 计算特征值：此时的W矩阵的最大特征值为0.8，那么下面演示的是最大特征值小于1时的矩阵相乘
>>> eigenvalues
<tf.Tensor: id=188, shape=(2,), dtype=float32, numpy=array([0. , 0.8], dtype=float32)>
>>> val = [W]
>>> for i in range(10):
...  val.append([val[-1]@W])
...
>>> val
[<tf.Tensor: id=185, shape=(2, 2), dtype=float32, numpy=
array([[0.4, 0.4],
       [0.4, 0.4]], dtype=float32)>, 
[<tf.Tensor: id=190, shape=(2, 2), dtype=float32, numpy=
array([[0.32000002, 0.32000002],
       [0.32000002, 0.32000002]], dtype=float32)>], 
[<tf.Tensor: id=192, shape=(1, 2, 2), dtype=float32, numpy=
array([[[0.256, 0.256],
        [0.256, 0.256]]], dtype=float32)>], 
[<tf.Tensor: id=194, shape=(1, 1, 2, 2), dtype=float32, numpy=
array([[[[0.20480001, 0.20480001],
         [0.20480001, 0.20480001]]]], dtype=float32)>], 
[<tf.Tensor: id=196, shape=(1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[0.16384001, 0.16384001],
          [0.16384001, 0.16384001]]]]], dtype=float32)>], 
[<tf.Tensor: id=198, shape=(1, 1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[[0.13107201, 0.13107201],
           [0.13107201, 0.13107201]]]]]], dtype=float32)>], 
[<tf.Tensor: id=200, shape=(1, 1, 1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[[[0.10485762, 0.10485762],
            [0.10485762, 0.10485762]]]]]]], dtype=float32)>], 
[<tf.Tensor: id=202, shape=(1, 1, 1, 1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[[[[0.08388609, 0.08388609],
             [0.08388609, 0.08388609]]]]]]]], dtype=float32)>], 
[<tf.Tensor: id=204, shape=(1, 1, 1, 1, 1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[[[[[0.06710888, 0.06710888],
              [0.06710888, 0.06710888]]]]]]]]], dtype=float32)>], 
[<tf.Tensor: id=206, shape=(1, 1, 1, 1, 1, 1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[[[[[[0.0536871, 0.0536871],
               [0.0536871, 0.0536871]]]]]]]]]], dtype=float32)>], 
[<tf.Tensor: id=208, shape=(1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2), dtype=float32, numpy=
array([[[[[[[[[[[0.04294968, 0.04294968],
                [0.04294968, 0.04294968]]]]]]]]]]], dtype=float32)>]]
>>> norm = list(map(lambda x:tf.norm(x).numpy(),val))
>>> norm
[0.8, 0.64000005, 0.512, 0.40960002, 0.32768002, 0.26214403, 0.20971523, 0.16777219, 0.13421775, 0.107374206, 0.08589937]
>>> plt.plot(range(1,12),norm)
[<matplotlib.lines.Line2D object at 0x000002577AACD888>]
>>> plt.show()

梯度裁剪(clip_by_value、clip_by_norm、clip_by_global_norm)

>>> a=tf.random.uniform([2,2]) * 5
>>> a
<tf.Tensor: id=356, shape=(2, 2), dtype=float32, numpy=
array([[1.5947735, 2.6630645],
       [3.2025905, 3.669492 ]], dtype=float32)>
>>> tf.norm(a)
<tf.Tensor: id=361, shape=(), dtype=float32, numpy=5.7755494>
>>> # 按范数方式裁剪
... b = tf.clip_by_norm(a, 5) #tf.clip_by_norm(梯度张量W, max) 等同于 梯度张量W/tf.norm(梯度张量W)*max
>>> b
<tf.Tensor: id=378, shape=(2, 2), dtype=float32, numpy=
array([[1.3806249, 2.3054643],
       [2.772542 , 3.176747 ]], dtype=float32)>
>>> tf.norm(b)
<tf.Tensor: id=383, shape=(), dtype=float32, numpy=5.0>
>>> a1 = a/tf.norm(a)*5 #梯度张量W/tf.norm(梯度张量W)*max 等同于 tf.clip_by_norm(梯度张量W, max) 
>>> tf.norm(a1)
<tf.Tensor: id=409, shape=(), dtype=float32, numpy=5.0000005>

>>> import tensorflow as tf
>>> w1=tf.random.normal([3,3]) # 创建梯度张量 1
>>> w2=tf.random.normal([3,3]) # 创建梯度张量 2
>>> w1
<tf.Tensor: id=480, shape=(3, 3), dtype=float32, numpy=
array([[-0.6576707 , -0.90442616,  0.34582302],
       [-1.7328235 , -2.1596272 ,  0.4980505 ],
       [-0.05848425,  0.3528551 ,  0.11080291]], dtype=float32)>
>>> w2
<tf.Tensor: id=486, shape=(3, 3), dtype=float32, numpy=
array([[ 0.08453857, -0.2454122 , -0.67583424],
       [ 0.32974792, -1.3895415 , -0.3052706 ],
       [-0.37487552, -0.9419025 , -0.86943924]], dtype=float32)>
>>> # 计算“还没实现等比例缩放的”所有参数的梯度W的范数global_norm
... global_norm = tf.math.sqrt(tf.norm(w1)**2+tf.norm(w2)**2)
>>> global_norm
<tf.Tensor: id=502, shape=(), dtype=float32, numpy=3.7236474>
>>> # 根据 max_norm=2 裁剪，返回“对所有参数的梯度W的范数实现了等比例缩放的新的”梯度W 和 原有的“还没实现等比例缩放的”所有参数的梯度W的范数global_norm
... (ww1,ww2),global_norm = tf.clip_by_global_norm([w1,w2],2)
>>> ww1
<tf.Tensor: id=522, shape=(3, 3), dtype=float32, numpy=
array([[-0.35324   , -0.4857743 ,  0.18574423],
       [-0.93071294, -1.1599525 ,  0.2675068 ],
       [-0.03141234,  0.18952121,  0.0595131 ]], dtype=float32)>
>>> ww2
<tf.Tensor: id=523, shape=(3, 3), dtype=float32, numpy=
array([[ 0.04540632, -0.1318128 , -0.3629958 ],
       [ 0.17711017, -0.74633354, -0.16396321],
       [-0.20134856, -0.5059032 , -0.46698257]], dtype=float32)>
>>> global_norm
<tf.Tensor: id=510, shape=(), dtype=float32, numpy=3.7236474>
>>> # 计算"实现了等比例缩放(裁剪后)的"所有参数的梯度W的范数global_norm
... global_norm2 = tf.math.sqrt(tf.norm(ww1)**2+tf.norm(ww2)**2)
>>> print(global_norm, global_norm2) #tf.Tensor(4.073522, shape=(), dtype=float32) tf.Tensor(2.0000002, shape=(), dtype=float32)
>>> # 计算“还没实现等比例缩放的”所有参数的梯度W的范数global_norm
... global_norm = tf.math.sqrt(tf.norm(w1)**2+tf.norm(w2)**2)
>>> global_norm
<tf.Tensor: id=502, shape=(), dtype=float32, numpy=3.7236474>
>>> max_norm = 2 
#tf.clip_by_global_norm([w1,w2],max_norm) 等同于 w1 * max_norm / max(global_norm, max_norm) 和 w2 * max_norm / max(global_norm, max_norm) 的效果
>>> ww1 = w1 * max_norm / max(global_norm, max_norm) 
>>> ww2 = w2 * max_norm / max(global_norm, max_norm)  
>>> ww1
<tf.Tensor: id=586, shape=(3, 3), dtype=float32, numpy=
array([[-0.35324004, -0.48577434,  0.18574424],
       [-0.930713  , -1.1599526 ,  0.2675068 ],
       [-0.03141235,  0.18952121,  0.05951311]], dtype=float32)>
>>> ww2
<tf.Tensor: id=591, shape=(3, 3), dtype=float32, numpy=
array([[ 0.04540633, -0.13181281, -0.36299583],
       [ 0.17711018, -0.74633354, -0.16396323],
       [-0.20134856, -0.5059032 , -0.4669826 ]], dtype=float32)>
>>> global_norm2 = tf.math.sqrt(tf.norm(ww1)**2+tf.norm(ww2)**2)
>>> global_norm2
<tf.Tensor: id=607, shape=(), dtype=float32, numpy=1.9999998>
>>> import tensorflow as tf
>>> w1=tf.random.normal([3,3])
>>> w2=tf.random.normal([3,3])
>>> w1
<tf.Tensor: id=5, shape=(3, 3), dtype=float32, numpy=
array([[-0.3745235 , -0.54776704, -0.6978908 ],
       [-0.48667282, -1.9662677 ,  1.2693951 ],
       [-1.6218463 , -1.3147658 ,  1.1985897 ]], dtype=float32)>
#tf.norm(a)默认为执行L2范数的tf.norm(a. ord=2)，等同于tf.sqrt(tf.reduce_sum(tf.square(a)))
#计算裁剪前的网络参数θ的总范数global_norm：所有裁剪前的网络参数θ的L2范数tf.norm(θ)的平方和，然后开方sqrt
>>> global_norm=tf.math.sqrt(tf.norm(w1)**2+tf.norm(w2)**2)
>>> global_norm #4.7265425
<tf.Tensor: id=27, shape=(), dtype=float32, numpy=4.7265425>
#通过tf.clip_by_global_norm([θ],MAX_norm)裁剪后，网络参数的梯度组的总范数缩减到MAX_norm=2
#clip_by_global_norm返回裁剪后的List[参数θ] 和 裁剪前的梯度总范数和global_norm 的2个对象
>>> (ww1,ww2),global_norm=tf.clip_by_global_norm([w1,w2],2) #梯度裁剪一般在计算出梯度后、梯度更新之前进行
>>> ww1
<tf.Tensor: id=47, shape=(3, 3), dtype=float32, numpy=
array([[-0.15847673, -0.2317834 , -0.29530713],
       [-0.20593184, -0.832011  ,  0.53713477],
       [-0.6862717 , -0.556333  ,  0.50717396]], dtype=float32)>
>>> ww2
<tf.Tensor: id=48, shape=(3, 3), dtype=float32, numpy=
array([[ 0.03117203, -0.7264457 ,  0.32293826],
       [ 0.5894358 ,  0.87403387,  0.04680141],
       [ 0.0015509 ,  0.15240058,  0.05759645]], dtype=float32)>
>>> global_norm
<tf.Tensor: id=35, shape=(), dtype=float32, numpy=4.7265425>
#计算裁剪后的网络参数θ的总范数global_norm：所有裁剪后的网络参数θ的L2范数tf.norm(θ)的平方和，然后开方sqrt
>>> global_norm2 = tf.math.sqrt(tf.norm(ww1)**2+tf.norm(ww2)**2)
>>> global_norm2
<tf.Tensor: id=64, shape=(), dtype=float32, numpy=1.9999998>
>>> print(global_norm, global_norm2)
tf.Tensor(4.7265425, shape=(), dtype=float32) tf.Tensor(1.9999998, shape=(), dtype=float32)

import  tensorflow as tf
from    tensorflow import keras
from    tensorflow.keras import datasets, layers, optimizers
import  os
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
print(tf.__version__)
(x, y), _ = datasets.mnist.load_data()
x = tf.convert_to_tensor(x, dtype=tf.float32) / 50. #标准化/归一化
y = tf.convert_to_tensor(y)
y = tf.one_hot(y, depth=10) #真实标签one-hot化
print('x:', x.shape, 'y:', y.shape)
#构建批量大小和epoch训练次数
train_db = tf.data.Dataset.from_tensor_slices((x,y)).batch(128).repeat(30)
x,y = next(iter(train_db))  #获取生成器并调用next遍历第一个批量大小的数据
print('sample:', x.shape, y.shape)
# print(x[0], y[0])
def main():
    # 784 => 512 第一层权重[输入神经元数, 输出神经元数]，偏置[输出神经元数]
    w1, b1 = tf.Variable(tf.random.truncated_normal([784, 512], stddev=0.1)), tf.Variable(tf.zeros([512]))
    # 512 => 256 第二层权重[输入神经元数, 输出神经元数]，偏置[输出神经元数]
    w2, b2 = tf.Variable(tf.random.truncated_normal([512, 256], stddev=0.1)), tf.Variable(tf.zeros([256]))
    # 256 => 10 第三层权重[输入神经元数, 输出神经元数]，偏置[输出神经元数]
    w3, b3 = tf.Variable(tf.random.truncated_normal([256, 10], stddev=0.1)), tf.Variable(tf.zeros([10]))
    optimizer = optimizers.SGD(lr=0.01) #SGD随机梯度下降优化算法
    #每次遍历训练集生成器的一个批量大小数据
    for step, (x,y) in enumerate(train_db):
        # [b, 28, 28] => [b, 784] 展平
        x = tf.reshape(x, (-1, 784))
        #构建梯度记录环境
        with tf.GradientTape() as tape:
            # layer1.
            h1 = x @ w1 + b1
            h1 = tf.nn.relu(h1)
            # layer2
            h2 = h1 @ w2 + b2
            h2 = tf.nn.relu(h2)
            # output
            out = h2 @ w3 + b3
            # out = tf.nn.relu(out)
            # compute loss
            # [b, 10] - [b, 10] 均方差mse = mean(sum(y-out)^2) 预测值与真实值之差的平方的平均值
            loss = tf.square(y-out)
            # [b, 10] => [b] 计算每个样本的平均误差
            loss = tf.reduce_mean(loss, axis=1)
            # [b] => scalar 总误差除以样本数
            loss = tf.reduce_mean(loss)
        #1.求导，tape.gradient(y,[参数θ])求参数θ相对于y的梯度信息
        #  dy_dw = tape.gradient(y, [w])
        #2.通过tape.gradient(loss,[参数θ])函数求得网络参数θ的梯度信息
        #  grads = tape.gradient(loss, [w1, b1, w2, b2, w3, b3]) 
        #  compute gradients。根据loss 求w1, w2, w3, b1, b2, b3的梯度值 用于后面继续更新对应的模型参数θ。
        grads = tape.gradient(loss, [w1, b1, w2, b2, w3, b3])
        # print('==before==')
        # for g in grads:  #计算所有裁剪前的网络参数θ的梯度值的L2范数tf.norm(a)，等同于tf.norm(a. ord=2)、tf.sqrt(tf.reduce_sum(tf.square(a)))
        #     print(tf.norm(g)) #tf.norm(a)默认为执行L2范数tf.norm(a. ord=2)，等同于tf.sqrt(tf.reduce_sum(tf.square(a)))
        #通过tf.clip_by_global_norm([θ],MAX_norm)裁剪后，网络参数的梯度组的总范数缩减到MAX_norm=15
        #clip_by_global_norm返回裁剪后的List[参数θ] 和 裁剪前的梯度总范数和global_norm 的2个对象
        grads,  _ = tf.clip_by_global_norm(grads, 15) #梯度裁剪一般在计算出梯度后、梯度更新之前进行
        # print('==after==')
        # for g in grads:  #计算所有裁剪后的网络参数θ的梯度值的L2范数tf.norm(a)，等同于tf.norm(a. ord=2)、tf.sqrt(tf.reduce_sum(tf.square(a)))
        #     print(tf.norm(g)) #tf.norm(a)默认为执行L2的tf.norm(a. ord=2)，等同于tf.sqrt(tf.reduce_sum(tf.square(a)))
        #优化器规则，根据 模型参数θ = θ - lr * grad 更新网络参数
        # update w' = w - lr*grad
        optimizer.apply_gradients(zip(grads, [w1, b1, w2, b2, w3, b3]))
        if step % 100 == 0:
            print(step, 'loss:', float(loss))
if __name__ == '__main__':
    main()
W = tf.ones([2,2]) #任意创建某矩阵
eigenvalues = tf.linalg.eigh(W)[0] # 计算特征值
eigenvalues
val = [W]
# 矩阵相乘n次方
for i in range(10): 
    val.append([val[-1]@W])
# 计算L2范数
norm = list(map(lambda x:tf.norm(x).numpy(),val))
plt.plot(range(1,12),norm)
plt.xlabel('n times')
plt.ylabel('L2-norm')
plt.savefig('w_n_times_1.svg')
W = tf.ones([2,2])*0.4 # 任意创建某矩阵
eigenvalues = tf.linalg.eigh(W)[0] # 计算特征值
print(eigenvalues)
val = [W]
for i in range(10):
    val.append([val[-1]@W])
norm = list(map(lambda x:tf.norm(x).numpy(),val))
plt.plot(range(1,12),norm)
plt.xlabel('n times')
plt.ylabel('L2-norm')
plt.savefig('w_n_times_0.svg')
a=tf.random.uniform([2,2])
tf.clip_by_value(a,0.4,0.6) # 梯度值裁剪
a=tf.random.uniform([2,2]) * 5
# 按范数方式裁剪
b = tf.clip_by_norm(a, 5)
tf.norm(a),tf.norm(b)
w1=tf.random.normal([3,3]) # 创建梯度张量1
w2=tf.random.normal([3,3]) # 创建梯度张量2
# 计算global norm
global_norm=tf.math.sqrt(tf.norm(w1)**2+tf.norm(w2)**2) 
# 根据global norm和max norm=2裁剪
(ww1,ww2),global_norm=tf.clip_by_global_norm([w1,w2],2)
# 计算裁剪后的张量组的global norm
global_norm2 = tf.math.sqrt(tf.norm(ww1)**2+tf.norm(ww2)**2)
print(global_norm, global_norm2)
with tf.GradientTape() as tape:
  logits = model(x) # 前向传播
  loss = criteon(y, logits) # 误差计算
# 计算梯度值
grads = tape.gradient(loss, model.trainable_variables)
grads, _ = tf.clip_by_global_norm(grads, 25) # 全局梯度裁剪
# 利用裁剪后的梯度张量更新参数
optimizer.apply_gradients(zip(grads, model.trainable_variables))

各种循环神经网络

" class="reference-link">SimpleRNNCell

import  tensorflow as tf
from    tensorflow import keras
from    tensorflow.keras import datasets, layers, optimizers
#状态向量特征长度h=3
cell = layers.SimpleRNNCell(3)  
#不需要指定序列长度s，默认为None即可。时间戳t的输入特征长度f/词向量特征长度f=4
cell.build(input_shape=(None,4)) 
#SimpleRNNCell内部维护了3个张量，kernel为Wxh(时间戳t的输入特征长度f, 状态向量特征长度h)，shape为(4, 3)，
#recurrent_kernel为Whh(状态向量特征长度h, 状态向量特征长度h)，shape为(3, 3)，bias为偏置b(状态向量特征长度h,)，shape为(3,)
#前向计算：ht = Wxh * Xt + Whh * ht-1 + b = (4, 3) * Xt + (3, 3) * ht-1 + (3,)
cell.trainable_variables 
#[<tf.Variable 'kernel:0' shape=(4, 3) dtype=float32, numpy=
#            array([[ 0.08301067, -0.46076387,  0.15347064],
#                   [-0.1890313 ,  0.60186946, -0.85642815],
#                   [ 0.5622045 , -0.07674742, -0.80009407],
#                   [ 0.5321919 , -0.44575736, -0.3680237 ]], dtype=float32)>, 
# <tf.Variable 'recurrent_kernel:0' shape=(3, 3) dtype=float32, numpy=
#            array([[ 0.9972923 ,  0.07062273, -0.02049942],
#                   [-0.06413236,  0.69885963, -0.7123779 ],
#                   [ 0.03598386, -0.71176374, -0.7014966 ]], dtype=float32)>, 
# <tf.Variable 'bias:0' shape=(3,) dtype=float32, numpy=
#            array([0., 0., 0.], dtype=float32)>]
#初始化状态向量h0(批量大小b,状态向量特征长度h)，shape=(4, 64)
h0 = [tf.zeros([4, 64])] 
#输入序列数据(b,s,f)，即序列数量b=4，序列长度s=80，时间戳t的输入特征长度f/词向量特征长度f=100，shape=(4, 80, 100)
x = tf.random.normal([4, 80, 100]) 
#按 x[:,i,:] 或 for xt in tf.unstack(x, axis=1) 方式可以从[序列数量b, 序列长度s, 词向量特征长度f]中获取 s个[序列数量b, 词向量特征长度f]，
#即一个序列中有N个单词数，那么就遍历N次，每次遍历同时获取批量多个序列中的同一个时间戳上的单词，当遍历次数等于序列长度s时才算完成一次网络层的前向计算
#即每次需要把(批量大小b, 词向量特征长度f)输入到SimpleRNNCell中，序列数量b=4，输入特征f=100，shape=(4, 100)
#即每次输入数据到SimpleRNNCell中时，输入所有序列数据中同一个时间戳t的数据，对于长度为t+1的训练来说，需要遍历等同于序列长度s的次数才算完成一次网络层的前向计算
xt = x[:,0,:]  #取第一个时间戳的输入x0(4, 100) 即xt为(批量大小b, 词向量特征长度f)
# 构建输入特征 f=100，序列长度 s=80，状态向量长度h=64 的Cell
cell = layers.SimpleRNNCell(64)
#SimpleRNNCell内部维护了3个张量，kernel为Wxh(时间戳t的输入特征长度f, 状态向量特征长度h)，shape为(100, 64)，
#recurrent_kernel为Whh(状态向量特征长度h, 状态向量特征长度h)，shape为(64, 64)，bias为偏置b(状态向量特征长度h,)，shape为(64,)
#前向计算，输入批量序列中同一个时间戳t的字符数据xt和状态向量h0：ht = Wxh * Xt + Whh * ht-1 + b = (100, 64) * (4, 100) + (64, 64) * (4, 64) + (64,)
#对于长度为t+1的训练来说，需要遍历等同于序列长度s的次数才算完成一次网络层的前向计算
out, h1 = cell(xt, h0) 
#输出ot和状态向量ht的shape都为(批量大小b, 状态向量特征长度h)，即(4, 64)。
#并且两者id一致，代表两者都指向为同一个内存空间的变量值，即状态向量ht直接作为输出ot。
print(out.shape, h1[0].shape) #(4, 64) (4, 64)
print(id(out), id(h1[0])) #1863500704760 1863500704760
h = h0
# 在序列长度的维度解开输入，得到 xt:[批量大小b, 时间戳t的输入特征长度f]
# 即一个序列中有N个单词数，那么就遍历N次，每次遍历同时获取批量多个序列中的同一个时间戳上的单词，当遍历次数等于序列长度s时才算完成一次网络层的前向计算
# 对于长度为t+1的训练来说，需要遍历等同于序列长度s的次数才算完成一次网络层的前向计算
for xt in tf.unstack(x, axis=1):
    # 前向计算，输入批量序列中同一个时间戳t的字符数据xt和状态向量h：ht = Wxh * Xt + Whh * ht-1 + b 
    out, h = cell(xt, h) 
# 最终输出可以聚合每个时间戳上的输出，也可以只取最后时间戳的输出
# 最终输出可以把每个时间戳t上的输出ot进行聚合再输出，或者只输出最后一个时间戳t的输出ot
out = out

多层SimpleRNNCell网络

import  tensorflow as tf
from    tensorflow import keras
from    tensorflow.keras import datasets, layers, optimizers
x = tf.random.normal([4,80,100])
# 构建 2 个 Cell,先 cell0,后 cell1
cell0 = layers.SimpleRNNCell(64)
cell1 = layers.SimpleRNNCell(64)
h0 = [tf.zeros([4,64])] # cell0 的初始状态向量
h1 = [tf.zeros([4,64])] # cell1 的初始状态向量
for xt in tf.unstack(x, axis=1):
    print("xt:",xt.shape) #(4, 100) [序列数量b, 词向量特征长度f]
    #第一层RNN 每次接收的是 [序列数量b, 词向量特征长度f]的序列数据，也即是批量序列数据中同一个时间戳的单词的词向量特征长度f
    #第二层RNN 每次接收的是 第一层RNN输出的 [序列数量b, 状态向量特征长度h]，也即是每个时间戳上的网络输出Ot，而网络输出Ot可以是直接等于状态向量ht，
    #或者也可以是对状态向量ht做一个线性变换Ot=Who*ht得到每个时间戳上的网络输出Ot。
    # xtw 作为输入，输出为 out0 
    out0, h0 = cell0(xt, h0)
    print("out0:",out0.shape, h0[0].shape) #(4, 64) (4, 64) [序列数量b, 状态向量特征长度h]
    # 上一个 cell 的输出 out0 作为本 cell 的输入
    out1, h1 = cell1(out0, h1)
    print("out1:",out1.shape, h1[0].shape) #(4, 64) (4, 64) [序列数量b, 状态向量特征长度h]

import  os
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
import  tensorflow as tf
import  numpy as np
from    tensorflow import keras
from    tensorflow.keras import layers
tf.random.set_seed(22)
np.random.seed(22) 
assert tf.__version__.startswith('2.')
batchsz = 128
total_words = 10000
max_review_len = 80
embedding_len = 100
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
# x_train:[b, 80]
# x_test: [b, 80]
x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True)
db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batchsz, drop_remainder=True)
print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)
class MyRNN(keras.Model):
    def __init__(self, units):
        super(MyRNN, self).__init__()
        # [b, 64]
        self.state0 = [tf.zeros([batchsz, units])]
        self.state1 = [tf.zeros([batchsz, units])]
        # 将文本转换为嵌入表示
        # [b, 80] => [b, 80, 100]
        self.embedding = layers.Embedding(total_words, embedding_len, input_length=max_review_len)
        # [b, 80, 100] , h_dim: 64
        # RNN: cell1 ,cell2, cell3
        # SimpleRNN
        self.rnn_cell0 = layers.SimpleRNNCell(units, dropout=0.5)
        self.rnn_cell1 = layers.SimpleRNNCell(units, dropout=0.5)
        # fc, [b, 80, 100] => [b, 64] => [b, 1]
        self.outlayer = layers.Dense(1)
    def call(self, inputs, training=None):
        """
        net(x) net(x, training=True) :train mode
        net(x, training=False): test
        :param inputs: [b, 80]
        :param training:
        :return:
        """
        # [b, 80]
        x = inputs
        # embedding: [b, 80] => [b, 80, 100]
        x = self.embedding(x)
        # rnn cell compute
        # [b, 80, 100] => [b, 64]
        state0 = self.state0
        state1 = self.state1
        for word in tf.unstack(x, axis=1): # word: [b, 100]
            # h1 = x*wxh+h0*whh
            # out0: [b, 64]
            out0, state0 = self.rnn_cell0(word, state0, training)
            # out1: [b, 64]
            out1, state1 = self.rnn_cell1(out0, state1, training)
        # out: [b, 64] => [b, 1]
        x = self.outlayer(out1)
        # p(y is pos|x)
        prob = tf.sigmoid(x)
        return prob
def main():
    units = 64
    epochs = 4
    model = MyRNN(units)
    model.compile(optimizer = keras.optimizers.Adam(0.001),
                  loss = tf.losses.BinaryCrossentropy(),
                  metrics=['accuracy'], experimental_run_tf_function=False)
    model.fit(db_train, epochs=epochs, validation_data=db_test)
    model.evaluate(db_test)
if __name__ == '__main__':
    main()
import  os
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
import  tensorflow as tf
import  numpy as np
from    tensorflow import keras
from    tensorflow.keras import layers
tf.random.set_seed(22)
np.random.seed(22) 
assert tf.__version__.startswith('2.')
batchsz = 128
total_words = 10000
max_review_len = 80
embedding_len = 100
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
# x_train:[b, 80]
# x_test: [b, 80]
x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True)
db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batchsz, drop_remainder=True)
print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)
class MyRNN(keras.Model):
    def __init__(self, units):
        super(MyRNN, self).__init__()
        # 将文本转换为嵌入表示
        # [b, 80] => [b, 80, 100]
        self.embedding = layers.Embedding(total_words, embedding_len, input_length=max_review_len)
        # [b, 80, 100] , h_dim: 64
        self.rnn = keras.Sequential([
            layers.SimpleRNN(units, dropout=0.5, return_sequences=True, unroll=True),
            layers.SimpleRNN(units, dropout=0.5, unroll=True)
        ])
        # fc, [b, 80, 100] => [b, 64] => [b, 1]
        self.outlayer = layers.Dense(1)
    def call(self, inputs, training=None):
        """
        net(x) net(x, training=True) :train mode
        net(x, training=False): test
        :param inputs: [b, 80]
        :param training:
        :return:
        """
        # [b, 80]
        x = inputs
        # embedding: [b, 80] => [b, 80, 100]
        x = self.embedding(x)
        # rnn cell compute
        # x: [b, 80, 100] => [b, 64]
        x = self.rnn(x,training=training)
        # out: [b, 64] => [b, 1]
        x = self.outlayer(x)
        # p(y is pos|x)
        prob = tf.sigmoid(x)
        return prob
def main():
    units = 64
    epochs = 4
    model = MyRNN(units)
    # model.build(input_shape=(4,80))
    # model.summary()
    model.compile(optimizer = keras.optimizers.Adam(0.001),
                  loss = tf.losses.BinaryCrossentropy(),
                  metrics=['accuracy'])
    model.fit(db_train, epochs=epochs, validation_data=db_test)
    model.evaluate(db_test)
if __name__ == '__main__':
    main()

SimpleRNN

情感分类任务的网络结构

#初始化状态向量h0(批量大小b,状态向量特征长度h/units)，此处即[b, 64]
[tf.zeros([batchsz, units])]
# 词向量编码 [b, 80] => [b, 80, 100]
layers.Embedding(total_words, embedding_len, input_length=max_review_len)
    total_words = 10000：词汇表大小
    input_length = max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充
    embedding_len = 100 # 词向量特征长度f，即时间戳t的输入特征长度f
#状态向量特征长度h/units=64
#xt为(批量大小b, 词向量特征长度f)。SimpleRNNCell内部维护了3个张量，kernel为Wxh(词向量特征长度f, 状态向量特征长度h)，shape为(100, 64)，
#recurrent_kernel为Whh(状态向量特征长度h, 状态向量特征长度h)，shape为(64, 64)，bias为偏置b(状态向量特征长度h,)，shape为(64,)
#前向计算，输入批量序列中同一个时间戳t的字符数据xt和状态向量h0：ht = Wxh * Xt + Whh * ht-1 + b = (100, 64)*(b, 100)+(64, 64)*[b, 64]+(64,)=>[b, 64]  
layers.SimpleRNNCell(units, dropout=0.5) 
#构建分类网络，用于将 CELL的输出特征进行分类，2分类
# [b, 80, 100] => [b, 64] => [b, 1]
 self.outlayer = layers.Dense(1)

1.按照 x[:,i,:] 或 for xt in tf.unstack(x, axis=1) 方式对[序列数量b, 序列长度s, 词向量特征长度f]数据进行遍历时，
  一共可以遍历出 s个[序列数量b, 词向量特征长度f]，即一个序列中有N个单词数，那么就遍历N次，每次遍历同时获取批量多个序列中的同一个时间戳上的单词，
  当遍历次数等于序列长度s时才算完成一次网络层的前向计算。
2.第一层RNN 每次接收的是 [序列数量b, 词向量特征长度f]的序列数据，也即是批量序列数据中同一个时间戳的单词的词向量特征长度f
  第二层RNN 每次接收的是 第一层RNN输出的 [序列数量b, 状态向量特征长度h]，也即是每个时间戳上的网络输出Ot，而网络输出Ot可以是直接等于状态向量ht，
  或者也可以是对状态向量ht做一个线性变换Ot=Who*ht得到每个时间戳上的网络输出Ot。

LSTM

import  tensorflow as tf
from    tensorflow import keras
from    tensorflow.keras import datasets, layers, optimizers
#[序列数量b, 序列长度s, 词向量特征长度f]
x = tf.random.normal([2,80,100])
# 得到一个时间戳的输入，从[序列数量b, 序列长度s, 词向量特征长度f]中获取 s个[序列数量b, 词向量特征长度f]
xt = x[:,0,:] 
xt.shape #TensorShape([2, 100])
# 创建 Cell
cell = layers.LSTMCell(64) 
# 初始化状态和输出 List[h,c]，状态向量h和c的形状均都是[批量大小b, 状态向量特征长度h]
state = [tf.zeros([2,64]),tf.zeros([2,64])]
# 前向计算
out, state = cell(xt, state) #state为List[h,c]，out和List的第一个元素h都是相同的
id(out),id(state[0]),id(state[1])
#遍历批量序列中的每一个时间戳，遍历次数和时间戳数量一致，每次遍历的时间戳的形状为[序列数量b, 词向量特征长度f]
for xt in tf.unstack(x, axis=1): 
 out, state = cell(xt, state)
layer = layers.LSTM(64)
out = layer(x) #[批量大小b,状态向量特征长度h]
out.shape #TensorShape([2, 64])
layer = layers.LSTM(64, return_sequences=True)
out = layer(x) #[批量大小b,序列长度s,状态向量特征长度h]
out.shape #TensorShape([2, 80, 64])

" class="reference-link">GRU

LSTM/GRU情感分类问题

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import matplotlib.pyplot as plt
x = tf.range(10)
x = tf.random.shuffle(x)
# 创建共10个单词，每个单词用长度为4的向量表示的层
net = layers.Embedding(10, 4) #[词汇表大小, 词向量特征长度f]
out = net(x)
out
out.shape #TensorShape([10, 4]) 即[词汇表大小, 词向量特征长度f]
net.embeddings
net.embeddings.shape #TensorShape([10, 4]) 即[词汇表大小, 词向量特征长度f]
net.embeddings.trainable #True
net.trainable = False
# 从预训练模型中加载词向量表
embed_glove = load_embed('glove.6B.50d.txt')
# 直接利用预训练的词向量表初始化Embedding层
net.set_weights([embed_glove])
cell = layers.SimpleRNNCell(3) #状态向量特征长度h
cell.build(input_shape=(None,4)) #input_shape=(序列数量,序列长度)
#SimpleRNNCell内部维护了3个张量，kernel为Wxh(词向量特征长度f, 状态向量特征长度h)，shape为(4, 3)，
#recurrent_kernel为Whh(状态向量特征长度h, 状态向量特征长度h)，shape为(3, 3)，bias为偏置b(状态向量特征长度h,)，shape为(3,)
cell.trainable_variables
#[<tf.Variable 'kernel:0' shape=(4, 3) dtype=float32, numpy=
#array(。。。, dtype=float32)>, 
#<tf.Variable 'recurrent_kernel:0' shape=(3, 3) dtype=float32, numpy=
#array(。。。, dtype=float32)>, 
#<tf.Variable 'bias:0' shape=(3,) dtype=float32, numpy=array([0., 0., 0.], dtype=float32)>]
# 初始化状态向量
h0 = [tf.zeros([4, 64])] #[词向量特征长度f, 状态向量特征长度h]
x = tf.random.normal([4, 80, 100]) #[序列数量b,序列长度s,词向量特征长度f]
xt = x[:,0,:] #批量序列的第一个时间戳/单词的形状 [序列数量b,词向量特征长度f]
# 构建词向量特征长度f=100，序列长度s=80，状态向量长度h=64的 Cell
cell = layers.SimpleRNNCell(64) 
out, h1 = cell(xt, h0) # 前向计算，out和h1均为同一个状态向量
print(out.shape, h1[0].shape) #(4, 64) (4, 64)
print(id(out), id(h1[0]))  #2310103532472 2310103532472
h = h0
# 在序列长度的维度解开输入，得到xt:[b,f] [序列数量b,词向量特征长度f]
for xt in tf.unstack(x, axis=1):
    out, h = cell(xt, h) # 前向计算
# 最终输出可以聚合每个时间戳上的输出，也可以只取最后时间戳的输出
out = out
x = tf.random.normal([4,80,100])  #[序列数量b,序列长度s,词向量特征长度f]
xt = x[:,0,:] # 取第一个时间戳的输入x0 [序列数量b,词向量特征长度f]
# 构建2个Cell,先cell0,后cell1
cell0 = layers.SimpleRNNCell(64) #状态向量长度h=64
cell1 = layers.SimpleRNNCell(64)
h0 = [tf.zeros([4,64])] # cell0的初始状态向量 [批量大小b,状态向量特征长度h]
h1 = [tf.zeros([4,64])] # cell1的初始状态向量
out0, h0 = cell0(xt, h0)
out1, h1 = cell1(out0, h1)
for xt in tf.unstack(x, axis=1):
    # xtw作为输入，输出为out0
    out0, h0 = cell0(xt, h0)
    # 上一个cell的输出out0作为本cell的输入
    out1, h1 = cell1(out0, h1)
print(x.shape)
# 保存上一层的所有时间戳上面的输出
middle_sequences = []
# 计算第一层的所有时间戳上的输出，并保存
for xt in tf.unstack(x, axis=1):
    out0, h0 = cell0(xt, h0)
    middle_sequences.append(out0)
# 计算第二层的所有时间戳上的输出
# 如果不是末层，需要保存所有时间戳上面的输出
for xt in middle_sequences:
    out1, h1 = cell1(xt, h1)
layer = layers.SimpleRNN(64) #状态向量长度h=64
x = tf.random.normal([4, 80, 100]) #[序列数量b,序列长度s,词向量特征长度f]
out = layer(x)
out.shape
layer = layers.SimpleRNN(64, return_sequences=True) #返回所有时间戳上的输出
out = layer(x) 
out
# 构建2层RNN网络
# 除最末层外，都需要返回所有时间戳的输出
net = keras.Sequential([ 
layers.SimpleRNN(64, return_sequences=True),
layers.SimpleRNN(64),
])
out = net(x)
x = tf.random.normal([2,80,100])#[序列数量b,序列长度s,词向量特征长度f]
xt = x[:,0,:] # 得到一个时间戳的输入  [序列数量b,词向量特征长度f]
cell = layers.LSTMCell(64) # 创建Cell #状态向量长度h=64
# 初始化状态和输出List,[h,c]
state = [tf.zeros([2,64]),tf.zeros([2,64])]  #[批量大小b,状态向量特征长度h]
out, state = cell(xt, state) # 前向计算
id(out),id(state[0]),id(state[1]) #(2316639101352, 2316639101352, 2316639102232) out和List的第一个元素h都是相同的
net = layers.LSTM(4) #状态向量长度h=4
net.build(input_shape=(None,5,3)) #input_shape=[序列数量b,序列长度s,词向量特征长度f]
#LSTM内部维护了3个张量，kernel为Wxh(词向量特征长度f, 状态向量特征长度h*4)，shape为(3, 16)，
#recurrent_kernel为Whh(状态向量特征长度h, 状态向量特征长度h*4)，shape为(4, 16)，bias为偏置b(状态向量特征长度h*4,)，shape为(16,)
net.trainable_variables
#[<tf.Variable 'kernel:0' shape=(3, 16) dtype=float32, numpy=
#array(。。。, dtype=float32)>, 
#<tf.Variable 'recurrent_kernel:0' shape=(4, 16) dtype=float32, numpy=
#array(。。。, dtype=float32)>, 
#<tf.Variable 'bias:0' shape=(16,) dtype=float32, numpy=
#array(。。。, dtype=float32)>]
net = layers.GRU(4) #状态向量长度h=4
net.build(input_shape=(None,5,3)) #input_shape=[序列数量b,序列长度s,词向量特征长度f]
#GRU内部维护了3个张量，kernel为Wxh(词向量特征长度f, 状态向量特征长度h*3)，shape为(3, 12)，
#recurrent_kernel为Whh(状态向量特征长度h, 状态向量特征长度h*3)，shape为(4, 12)，bias为偏置b(状态向量特征长度h*3,)，shape为(12,)
net.trainable_variables
[<tf.Variable 'kernel:0' shape=(3, 12) dtype=float32, numpy=
array(。。。, dtype=float32)>, 
<tf.Variable 'recurrent_kernel:0' shape=(4, 12) dtype=float32, numpy=
array(。。。, dtype=float32)>, 
<tf.Variable 'bias:0' shape=(2, 12) dtype=float32, numpy=
array(。。。, dtype=float32)>]
# 初始化状态向量
h = [tf.zeros([2,64])]
cell = layers.GRUCell(64) # 新建GRU Cell
for xt in tf.unstack(x, axis=1):
    out, h = cell(xt, h)
out.shape
import  os
import  tensorflow as tf
import  numpy as np
from    tensorflow import keras
from    tensorflow.keras import layers, losses, optimizers, Sequential
tf.random.set_seed(22)
np.random.seed(22)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
assert tf.__version__.startswith('2.')
batchsz = 128 # 批量大小
total_words = 10000 # 词汇表大小N_vocab
max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充
embedding_len = 100 # 词向量特征长度f
# 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词
#x_train/x_test实际为25000个list列表，x_train中的每个list列表中有218个数字(单词)，x_test中的每个list列表中有218个数字(单词)
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
print(x_train.shape, len(x_train[0]), y_train.shape) #(25000,) 218 (25000,)
print(x_test.shape, len(x_test[0]), y_test.shape) #(25000,) 68 (25000,)
x_train[0] #这个list列表中有218个数字(单词)
# 数字编码表：字典中的key为单词，value为索引
word_index = keras.datasets.imdb.get_word_index()
# for k,v in word_index.items():
#     print(k,v)
#把数字编码表(字典)中的每个key(单词)对应的value(索引值)加3
word_index = {k:(v+3) for k,v in word_index.items()}
word_index["<PAD>"] = 0
word_index["<START>"] = 1
word_index["<UNK>"] = 2  # unknown
word_index["<UNUSED>"] = 3
# 翻转编码表：key(单词)换到value的位置，value(索引值)换到key的位置
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
#根据翻转的数字编码表(字典) 通过序列数据(每个list列表)中的每个数字(索引值) 获取其对应的单词
def decode_review(text):
    return ' '.join([reverse_word_index.get(i, '?') for i in text])
#根据该list列表中的所有数字(索引值)获取其对应的单词
decode_review(x_train[8])
print('Indexing word vectors.')
embeddings_index = {}
#GLOVE_DIR = r'C:\Users\z390\Downloads\glove6b50dtxt'
GLOVE_DIR = r'F:\人工智能\学习资料\GloVe 词嵌入'
with open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt'),encoding='utf-8') as f:
    for line in f:
        values = line.split()
        word = values[0] #获取单词
        coefs = np.asarray(values[1:], dtype='float32')#获取单词对应的训练好的词向量特征值
        embeddings_index[word] = coefs #单词作为key，单词对应的权重矩阵值作为value
print('Found %s word vectors.' % len(embeddings_index)) #Found 400000 word vectors.
len(embeddings_index.keys()) #400000
len(word_index.keys()) #88588
MAX_NUM_WORDS = total_words # 词汇表大小N_vocab
# prepare embedding matrix
num_words = min(MAX_NUM_WORDS, len(word_index))
embedding_matrix = np.zeros((num_words, embedding_len)) #[词汇表大小, 词向量特征长度f]
applied_vec_count = 0
for word, i in word_index.items(): #遍历每个key(单词)和对应的value(索引值)
    if i >= MAX_NUM_WORDS:
        continue
    embedding_vector = embeddings_index.get(word) #根据key(单词)获取对应的训练好的词向量特征值
    # print(word,embedding_vector)
    if embedding_vector is not None:
        # words not found in embedding index will be all-zeros.
        embedding_matrix[i] = embedding_vector #key为单词对应的索引值，value为单词对应的训练好的词向量特征值
        applied_vec_count += 1
#9793 (10000, 100) 即[词汇表大小, 词向量特征长度f]
print(applied_vec_count, embedding_matrix.shape)
# x_train:[b, 80]
# x_test: [b, 80]
# 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充
x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)
# 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True)
db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batchsz, drop_remainder=True)
#x_train shape: (25000, 80) tf.Tensor(1, shape=(), dtype=int64) tf.Tensor(0, shape=(), dtype=int64)
print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
#x_test shape: (25000, 80)
print('x_test shape:', x_test.shape)
class MyRNN(keras.Model):
    # Cell方式构建多层网络
    def __init__(self, units):
        super(MyRNN, self).__init__() 
        # total_words 词汇表大小
        # input_length = max_review_len 句子最大长度s，大于的句子部分将截断，小于的将填充
        # embedding_len 词向量特征长度f，即时间戳t的输入特征长度f
        # 在获得了词汇表后，利用词汇表初始化 Embedding 层即可，并设置 Embedding 层不参与梯度优化，设置trainable=False不参与梯度更新
        # 词向量编码 [b, 80] => [b, 80, 100]
        self.embedding = layers.Embedding(total_words, embedding_len, input_length=max_review_len, trainable=False)
        # input_shape=[序列数量b,序列长度s]
        self.embedding.build(input_shape=(None,max_review_len))
        # 直接利用预训练的词向量表初始化 Embedding 层
        self.embedding.set_weights([embedding_matrix]) 
        # 构建RNN
        self.rnn = keras.Sequential([
            layers.LSTM(units, dropout=0.5, return_sequences=True),
            layers.LSTM(units, dropout=0.5)
        ])
        # 构建分类网络，用于将CELL的输出特征进行分类，2分类
        # [b, 80, 100] => [b, 64] => [b, 1]
        self.outlayer = Sequential([
            layers.Dense(32),
            layers.Dropout(rate=0.5),
            layers.ReLU(),
            layers.Dense(1)])
    def call(self, inputs, training=None):
        x = inputs # [b, 80]
        # embedding: [b, 80] => [b, 80, 100]  [序列数量b,序列长度s,词向量特征长度f]
        x = self.embedding(x)
        # rnn cell compute,[b, 80, 100] => [b, 64]
        x = self.rnn(x)
        # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1]
        x = self.outlayer(x,training)
        # p(y is pos|x)
        prob = tf.sigmoid(x)
        return prob
def main():
    units = 512 # RNN状态向量长度f
    epochs = 50 # 训练epochs
    model = MyRNN(units)
    # 装配
    model.compile(optimizer = optimizers.Adam(0.001),
                  loss = losses.BinaryCrossentropy(),
                  metrics=['accuracy'])
    # 训练和验证
    model.fit(db_train, epochs=epochs, validation_data=db_test)
    # 测试
    model.evaluate(db_test)
if __name__ == '__main__':
    main()
import  os
import  tensorflow as tf
import  numpy as np
from    tensorflow import keras
from    tensorflow.keras import layers, losses, optimizers, Sequential
tf.random.set_seed(22)
np.random.seed(22)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
assert tf.__version__.startswith('2.')
batchsz = 128 # 批量大小
total_words = 10000 # 词汇表大小N_vocab
max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充
embedding_len = 100 # 词向量特征长度f
# 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
print(x_train.shape, len(x_train[0]), y_train.shape)
print(x_test.shape, len(x_test[0]), y_test.shape)
x_train[0]
# 数字编码表
word_index = keras.datasets.imdb.get_word_index()
# for k,v in word_index.items():
#     print(k,v)
word_index = {k:(v+3) for k,v in word_index.items()}
word_index["<PAD>"] = 0
word_index["<START>"] = 1
word_index["<UNK>"] = 2  # unknown
word_index["<UNUSED>"] = 3
# 翻转编码表
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
def decode_review(text):
    return ' '.join([reverse_word_index.get(i, '?') for i in text])
decode_review(x_train[8])
# x_train:[b, 80]
# x_test: [b, 80]
# 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充
x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)
# 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True)
db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batchsz, drop_remainder=True)
print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)
class MyRNN(keras.Model):
    # Cell方式构建多层网络
    def __init__(self, units):
        super(MyRNN, self).__init__() 
        # 词向量编码 [b, 80] => [b, 80, 100]
        self.embedding = layers.Embedding(total_words, embedding_len, input_length=max_review_len)
        # 构建RNN
        self.rnn = keras.Sequential([
            layers.LSTM(units, dropout=0.5, return_sequences=True),
            layers.LSTM(units, dropout=0.5)
        ])
        # 构建分类网络，用于将CELL的输出特征进行分类，2分类
        # [b, 80, 100] => [b, 64] => [b, 1]
        self.outlayer = Sequential([
            layers.Dense(32),
            layers.Dropout(rate=0.5),
            layers.ReLU(),
            layers.Dense(1)])
    def call(self, inputs, training=None):
        x = inputs # [b, 80]
        # embedding: [b, 80] => [b, 80, 100]
        x = self.embedding(x)
        # rnn cell compute,[b, 80, 100] => [b, 64]
        x = self.rnn(x)
        # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1]
        x = self.outlayer(x,training)
        # p(y is pos|x)
        prob = tf.sigmoid(x)
        return prob
def main():
    units = 32 # RNN状态向量长度f
    epochs = 50 # 训练epochs
    model = MyRNN(units)
    # 装配
    model.compile(optimizer = optimizers.Adam(0.001),
                  loss = losses.BinaryCrossentropy(),
                  metrics=['accuracy'])
    # 训练和验证
    model.fit(db_train, epochs=epochs, validation_data=db_test)
    # 测试
    model.evaluate(db_test)
if __name__ == '__main__':
    main()
import  os
import  tensorflow as tf
import  numpy as np
from    tensorflow import keras
from    tensorflow.keras import layers, losses, optimizers, Sequential
tf.random.set_seed(22)
np.random.seed(22)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
assert tf.__version__.startswith('2.')
batchsz = 128 # 批量大小
total_words = 10000 # 词汇表大小N_vocab
max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充
embedding_len = 100 # 词向量特征长度f
# 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
print(x_train.shape, len(x_train[0]), y_train.shape)
print(x_test.shape, len(x_test[0]), y_test.shape)
x_train[0]
# 数字编码表
word_index = keras.datasets.imdb.get_word_index()
# for k,v in word_index.items():
#     print(k,v)
word_index = {k:(v+3) for k,v in word_index.items()}
word_index["<PAD>"] = 0
word_index["<START>"] = 1
word_index["<UNK>"] = 2  # unknown
word_index["<UNUSED>"] = 3
# 翻转编码表
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
def decode_review(text):
    return ' '.join([reverse_word_index.get(i, '?') for i in text])
decode_review(x_train[8])
# x_train:[b, 80]
# x_test: [b, 80]
# 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充
x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)
# 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True)
db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batchsz, drop_remainder=True)
print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)
class MyRNN(keras.Model):
    # Cell方式构建多层网络
    def __init__(self, units):
        super(MyRNN, self).__init__() 
        # 词向量编码 [b, 80] => [b, 80, 100]
        self.embedding = layers.Embedding(total_words, embedding_len, input_length=max_review_len)
        # 构建RNN
        self.rnn = keras.Sequential([
            layers.GRU(units, dropout=0.5, return_sequences=True),
            layers.GRU(units, dropout=0.5)
        ])
        # 构建分类网络，用于将CELL的输出特征进行分类，2分类
        # [b, 80, 100] => [b, 64] => [b, 1]
        self.outlayer = Sequential([
            layers.Dense(32),
            layers.Dropout(rate=0.5),
            layers.ReLU(),
            layers.Dense(1)])
    def call(self, inputs, training=None):
        x = inputs # [b, 80]
        # embedding: [b, 80] => [b, 80, 100]
        x = self.embedding(x)
        # rnn cell compute,[b, 80, 100] => [b, 64]
        x = self.rnn(x)
        # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1]
        x = self.outlayer(x,training)
        # p(y is pos|x)
        prob = tf.sigmoid(x)
        return prob
def main():
    units = 32 # RNN状态向量长度f
    epochs = 50 # 训练epochs
    model = MyRNN(units)
    # 装配
    model.compile(optimizer = optimizers.Adam(0.001),
                  loss = losses.BinaryCrossentropy(),
                  metrics=['accuracy'])
    # 训练和验证
    model.fit(db_train, epochs=epochs, validation_data=db_test)
    # 测试
    model.evaluate(db_test)
if __name__ == '__main__':
    main()
import  os
import  tensorflow as tf
import  numpy as np
from    tensorflow import keras
from    tensorflow.keras import layers, losses, optimizers, Sequential
tf.random.set_seed(22)
np.random.seed(22)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
assert tf.__version__.startswith('2.')
batchsz = 512 # 批量大小
total_words = 10000 # 词汇表大小N_vocab
max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充
embedding_len = 100 # 词向量特征长度f
# 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
print(x_train.shape, len(x_train[0]), y_train.shape)
print(x_test.shape, len(x_test[0]), y_test.shape)
x_train[0]
# 数字编码表
word_index = keras.datasets.imdb.get_word_index()
# for k,v in word_index.items():
#     print(k,v)
word_index = {k:(v+3) for k,v in word_index.items()}
word_index["<PAD>"] = 0
word_index["<START>"] = 1
word_index["<UNK>"] = 2  # unknown
word_index["<UNUSED>"] = 3
# 翻转编码表
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
def decode_review(text):
    return ' '.join([reverse_word_index.get(i, '?') for i in text])
decode_review(x_train[8])
# x_train:[b, 80]
# x_test: [b, 80]
# 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充
x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)
# 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True)
db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batchsz, drop_remainder=True)
print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)
class MyRNN(keras.Model):
    # Cell方式构建多层网络
    def __init__(self, units):
        super(MyRNN, self).__init__() 
        # 词向量编码 [b, 80] => [b, 80, 100]
        self.embedding = layers.Embedding(total_words, embedding_len, input_length=max_review_len)
        # 构建RNN
        self.rnn = keras.Sequential([
            layers.SimpleRNN(units, dropout=0.5, return_sequences=True),
            layers.SimpleRNN(units, dropout=0.5)
        ])
        # 构建分类网络，用于将CELL的输出特征进行分类，2分类
        # [b, 80, 100] => [b, 64] => [b, 1]
        self.outlayer = Sequential([
            layers.Dense(32),
            layers.Dropout(rate=0.5),
            layers.ReLU(),
            layers.Dense(1)])
    def call(self, inputs, training=None):
        x = inputs # [b, 80]
        # embedding: [b, 80] => [b, 80, 100]
        x = self.embedding(x)
        # rnn cell compute,[b, 80, 100] => [b, 64]
        x = self.rnn(x)
        # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1]
        x = self.outlayer(x,training)
        # p(y is pos|x)
        prob = tf.sigmoid(x)
        return prob
def main():
    units = 64 # RNN状态向量长度f
    epochs = 50 # 训练epochs
    model = MyRNN(units)
    # 装配
    model.compile(optimizer = optimizers.Adam(0.001),
                  loss = losses.BinaryCrossentropy(),
                  metrics=['accuracy'])
    # 训练和验证
    model.fit(db_train, epochs=epochs, validation_data=db_test)
    # 测试
    model.evaluate(db_test)
if __name__ == '__main__':
    main()
import  os
import  tensorflow as tf
import  numpy as np
from    tensorflow import keras
from    tensorflow.keras import layers, losses, optimizers, Sequential
tf.random.set_seed(22)
np.random.seed(22)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
assert tf.__version__.startswith('2.')
batchsz = 128 # 批量大小
total_words = 10000 # 词汇表大小N_vocab
max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充
embedding_len = 100 # 词向量特征长度f
# 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
print(x_train.shape, len(x_train[0]), y_train.shape)
print(x_test.shape, len(x_test[0]), y_test.shape)
x_train[0]
# 数字编码表
word_index = keras.datasets.imdb.get_word_index()
# for k,v in word_index.items():
#     print(k,v)
word_index = {k:(v+3) for k,v in word_index.items()}
word_index["<PAD>"] = 0
word_index["<START>"] = 1
word_index["<UNK>"] = 2  # unknown
word_index["<UNUSED>"] = 3
# 翻转编码表
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
def decode_review(text):
    return ' '.join([reverse_word_index.get(i, '?') for i in text])
decode_review(x_train[8])
# x_train:[b, 80]
# x_test: [b, 80]
# 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充
x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)
# 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True)
db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batchsz, drop_remainder=True)
print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)
class MyRNN(keras.Model):
    # Cell方式构建多层网络
    def __init__(self, units):
        super(MyRNN, self).__init__()
        # [b, 64]，构建Cell初始化状态向量，重复使用
        self.state0 = [tf.zeros([batchsz, units])]
        self.state1 = [tf.zeros([batchsz, units])]
        # 词向量编码 [b, 80] => [b, 80, 100]
        self.embedding = layers.Embedding(total_words, embedding_len,
                                          input_length=max_review_len)
        # 构建2个Cell
        self.rnn_cell0 = layers.GRUCell(units, dropout=0.5)
        self.rnn_cell1 = layers.GRUCell(units, dropout=0.5)
        # 构建分类网络，用于将CELL的输出特征进行分类，2分类
        # [b, 80, 100] => [b, 64] => [b, 1]
        self.outlayer = Sequential([
            layers.Dense(units),
            layers.Dropout(rate=0.5),
            layers.ReLU(),
            layers.Dense(1)])
    def call(self, inputs, training=None):
        x = inputs # [b, 80]
        # embedding: [b, 80] => [b, 80, 100]
        x = self.embedding(x)
        # rnn cell compute,[b, 80, 100] => [b, 64]
        state0 = self.state0
        state1 = self.state1
        for word in tf.unstack(x, axis=1): # word: [b, 100] 
            out0, state0 = self.rnn_cell0(word, state0, training) 
            out1, state1 = self.rnn_cell1(out0, state1, training)
        # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1]
        x = self.outlayer(out1, training)
        # p(y is pos|x)
        prob = tf.sigmoid(x)
        return prob
def main():
    units = 64 # RNN状态向量长度f
    epochs = 50 # 训练epochs
    model = MyRNN(units)
    # 装配
    model.compile(optimizer = optimizers.RMSprop(0.001),
                  loss = losses.BinaryCrossentropy(),
                  metrics=['accuracy'])
    # 训练和验证
    model.fit(db_train, epochs=epochs, validation_data=db_test)
    # 测试
    model.evaluate(db_test)
if __name__ == '__main__':
    main()
import  os
import  tensorflow as tf
import  numpy as np
from    tensorflow import keras
from    tensorflow.keras import layers, losses, optimizers, Sequential
tf.random.set_seed(22)
np.random.seed(22)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
assert tf.__version__.startswith('2.')
batchsz = 128 # 批量大小
total_words = 10000 # 词汇表大小N_vocab
max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充
embedding_len = 100 # 词向量特征长度f
# 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
print(x_train.shape, len(x_train[0]), y_train.shape)
print(x_test.shape, len(x_test[0]), y_test.shape)
x_train[0]
# 数字编码表
word_index = keras.datasets.imdb.get_word_index()
# for k,v in word_index.items():
#     print(k,v)
word_index = {k:(v+3) for k,v in word_index.items()}
word_index["<PAD>"] = 0
word_index["<START>"] = 1
word_index["<UNK>"] = 2  # unknown
word_index["<UNUSED>"] = 3
# 翻转编码表
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
def decode_review(text):
    return ' '.join([reverse_word_index.get(i, '?') for i in text])
decode_review(x_train[8])
# x_train:[b, 80]
# x_test: [b, 80]
# 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充
x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)
# 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True)
db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batchsz, drop_remainder=True)
print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)
class MyRNN(keras.Model):
    # Cell方式构建多层网络
    def __init__(self, units):
        super(MyRNN, self).__init__()
        # [b, 64]，构建Cell初始化状态向量，重复使用
        self.state0 = [tf.zeros([batchsz, units]),tf.zeros([batchsz, units])]
        self.state1 = [tf.zeros([batchsz, units]),tf.zeros([batchsz, units])]
        # 词向量编码 [b, 80] => [b, 80, 100]
        self.embedding = layers.Embedding(total_words, embedding_len, input_length=max_review_len)
        # 构建2个Cell
        self.rnn_cell0 = layers.LSTMCell(units, dropout=0.5)
        self.rnn_cell1 = layers.LSTMCell(units, dropout=0.5)
        # 构建分类网络，用于将CELL的输出特征进行分类，2分类
        # [b, 80, 100] => [b, 64] => [b, 1]
        self.outlayer = Sequential([
            layers.Dense(units),
            layers.Dropout(rate=0.5),
            layers.ReLU(),
            layers.Dense(1)])
    def call(self, inputs, training=None):
        x = inputs # [b, 80]
        # embedding: [b, 80] => [b, 80, 100]
        x = self.embedding(x)
        # rnn cell compute,[b, 80, 100] => [b, 64]
        state0 = self.state0
        state1 = self.state1
        for word in tf.unstack(x, axis=1): # word: [b, 100] 
            out0, state0 = self.rnn_cell0(word, state0, training) 
            out1, state1 = self.rnn_cell1(out0, state1, training)
        # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1]
        x = self.outlayer(out1,training)
        # p(y is pos|x)
        prob = tf.sigmoid(x)
        return prob
def main():
    units = 64 # RNN状态向量长度f
    epochs = 50 # 训练epochs
    model = MyRNN(units)
    # 装配
    model.compile(optimizer = optimizers.RMSprop(0.001),
                  loss = losses.BinaryCrossentropy(),
                  metrics=['accuracy'])
    # 训练和验证
    model.fit(db_train, epochs=epochs, validation_data=db_test)
    # 测试
    model.evaluate(db_test)
if __name__ == '__main__':
    main()
import  os
import  tensorflow as tf
import  numpy as np
from    tensorflow import keras
from    tensorflow.keras import layers, losses, optimizers, Sequential
tf.random.set_seed(22)
np.random.seed(22)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
assert tf.__version__.startswith('2.')
batchsz = 128 # 批量大小
total_words = 10000 # 词汇表大小N_vocab
max_review_len = 80 # 句子最大长度s，大于的句子部分将截断，小于的将填充
embedding_len = 100 # 词向量特征长度f
# 加载IMDB数据集，此处的数据采用数字编码，一个数字代表一个单词
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)
print(x_train.shape, len(x_train[0]), y_train.shape)
print(x_test.shape, len(x_test[0]), y_test.shape)
x_train[0]
# 数字编码表
word_index = keras.datasets.imdb.get_word_index()
# for k,v in word_index.items():
#     print(k,v)
word_index = {k:(v+3) for k,v in word_index.items()}
word_index["<PAD>"] = 0
word_index["<START>"] = 1
word_index["<UNK>"] = 2  # unknown
word_index["<UNUSED>"] = 3
# 翻转编码表
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
def decode_review(text):
    return ' '.join([reverse_word_index.get(i, '?') for i in text])
decode_review(x_train[8])
# x_train:[b, 80]
# x_test: [b, 80]
# 截断和填充句子，使得等长，此处长句子保留句子后面的部分，短句子在前面填充
x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)
# 构建数据集，打散，批量，并丢掉最后一个不够batchsz的batch
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train = db_train.shuffle(1000).batch(batchsz, drop_remainder=True)
db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batchsz, drop_remainder=True)
print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)
class MyRNN(keras.Model):
    # Cell方式构建多层网络
    def __init__(self, units):
        super(MyRNN, self).__init__()
        # [b, 64]，构建Cell初始化状态向量，重复使用
        self.state0 = [tf.zeros([batchsz, units])]
        self.state1 = [tf.zeros([batchsz, units])]
        # 词向量编码 [b, 80] => [b, 80, 100]
        self.embedding = layers.Embedding(total_words, embedding_len,
                                          input_length=max_review_len)
        # 构建2个Cell
        self.rnn_cell0 = layers.SimpleRNNCell(units, dropout=0.5)
        self.rnn_cell1 = layers.SimpleRNNCell(units, dropout=0.5)
        # 构建分类网络，用于将CELL的输出特征进行分类，2分类
        # [b, 80, 100] => [b, 64] => [b, 1]
        self.outlayer = Sequential([
            layers.Dense(units),
            layers.Dropout(rate=0.5),
            layers.ReLU(),
            layers.Dense(1)])
    def call(self, inputs, training=None):
        x = inputs # [b, 80]
        # embedding: [b, 80] => [b, 80, 100]
        x = self.embedding(x)
        # rnn cell compute,[b, 80, 100] => [b, 64]
        state0 = self.state0
        state1 = self.state1
        for word in tf.unstack(x, axis=1): # word: [b, 100] 
            out0, state0 = self.rnn_cell0(word, state0, training) 
            out1, state1 = self.rnn_cell1(out0, state1, training)
        # 末层最后一个输出作为分类网络的输入: [b, 64] => [b, 1]
        x = self.outlayer(out1, training)
        # p(y is pos|x)
        prob = tf.sigmoid(x)
        return prob
def main():
    units = 64 # RNN状态向量长度f
    epochs = 50 # 训练epochs
    model = MyRNN(units)
    # 装配
    model.compile(optimizer = optimizers.RMSprop(0.001),
                  loss = losses.BinaryCrossentropy(),
                  metrics=['accuracy'])
    # 训练和验证
    model.fit(db_train, epochs=epochs, validation_data=db_test)
    # 测试
    model.evaluate(db_test)
if __name__ == '__main__':
    main()
import  os
import  tensorflow as tf
import  numpy as np
from    tensorflow import keras
tf.random.set_seed(22)
np.random.seed(22)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
assert tf.__version__.startswith('2.')
# 确定随机种子的可重复性
np.random.seed(7)
# 加载数据集，但只保留前n个单词，其余的为零
top_words = 10000
# 截断和填充输入序列
max_review_length = 80
(X_train, y_train), (X_test, y_test) = keras.datasets.imdb.load_data(num_words=top_words)
# X_train = tf.convert_to_tensor(X_train)
# y_train = tf.one_hot(y_train, depth=2)
print('Pad sequences (samples x time)')
x_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=max_review_length)
x_test = keras.preprocessing.sequence.pad_sequences(X_test, maxlen=max_review_length)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
class RNN(keras.Model):
    def __init__(self, units, num_classes, num_layers):
        super(RNN, self).__init__()
        # self.cells = [keras.layers.LSTMCell(units) for _ in range(num_layers)]
        # self.rnn = keras.layers.RNN(self.cells, unroll=True)
        self.rnn = keras.layers.LSTM(units, return_sequences=True)
        self.rnn2 = keras.layers.LSTM(units)
        # self.cells = (keras.layers.LSTMCell(units) for _ in range(num_layers))
        # self.rnn = keras.layers.RNN(self.cells, return_sequences=True, return_state=True)
        # self.rnn = keras.layers.LSTM(units, unroll=True)
        # self.rnn = keras.layers.StackedRNNCells(self.cells)
        # 总共有1000个单词，每个单词都将嵌入到100长度的向量中，最大句子长度是80字
        self.embedding = keras.layers.Embedding(top_words, 100, input_length=max_review_length)
        self.fc = keras.layers.Dense(1)
    def call(self, inputs, training=None, mask=None):
        # print('x', inputs.shape)
        # [b, sentence len] => [b, sentence len, word embedding]
        x = self.embedding(inputs)
        # print('embedding', x.shape)
        x = self.rnn(x) 
        x = self.rnn2(x) 
        # print('rnn', x.shape)
        x = self.fc(x)
        print(x.shape)
        return x
def main():
    units = 64
    num_classes = 2
    batch_size = 32
    epochs = 20
    model = RNN(units, num_classes, num_layers=2)
    model.compile(optimizer=keras.optimizers.Adam(0.001),
                  loss=keras.losses.BinaryCrossentropy(from_logits=True),
                  metrics=['accuracy'])
    # train
    model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs,
              validation_data=(x_test, y_test), verbose=1)
    # evaluate on test set
    scores = model.evaluate(x_test, y_test, batch_size, verbose=1)
    print("Final test loss and accuracy :", scores)
if __name__ == '__main__':
    main()