tensor2tensor get_timing_signal_1d

朴灿烈づ我的快乐病毒、 2022-02-04 01:43 158阅读 0赞

[get\_timing\_signal\_1d][get_timing_signal_1d]

''' Created on 2019年5月6日 '''
    
    import tensorflow as tf
    import math
    import cv2
    
    def get_timing_signal_1d(length,
                             channels,
                             min_timescale=1.0,
                             max_timescale=1.0e4,
                             start_index=0):
        """Gets a bunch of sinusoids of different frequencies. Each channel of the input Tensor is incremented by a sinusoid of a different frequency and phase. This allows attention to learn to use absolute and relative positions. Timing signals should be added to some precursors of both the query and the memory inputs to attention. The use of relative position is possible because sin(x+y) and cos(x+y) can be expressed in terms of y, sin(x) and cos(x). In particular, we use a geometric sequence of timescales starting with min_timescale and ending with max_timescale. The number of different timescales is equal to channels / 2. For each timescale, we generate the two sinusoidal signals sin(timestep/timescale) and cos(timestep/timescale). All of these sinusoids are concatenated in the channels dimension. Args: length: scalar, length of timing signal sequence. channels: scalar, size of timing embeddings to create. The number of different timescales is equal to channels / 2. min_timescale: a float max_timescale: a float start_index: index of first position Returns: a Tensor of timing signals [1, length, channels] """
        position = tf.to_float(tf.range(length) + start_index)
        num_timescales = channels // 2
        #时间尺度的增量
        log_timescale_increment = (
            math.log(float(max_timescale) / float(min_timescale)) /
            (tf.to_float(num_timescales) - 1))
        #按增量生成(1,256)
        inv_timescales = min_timescale * tf.exp(
            tf.to_float(tf.range(num_timescales)) * -log_timescale_increment)
        #(100)->(100,1),(256)->(1,256),(100,1)*(1,256)->(100,256)
        scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0)
        #(100,512)
        signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1)
        signal = tf.pad(signal, [[0, 0], [0, tf.mod(channels, 2)]])
        signal = tf.reshape(signal, [1, length, channels])
        return signal
    
    signal = get_timing_signal_1d(100,512)
    
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        result = sess.run(signal)
        result = result.reshape(result.shape[1],result.shape[2])
        cv2.imshow("result.exe",result) 
        cv2.waitKey(0)
    
    
    # length = tf.constant(20)
    # channels=512
    # min_timescale=1.0
    # max_timescale=1.0e4
    # start_index=tf.constant(0)
    # 
    # 
    # position = tf.to_float(tf.range(length) + start_index)
    # num_timescales = channels // 2
    # #时间尺度的增量
    # log_timescale_increment = (
    # math.log(float(max_timescale) / float(min_timescale)) /
    # (tf.to_float(num_timescales) - 1))
    # #按num_timescales生成，(1,256)
    # inv_timescales = min_timescale * tf.exp(
    # tf.to_float(tf.range(num_timescales)) * -log_timescale_increment)
    # #(20)->(20,1),(256)->(1,256),(20,1)*(1,256)->(20,256)
    # scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0)
    # #(20,512)
    # signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1)
    # # signal = tf.pad(signal, [[0, 0], [0, tf.mod(channels, 2)]])
    # #(1,20,512)
    # # signal = tf.reshape(signal, [1, length, channels])
    # with tf.Session() as sess:
    # sess.run(tf.global_variables_initializer())
    # print(sess.run(position))
    # print(sess.run(log_timescale_increment))
    # print(sess.run(inv_timescales))
    # print(sess.run(scaled_time))
    # print(sess.run(signal)[:10])
    # result = sess.run(signal)
    # cv2.imshow("notepad.exe",result) 
    # cv2.waitKey(0)

![在这里插入图片描述][20190506165953267.PNG]

[get_timing_signal_1d]: https://github.com/tensorflow/tensor2tensor/blob/23bd23b9830059fbc349381b70d9429b5c40a139/tensor2tensor/layers/common_attention.py
[20190506165953267.PNG]: /images/20220204/4401b7119ba7441dacda22572b4bcdd2.png