影評分析-TensorFlow和TensorBoard自然語言分析動手實驗

max_review_length = 200

train_pad = pad_sequences(train_seq, maxlen = max_review_length)
print("train_pad is complete.")

test_pad = pad_sequences(test_seq, maxlen = max_review_length)
print("test_pad is complete.")

np.percentile(lengths.counts, 80)

對於長度為200時，基本上80%的影評數據所有的單詞都會被包括。對於那些長度超出200的記錄，多餘的部分會被去除。相反，那些不足的影評則會被填充令牌補滿剩餘部分。大家需要考慮如何對數據進行更有效的填充和截取。這裏我不做詳細的討論，但是大家可以想想自己有哪些更好的想法。

至此，我們可以把數據分成訓練集和驗證集。

x_train, x_valid, y_train, y_valid = train_test_split(train_pad, train.sentiment, test_size = 0.15, random_state = 2)

def get_batches(x, y, batch_size):
    '''Create the batches for the training and validation data'''
    n_batches = len(x)//batch_size
    x, y = x[:n_batches*batch_size], y[:n_batches*batch_size]
    for ii in range(0, len(x), batch_size):
        yield x[ii:ii+batch_size], y[ii:ii+batch_size]
def get_test_batches(x, batch_size):
    '''Create the batches for the testing data'''
    n_batches = len(x)//batch_size
    x = x[:n_batches*batch_size]
    for ii in range(0, len(x), batch_size):
        yield x[ii:ii+batch_size]

上麵的這些方法會把數據分成數據量相同的組。這裏需要注意的是如果最後一個組的數據量小於batch_size，那它將被忽略。因此，batch_size應該是你的數據集長度的整數倍，否則，你可能會在後麵上傳預測數據時遇到一些問題。下麵我們就來開始一步一步進行循環神經網絡的編寫。

def build_rnn(n_words, embed_size, batch_size, lstm_size, num_layers, dropout, learning_rate, multiple_fc, fc_units):
    '''Build the Recurrent Neural Network'''

    tf.reset_default_graph()

    # Declare placeholders we'll feed into the graph
    with tf.name_scope('inputs'):
        inputs = tf.placeholder(tf.int32, [None, None], name='inputs')

    with tf.name_scope('labels'):
        labels = tf.placeholder(tf.int32, [None, None], name='labels')

    keep_prob = tf.placeholder(tf.float32, name='keep_prob')

    # Create the embeddings
    with tf.name_scope("embeddings"):
        embedding = tf.Variable(tf.random_uniform((n_words, 
                                    embed_size), -1, 1))
        embed = tf.nn.embedding_lookup(embedding, inputs)

    # Build the RNN layers
    with tf.name_scope("RNN_layers"):
        lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size)
        drop = tf.contrib.rnn.DropoutWrapper(lstm, 
                                         output_keep_prob=keep_prob)
        cell = tf.contrib.rnn.MultiRNNCell([drop] * num_layers)
    
    # Set the initial state
    with tf.name_scope("RNN_init_state"):
        initial_state = cell.zero_state(batch_size, tf.float32)

    # Run the data through the RNN layers
    with tf.name_scope("RNN_forward"):
        outputs, final_state = tf.nn.dynamic_rnn(
                                        cell,         
                                        embed,
                                        initial_state=initial_state)    
    
    # Create the fully connected layers
    with tf.name_scope("fully_connected"):
        
        # Initialize the weights and biases
        weights = tf.truncated_normal_initializer(stddev=0.1)
        biases = tf.zeros_initializer()
        
        dense = tf.contrib.layers.fully_connected(outputs[:, -1],
                    num_outputs = fc_units,
                    activation_fn = tf.sigmoid,
                    weights_initializer = weights,
                    biases_initializer = biases)
        
        dense = tf.contrib.layers.dropout(dense, keep_prob)
        
        # Depending on the iteration, use a second fully connected 
          layer
        if multiple_fc == True:
            dense = tf.contrib.layers.fully_connected(dense,
                        num_outputs = fc_units,
                        activation_fn = tf.sigmoid,
                        weights_initializer = weights,
                        biases_initializer = biases)
            
            dense = tf.contrib.layers.dropout(dense, keep_prob)
    
    # Make the predictions
    with tf.name_scope('predictions'):
        predictions = tf.contrib.layers.fully_connected(dense, 
                          num_outputs = 1, 
                          activation_fn=tf.sigmoid,
                          weights_initializer = weights,
                          biases_initializer = biases)
        
        tf.summary.histogram('predictions', predictions)
    
    # Calculate the cost
    with tf.name_scope('cost'):
        cost = tf.losses.mean_squared_error(labels, predictions)
        tf.summary.scalar('cost', cost)
    
    # Train the model
    with tf.name_scope('train'):    
        optimizer = 
            tf.train.AdamOptimizer(learning_rate).minimize(cost)

    # Determine the accuracy
    with tf.name_scope("accuracy"):
        correct_pred = tf.equal(tf.cast(tf.round(predictions), 
                                        tf.int32), 
                                        labels)
        accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
        tf.summary.scalar('accuracy', accuracy)
    
    # Merge all of the summaries
    merged = tf.summary.merge_all()    

    # Export the nodes 
    export_nodes = ['inputs', 'labels', 'keep_prob','initial_state',        
                    'final_state','accuracy', 'predictions', 'cost', 
                    'optimizer', 'merged']
    Graph = namedtuple('Graph', export_nodes)
    local_dict = locals()
    graph = Graph(*[local_dict[each] for each in export_nodes])
    
    return graph

如果之前你沒有用過TensorBoard, 我建議可以看下Siraj Raval的這個視頻https://www.youtube.com/watch?v=fBVEXKp4DIc, 同時他還有很多相關的視頻教程，同樣非常出色。

tf.reset_default_graph()

在開始訓練前，需要把我們的圖重置一下，以保證訓練數據的質量。

# Declare placeholders we'll feed into the graph
    with tf.name_scope('inputs'):
        inputs = tf.placeholder(tf.int32, [None, None],  
                                    name='inputs')

    with tf.name_scope('labels'):
        labels = tf.placeholder(tf.int32, [None, None], 
                                    name='labels')

    keep_prob = tf.placeholder(tf.float32, name='keep_prob')

這些是我們數據的占位符。當我們用TensorBoard 圖形化我們的數據後，tf.name_scope()可以用來標記圖中的特定部分。

# Create the embeddings
    with tf.name_scope("embeddings"):
        embedding = tf.Variable(tf.random_uniform((n_words, 
                                  embed_size), -1, 1))
        embed = tf.nn.embedding_lookup(embedding, inputs)

通過映射(embedding)，我們把準備的單詞表轉換成向量，embed_size表示這個向量的維度。這裏有更詳細的討論：https://www.quora.com/What-does-the-word-embedding-mean-in-the-context-of-Machine-Learning

雖然這裏我用了隨機均勻分布的方法，其實還有其他很多方法建立這樣映射。比如用帶有較小標準差的截斷正態分布也是不錯的選擇，那代碼就是這樣的：

embedding = tf.Variable(tf.truncated_normal((n_words, embed_size), -0.1, 0.1))

讓我們自己動手試試！

# Build the RNN layers
with tf.name_scope("RNN_layers"):
    lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size)
    drop = tf.contrib.rnn.DropoutWrapper(lstm, 
                                         output_keep_prob=keep_prob)
    cell = tf.contrib.rnn.MultiRNNCell([drop] * num_layers)

# Set the initial state
with tf.name_scope("RNN_init_state"):
    initial_state = cell.zero_state(batch_size, tf.float32)

這步創建了圖的初始狀態。

# Run the data through the RNN layers
with tf.name_scope("RNN_forward"):
    outputs, final_state = tf.nn.dynamic_rnn(cell, embed,
                                        initial_state=initial_state)

# Create the fully connected layers
with tf.name_scope("fully_connected"):
        
    # Initialize the weights and biases
    weights = tf.truncated_normal_initializer(stddev=0.1)
    biases = tf.zeros_initializer()
        
    dense = tf.contrib.layers.fully_connected(outputs[:, -1],
                num_outputs = fc_units,
                activation_fn = tf.sigmoid,
                weights_initializer = weights,
                biases_initializer = biases)
        
    dense = tf.contrib.layers.dropout(dense, keep_prob)
        
    # Depending on the iteration, use a second fully connected layer
    if multiple_fc == True:
        dense = tf.contrib.layers.fully_connected(dense,
                    num_outputs = fc_units,
                    activation_fn = tf.sigmoid,
                    weights_initializer = weights,
                    biases_initializer = biases)
            
        dense = tf.contrib.layers.dropout(dense, keep_prob)

這步就是我們添加第一層和可能完全鏈接的第二層網絡。他們的權重(weight)和偏置值(bias)會在之前提到過的映射(embedding)方法中進行初始化。這裏multiple_fc 是方法的一個參數。它使我們能夠測試這個模型的架構。用這個方法，你可以對不同的內容進行測試，比如如何初始化權重和偏置值，是否使用LSTM（Long Short Term Memory）或GRU（Gated Recurrent Unit），等等

# Make the predictions
with tf.name_scope('predictions'):
    predictions = tf.contrib.layers.fully_connected(dense, 
                      num_outputs = 1, 
                      activation_fn = tf.sigmoid,
                      weights_initializer = weights,
                      biases_initializer = biases)
        
    tf.summary.histogram('predictions', predictions)

我們這裏隻有關於影評態度(0到1)的一個輸出。Sigmoid方法把最終完全連接的輸出層映射到了這個區間。tf.summary.histogram() 記錄了我們的預測結果，並輸出成TensorBoard中的柱狀圖。這能使我們清楚的看到，在訓練我們的模型過程中，預測分布的是如何改變的。同時，我們也可以看到訓練集和驗證集是如何進行比較的。

# Calculate the cost
with tf.name_scope('cost'):
    cost = tf.losses.mean_squared_error(labels, predictions)
    tf.summary.scalar('cost', cost)

這裏計算的是訓練過程中的花費。我們用到了tf.summary.scalar()方法，這裏花費是一個沒有區間的標量值。

# Train the model
with tf.name_scope('train'):    
    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)

Adam是一個通用的優化器，它能使模塊訓練更有效率。你還可以用其他的算法。這裏我們就不展開討論了。

# Determine the accuracy
with tf.name_scope("accuracy"):
    correct_pred = tf.equal(tf.cast(tf.round(predictions), 
                                        tf.int32), 
                                        labels)
    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
    tf.summary.scalar('accuracy', accuracy)

# Merge all of the summaries
merged = tf.summary.merge_all()

這裏，我們進行匯總，簡化數據保存的過程。

# Export the nodes 
export_nodes = ['inputs', 'labels', 'keep_prob','initial_state',        
                'final_state','accuracy', 'predictions', 'cost', 
                'optimizer', 'merged']
Graph = namedtuple('Graph', export_nodes)
local_dict = locals()
graph = Graph(*[local_dict[each] for each in export_nodes])

導出所有節點，為後續的訓練函數做準備。至此，我們完成了循環神經網絡的創建，接下來我們就能用其進行訓練了。

def train(model, epochs, log_string):
    '''Train the RNN'''

    saver = tf.train.Saver()
    
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        # Used to determine when to stop the training early
        valid_loss_summary = []
        
        # Keep track of which batch iteration is being trained
        iteration = 0

        print()
        print("Training Model: {}".format(log_string))

        train_writer = tf.summary.FileWriter('./logs/3/train/{}'.format(log_string), sess.graph)
        valid_writer = tf.summary.FileWriter('./logs/3/valid/{}'.format(log_string))

        for e in range(epochs):
            state = sess.run(model.initial_state)
            
            # Record progress with each epoch
            train_loss = []
            train_acc = []
            val_acc = []
            val_loss = []

            with tqdm(total=len(x_train)) as pbar:
                for _, (x, y) in enumerate(get_batches(x_train,       
                                               y_train, 
                                               batch_size), 1):
                    feed = {model.inputs: x,
                            model.labels: y[:, None],
                            model.keep_prob: dropout,
                            model.initial_state: state}
                    summary, loss, acc, state, _ =     
                                          sess.run([model.merged, 
                                                  model.cost, 
                                                  model.accuracy, 
                                                  model.final_state, 
                                                  model.optimizer], 
                                                  feed_dict=feed)                
                    
                    # Record the loss and accuracy of each training  
                      batch
                    
                    train_loss.append(loss)
                    train_acc.append(acc)
                    
                    # Record the progress of training
                    train_writer.add_summary(summary, iteration)
                    
                    iteration += 1
                    pbar.update(batch_size)
            
            # Average the training loss and accuracy of each epoch
            avg_train_loss = np.mean(train_loss)
            avg_train_acc = np.mean(train_acc) 

            val_state = sess.run(model.initial_state)
            with tqdm(total=len(x_valid)) as pbar:
                for x, y in get_batches(x_valid,y_valid,batch_size):
                    feed = {model.inputs: x,
                            model.labels: y[:, None],
                            model.keep_prob: 1,
                            model.initial_state: val_state}
                    summary, batch_loss, batch_acc, val_state =     
                                 sess.run([model.merged, 
                                           model.cost, 
                                           model.accuracy, 
                                           model.final_state], 
                                           feed_dict=feed)
                    
                    # Record the validation loss and accuracy of 
                      each epoch
                    
                    val_loss.append(batch_loss)
                    val_acc.append(batch_acc)
                    pbar.update(batch_size)
            
            # Average the validation loss and accuracy of each epoch
            avg_valid_loss = np.mean(val_loss)    
            avg_valid_acc = np.mean(val_acc)
            valid_loss_summary.append(avg_valid_loss)
            
            # Record the validation data's progress
            valid_writer.add_summary(summary, iteration)

            # Print the progress of each epoch
            print("Epoch: {}/{}".format(e, epochs),
                  "Train Loss: {:.3f}".format(avg_train_loss),
                  "Train Acc: {:.3f}".format(avg_train_acc),
                  "Valid Loss: {:.3f}".format(avg_valid_loss),
                  "Valid Acc: {:.3f}".format(avg_valid_acc))

            # Stop training if the validation loss does not decrease 
              after 3 epochs
            
            if avg_valid_loss > min(valid_loss_summary):
                print("No Improvement.")
                stop_early += 1
                if stop_early == 3:
                    break   
            
            # Reset stop_early if the validation loss finds a new low
            # Save a checkpoint of the model
            else:
                print("New Record!")
                stop_early = 0
                checkpoint ="./sentiment_{}.ckpt".format(log_string)
                saver.save(sess, checkpoint)

下麵讓我們一步步來解釋上麵的代碼

saver = tf.train.Saver()

這裏是為了保存每個檢查點

train_writer = tf.summary.FileWriter('./logs/3/train/{}'.format(log_string), sess.graph)
valid_writer = tf.summary.FileWriter('./logs/3/valid/{}'.format(log_string))

這段代碼把訓練的匯總作為日誌存在本地。建議把它們存放在同一個logs文件夾下,同時訓練和驗證匯總分別放在不同的子文件夾。這樣能更方便的在TensorBoard上進行比對。

tqdm() （https://pypi.python.org/pypi/tqdm）則會記錄每個過程的時間，這樣我就能了解每次過程中還有大致多少剩餘時間。

# Record the validation data's progress
valid_writer.add_summary(summary, iteration)

# Reset stop_early if the validation loss finds a new low
# Save a checkpoint of the model
else:
    print("New Record!")
    stop_early = 0
    checkpoint = "./sentiment_{}.ckpt".format(log_string)
    saver.save(sess, checkpoint)

n_words = len(word_index)
embed_size = 300
batch_size = 250
lstm_size = 128
num_layers = 2
dropout = 0.5
learning_rate = 0.001
epochs = 100
multiple_fc = False
fc_units = 256

# Train the model with the desired tuning parameters
for lstm_size in [64,128]:
    for multiple_fc in [True, False]:
        for fc_units in [128, 256]:
            log_string = 'ru={},fcl={},fcu={}'.format(lstm_size,
                                                      multiple_fc,
                                                      fc_units)
            model = build_rnn(n_words = n_words, 
                              embed_size = embed_size,
                              batch_size = batch_size,
                              lstm_size = lstm_size,
                              num_layers = num_layers,
                              dropout = dropout,
                              learning_rate = learning_rate,
                              multiple_fc = multiple_fc,
                              fc_units = fc_units)            
            train(model, epochs, log_string)

這裏我可以調整不同的lstm_size, multiple_fc, 和 fc_units參數值。用這種結構，你可以設置任何你想調優的值。注意，這裏需要在日誌中記錄這些數值。

def make_predictions(lstm_size, multiple_fc, fc_units, checkpoint):
    '''Predict the sentiment of the testing data'''
    
    # Record all of the predictions
    all_preds = []

    model = build_rnn(n_words = n_words, 
                      embed_size = embed_size,
                      batch_size = batch_size,
                      lstm_size = lstm_size,
                      num_layers = num_layers,
                      dropout = dropout,
                      learning_rate = learning_rate,
                      multiple_fc = multiple_fc,
                      fc_units = fc_units) 
    
    with tf.Session() as sess:
        saver = tf.train.Saver()
        # Load the model
        saver.restore(sess, checkpoint)
        test_state = sess.run(model.initial_state)
        for _, x in enumerate(get_test_batches(x_test, 
                                               batch_size), 1):
            feed = {model.inputs: x,
                    model.keep_prob: 1,
                    model.initial_state: test_state}
            predictions = sess.run(model.predictions,feed_dict=feed)
            for pred in predictions:
                all_preds.append(float(pred))
                
    return all_preds

這個方法會得到訓練數據的預測結果。這裏要注意的是參數的設置需要與你的調優能夠匹配。否則，你的預測結果可能會使用默認的參數值，而導致最終的結果並不能達到你的預期。

這就是整個項目的大致內容，這是在Github（https://github.com/Currie32/Movie-Reviews-Sentiment）上的源代碼。如果對代碼有任何問題或改進意見，我們也可以進行進一步的探討。

本文由北郵@愛可可-愛生活 老師推薦，阿裏雲雲棲社區組織翻譯。

Predicting Movie Review Sentiment with TensorFlow and TensorBoardDave Currie

作者介紹：Dave Currie 致力於機器學習(自然語言處理方向)與數據科學研究的軟件工程師。

Linkedinhttps://www.linkedin.com/in/davidcurrie32

原文