AI比赛-推荐系统（一）-新闻推荐02：训练item词向量【每篇新闻文章对应一个词向量】【word2vec：将每个用户点击的新闻文章序列作为输入（类似文本序列），训练出每篇文章的向量表示】【天池】

末蓝、 2023-09-25 14:54 43阅读 0赞

安装gensim，使用gensim中的word2vec模型

# 安装gensim
    !pip install gensim

import pandas as pd
    import numpy as np
    from sklearn.preprocessing import MinMaxScaler
    import matplotlib.pyplot as plt
    from gensim.models import Word2Vec
    import logging

## 一、读取原始数据 ##

#### 1、加载原始数据 ####

path = './data_raw/'
    save_path = './temp_results/'

#### 2、读取训练数据集 ####

# 训练数据集
    trn_click = pd.read_csv(path + 'train_click_log.csv')
    trn_click = trn_click.sort_values('user_id')
    trn_click

<table> 
 <thead> 
  <tr> 
   <th></th> 
   <th>user_id</th> 
   <th>click_article_id</th> 
   <th>click_timestamp</th> 
   <th>click_environment</th> 
   <th>click_deviceGroup</th> 
   <th>click_os</th> 
   <th>click_country</th> 
   <th>click_region</th> 
   <th>click_referrer_type</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <th>1112620</th> 
   <td>0</td> 
   <td>157507</td> 
   <td>1508211702520</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>25</td> 
   <td>2</td> 
  </tr> 
  <tr> 
   <th>1112619</th> 
   <td>0</td> 
   <td>30760</td> 
   <td>1508211672520</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>25</td> 
   <td>2</td> 
  </tr> 
  <tr> 
   <th>1112602</th> 
   <td>1</td> 
   <td>63746</td> 
   <td>1508211346889</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>25</td> 
   <td>6</td> 
  </tr> 
  <tr> 
   <th>1112601</th> 
   <td>1</td> 
   <td>289197</td> 
   <td>1508211316889</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>25</td> 
   <td>6</td> 
  </tr> 
  <tr> 
   <th>1112600</th> 
   <td>2</td> 
   <td>168401</td> 
   <td>1508211468695</td> 
   <td>4</td> 
   <td>3</td> 
   <td>20</td> 
   <td>1</td> 
   <td>25</td> 
   <td>2</td> 
  </tr> 
  <tr> 
   <th>...</th> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
  </tr> 
  <tr> 
   <th>1081895</th> 
   <td>199999</td> 
   <td>218355</td> 
   <td>1508176867088</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>13</td> 
   <td>1</td> 
  </tr> 
  <tr> 
   <th>660731</th> 
   <td>199999</td> 
   <td>161191</td> 
   <td>1507665351186</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>13</td> 
   <td>1</td> 
  </tr> 
  <tr> 
   <th>660732</th> 
   <td>199999</td> 
   <td>42223</td> 
   <td>1507665381186</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>13</td> 
   <td>1</td> 
  </tr> 
  <tr> 
   <th>211041</th> 
   <td>199999</td> 
   <td>123909</td> 
   <td>1507226987864</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>13</td> 
   <td>1</td> 
  </tr> 
  <tr> 
   <th>0</th> 
   <td>199999</td> 
   <td>160417</td> 
   <td>1507029570190</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>13</td> 
   <td>1</td> 
  </tr> 
 </tbody> 
</table>

1112623 rows × 9 columns

#### 3、读取测试数据集 ####

# 读取测试数据集
    tst_click = pd.read_csv(path + 'testA_click_log.csv')
    tst_click = tst_click.sort_values('user_id')
    tst_click

<table> 
 <thead> 
  <tr> 
   <th></th> 
   <th>user_id</th> 
   <th>click_article_id</th> 
   <th>click_timestamp</th> 
   <th>click_environment</th> 
   <th>click_deviceGroup</th> 
   <th>click_os</th> 
   <th>click_country</th> 
   <th>click_region</th> 
   <th>click_referrer_type</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <th>138222</th> 
   <td>200000</td> 
   <td>195839</td> 
   <td>1507030363999</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
  </tr> 
  <tr> 
   <th>138223</th> 
   <td>200000</td> 
   <td>191971</td> 
   <td>1507030393999</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
  </tr> 
  <tr> 
   <th>378656</th> 
   <td>200000</td> 
   <td>194300</td> 
   <td>1507651461280</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
  </tr> 
  <tr> 
   <th>138221</th> 
   <td>200001</td> 
   <td>175040</td> 
   <td>1507029536442</td> 
   <td>4</td> 
   <td>3</td> 
   <td>2</td> 
   <td>1</td> 
   <td>18</td> 
   <td>7</td> 
  </tr> 
  <tr> 
   <th>138219</th> 
   <td>200002</td> 
   <td>297906</td> 
   <td>1507029946064</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>8</td> 
   <td>1</td> 
  </tr> 
  <tr> 
   <th>...</th> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
  </tr> 
  <tr> 
   <th>155995</th> 
   <td>249999</td> 
   <td>313431</td> 
   <td>1507052560685</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>13</td> 
   <td>2</td> 
  </tr> 
  <tr> 
   <th>178071</th> 
   <td>249999</td> 
   <td>214800</td> 
   <td>1507117287497</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>13</td> 
   <td>2</td> 
  </tr> 
  <tr> 
   <th>178070</th> 
   <td>249999</td> 
   <td>233717</td> 
   <td>1507117257497</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>13</td> 
   <td>2</td> 
  </tr> 
  <tr> 
   <th>188451</th> 
   <td>249999</td> 
   <td>233717</td> 
   <td>1507133510213</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>13</td> 
   <td>2</td> 
  </tr> 
  <tr> 
   <th>0</th> 
   <td>249999</td> 
   <td>160974</td> 
   <td>1506959142820</td> 
   <td>4</td> 
   <td>1</td> 
   <td>17</td> 
   <td>1</td> 
   <td>13</td> 
   <td>2</td> 
  </tr> 
 </tbody> 
</table>

518010 rows × 9 columns

#### 4、读取文章信息 ####

# 读取文章信息
    item_df = pd.read_csv(path + 'articles.csv')
    item_df = item_df.sort_values('article_id')
    item_df

<table> 
 <thead> 
  <tr> 
   <th></th> 
   <th>article_id</th> 
   <th>category_id</th> 
   <th>created_at_ts</th> 
   <th>words_count</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <th>0</th> 
   <td>0</td> 
   <td>0</td> 
   <td>1513144419000</td> 
   <td>168</td> 
  </tr> 
  <tr> 
   <th>1</th> 
   <td>1</td> 
   <td>1</td> 
   <td>1405341936000</td> 
   <td>189</td> 
  </tr> 
  <tr> 
   <th>2</th> 
   <td>2</td> 
   <td>1</td> 
   <td>1408667706000</td> 
   <td>250</td> 
  </tr> 
  <tr> 
   <th>3</th> 
   <td>3</td> 
   <td>1</td> 
   <td>1408468313000</td> 
   <td>230</td> 
  </tr> 
  <tr> 
   <th>4</th> 
   <td>4</td> 
   <td>1</td> 
   <td>1407071171000</td> 
   <td>162</td> 
  </tr> 
  <tr> 
   <th>...</th> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
  </tr> 
  <tr> 
   <th>364042</th> 
   <td>364042</td> 
   <td>460</td> 
   <td>1434034118000</td> 
   <td>144</td> 
  </tr> 
  <tr> 
   <th>364043</th> 
   <td>364043</td> 
   <td>460</td> 
   <td>1434148472000</td> 
   <td>463</td> 
  </tr> 
  <tr> 
   <th>364044</th> 
   <td>364044</td> 
   <td>460</td> 
   <td>1457974279000</td> 
   <td>177</td> 
  </tr> 
  <tr> 
   <th>364045</th> 
   <td>364045</td> 
   <td>460</td> 
   <td>1515964737000</td> 
   <td>126</td> 
  </tr> 
  <tr> 
   <th>364046</th> 
   <td>364046</td> 
   <td>460</td> 
   <td>1505811330000</td> 
   <td>479</td> 
  </tr> 
 </tbody> 
</table>

364047 rows × 4 columns

## 二、数据预处理 ##

#### 1、重命名item\_df，与trn\_click、tst\_click中保持一致方便后续match ####

# 重命名item_df，与trn_click、tst_click中保持一致方便后续match  
    item_df = item_df.rename(columns={'article_id': 'click_article_id'})  
    item_df

<table> 
 <thead> 
  <tr> 
   <th></th> 
   <th>click_article_id</th> 
   <th>category_id</th> 
   <th>created_at_ts</th> 
   <th>words_count</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <th>0</th> 
   <td>0</td> 
   <td>0</td> 
   <td>1513144419000</td> 
   <td>168</td> 
  </tr> 
  <tr> 
   <th>1</th> 
   <td>1</td> 
   <td>1</td> 
   <td>1405341936000</td> 
   <td>189</td> 
  </tr> 
  <tr> 
   <th>2</th> 
   <td>2</td> 
   <td>1</td> 
   <td>1408667706000</td> 
   <td>250</td> 
  </tr> 
  <tr> 
   <th>3</th> 
   <td>3</td> 
   <td>1</td> 
   <td>1408468313000</td> 
   <td>230</td> 
  </tr> 
  <tr> 
   <th>4</th> 
   <td>4</td> 
   <td>1</td> 
   <td>1407071171000</td> 
   <td>162</td> 
  </tr> 
  <tr> 
   <th>...</th> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
   <td>...</td> 
  </tr> 
  <tr> 
   <th>364042</th> 
   <td>364042</td> 
   <td>460</td> 
   <td>1434034118000</td> 
   <td>144</td> 
  </tr> 
  <tr> 
   <th>364043</th> 
   <td>364043</td> 
   <td>460</td> 
   <td>1434148472000</td> 
   <td>463</td> 
  </tr> 
  <tr> 
   <th>364044</th> 
   <td>364044</td> 
   <td>460</td> 
   <td>1457974279000</td> 
   <td>177</td> 
  </tr> 
  <tr> 
   <th>364045</th> 
   <td>364045</td> 
   <td>460</td> 
   <td>1515964737000</td> 
   <td>126</td> 
  </tr> 
  <tr> 
   <th>364046</th> 
   <td>364046</td> 
   <td>460</td> 
   <td>1505811330000</td> 
   <td>479</td> 
  </tr> 
 </tbody> 
</table>

364047 rows × 4 columns

#### 2、计算用户点击文章的次数，并添加新的一列count ####

# 计算用户点击文章的次数，并添加新的一列count
    trn_click['click_cnts'] = trn_click.groupby(['user_id'])['click_timestamp'].transform('count')
    trn_click