个性化推荐系统——2. 数据预处理

矫情吗；* 2023-07-03 07:05 21阅读 0赞

# 数据预处理 #

*  各种ID不用变,UserID,OccupationID,MovieID
 *  类别数据用字典转化为数字类型，Gender,Age,Genres
 *  Title进行word2vec的转换

import pandas as pd
    import numpy as np
    import pickle
    import re

def load_data():
        #读取User数据
        users_title=['UserID','Gender','Age','JobID','Zip-code']
        users=pd.read_csv('./ml-1m/users.dat',sep='::',header=None,names=users_title,engine='python')
        users=users.filter(regex='UserID|Gender|Age|JobID')#去掉zip-code
        #filter过滤列表中的元素，并且返回⼀个由所有符合要求的元素所构成的列表，符合要求即函数映射到该元素时返回值为True.这个filter类似于⼀个for循环，但它是⼀个内置函数，并且更快。
        users_orig=users.values
        #改变类别数据——性别&年龄
        gender_map={ 'F':0,'M':1}
        users['Gender']=users['Gender'].map(gender_map)
        
        age_map={ val:ii for ii,val in enumerate(set(users['Age']))}
        users['Age']=users['Age'].map(age_map)
        
        #读取Movie数据
        movies_title = ['MovieID', 'Title', 'Genres']
        movies = pd.read_csv('./ml-1m/movies.dat', sep='::', header=None, names=movies_title, engine = 'python')
        movies_orig = movies.values
        #将Title中的年份去掉
        pattern = re.compile(r'^(.*)$(\d+)$$')
    
        title_map = { val:pattern.match(val).group(1) for ii,val in enumerate(set(movies['Title']))}
        movies['Title'] = movies['Title'].map(title_map)
        
        #将电影的分类转化为数字字典
        genres_set=set()
        for val in movies['Genres'].str.split('|'):
            genres_set.update(val)
            
        genres_set.add('<PAD>')#空白填充为'<PAD>',统一长度所需，为神经网络模型训练提供便利
        genres2int={ val:ii for ii,val in enumerate(genres_set)}
        
        #将电影类型转成等长数字列表，长度是18
        genres_map = { val:[genres2int[row] for row in val.split('|')] for ii,val in enumerate(set(movies['Genres']))}
    
        for key in genres_map:
            for cnt in range(max(genres2int.values()) - len(genres_map[key])):
                genres_map[key].insert(len(genres_map[key]) + cnt,genres2int['<PAD>'])
        
        movies['Genres'] = movies['Genres'].map(genres_map)
    
        #电影Title转数字字典
        title_set = set()
        for val in movies['Title'].str.split():
            title_set.update(val)
        
        title_set.add('<PAD>')
        title2int = { val:ii for ii, val in enumerate(title_set)}
    
        #将电影Title转成等长数字列表，长度是15
        title_count = 15
        title_map = { val:[title2int[row] for row in val.split()] for ii,val in enumerate(set(movies['Title']))}
        
        for key in title_map:
            for cnt in range(title_count - len(title_map[key])):
                title_map[key].insert(len(title_map[key]) + cnt,title2int['<PAD>'])
        
        movies['Title'] = movies['Title'].map(title_map)
    
        #读取评分数据集
        ratings_title = ['UserID','MovieID', 'ratings', 'timestamps']
        ratings = pd.read_csv('./ml-1m/ratings.dat', sep='::', header=None, names=ratings_title, engine = 'python')
        ratings = ratings.filter(regex='UserID|MovieID|ratings')
    
        #合并三个表
        data = pd.merge(pd.merge(ratings, users), movies)
        
        #将数据分成X和y两张表
        target_fields = ['ratings']
        features_pd, targets_pd = data.drop(target_fields, axis=1), data[target_fields]
        
        features = features_pd.values
        targets_values = targets_pd.values
        
        return title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig

# 加载数据并保存到本地 #

*  title\_count：Title字段的长度（15）
 *  title\_set：Title文本的集合
 *  genres2int：电影类型转数字的字典
 *  features：是输入X
 *  targets\_values：是学习目标y
 *  ratings：评分数据集的Pandas对象
 *  users：用户数据集的Pandas对象
 *  movies：电影数据的Pandas对象
 *  data：三个数据集组合在一起的Pandas对象
 *  movies\_orig：没有做数据处理的原始电影数据
 *  users\_orig：没有做数据处理的原始用户数据

title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig = load_data()
    #pickle保存到本地后可以快速加载
    pickle.dump((title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig), open('preprocess.p', 'wb'))

# 预处理后的数据 #

users.head()

<table> 
 <thead> 
  <tr> 
   <th></th> 
   <th>UserID</th> 
   <th>Gender</th> 
   <th>Age</th> 
   <th>JobID</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <th>0</th> 
   <td>1</td> 
   <td>0</td> 
   <td>0</td> 
   <td>10</td> 
  </tr> 
  <tr> 
   <th>1</th> 
   <td>2</td> 
   <td>1</td> 
   <td>5</td> 
   <td>16</td> 
  </tr> 
  <tr> 
   <th>2</th> 
   <td>3</td> 
   <td>1</td> 
   <td>6</td> 
   <td>15</td> 
  </tr> 
  <tr> 
   <th>3</th> 
   <td>4</td> 
   <td>1</td> 
   <td>2</td> 
   <td>7</td> 
  </tr> 
  <tr> 
   <th>4</th> 
   <td>5</td> 
   <td>1</td> 
   <td>6</td> 
   <td>20</td> 
  </tr> 
 </tbody> 
</table>

movies.head()

<table> 
 <thead> 
  <tr> 
   <th></th> 
   <th>MovieID</th> 
   <th>Title</th> 
   <th>Genres</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <th>0</th> 
   <td>1</td> 
   <td>[2194, 4563, 2402, 2402, 2402, 2402, 2402, 240...</td> 
   <td>[16, 18, 5, 13, 13, 13, 13, 13, 13, 13, 13, 13...</td> 
  </tr> 
  <tr> 
   <th>1</th> 
   <td>2</td> 
   <td>[2558, 2402, 2402, 2402, 2402, 2402, 2402, 240...</td> 
   <td>[10, 18, 1, 13, 13, 13, 13, 13, 13, 13, 13, 13...</td> 
  </tr> 
  <tr> 
   <th>2</th> 
   <td>3</td> 
   <td>[1335, 4290, 3288, 2402, 2402, 2402, 2402, 240...</td> 
   <td>[5, 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13...</td> 
  </tr> 
  <tr> 
   <th>3</th> 
   <td>4</td> 
   <td>[2423, 5164, 3171, 2402, 2402, 2402, 2402, 240...</td> 
   <td>[5, 17, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13...</td> 
  </tr> 
  <tr> 
   <th>4</th> 
   <td>5</td> 
   <td>[4573, 2552, 1568, 2808, 2806, 1319, 2402, 240...</td> 
   <td>[5, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13...</td> 
  </tr> 
 </tbody> 
</table>

movies.values[0]

array([1,
           list([2194, 4563, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402, 2402]),
           list([16, 18, 5, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13])],
          dtype=object)