-
Notifications
You must be signed in to change notification settings - Fork 0
/
Generator.py
162 lines (120 loc) · 4.68 KB
/
Generator.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
#!/usr/bin/env python
# _*_ coding:utf-8 _*_
from tensorflow.keras.utils import Sequence
import numpy as np
FLAG_CTR = 1
def fetch_ctr_dim3(News, docids, bucket, flag=1):
batch_size, doc_num = docids.shape
doc_imp = News.news_stat_imp[docids]
doc_click = News.news_stat_click[docids]
ctr = np.zeros(docids.shape)
for i in range(batch_size):
for j in range(doc_num):
b = bucket[i, j] - 1
if b < 0:
b = 0
ctr[i, j] = doc_click[i, j, b] / (doc_imp[i, j, b] + 0.01)
ctr = ctr * 200
ct = np.ceil(ctr)
ctr = np.array(ct, dtype='int32')
return ctr
class TrainGenerator(Sequence):
def __init__(self, News, Users, news_id, user_ids, buckets, news_exist_times, label, uid, batch_size):
self.News = News
self.Users = Users
self.user_ids = user_ids
self.doc_id = news_id
self.buckets = buckets
self.news_exist_times = news_exist_times
self.label = label
self.uid = uid
self.batch_size = batch_size
self.ImpNum = self.label.shape[0]
def __len__(self):
return int(np.ceil(self.ImpNum / float(self.batch_size)))
def __getitem__(self, idx):
start = idx * self.batch_size
ed = (idx + 1) * self.batch_size
if ed > self.ImpNum:
ed = self.ImpNum
doc_ids = self.doc_id[start:ed]
news_feature = self.News.fetch_news(doc_ids)
user_ids = self.user_ids[start:ed]
userid = self.uid[start:ed]
clicked_ids = self.Users.click[user_ids]
user_feature = self.News.fetch_news(clicked_ids)
user_feature_id = userid
bucket = self.buckets[start:ed]
news_exist_time = self.news_exist_times[start:ed]
click_bucket = self.Users.click_bucket[user_ids]
click_ctr = fetch_ctr_dim3(self.News, clicked_ids, click_bucket, FLAG_CTR)
label = self.label[start:ed]
return [news_feature, user_feature, bucket, news_exist_time, user_feature_id, click_ctr], [label]
class UserGenerator(Sequence):
def __init__(self, News, Users, uid, batch_size):
self.News = News
self.Users = Users
self.uid = uid
self.batch_size = batch_size
self.ImpNum = self.Users.click.shape[0]
def __len__(self):
return int(np.ceil(self.ImpNum / float(self.batch_size)))
def __getitem__(self, idx):
start = idx * self.batch_size
ed = (idx + 1) * self.batch_size
if ed > self.ImpNum:
ed = self.ImpNum
uid = self.uid[start:ed]
clicked_ids = self.Users.click[start:ed]
user_feature = self.News.fetch_news(clicked_ids)
user_feature_id = uid
click_bucket = self.Users.click_bucket[start:ed]
click_ctr = fetch_ctr_dim3(self.News, clicked_ids, click_bucket, FLAG_CTR)
return [user_feature, click_ctr]
class PopularityGenerator(Sequence):
def __init__(self, Impressions, batch_size):
self.Impressions = Impressions
self.batch_size = batch_size
self.ImpNum = len(self.Impressions)
def __len__(self):
return int(np.ceil(self.ImpNum / float(self.batch_size)))
def __getitem__(self, idx):
start = idx * self.batch_size
# ed = (idx + 1) * self.batch_size
if start > self.ImpNum:
start = self.ImpNum
bucket = self.Impressions[start]['tsp']
bucket = np.array(bucket)
news_exist_time = self.Impressions[start]['news_exist_time']
news_exist_time = np.array(news_exist_time)
news_exist_time = news_exist_time.astype(np.int)
return [bucket, news_exist_time]
class PopularityBiasGenerator(Sequence):
def __init__(self, uid, batch_size):
self.uid = uid
self.batch_size = batch_size
self.ImpNum = len(self.uid)
def __len__(self):
return int(np.ceil(self.ImpNum / float(self.batch_size)))
def __getitem__(self, idx):
start = idx * self.batch_size
ed = (idx + 1) * self.batch_size
if start > self.ImpNum:
start = self.ImpNum
uid = self.uid[start:ed]
return uid
class NewsGenerator(Sequence):
def __init__(self, News, batch_size):
self.News = News
self.batch_size = batch_size
self.ImpNum = self.News.title.shape[0]
def __len__(self):
return int(np.ceil(self.ImpNum / float(self.batch_size)))
def __getitem__(self, idx):
start = idx * self.batch_size
ed = (idx + 1) * self.batch_size
if ed > self.ImpNum:
ed = self.ImpNum
doc_ids = np.array([i for i in range(start, ed)])
news_feature = self.News.fetch_news(doc_ids)
return news_feature