Skip to content

Commit

Permalink
version compatibility
Browse files Browse the repository at this point in the history
  • Loading branch information
@shenweichen
shenweichen authored May 17, 2020
1 parent 3e099bc commit fe1e7b3
Show file tree
Hide file tree
Showing 24 changed files with 307 additions and 101 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
strategy:
matrix:
python-version: [3.5,3.6,3.7]
tf-version: [1.4.0,1.14.0,2.1.0]
tf-version: [1.4.0,1.14.0,2.1.0,2.2.0]

exclude:
- python-version: 3.7
Expand Down
3 changes: 2 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ Let's [**Get Started!**](https://deepmatch.readthedocs.io/en/latest/Quick-Start.
| DSSM | [CIKM 2013][Deep Structured Semantic Models for Web Search using Clickthrough Data](https://www.microsoft.com/en-us/research/publication/learning-deep-structured-semantic-models-for-web-search-using-clickthrough-data/) |
| YoutubeDNN | [RecSys 2016][Deep Neural Networks for YouTube Recommendations](https://www.researchgate.net/publication/307573656_Deep_Neural_Networks_for_YouTube_Recommendations) |
| NCF | [WWW 2017][Neural Collaborative Filtering](https://arxiv.org/abs/1708.05031) |
| MIND | [CIKM 2019][Multi-interest network with dynamic routing for recommendation at Tmall](https://arxiv.org/pdf/1904.08030) |
| SDM | [CIKM 2019][SDM: Sequential Deep Matching Model for Online Large-scale Recommender System](https://arxiv.org/abs/1909.00385) |
| MIND | [CIKM 2019][Multi-interest network with dynamic routing for recommendation at Tmall](https://arxiv.org/pdf/1904.08030) |

## Contributors([welcome to join us!](./CONTRIBUTING.md))

Expand Down
2 changes: 1 addition & 1 deletion deepmatch/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from .utils import check_version

__version__ = '0.1.2'
__version__ = '0.1.3'
check_version(__version__)
61 changes: 39 additions & 22 deletions deepmatch/layers/interaction.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
import itertools

import tensorflow as tf
from tensorflow.python.keras import backend as K
from tensorflow.python.keras.initializers import TruncatedNormal
from tensorflow.python.keras.layers import Layer, Dense, Dropout
from deepctr.layers.normalization import LayerNormalization
from deepctr.layers.utils import softmax, reduce_mean
from tensorflow.python.keras.initializers import TruncatedNormal
from tensorflow.python.keras.layers import Layer, Dense, Dropout


class DotAttention(Layer):
Expand All @@ -14,6 +11,7 @@ class DotAttention(Layer):
:param key: [batch_size, T, C]
:return: [batch_size, 1, T]
"""

def __init__(self, scale=True, **kwargs):
self.scale = scale
super(DotAttention, self).__init__(**kwargs)
Expand Down Expand Up @@ -47,6 +45,7 @@ class ConcatAttention(Layer):
:return: [batch_size, 1, T]
query_size should keep the same dim with key_size
"""

def __init__(self, scale=True, **kwargs):
self.scale = scale
super(ConcatAttention, self).__init__(**kwargs)
Expand Down Expand Up @@ -85,6 +84,7 @@ class SoftmaxWeightedSum(Layer):
:return: weighted sum vector
[batch_size, 1, units]
"""

def __init__(self, dropout_rate=0.2, future_binding=False, seed=2020, **kwargs):
self.dropout_rate = dropout_rate
self.future_binding = future_binding
Expand Down Expand Up @@ -137,14 +137,15 @@ class AttentionSequencePoolingLayer(Layer):
:param keys_length: [batch_size, 1]
:return: [batch_size, 1, C_k]
"""

def __init__(self, dropout_rate=0, **kwargs):
self.dropout_rate = dropout_rate
super(AttentionSequencePoolingLayer, self).__init__(**kwargs)

def build(self, input_shape):
if not isinstance(input_shape, list) or len(input_shape) != 3:
raise ValueError('A `SequenceFeatureMask` layer should be called '
'on a list of 3 inputs')
'on a list of 3 inputs')
self.concat_att = ConcatAttention()
self.softmax_weight_sum = SoftmaxWeightedSum(dropout_rate=self.dropout_rate, future_binding=False)
super(AttentionSequencePoolingLayer, self).build(input_shape)
Expand Down Expand Up @@ -178,6 +179,7 @@ class SelfAttention(Layer):
:param key_masks: A 3d tensor with shape of [batch_size, 1]
:return: A 3d tensor with shape of [batch_size, 1]
"""

def __init__(self, scale=True, dropout_rate=0.2, future_binding=True, use_layer_norm=True, seed=2020, **kwargs):
self.scale = scale
self.dropout_rate = dropout_rate
Expand All @@ -192,7 +194,8 @@ def build(self, input_shape):
'on a list of 2 tensors')
self.layer_norm = LayerNormalization()
self.attention = DotAttention(scale=self.scale)
self.softmax_weight_sum = SoftmaxWeightedSum(dropout_rate=self.dropout_rate, future_binding=self.future_binding, seed=self.seed)
self.softmax_weight_sum = SoftmaxWeightedSum(dropout_rate=self.dropout_rate, future_binding=self.future_binding,
seed=self.seed)
super(SelfAttention, self).build(input_shape)

def call(self, inputs, mask=None, **kwargs):
Expand All @@ -217,7 +220,9 @@ class SelfMultiHeadAttention(Layer):
:param key_masks: A 3d tensor with shape of [batch_size, 1]
:return: A 3d tensor with shape of [batch_size, T, C]
"""
def __init__(self, num_units=8, head_num=4, scale=True, dropout_rate=0.2, future_binding=True, use_layer_norm=True, use_res=True,

def __init__(self, num_units=8, head_num=4, scale=True, dropout_rate=0.2, future_binding=True, use_layer_norm=True,
use_res=True,
seed=2020, **kwargs):
if head_num <= 0:
raise ValueError('head_num must be a int > 0')
Expand All @@ -240,16 +245,17 @@ def build(self, input_shape):
embedding_size = int(input_shape[0][-1])
if self.num_units == None:
self.num_units = embedding_size
self.W = self.add_weight(name='Q_K_V', shape=[embedding_size, self.num_units*3],
dtype=tf.float32,
initializer=TruncatedNormal(seed=self.seed))
self.W = self.add_weight(name='Q_K_V', shape=[embedding_size, self.num_units * 3],
dtype=tf.float32,
initializer=TruncatedNormal(seed=self.seed))
self.W_output = self.add_weight(name='output_W', shape=[self.num_units, self.num_units],
dtype=tf.float32,
initializer=TruncatedNormal(seed=self.seed))
dtype=tf.float32,
initializer=TruncatedNormal(seed=self.seed))

self.layer_norm = LayerNormalization()
self.attention = DotAttention(scale=self.scale)
self.softmax_weight_sum = SoftmaxWeightedSum(dropout_rate=self.dropout_rate, future_binding=self.future_binding, seed=self.seed)
self.softmax_weight_sum = SoftmaxWeightedSum(dropout_rate=self.dropout_rate, future_binding=self.future_binding,
seed=self.seed)
self.dropout = Dropout(self.dropout_rate, seed=self.seed)
self.seq_len_max = int(input_shape[0][1])
# Be sure to call this somewhere!
Expand All @@ -267,7 +273,7 @@ def call(self, inputs, mask=None, training=None, **kwargs):

# head_num None F D
querys = tf.concat(tf.split(querys, self.head_num, axis=2), axis=0) # (h*N, T_q, C/h)
keys = tf.concat(tf.split(keys, self.head_num, axis=2), axis=0) # (h*N, T_k, C/h)
keys = tf.concat(tf.split(keys, self.head_num, axis=2), axis=0) # (h*N, T_k, C/h)
values = tf.concat(tf.split(values, self.head_num, axis=2), axis=0) # (h*N, T_k, C/h)

# (h*N, T_q, T_k)
Expand All @@ -276,10 +282,10 @@ def call(self, inputs, mask=None, training=None, **kwargs):
key_masks = tf.tile(key_masks, [self.head_num, 1]) # (h*N, T_k)
key_masks = tf.tile(tf.expand_dims(key_masks, 1), [1, tf.shape(input_info)[1], 1]) # (h*N, T_q, T_k)

outputs = self.softmax_weight_sum([align, values, key_masks]) # (h*N, T_q, C/h)
outputs = tf.concat(tf.split(outputs, self.head_num, axis=0), axis=2) # (N, T_q, C)
outputs = self.softmax_weight_sum([align, values, key_masks]) # (h*N, T_q, C/h)
outputs = tf.concat(tf.split(outputs, self.head_num, axis=0), axis=2) # (N, T_q, C)

outputs = tf.tensordot(outputs, self.W_output, axes=(-1, 0))# (N, T_q, C)
outputs = tf.tensordot(outputs, self.W_output, axes=(-1, 0)) # (N, T_q, C)
outputs = self.dropout(outputs, training=training)
if self.use_res:
outputs += input_info
Expand All @@ -292,8 +298,10 @@ def compute_output_shape(self, input_shape):
return (None, input_shape[0][1], self.num_units)

def get_config(self, ):
config = {'num_units':self.num_units, 'head_num':self.head_num, 'scale': self.scale, 'dropout_rate': self.dropout_rate,
'future_binding': self.future_binding, 'use_layer_norm':self.use_layer_norm, 'use_res': self.use_res,'seed': self.seed}
config = {'num_units': self.num_units, 'head_num': self.head_num, 'scale': self.scale,
'dropout_rate': self.dropout_rate,
'future_binding': self.future_binding, 'use_layer_norm': self.use_layer_norm, 'use_res': self.use_res,
'seed': self.seed}
base_config = super(SelfMultiHeadAttention, self).get_config()
return dict(list(base_config.items()) + list(config.items()))

Expand All @@ -308,7 +316,9 @@ class UserAttention(Layer):
:param key_masks: A 3d tensor with shape of [batch_size, 1]
:return: A 3d tensor with shape of [batch_size, 1, C]
"""
def __init__(self, num_units=None, activation='tanh', use_res=True, dropout_rate=0, scale=True, seed=2020, **kwargs):

def __init__(self, num_units=None, activation='tanh', use_res=True, dropout_rate=0, scale=True, seed=2020,
**kwargs):
self.scale = scale
self.num_units = num_units
self.activation = activation
Expand Down Expand Up @@ -346,4 +356,11 @@ def compute_output_shape(self, input_shape):
return (None, 1, input_shape[1][2])

def compute_mask(self, inputs, mask):
return mask
return mask

def get_config(self, ):
config = {'num_units': self.num_units, 'activation': self.activation, 'use_res': self.use_res,
'dropout_rate': self.dropout_rate,
'scale': self.scale, 'seed': self.seed, }
base_config = super(UserAttention, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
21 changes: 11 additions & 10 deletions deepmatch/layers/sequence.py
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@

import itertools

import tensorflow as tf
from tensorflow.python.keras.layers import Layer


class DynamicMultiRNN(Layer):
def __init__(self, num_units=None, rnn_type='LSTM', return_sequence=True, num_layers=2, num_residual_layers=1, dropout_rate=0.2,
def __init__(self, num_units=None, rnn_type='LSTM', return_sequence=True, num_layers=2, num_residual_layers=1,
dropout_rate=0.2,
forget_bias=1.0, **kwargs):

self.num_units = num_units
Expand Down Expand Up @@ -62,13 +60,17 @@ def build(self, input_shape):

def call(self, input_list, mask=None, training=None):
rnn_input, sequence_length = input_list

try:
with tf.name_scope("rnn"), tf.variable_scope("rnn", reuse=tf.AUTO_REUSE):
rnn_output, hidden_state = tf.nn.dynamic_rnn(self.final_cell, inputs=rnn_input, sequence_length=tf.squeeze(sequence_length),
rnn_output, hidden_state = tf.nn.dynamic_rnn(self.final_cell, inputs=rnn_input,
sequence_length=tf.squeeze(sequence_length),
dtype=tf.float32, scope=self.name)
except:
rnn_output, hidden_state = tf.compat.v1.nn.dynamic_rnn(self.final_cell, inputs=rnn_input,sequence_length=tf.squeeze(sequence_length),
dtype=tf.float32, scope=self.name)
with tf.name_scope("rnn"), tf.compat.v1.variable_scope("rnn", reuse=tf.compat.v1.AUTO_REUSE):
rnn_output, hidden_state = tf.compat.v1.nn.dynamic_rnn(self.final_cell, inputs=rnn_input,
sequence_length=tf.squeeze(sequence_length),
dtype=tf.float32, scope=self.name)
if self.return_sequence:
return rnn_output
else:
Expand All @@ -82,9 +84,8 @@ def compute_output_shape(self, input_shape):
return (None, 1, rnn_input_shape[2])

def get_config(self, ):
config = {'num_units': self.num_units, 'rnn_type': self.rnn_type, 'return_sequence':self.return_sequence, 'num_layers': self.num_layers,
config = {'num_units': self.num_units, 'rnn_type': self.rnn_type, 'return_sequence': self.return_sequence,
'num_layers': self.num_layers,
'num_residual_layers': self.num_residual_layers, 'dropout_rate': self.dropout}
base_config = super(DynamicMultiRNN, self).get_config()
return dict(list(base_config.items()) + list(config.items()))


19 changes: 10 additions & 9 deletions deepmatch/models/mind.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

from deepmatch.utils import get_item_embedding
from ..inputs import create_embedding_matrix
from ..layers.core import CapsuleLayer, PoolingLayer, LabelAwareAttention,SampledSoftmaxLayer,EmbeddingIndex
from ..layers.core import CapsuleLayer, PoolingLayer, LabelAwareAttention, SampledSoftmaxLayer, EmbeddingIndex


def shape_target(target_emb_tmp, target_emb_size):
Expand Down Expand Up @@ -59,7 +59,7 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1
item_feature_name = item_feature_column.name
item_vocabulary_size = item_feature_columns[0].vocabulary_size
item_embedding_dim = item_feature_columns[0].embedding_dim
#item_index = Input(tensor=tf.constant([list(range(item_vocabulary_size))]))
# item_index = Input(tensor=tf.constant([list(range(item_vocabulary_size))]))

history_feature_list = [item_feature_name]

Expand All @@ -82,8 +82,9 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1
seq_max_len = history_feature_columns[0].maxlen
inputs_list = list(features.values())

embedding_matrix_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding, init_std,
seed, prefix="")
embedding_matrix_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding,
init_std,
seed, prefix="")

item_features = build_input_features(item_feature_columns)

Expand All @@ -108,8 +109,8 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1
history_emb = PoolingLayer()(NoMask()(keys_emb_list))
target_emb = PoolingLayer()(NoMask()(query_emb_list))

#target_emb_size = target_emb.get_shape()[-1].value
#max_len = history_emb.get_shape()[1].value
# target_emb_size = target_emb.get_shape()[-1].value
# max_len = history_emb.get_shape()[1].value
hist_len = features['hist_len']

high_capsule = CapsuleLayer(input_units=item_embedding_dim,
Expand All @@ -125,7 +126,6 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1
else:
user_deep_input = high_capsule


user_embeddings = DNN(user_dnn_hidden_units, dnn_activation, l2_reg_dnn,
dnn_dropout, dnn_use_bn, seed, name="user_embedding")(user_deep_input)
item_inputs_list = list(item_features.values())
Expand All @@ -144,13 +144,14 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1
user_embedding_final = LabelAwareAttention(k_max=k_max, pow_p=p, )((user_embeddings, target_emb))

output = SampledSoftmaxLayer(num_sampled=num_sampled)(
inputs=(pooling_item_embedding_weight,user_embedding_final, item_features[item_feature_name]))
[pooling_item_embedding_weight, user_embedding_final, item_features[item_feature_name]])
model = Model(inputs=inputs_list + item_inputs_list, outputs=output)

model.__setattr__("user_input", inputs_list)
model.__setattr__("user_embedding", user_embeddings)

model.__setattr__("item_input", item_inputs_list)
model.__setattr__("item_embedding", get_item_embedding(pooling_item_embedding_weight, item_features[item_feature_name]))
model.__setattr__("item_embedding",
get_item_embedding(pooling_item_embedding_weight, item_features[item_feature_name]))

return model
Loading

0 comments on commit fe1e7b3

Please sign in to comment.