Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactored PMTSNE to include steps #131

Merged
merged 1 commit into from
May 31, 2019
Merged

Refactored PMTSNE to include steps #131

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 92 additions & 44 deletions src/Math-TSNE/PMTSNE.class.st
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ Class {
'learningRate',
'minGain',
'job',
'computeErrorEvery'
'computeErrorEvery',
'stateVariables'
],
#category : #'Math-TSNE'
}
Expand Down Expand Up @@ -86,17 +87,21 @@ PMTSNE >> computeErrorEveryDefaultValue [
]

{ #category : #'stepping and presenter' }
PMTSNE >> computeGradient: p withProgressUpdate: iteration [
PMTSNE >> computeGradient [
"Computes gradient of KL divergence"

| num sumNum q pq dY tmp yiDiff error |
| num sumNum p q pq dY tmp yiDiff |
"Calculates num and q"
num := self computeLowDimensionalStudentT.
sumNum := num sum sum max: (Float epsilon).
q := num collect: [:element |
(element / sumNum) max: (Float epsilon)
].
stateVariables add: 'q'->q.

p := stateVariables at: 'p'.
pq := p - q.

dY := OrderedCollection new.
1 to: (x dimension x) do: [ :i |
tmp := (pq rowAt: i) hadamardProduct: (num rowAt: i).
Expand All @@ -106,20 +111,6 @@ PMTSNE >> computeGradient: p withProgressUpdate: iteration [
dY add: (tmp hadamardProduct: yiDiff) sum
].
dY := PMMatrix rows: dY.

(iteration % computeErrorEvery = 0) ifTrue: [
error := PMMatrix rows: (x dimension x) columns: (x dimension x).
1 to: (x dimension x) do: [ :i |
1 to: (x dimension x) do: [ :j |
error rowAt: i columnAt: j put: (
(p rowAt: i columnAt: j) * (((p rowAt: i columnAt: j) / (q rowAt: i columnAt: j)) ln)
).
].
].
error := error sum sum.
job title: ('' join: { 'Step 3/3: Performing gradient descent '. iteration. '/'. maxIter.' error = '. error }).
job progress: (iteration/maxIter).
].

^ dY
]
Expand Down Expand Up @@ -269,35 +260,15 @@ PMTSNE >> finalMomentumDefaultValue [
PMTSNE >> gradientDescent [
"Tries to minimize the cost, which is KL divergence"

| p gains iY momentum dY yMeanAccumulator |
job title: 'Step 3/3: Performing gradient descent'.
p := self computePValues.
gains := PMMatrix onesRows: x dimension x cols: outputDims.
iY := PMMatrix zerosRows: x dimension x cols: outputDims.
momentum := initialMomentum.
job progress: 0.0.
stateVariables add: 'p'->(self computePValues).
stateVariables add: 'gains'->(PMMatrix onesRows: x dimension x cols: outputDims).
stateVariables add: 'iY'->(PMMatrix zerosRows: x dimension x cols: outputDims).
1 to: maxIter do: [ :iteration |
dY := self computeGradient: p withProgressUpdate: iteration.
momentum := iteration < 20
ifTrue: [ initialMomentum ]
ifFalse: [ finalMomentum ].
1 to: (x dimension x) do: [ :i |
1 to: outputDims do: [ :j |
((dY rowAt: i columnAt: j) > 0) = ((iY rowAt: i columnAt: j) > 0)
ifTrue: [ gains rowAt: i columnAt: j put: (((gains rowAt: i columnAt: j) * 0.8) max: minGain) ]
ifFalse: [ gains rowAt: i columnAt: j put: (gains rowAt: i columnAt: j) + 0.2 ].
]
].
iY := iY * momentum - ((dY hadamardProduct: gains) * learningRate).
y := y + iY.
yMeanAccumulator := PMVectorAccumulator new: outputDims.
y rowsDo: [ :row |
yMeanAccumulator accumulate: row.
].
y := PMMatrix rows: (y rowsCollect: [ :row |
row - (yMeanAccumulator average)
]).
"Stop exaggeration"
(iteration = 100) ifTrue: [ p := p * (1/4) ].
stateVariables add: 'iteration'->iteration.
self step.
self postStep.
].
]

Expand Down Expand Up @@ -340,6 +311,7 @@ PMTSNE >> initialize [
learningRate := self learningRateDefaultValue.
minGain := self minGainDefaultValue.
computeErrorEvery := self computeErrorEveryDefaultValue.
stateVariables := Dictionary new.
self initializeJob.

]
Expand Down Expand Up @@ -454,6 +426,12 @@ PMTSNE >> perplexityDefaultValue [
^ 30.0
]

{ #category : #'stepping and presenter' }
PMTSNE >> postStep [
"Here will be all the calls for visualization and debugging methods"
self updateProgressWithError
]

{ #category : #running }
PMTSNE >> reduceXToInputDims [
"Runs PCA on X in order to reduce its dimensionality to initialDims dimensions."
Expand All @@ -478,6 +456,76 @@ PMTSNE >> start [
job run.
]

{ #category : #'stepping and presenter' }
PMTSNE >> step [
| iteration gains iY dY momentum yMeanAccumulator p |
iteration := stateVariables at: 'iteration'.
gains := stateVariables at: 'gains'.
iY := stateVariables at: 'iY'.
p := stateVariables at: 'p'.

dY := self computeGradient.

"Momentum accelerates gradient descent by dampening one direction"
momentum := iteration < 20
ifTrue: [ initialMomentum ]
ifFalse: [ finalMomentum ].

"Compute gain based on direction of descent"
1 to: x dimension x do: [ :i |
1 to: outputDims do: [ :j |
(dY rowAt: i columnAt: j) > 0 = ((iY rowAt: i columnAt: j) > 0)
ifTrue: [ gains
rowAt: i
columnAt: j
put: ((gains rowAt: i columnAt: j) * 0.8 max: minGain) ]
ifFalse: [ gains rowAt: i columnAt: j put: (gains rowAt: i columnAt: j) + 0.2 ] ] ].

"Update y according to gradient, momentum and gain"
iY := iY * momentum - ((dY hadamardProduct: gains) * learningRate).
y := y + iY.

"Center y by subtracting mean"
yMeanAccumulator := PMVectorAccumulator new: outputDims.
y rowsDo: [ :row | yMeanAccumulator accumulate: row ].
y := PMMatrix
rows: (y rowsCollect: [ :row | row - yMeanAccumulator average ]).

"Update state variables for the next step"
stateVariables add: 'gains'->gains.
stateVariables add: 'iY'->iY.

"Stop early exaggeration on 100th iteration"
iteration = 100
ifFalse: [ ^ self ].
p := p * (1 / 4).
stateVariables add: 'p' -> p
]

{ #category : #'stepping and presenter' }
PMTSNE >> updateProgressWithError [
| error p q iteration |
iteration := stateVariables at: 'iteration'.
p := stateVariables at: 'p'.
q := stateVariables at: 'q'.

iteration % computeErrorEvery = 0
ifFalse: [ ^ self ].

error := PMMatrix rows: x dimension x columns: x dimension x.
1 to: x dimension x do: [ :i |
1 to: x dimension x do: [ :j |
error rowAt: i columnAt: j
put: (p rowAt: i columnAt: j) * ((p rowAt: i columnAt: j) / (q rowAt: i columnAt: j)) ln
] ].
error := error sum sum.

job title: ('' join:
{'Step 3/3: Performing gradient descent '. iteration. '/'. maxIter. ' error = '. error}
).
job progress: iteration / maxIter.
]

{ #category : #accessing }
PMTSNE >> x [
^ x
Expand Down