Repository navigation
Expand file tree
/
Copy pathmodel.js
More file actions
83 lines (68 loc) · 2.44 KB
/
Copy pathmodel.js
File metadata and controls
83 lines (68 loc) · 2.44 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
function randomMatrix(rows, cols) {
return Array.from({ length: rows }, () =>
Array.from({ length: cols }, () => Math.random()),
);
}
function softmax(logits) {
// 수치 안정성 개선
const maxLogit = Math.max(...logits);
const exps = logits.map((value) => Math.exp(value - maxLogit));
const sumExp = exps.reduce((sum, value) => sum + value, 0);
return exps.map((value) => value / sumExp);
}
function forward(inputIndex, W1, W2, vocabSize, embeddingSize) {
const hidden = W1[inputIndex];
const scores = new Array(vocabSize).fill(0);
for (let vocabIndex = 0; vocabIndex < vocabSize; vocabIndex += 1) {
let score = 0;
for (let dim = 0; dim < embeddingSize; dim += 1) {
score += hidden[dim] * W2[dim][vocabIndex];
}
scores[vocabIndex] = score;
}
return { hidden, scores };
}
function getLoss(predictedProbabilities, targetIndex) {
return -Math.log(predictedProbabilities[targetIndex] + 1e-12);
}
function backward(inputIndex, targetIndex, hidden, probs, W1, W2, learningRate) {
// 1) dScores 계산
const dScores = [...probs];
dScores[targetIndex] -= 1;
// 2) dHidden 계산 (W2 업데이트 전에 계산)
const dHidden = new Array(hidden.length).fill(0);
for (let dim = 0; dim < hidden.length; dim += 1) {
for (let vocabIndex = 0; vocabIndex < dScores.length; vocabIndex += 1) {
dHidden[dim] += W2[dim][vocabIndex] * dScores[vocabIndex];
}
}
// 3) W2 업데이트
for (let dim = 0; dim < hidden.length; dim += 1) {
for (let vocabIndex = 0; vocabIndex < dScores.length; vocabIndex += 1) {
W2[dim][vocabIndex] -= learningRate * hidden[dim] * dScores[vocabIndex];
}
}
// 4) W1 업데이트 (input row only)
for (let dim = 0; dim < hidden.length; dim += 1) {
W1[inputIndex][dim] -= learningRate * dHidden[dim];
}
}
function train(trainingData, epochs, learningRate, W1, W2, vocabSize, embeddingSize) {
for (let epoch = 0; epoch < epochs; epoch += 1) {
let totalLoss = 0;
for (const { input, target } of trainingData) {
const { hidden, scores } = forward(input, W1, W2, vocabSize, embeddingSize);
const probs = softmax(scores);
totalLoss += getLoss(probs, target);
backward(input, target, hidden, probs, W1, W2, learningRate);
}
if (epoch % 100 === 0) {
const avgLoss = totalLoss / trainingData.length;
console.log(`epoch ${epoch} loss: ${avgLoss.toFixed(4)}`);
}
}
}
module.exports = {
randomMatrix,
train,
};