RecurrentJava

是 否 +2 论坛币

k人 参与回答

经管之家送您一份

应届毕业生专属福利!

求职就业群

赵安豆老师微信：zhaoandou666

经管之家联合CDA

送您一个全额奖学金名额~ !

立即领取

感谢您参与论坛问题回答

经管之家送您两个论坛币！

+2 论坛币

RecurrentJava

RecurrentJava is a reimplementation of Andrej Karpathy's RecurrentJS, in Java.

It currently features:

• Deep Recurrent Neural Networks
• Long Short-Term Memory Networks
• Gated Recurrent Unit Neural Networks
• Backpropagation Through Time handled via Automatic Differentiation.
  

ExamplePaulGraham.java shows how to do character-by-character sentence prediction and generation.

Sample output:

========================================REPORT:calculating perplexity over entire data set...Median Perplexity = 1.4959Temperature 1.0 prediction:    "there's a more kemmaces of meanness that hade? tagh o; mool"    "it fart dect about twish i could see gve..."Temperature 0.75 prediction:    "that's not absolutely note a lot of the startup? path they'll should owt"    "i realize how crazy all thi..."Temperature 0.5 prediction:    "the most stripiess to more here that happens never get them"    "if you do that role kropate that's the w..."Temperature 0.25 prediction:    "the person who needs something making the same spignf befart"    "the startup founders who never about wh..."Temperature 0.1 prediction:    "the startup founders who never about which in your expanding, it's a sign when idea way we don't the..."Argmax prediction:    "the problem is not that most towns kill startups"    "the problem is not that most towns kill startups"    "th..."========================================License

MIT

本帖隐藏的内容

RecurrentJava-master.zip (11.5 MB)

2016-9-3 08:18:20 上传

扫码加我 拉你入群

请注明：姓名-公司-职位

以便审核进群资格，未注明则拒绝

分享0 收藏0 回帖

关键词：Recurrent CURRENT curr Java jav prediction currently features Memory entire

1. package model;
   
2. import java.util.ArrayList;
   
3. import java.util.List;
   
4. import java.util.Random;
   
5. 
   
6. import matrix.Matrix;
   
7. import autodiff.Graph;
   
8. 
   
9. 
   
10. public class FeedForwardLayer implements Model {
    
11. 
    
12.         private static final long serialVersionUID = 1L;
    
13.         Matrix W;
    
14.         Matrix b;
    
15.         Nonlinearity f;
    
16.        
    
17.         public FeedForwardLayer(int inputDimension, int outputDimension, Nonlinearity f, double initParamsStdDev, Random rng) {
    
18.                 W = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng);
    
19.                 b = new Matrix(outputDimension);
    
20.                 this.f = f;
    
21.         }
    
22.        
    
23.         @Override
    
24.         public Matrix forward(Matrix input, Graph g) throws Exception {
    
25.                 Matrix sum = g.add(g.mul(W, input), b);
    
26.                 Matrix out = g.nonlin(f, sum);
    
27.                 return out;
    
28.         }
    
29. 
    
30.         @Override
    
31.         public void resetState() {
    
32. 
    
33.         }
    
34. 
    
35.         @Override
    
36.         public List<Matrix> getParameters() {
    
37.                 List<Matrix> result = new ArrayList<>();
    
38.                 result.add(W);
    
39.                 result.add(b);
    
40.                 return result;
    
41.         }
    
42. }

复制代码

1. package model;
   
2. import java.util.ArrayList;
   
3. import java.util.List;
   
4. import java.util.Random;
   
5. 
   
6. import matrix.Matrix;
   
7. import autodiff.Graph;
   
8. 
   
9. /*
   
10. * As described in:
    
11. *         "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation"
    
12. *         http://arxiv.org/abs/1406.1078
    
13. */
    
14. 
    
15. public class GruLayer implements Model {
    
16. 
    
17.         private static final long serialVersionUID = 1L;
    
18.         int inputDimension;
    
19.         int outputDimension;
    
20.        
    
21.         Matrix IHmix, HHmix, Bmix;
    
22.         Matrix IHnew, HHnew, Bnew;
    
23.         Matrix IHreset, HHreset, Breset;
    
24.        
    
25.         Matrix context;
    
26.        
    
27.         Nonlinearity fMix = new SigmoidUnit();
    
28.         Nonlinearity fReset = new SigmoidUnit();
    
29.         Nonlinearity fNew = new TanhUnit();
    
30.        
    
31.         public GruLayer(int inputDimension, int outputDimension, double initParamsStdDev, Random rng) {
    
32.                 this.inputDimension = inputDimension;
    
33.                 this.outputDimension = outputDimension;
    
34.                 IHmix = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng);
    
35.                 HHmix = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng);
    
36.                 Bmix = new Matrix(outputDimension);
    
37.                 IHnew = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng);
    
38.                 HHnew = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng);
    
39.                 Bnew = new Matrix(outputDimension);
    
40.                 IHreset = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng);
    
41.                 HHreset = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng);
    
42.                 Breset= new Matrix(outputDimension);
    
43.         }
    
44.        
    
45.         @Override
    
46.         public Matrix forward(Matrix input, Graph g) throws Exception {
    
47.                
    
48.                 Matrix sum0 = g.mul(IHmix, input);
    
49.                 Matrix sum1 = g.mul(HHmix, context);
    
50.                 Matrix actMix = g.nonlin(fMix, g.add(g.add(sum0, sum1), Bmix));
    
51. 
    
52.                 Matrix sum2 = g.mul(IHreset, input);
    
53.                 Matrix sum3 = g.mul(HHreset, context);
    
54.                 Matrix actReset = g.nonlin(fReset, g.add(g.add(sum2, sum3), Breset));
    
55.                
    
56.                 Matrix sum4 = g.mul(IHnew, input);
    
57.                 Matrix gatedContext = g.elmul(actReset, context);
    
58.                 Matrix sum5 = g.mul(HHnew, gatedContext);
    
59.                 Matrix actNewPlusGatedContext = g.nonlin(fNew, g.add(g.add(sum4, sum5), Bnew));
    
60.                
    
61.                 Matrix memvals = g.elmul(actMix, context);
    
62.                 Matrix newvals = g.elmul(g.oneMinus(actMix), actNewPlusGatedContext);
    
63.                 Matrix output = g.add(memvals, newvals);
    
64.                
    
65.                 //rollover activations for next iteration
    
66.                 context = output;
    
67.                
    
68.                 return output;
    
69.         }
    
70. 
    
71.         @Override
    
72.         public void resetState() {
    
73.                 context = new Matrix(outputDimension);
    
74.         }
    
75. 
    
76.         @Override
    
77.         public List<Matrix> getParameters() {
    
78.                 List<Matrix> result = new ArrayList<>();
    
79.                 result.add(IHmix);
    
80.                 result.add(HHmix);
    
81.                 result.add(Bmix);
    
82.                 result.add(IHnew);
    
83.                 result.add(HHnew);
    
84.                 result.add(Bnew);
    
85.                 result.add(IHreset);
    
86.                 result.add(HHreset);
    
87.                 result.add(Breset);
    
88.                 return result;
    
89.         }
    
90. 
    
91. }

复制代码

1. package model;
   
2. import java.util.ArrayList;
   
3. import java.util.List;
   
4. import java.util.Random;
   
5. 
   
6. import matrix.Matrix;
   
7. import autodiff.Graph;
   
8. 
   
9. 
   
10. public class LinearLayer implements Model {
    
11. 
    
12.         private static final long serialVersionUID = 1L;
    
13.         Matrix W;
    
14.         //no biases
    
15.        
    
16.         public LinearLayer(int inputDimension, int outputDimension, double initParamsStdDev, Random rng) {
    
17.                 W = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng);
    
18.         }
    
19.        
    
20.         @Override
    
21.         public Matrix forward(Matrix input, Graph g) throws Exception {
    
22.                 Matrix out = g.mul(W, input);
    
23.                 return out;
    
24.         }
    
25. 
    
26.         @Override
    
27.         public void resetState() {
    
28. 
    
29.         }
    
30. 
    
31.         @Override
    
32.         public List<Matrix> getParameters() {
    
33.                 List<Matrix> result = new ArrayList<>();
    
34.                 result.add(W);
    
35.                 return result;
    
36.         }
    
37. }

复制代码

1. package model;
   
2. import java.util.ArrayList;
   
3. import java.util.List;
   
4. import java.util.Random;
   
5. 
   
6. import matrix.Matrix;
   
7. import autodiff.Graph;
   
8. 
   
9. public class LstmLayer implements Model {
   
10.        
    
11.         private static final long serialVersionUID = 1L;
    
12.         int inputDimension;
    
13.         int outputDimension;
    
14.        
    
15.         Matrix Wix, Wih, bi;
    
16.         Matrix Wfx, Wfh, bf;
    
17.         Matrix Wox, Woh, bo;
    
18.         Matrix Wcx, Wch, bc;
    
19.        
    
20.         Matrix hiddenContext;
    
21.         Matrix cellContext;
    
22.        
    
23.         Nonlinearity fInputGate = new SigmoidUnit();
    
24.         Nonlinearity fForgetGate = new SigmoidUnit();
    
25.         Nonlinearity fOutputGate = new SigmoidUnit();
    
26.         Nonlinearity fCellInput = new TanhUnit();
    
27.         Nonlinearity fCellOutput = new TanhUnit();
    
28.        
    
29.         public LstmLayer(int inputDimension, int outputDimension, double initParamsStdDev, Random rng) {
    
30.                 this.inputDimension = inputDimension;
    
31.                 this.outputDimension = outputDimension;
    
32.                 Wix = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng);
    
33.                 Wih = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng);
    
34.                 bi = new Matrix(outputDimension);
    
35.                 Wfx = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng);
    
36.                 Wfh = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng);
    
37.                 //set forget bias to 1.0, as described here: http://jmlr.org/proceedings/papers/v37/jozefowicz15.pdf
    
38.                 bf = Matrix.ones(outputDimension, 1);
    
39.                 Wox = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng);
    
40.                 Woh = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng);
    
41.                 bo = new Matrix(outputDimension);
    
42.                 Wcx = Matrix.rand(outputDimension, inputDimension, initParamsStdDev, rng);
    
43.                 Wch = Matrix.rand(outputDimension, outputDimension, initParamsStdDev, rng);
    
44.                 bc = new Matrix(outputDimension);
    
45.         }
    
46. 
    
47.         @Override
    
48.         public Matrix forward(Matrix input, Graph g) throws Exception {
    
49.                
    
50.                 //input gate
    
51.                 Matrix sum0 = g.mul(Wix, input);
    
52.                 Matrix sum1 = g.mul(Wih, hiddenContext);
    
53.                 Matrix inputGate = g.nonlin(fInputGate, g.add(g.add(sum0, sum1), bi));
    
54.                
    
55.                 //forget gate
    
56.                 Matrix sum2 = g.mul(Wfx, input);
    
57.                 Matrix sum3 = g.mul(Wfh, hiddenContext);
    
58.                 Matrix forgetGate = g.nonlin(fForgetGate, g.add(g.add(sum2, sum3), bf));
    
59.                
    
60.                 //output gate
    
61.                 Matrix sum4 = g.mul(Wox, input);
    
62.                 Matrix sum5 = g.mul(Woh, hiddenContext);
    
63.                 Matrix outputGate = g.nonlin(fOutputGate, g.add(g.add(sum4, sum5), bo));
    
64. 
    
65.                 //write operation on cells
    
66.                 Matrix sum6 = g.mul(Wcx, input);
    
67.                 Matrix sum7 = g.mul(Wch, hiddenContext);
    
68.                 Matrix cellInput = g.nonlin(fCellInput, g.add(g.add(sum6, sum7), bc));
    
69.                
    
70.                 //compute new cell activation
    
71.                 Matrix retainCell = g.elmul(forgetGate, cellContext);
    
72.                 Matrix writeCell = g.elmul(inputGate,  cellInput);
    
73.                 Matrix cellAct = g.add(retainCell,  writeCell);
    
74.                
    
75.                 //compute hidden state as gated, saturated cell activations
    
76.                 Matrix output = g.elmul(outputGate, g.nonlin(fCellOutput, cellAct));
    
77.                
    
78.                 //rollover activations for next iteration
    
79.                 hiddenContext = output;
    
80.                 cellContext = cellAct;
    
81.                
    
82.                 return output;
    
83.         }
    
84. 
    
85.         @Override
    
86.         public void resetState() {
    
87.                 hiddenContext = new Matrix(outputDimension);
    
88.                 cellContext = new Matrix(outputDimension);
    
89.         }
    
90. 
    
91.         @Override
    
92.         public List<Matrix> getParameters() {
    
93.                 List<Matrix> result = new ArrayList<>();
    
94.                 result.add(Wix);
    
95.                 result.add(Wih);
    
96.                 result.add(bi);
    
97.                 result.add(Wfx);
    
98.                 result.add(Wfh);
    
99.                 result.add(bf);
    
100.                 result.add(Wox);
     
101.                 result.add(Woh);
     
102.                 result.add(bo);
     
103.                 result.add(Wcx);
     
104.                 result.add(Wch);
     
105.                 result.add(bc);
     
106.                 return result;
     
107.         }
     
108. }

复制代码

1. package model;
   
2. 
   
3. import java.util.ArrayList;
   
4. import java.util.List;
   
5. 
   
6. import matrix.Matrix;
   
7. import autodiff.Graph;
   
8. 
   
9. public class NeuralNetwork implements Model {
   
10. 
    
11.         private static final long serialVersionUID = 1L;
    
12.         List<Model> layers = new ArrayList<>();
    
13.        
    
14.         public NeuralNetwork(List<Model> layers) {
    
15.                 this.layers = layers;
    
16.         }
    
17.        
    
18.         @Override
    
19.         public Matrix forward(Matrix input, Graph g) throws Exception {
    
20.                 Matrix prev = input;
    
21.                 for (Model layer : layers) {
    
22.                         prev = layer.forward(prev, g);
    
23.                 }
    
24.                 return prev;
    
25.         }
    
26. 
    
27.         @Override
    
28.         public void resetState() {
    
29.                 for (Model layer : layers) {
    
30.                         layer.resetState();
    
31.                 }
    
32.         }
    
33. 
    
34.         @Override
    
35.         public List<Matrix> getParameters() {
    
36.                 List<Matrix> result = new ArrayList<>();
    
37.                 for (Model layer : layers) {
    
38.                         result.addAll(layer.getParameters());
    
39.                 }
    
40.                 return result;
    
41.         }
    
42. }

复制代码