/** * Returns matrices of the right size for either binary or unary (terminal) classification */ SimpleMatrix randomClassificationMatrix() { SimpleMatrix score = new SimpleMatrix(numClasses, numHid + 1); double range = 1.0 / (Math.sqrt((double) numHid)); score.insertIntoThis(0, 0, SimpleMatrix.random(numClasses, numHid, -range, range, rand)); // bias column goes from 0 to 1 initially score.insertIntoThis(0, numHid, SimpleMatrix.random(numClasses, 1, 0.0, 1.0, rand)); return score.scale(op.trainOptions.scalingForInit); }
SimpleMatrix randomTransformMatrix() { SimpleMatrix binary = new SimpleMatrix(numHid, numHid * 2 + 1); // bias column values are initialized zero binary.insertIntoThis(0, 0, randomTransformBlock()); binary.insertIntoThis(0, numHid, randomTransformBlock()); return binary.scale(op.trainOptions.scalingForInit); }
/** * Creates a random context matrix. This will be numRows x * 2*numCols big. These can be appended to the end of either a * unary or binary transform matrix to get the transform matrix * which uses context words. */ private SimpleMatrix randomContextMatrix() { SimpleMatrix matrix = new SimpleMatrix(numRows, numCols * 2); matrix.insertIntoThis(0, 0, identity.scale(op.trainOptions.scalingForInit * 0.1)); matrix.insertIntoThis(0, numCols, identity.scale(op.trainOptions.scalingForInit * 0.1)); matrix = matrix.plus(SimpleMatrix.random(numRows,numCols * 2,-1.0/Math.sqrt((double)numCols * 100.0),1.0/Math.sqrt((double)numCols * 100.0),rand)); return matrix; }
public void addRandomUnaryMatrix(String childBasic) { if (unaryTransform.get(childBasic) != null) { return; } ++numUnaryMatrices; // scoring matrix SimpleMatrix score = SimpleMatrix.random(1, numCols, -1.0/Math.sqrt((double)numCols),1.0/Math.sqrt((double)numCols),rand); unaryScore.put(childBasic, score.scale(op.trainOptions.scalingForInit)); SimpleMatrix transform; if (op.trainOptions.useContextWords) { transform = new SimpleMatrix(numRows, numCols * 3 + 1); // leave room for bias term transform.insertIntoThis(0,numCols + 1, randomContextMatrix()); } else { transform = new SimpleMatrix(numRows, numCols + 1); } SimpleMatrix unary = randomTransformMatrix(); transform.insertIntoThis(0, 0, unary); unaryTransform.put(childBasic, transform.scale(op.trainOptions.scalingForInit)); }
/** * Concatenates several column vectors into one large column vector */ public static SimpleMatrix concatenate(SimpleMatrix ... vectors) { int size = 0; for (SimpleMatrix vector : vectors) { size += vector.numRows(); } SimpleMatrix result = new SimpleMatrix(size, 1); int index = 0; for (SimpleMatrix vector : vectors) { result.insertIntoThis(index, 0, vector); index += vector.numRows(); } return result; }
private static double scaleAndRegularize(Map<String, SimpleMatrix> derivatives, Map<String, SimpleMatrix> currentMatrices, double scale, double regCost, boolean activeMatricesOnly, boolean dropBiasColumn) { double cost = 0.0; // the regularization cost for (Map.Entry<String, SimpleMatrix> entry : currentMatrices.entrySet()) { SimpleMatrix D = derivatives.get(entry.getKey()); if (activeMatricesOnly && D == null) { // Fill in an emptpy matrix so the length of theta can match. // TODO: might want to allow for sparse parameter vectors derivatives.put(entry.getKey(), new SimpleMatrix(entry.getValue().numRows(), entry.getValue().numCols())); continue; } SimpleMatrix regMatrix = entry.getValue(); if (dropBiasColumn) { regMatrix = new SimpleMatrix(regMatrix); regMatrix.insertIntoThis(0, regMatrix.numCols() - 1, new SimpleMatrix(regMatrix.numRows(), 1)); } D = D.scale(scale).plus(regMatrix.scale(regCost)); derivatives.put(entry.getKey(), D); cost += regMatrix.elementMult(regMatrix).elementSum() * regCost / 2.0; } return cost; }
public void addRandomBinaryMatrix(String leftBasic, String rightBasic) { if (binaryTransform.get(leftBasic, rightBasic) != null) { return; } ++numBinaryMatrices; // scoring matrix SimpleMatrix score = SimpleMatrix.random(1, numCols, -1.0/Math.sqrt((double)numCols),1.0/Math.sqrt((double)numCols),rand); binaryScore.put(leftBasic, rightBasic, score.scale(op.trainOptions.scalingForInit)); SimpleMatrix binary; if (op.trainOptions.useContextWords) { binary = new SimpleMatrix(numRows, numCols * 4 + 1); // leave room for bias term binary.insertIntoThis(0,numCols*2+1, randomContextMatrix()); } else { binary = new SimpleMatrix(numRows, numCols * 2 + 1); } SimpleMatrix left = randomTransformMatrix(); SimpleMatrix right = randomTransformMatrix(); binary.insertIntoThis(0, 0, left); binary.insertIntoThis(0, numCols, right); binaryTransform.put(leftBasic, rightBasic, binary.scale(op.trainOptions.scalingForInit)); }
/** * Concatenates several column vectors into one large column * vector, adds a 1.0 at the end as a bias term */ public static SimpleMatrix concatenateWithBias(SimpleMatrix ... vectors) { int size = 0; for (SimpleMatrix vector : vectors) { size += vector.numRows(); } // one extra for the bias size++; SimpleMatrix result = new SimpleMatrix(size, 1); int index = 0; for (SimpleMatrix vector : vectors) { result.insertIntoThis(index, 0, vector); index += vector.numRows(); } result.set(index, 0, 1.0); return result; }
private static double scaleAndRegularize(TwoDimensionalMap<String, String, SimpleMatrix> derivatives, TwoDimensionalMap<String, String, SimpleMatrix> currentMatrices, double scale, double regCost, boolean dropBiasColumn) { double cost = 0.0; // the regularization cost for (TwoDimensionalMap.Entry<String, String, SimpleMatrix> entry : currentMatrices) { SimpleMatrix D = derivatives.get(entry.getFirstKey(), entry.getSecondKey()); SimpleMatrix regMatrix = entry.getValue(); if (dropBiasColumn) { regMatrix = new SimpleMatrix(regMatrix); regMatrix.insertIntoThis(0, regMatrix.numCols() - 1, new SimpleMatrix(regMatrix.numRows(), 1)); } D = D.scale(scale).plus(regMatrix.scale(regCost)); derivatives.put(entry.getFirstKey(), entry.getSecondKey(), D); cost += regMatrix.elementMult(regMatrix).elementSum() * regCost / 2.0; } return cost; }
/** * Returns matrices of the right size for either binary or unary (terminal) classification */ SimpleMatrix randomClassificationMatrix() { SimpleMatrix score = new SimpleMatrix(numClasses, numHid + 1); double range = 1.0 / (Math.sqrt((double) numHid)); score.insertIntoThis(0, 0, SimpleMatrix.random(numClasses, numHid, -range, range, rand)); // bias column goes from 0 to 1 initially score.insertIntoThis(0, numHid, SimpleMatrix.random(numClasses, 1, 0.0, 1.0, rand)); return score.scale(op.trainOptions.scalingForInit); }
SimpleMatrix randomTransformMatrix() { SimpleMatrix binary = new SimpleMatrix(numHid, numHid * 2 + 1); // bias column values are initialized zero binary.insertIntoThis(0, 0, randomTransformBlock()); binary.insertIntoThis(0, numHid, randomTransformBlock()); return binary.scale(op.trainOptions.scalingForInit); }
/** * Creates a random context matrix. This will be numRows x * 2*numCols big. These can be appended to the end of either a * unary or binary transform matrix to get the transform matrix * which uses context words. */ private SimpleMatrix randomContextMatrix() { SimpleMatrix matrix = new SimpleMatrix(numRows, numCols * 2); matrix.insertIntoThis(0, 0, identity.scale(op.trainOptions.scalingForInit * 0.1)); matrix.insertIntoThis(0, numCols, identity.scale(op.trainOptions.scalingForInit * 0.1)); matrix = matrix.plus(SimpleMatrix.random(numRows,numCols * 2,-1.0/Math.sqrt((double)numCols * 100.0),1.0/Math.sqrt((double)numCols * 100.0),rand)); return matrix; }
SimpleMatrix randomTransformMatrix() { SimpleMatrix binary = new SimpleMatrix(numHid, numHid * 2 + 1); // bias column values are initialized zero binary.insertIntoThis(0, 0, randomTransformBlock()); binary.insertIntoThis(0, numHid, randomTransformBlock()); return binary.scale(op.trainOptions.scalingForInit); }
/** * Creates a random context matrix. This will be numRows x * 2*numCols big. These can be appended to the end of either a * unary or binary transform matrix to get the transform matrix * which uses context words. */ private SimpleMatrix randomContextMatrix() { SimpleMatrix matrix = new SimpleMatrix(numRows, numCols * 2); matrix.insertIntoThis(0, 0, identity.scale(op.trainOptions.scalingForInit * 0.1)); matrix.insertIntoThis(0, numCols, identity.scale(op.trainOptions.scalingForInit * 0.1)); matrix = matrix.plus(SimpleMatrix.random(numRows,numCols * 2,-1.0/Math.sqrt((double)numCols * 100.0),1.0/Math.sqrt((double)numCols * 100.0),rand)); return matrix; }
/** * Creates a random context matrix. This will be numRows x * 2*numCols big. These can be appended to the end of either a * unary or binary transform matrix to get the transform matrix * which uses context words. */ private SimpleMatrix randomContextMatrix() { SimpleMatrix matrix = new SimpleMatrix(numRows, numCols * 2); matrix.insertIntoThis(0, 0, identity.scale(op.trainOptions.scalingForInit * 0.1)); matrix.insertIntoThis(0, numCols, identity.scale(op.trainOptions.scalingForInit * 0.1)); matrix = matrix.plus(SimpleMatrix.random(numRows,numCols * 2,-1.0/Math.sqrt((double)numCols * 100.0),1.0/Math.sqrt((double)numCols * 100.0),rand)); return matrix; }
/** * Returns matrices of the right size for either binary or unary (terminal) classification */ SimpleMatrix randomClassificationMatrix() { SimpleMatrix score = new SimpleMatrix(numClasses, numHid + 1); // Leave the bias column with 0 values double range = 1.0 / (Math.sqrt((double) numHid)); score.insertIntoThis(0, 0, SimpleMatrix.random(numClasses, numHid, -range, range, rand)); return score.scale(op.trainOptions.scalingForInit); }
/** * Concatenates several column vectors into one large column vector */ public static SimpleMatrix concatenate(SimpleMatrix ... vectors) { int size = 0; for (SimpleMatrix vector : vectors) { size += vector.numRows(); } SimpleMatrix result = new SimpleMatrix(size, 1); int index = 0; for (SimpleMatrix vector : vectors) { result.insertIntoThis(index, 0, vector); index += vector.numRows(); } return result; }
/** * Concatenates several column vectors into one large column vector */ public static SimpleMatrix concatenate(SimpleMatrix ... vectors) { int size = 0; for (SimpleMatrix vector : vectors) { size += vector.numRows(); } SimpleMatrix result = new SimpleMatrix(size, 1); int index = 0; for (SimpleMatrix vector : vectors) { result.insertIntoThis(index, 0, vector); index += vector.numRows(); } return result; }
/** * Concatenates several column vectors into one large column vector */ public static SimpleMatrix concatenate(SimpleMatrix ... vectors) { int size = 0; for (SimpleMatrix vector : vectors) { size += vector.numRows(); } SimpleMatrix result = new SimpleMatrix(size, 1); int index = 0; for (SimpleMatrix vector : vectors) { result.insertIntoThis(index, 0, vector); index += vector.numRows(); } return result; }
private static double scaleAndRegularize(TwoDimensionalMap<String, String, SimpleMatrix> derivatives, TwoDimensionalMap<String, String, SimpleMatrix> currentMatrices, double scale, double regCost, boolean dropBiasColumn) { double cost = 0.0; // the regularization cost for (TwoDimensionalMap.Entry<String, String, SimpleMatrix> entry : currentMatrices) { SimpleMatrix D = derivatives.get(entry.getFirstKey(), entry.getSecondKey()); SimpleMatrix regMatrix = entry.getValue(); if (dropBiasColumn) { regMatrix = new SimpleMatrix(regMatrix); regMatrix.insertIntoThis(0, regMatrix.numCols() - 1, new SimpleMatrix(regMatrix.numRows(), 1)); } D = D.scale(scale).plus(regMatrix.scale(regCost)); derivatives.put(entry.getFirstKey(), entry.getSecondKey(), D); cost += regMatrix.elementMult(regMatrix).elementSum() * regCost / 2.0; } return cost; }