/** * Sample a dataset * * @param numSamples the number of samples to getFromOrigin * @param rng the rng to use * @param withReplacement whether to allow duplicates (only tracked by example row number) * @return the sample dataset */ @Override public DataSet sample(int numSamples, org.nd4j.linalg.api.rng.Random rng, boolean withReplacement) { INDArray examples = Nd4j.create(numSamples, getFeatures().columns()); INDArray outcomes = Nd4j.create(numSamples, numOutcomes()); Set<Integer> added = new HashSet<>(); for (int i = 0; i < numSamples; i++) { int picked = rng.nextInt(numExamples()); if (!withReplacement) while (added.contains(picked)) picked = rng.nextInt(numExamples()); examples.putRow(i, get(picked).getFeatures()); outcomes.putRow(i, get(picked).getLabels()); } return new DataSet(examples, outcomes); }
miniBatchSize = next.numExamples(); for (int i = 0; i < next.numExamples(); i++) { DataSet currExample = next.get(i); if (!labelRootDirs.get(currExample.outcome()).exists()) labelRootDirs.get(currExample.outcome()).mkdirs();
DataSet example = filtered.get(i); int o2 = example.outcome(); Integer outcome = labelMap.get(o2);
INDArray newFeatures = result.get(i).getFeatureMatrix(); newFeatures.tensorAlongDimension(0, new int[] {0, 2, 3}).divi(255); newFeatures.tensorAlongDimension(1, new int[] {0, 2, 3}).subi(uMean).divi(uStd); newFeatures.tensorAlongDimension(2, new int[] {0, 2, 3}).subi(vMean).divi(vStd); result.get(i).setFeatures(newFeatures);
/** * Returns the total number of labels. * * @return the total number of labels */ @Override public int totalOutcomes() { return data.get(0).getLabels().columns(); }
DataSet D = input.get(i); if (rng.nextDouble() < (splitSize-inTrain)/(double)(input.numExamples()-i)) { train_features.putRow(inTrain, D.getFeatures());
public static void drawMnist(DataSet mnist,INDArray reconstruct) throws InterruptedException { for(int j = 0; j < mnist.numExamples(); j++) { INDArray draw1 = mnist.get(j).getFeatureMatrix().mul(255); INDArray reconstructed2 = reconstruct.getRow(j); INDArray draw2 = Nd4j.getDistributions().createBinomial(1,reconstructed2).sample(reconstructed2.shape()).mul(255); DrawReconstruction d = new DrawReconstruction(draw1); d.title = "REAL"; d.draw(); DrawReconstruction d2 = new DrawReconstruction(draw2,1000,1000); d2.title = "TEST"; d2.draw(); Thread.sleep(1000); d.frame.dispose(); d2.frame.dispose(); } }
/** * Returns the number of input columns. * * @return the number of input columns */ @Override public int inputColumns() { return data.get(0).getFeatures().columns(); }
/** * Sample a dataset * * @param numSamples the number of samples to getFromOrigin * @param rng the rng to use * @param withReplacement whether to allow duplicates (only tracked by example row number) * @return the sample dataset */ @Override public DataSet sample(int numSamples, org.nd4j.linalg.api.rng.Random rng, boolean withReplacement) { INDArray examples = Nd4j.create(numSamples, getFeatures().columns()); INDArray outcomes = Nd4j.create(numSamples, numOutcomes()); Set<Integer> added = new HashSet<>(); for (int i = 0; i < numSamples; i++) { int picked = rng.nextInt(numExamples()); if (!withReplacement) while (added.contains(picked)) picked = rng.nextInt(numExamples()); examples.putRow(i, get(picked).getFeatures()); outcomes.putRow(i, get(picked).getLabels()); } return new DataSet(examples, outcomes); }
public static void drawMnist(DataSet mnist, INDArray reconstruct) throws InterruptedException { for (int j = 0; j < mnist.numExamples(); j++) { INDArray draw1 = mnist.get(j).getFeatureMatrix().mul(255); INDArray reconstructed2 = reconstruct.getRow(j); INDArray draw2 = Nd4j.getDistributions().createBinomial(1, reconstructed2).sample(reconstructed2.shape()) .mul(255); DrawReconstruction d = new DrawReconstruction(draw1); d.title = "REAL"; d.draw(); DrawReconstruction d2 = new DrawReconstruction(draw2, 1000, 1000); d2.title = "TEST"; d2.draw(); Thread.sleep(1000); d.frame.dispose(); d2.frame.dispose(); } }
DataSet example = filtered.get(i); int o2 = example.outcome(); Integer outcome = labelMap.get(o2);
/** * Fetches the next dataset. You need to call this * to get a new dataset, otherwise {@link #next()} * just returns the last data applyTransformToDestination fetch * * @param numExamples the number of examples to fetch */ @Override public void fetch(int numExamples) { initializeCurrFromList(data.get(ArrayUtil.range(cursor, cursor + numExamples)).asList()); } }
INDArray outcomes = Nd4j.create(numSamples, next.numOutcomes(),'f'); for (int delegateIndex : selectedIndices) { DataSet dataSet = next.get(delegateIndex); examples.putRow(num, dataSet.getFeatures()); outcomes.putRow(num, dataSet.getLabels());
INDArray reconstructed = network.activate(test.getFeatureMatrix()); for (int i = 0; i < test.numExamples(); i++) { INDArray draw1 = test.get(i).getFeatureMatrix().mul(255); INDArray reconstructed2 = reconstructed.getRow(i); INDArray draw2 = Nd4j.getDistributions().createBinomial(1, reconstructed2)
/** * Fetches the next dataset. You need to call this * to getFromOrigin a new dataset, otherwise {@link #next()} * just returns the last data applyTransformToDestination fetch * * @param numExamples the number of examples to fetch */ @Override public void fetch(int numExamples) { initializeCurrFromList(data.get(ArrayUtil.range(cursor, cursor + numExamples)).asList()); } }
miniBatchSize = next.numExamples(); for (int i = 0; i < next.numExamples(); i++) { DataSet currExample = next.get(i); if (!labelRootDirs.get(currExample.outcome()).exists()) labelRootDirs.get(currExample.outcome()).mkdirs();
/** * Return a batch record based on a dataset * @param dataSet the dataset to get the batch record for * @return the batch record */ public static BatchRecord fromDataSet(DataSet dataSet) { BatchRecord batchRecord = new BatchRecord(); for (int i = 0; i < dataSet.numExamples(); i++) { batchRecord.add(CSVRecord.fromRow(dataSet.get(i))); } return batchRecord; }
/** * Return a batch record based on a dataset * @param dataSet the dataset to get the batch record for * @return the batch record */ public static BatchCSVRecord fromDataSet(DataSet dataSet) { BatchCSVRecord batchCSVRecord = new BatchCSVRecord(); for (int i = 0; i < dataSet.numExamples(); i++) { batchCSVRecord.add(SingleCSVRecord.fromRow(dataSet.get(i))); } return batchCSVRecord; }