public TestDataSetIterator(DataSet dataset, int batch) { this(dataset.asList(), batch); }
@Override public Iterator<DataSet> iterator() { return asList().iterator(); }
/** * Partitions the data transform by the specified number. * * @param num the number to split by * @return the partitioned data transform */ @Override public List<DataSet> dataSetBatches(int num) { List<List<DataSet>> list = Lists.partition(asList(), num); List<DataSet> ret = new ArrayList<>(); for (List<DataSet> l : list) ret.add(DataSet.merge(l)); return ret; }
/** * Partitions a dataset in to mini batches where * each dataset in each list is of the specified number of examples * * @param num the number to split by * @return the partitioned datasets */ @Override public List<DataSet> batchBy(int num) { List<DataSet> batched = Lists.newArrayList(); for (List<DataSet> splitBatch : Lists.partition(asList(), num)) { batched.add(DataSet.merge(splitBatch)); } return batched; }
/** * Strips the data transform of all but the passed in labels * * @param labels strips the data transform of all but the passed in labels * @return the dataset with only the specified labels */ @Override public DataSet filterBy(int[] labels) { List<DataSet> list = asList(); List<DataSet> newList = new ArrayList<>(); List<Integer> labelList = new ArrayList<>(); for (int i : labels) labelList.add(i); for (DataSet d : list) { int outcome = d.outcome(); if (labelList.contains(outcome)) { newList.add(d); } } return DataSet.merge(newList); }
public void sortByLabel() { Map<Integer, Queue<DataSet>> map = new HashMap<>(); List<DataSet> data = asList(); int numLabels = numOutcomes(); int examples = numExamples();
@Override protected List<DataSet> split(DataSet toSplit) { return toSplit.asList(); }
public TestDataSetIterator(DataSet dataset, int batch) { this(dataset.asList(), batch); }
@Override public Iterator<DataSet> iterator() { return asList().iterator(); }
/** * Partitions the data transform by the specified number. * * @param num the number to split by * @return the partitioned data transform */ @Override public List<DataSet> dataSetBatches(int num) { List<List<DataSet>> list = Lists.partition(asList(), num); List<DataSet> ret = new ArrayList<>(); for (List<DataSet> l : list) ret.add(DataSet.merge(l)); return ret; }
/** * Partitions a dataset in to mini batches where * each dataset in each list is of the specified number of examples * * @param num the number to split by * @return the partitioned datasets */ @Override public List<DataSet> batchBy(int num) { List<DataSet> batched = Lists.newArrayList(); for (List<DataSet> splitBatch : Lists.partition(asList(), num)) { batched.add(DataSet.merge(splitBatch)); } return batched; }
/** Create a DataSetIterator for training * @param x X values * @param function Function to evaluate * @param batchSize Batch size (number of examples for every call of DataSetIterator.next()) * @param rng Random number generator (for repeatability) */ private static DataSetIterator getTrainingData(final INDArray x, final MathFunction function, final int batchSize, final Random rng) { final INDArray y = function.getFunctionValues(x); final DataSet allData = new DataSet(x,y); final List<DataSet> list = allData.asList(); Collections.shuffle(list,rng); return new ListDataSetIterator(list,batchSize); }
public MovingWindowDataSetFetcher(DataSet data, int windowRows, int windowColumns) { this.data = data; this.windowRows = windowRows; this.windowColumns = windowColumns; List<DataSet> list = data.asList(); List<DataSet> flipped = new ArrayList<>(); for (int i = 0; i < list.size(); i++) { INDArray label = list.get(i).getLabels(); List<INDArray> windows = new MovingWindowMatrix(list.get(i).getFeatureMatrix(), windowRows, windowColumns, true) .windows(true); for (int j = 0; j < windows.size(); j++) { flipped.add(new DataSet(windows.get(j), label)); } flipped.add(list.get(i)); } this.data = DataSet.merge(flipped); }
public MovingWindowDataSetFetcher(DataSet data, int windowRows, int windowColumns) { this.data = data; this.windowRows = windowRows; this.windowColumns = windowColumns; List<DataSet> list = data.asList(); List<DataSet> flipped = new ArrayList<>(); for (int i = 0; i < list.size(); i++) { INDArray label = list.get(i).getLabels(); List<INDArray> windows = new MovingWindowMatrix(list.get(i).getFeatures(), windowRows, windowColumns, true) .windows(true); for (int j = 0; j < windows.size(); j++) { flipped.add(new DataSet(windows.get(j), label)); } flipped.add(list.get(i)); } this.data = DataSet.merge(flipped); }
/** * Fetches the next dataset. You need to call this * to getFromOrigin a new dataset, otherwise {@link #next()} * just returns the last data applyTransformToDestination fetch * * @param numExamples the number of examples to fetch */ @Override public void fetch(int numExamples) { initializeCurrFromList(data.get(ArrayUtil.range(cursor, cursor + numExamples)).asList()); } }
@Override public void train(FederatedDataSet federatedDataSet) { DataSet trainingData = (DataSet) federatedDataSet.getNativeDataSet(); List<DataSet> listDs = trainingData.asList(); DataSetIterator mnistTrain = new ListDataSetIterator(listDs, BATCH_SIZE); for (int i = 0; i < N_EPOCHS; i++) { model.fit(mnistTrain); } }
/** * Fetches the next dataset. You need to call this * to get a new dataset, otherwise {@link #next()} * just returns the last data applyTransformToDestination fetch * * @param numExamples the number of examples to fetch */ @Override public void fetch(int numExamples) { initializeCurrFromList(data.get(ArrayUtil.range(cursor, cursor + numExamples)).asList()); } }
@Override public String evaluate(FederatedDataSet federatedDataSet) { DataSet testData = (DataSet) federatedDataSet.getNativeDataSet(); List<DataSet> listDs = testData.asList(); DataSetIterator iterator = new ListDataSetIterator(listDs, BATCH_SIZE); return mNetwork.evaluate(iterator).stats(); }
@Override public void train(FederatedDataSet dataSource) { DataSet trainingData = (DataSet) dataSource.getNativeDataSet(); List<DataSet> listDs = trainingData.asList(); DataSetIterator iterator = new ListDataSetIterator(listDs, BATCH_SIZE); //Train the network on the full data set, and evaluate in periodically for (int i = 0; i < N_EPOCHS; i++) { iterator.reset(); mNetwork.fit(iterator); } }
@Override public String evaluate(FederatedDataSet federatedDataSet) { DataSet testData = (DataSet) federatedDataSet.getNativeDataSet(); List<DataSet> listDs = testData.asList(); DataSetIterator iterator = new ListDataSetIterator(listDs, BATCH_SIZE); Evaluation eval = new Evaluation(OUTPUT_NUM); //create an evaluation object with 10 possible classes while (iterator.hasNext()) { DataSet next = iterator.next(); INDArray output = model.output(next.getFeatureMatrix()); //get the networks prediction eval.eval(next.getLabels(), output); //check the prediction against the true class } return eval.stats(); }