/** * Shuffles the dataset and resets to the first fold * * @return void */ @Override public void reset() { //shuffle and return new k folds singleFold.shuffle(); kCursor = 0; }
@Override public void shuffle() { long seed = System.currentTimeMillis(); shuffle(seed); }
/** * Splits a dataset in to test and train randomly. * This will modify the dataset in place to shuffle it before splitting into test/train! * * @param numHoldout the number to hold out for training * @param rng Random Number Generator to use to shuffle the dataset * @return the pair of datasets for the train test split */ @Override public SplitTestAndTrain splitTestAndTrain(int numHoldout, Random rng) { long seed = rng.nextLong(); this.shuffle(seed); return splitTestAndTrain(numHoldout); }
/** * Shuffles the dataset and resets to the first fold * * @return void */ @Override public void reset() { //shuffle and return new k folds singleFold.shuffle(); kCursor = 0; }
@Override public void shuffle() { long seed = System.currentTimeMillis(); shuffle(seed); }
/** * Splits a dataset in to test and train randomly. * This will modify the dataset in place to shuffle it before splitting into test/train! * * @param numHoldout the number to hold out for training * @param rng Random Number Generator to use to shuffle the dataset * @return the pair of datasets for the train test split */ @Override public SplitTestAndTrain splitTestAndTrain(int numHoldout, Random rng) { long seed = rng.nextLong(); this.shuffle(seed); return splitTestAndTrain(numHoldout); }
@Override public FederatedDataSet getTrainingData() { Random rand = new Random(seed); double[] sum = new double[N_SAMPLES]; double[] input1 = new double[N_SAMPLES]; double[] input2 = new double[N_SAMPLES]; for (int i = 0; i < N_SAMPLES; i++) { input1[i] = MIN_RANGE + (MAX_RANGE - MIN_RANGE) * rand.nextDouble(); input2[i] = MIN_RANGE + (MAX_RANGE - MIN_RANGE) * rand.nextDouble(); sum[i] = input1[i] + input2[i]; } INDArray inputNDArray1 = Nd4j.create(input1, new int[]{N_SAMPLES, 1}); INDArray inputNDArray2 = Nd4j.create(input2, new int[]{N_SAMPLES, 1}); INDArray inputNDArray = Nd4j.hstack(inputNDArray1, inputNDArray2); INDArray outPut = Nd4j.create(sum, new int[]{N_SAMPLES, 1}); DataSet dataSet = new DataSet(inputNDArray, outPut); dataSet.shuffle(); return new FederatedDataSetImpl(dataSet); }
loadDS.shuffle(seed); loadDSIndex = 0;
result.shuffle(seed); return result;
allData.shuffle(seed);
private void createDataSource() throws IOException, InterruptedException { //First: get the dataset using the record reader. CSVRecordReader handles loading/parsing int numLinesToSkip = 0; String delimiter = ","; RecordReader recordReader = new CSVRecordReader(numLinesToSkip, delimiter); recordReader.initialize(new InputStreamInputSplit(dataFile)); //Second: the RecordReaderDataSetIterator handles conversion to DataSet objects, ready for use in neural network int labelIndex = 4; //5 values in each row of the iris.txt CSV: 4 input features followed by an integer label (class) index. Labels are the 5th value (index 4) in each row int numClasses = 3; //3 classes (types of iris flowers) in the iris data set. Classes have integer values 0, 1 or 2 DataSetIterator iterator = new RecordReaderDataSetIterator(recordReader, batchSize, labelIndex, numClasses); DataSet allData = iterator.next(); allData.shuffle(); SplitTestAndTrain testAndTrain = allData.splitTestAndTrain(0.80); //Use 80% of data for training trainingData = testAndTrain.getTrain(); testData = testAndTrain.getTest(); //We need to normalize our data. We'll use NormalizeStandardize (which gives us mean 0, unit variance): DataNormalization normalizer = new NormalizerStandardize(); normalizer.fit(trainingData); //Collect the statistics (mean/stdev) from the training data. This does not modify the input data normalizer.transform(trainingData); //Apply normalization to the training data normalizer.transform(testData); //Apply normalization to the test data. This is using statistics calculated from the *training* set }
trainingData.shuffle();