private static <S extends MessageSchema<S>> void processAvailData(TapeWriteStage<S> ss) { //only zero when we need to find the next block and have finished the previous if (0==ss.totalPrimaryCopy) { findStableCutPoint(ss); //we have established the point that we can read up to, this value is changed by the writer on the other side //get the start and stop locations for the copy //now find the point to start reading from, this is moved forward with each new read. if ((ss.totalPrimaryCopy = (ss.headPos - ss.cachedTail)) <= 0) { assert(ss.totalPrimaryCopy==0); return; //nothing to copy so come back later } setupBuffersToWriteFrom(ss); } copyToFile(ss); }
@Override public void run() { processAvailData(this);//spin here makes little difference, its the ammount of work done which is the problem. }
fileChannelWrite(ss, ss.slabBuffer1); if (ss.slabBuffer1.hasRemaining()) { return; fileChannelWrite(ss, ss.slabBuffer2); if (ss.slabBuffer2.hasRemaining()) { return; recordCopyComplete(ss);
public static File writeTapeToFileUsingPipe(Pipe<RawDataSchema> inputPipe, byte[] testData) throws IOException, FileNotFoundException { GraphManager gm = new GraphManager(); File f2 = File.createTempFile("fileTapeWriteTest", "dat"); f2.deleteOnExit(); new ByteArrayProducerStage(gm, testData, inputPipe); new TapeWriteStage(gm, inputPipe, new RandomAccessFile(f2,"rw")); GraphManager.enableBatching(gm); ThreadPerStageScheduler scheduler = new ThreadPerStageScheduler(gm); scheduler.startup(); scheduler.awaitTermination(3, TimeUnit.SECONDS); return f2; }
@Override public void shutdown() { //if we are in the middle of a partial copy push the data out, this is blocking while (0!=totalPrimaryCopy) { //if all the copies are done then record it as complete, does as much work as possible each time its called. copyToFile(this); } }
PronghornStage s2 = new TapeWriteStage(gm, loadedDataPipe, new RandomAccessFile(tempFile,"rw")); //NOTE: use rwd/rws to sync flush with every write (much slower)