/** * Return a replay stream, cued up to begining of content * * @throws IOException * @return An RIS. */ public ReplayInputStream getMessageBodyReplayInputStream() throws IOException { return getReplayInputStream(this.messageBodyBeginMark); }
protected URI writeRequest(final WARCWriter w, final String timestamp, final String mimetype, final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { WARCRecordInfo recordInfo = new WARCRecordInfo(); recordInfo.setType(WARCRecordType.request); recordInfo.setUrl(curi.toString()); recordInfo.setCreate14DigitDate(timestamp); recordInfo.setMimetype(mimetype); recordInfo.setExtraHeaders(namedFields); recordInfo.setContentLength(curi.getRecorder().getRecordedOutput().getSize()); recordInfo.setEnforceLength(true); final URI uid = qualifyRecordID(baseid, TYPE, WARCRecordType.request.toString()); recordInfo.setRecordId(uid); ReplayInputStream ris = curi.getRecorder().getRecordedOutput().getReplayInputStream(); recordInfo.setContentStream(ris); try { w.writeRecord(recordInfo); } finally { IOUtils.closeQuietly(ris); } return recordInfo.getRecordId(); }
/** * Create a new RecordingInputStream. * * @param bufferSize Size of buffer to use. * @param backingFilename Name of backing file. */ public RecordingInputStream(int bufferSize, String backingFilename) { this.recordingOutputStream = new RecordingOutputStream(bufferSize, backingFilename); }
int mark = findMessageBodyBeginMark(b, off, len); if (mark > 0) { if(recording) { record(b, off, mark - off); markMessageBodyBegin(); len = len - (mark - off); off = mark; record(b, off, len); checkLimits();
public void write(int b) throws IOException { if(position<maxPosition) { // revisiting previous content; do nothing but advance position position++; return; } if(recording) { record(b); } if (this.out != null) { this.out.write(b); } // see comment on int[] lastTwoBytes if (messageBodyBeginMark < 0l) { // looking for "\n\n" or "\n\r\n" if (b == '\n' && (lastTwoBytes[1] == '\n' || (lastTwoBytes[0] == '\n' && lastTwoBytes[1] == '\r'))) { markMessageBodyBegin(); } else { lastTwoBytes[0] = lastTwoBytes[1]; lastTwoBytes[1] = b; } } checkLimits(); }
long totalBytes = recordingOutputStream.position - recordingOutputStream.getMessageBodyBegin(); long bytesRead = -1L; long maxToRead = -1; : Math.min(drainBuffer.length, contentLength - totalBytes); maxToRead = Math.min(maxToRead, recordingOutputStream.getRemainingLength()); recordingOutputStream.checkLimits(); } catch (SocketException se) { throw se;
public long getSize() { return this.recordingOutputStream.getSize(); }
/** * Wrap the given stream, both recording and passing along any data written * to this RecordingOutputStream. * * @param wrappedStream Stream to wrap. May be null for case where we * want to write to a file backed stream only. * * @throws IOException If failed creation of backing file. */ public void open(OutputStream wrappedStream) throws IOException { if(isOpen()) { // error; should not be opening/wrapping in an unclosed // stream remains open throw new IOException("ROS already open for " +Thread.currentThread().getName()); } clearForReuse(); this.out = wrappedStream; startTime = System.currentTimeMillis(); }
: Math.min(drainBuffer.length, softMaxLength - totalBytes); maxToRead = Math.min(maxToRead, recordingOutputStream.getRemainingLength()); recordingOutputStream.checkLimits(); } catch (SocketException se) { throw se;
public void closeRecorder() throws IOException { this.recordingOutputStream.closeRecorder(); }
public void write(int b) throws IOException { if(position<maxPosition) { // revisiting previous content; do nothing but advance position position++; return; } if(recording) { record(b); } if (this.out != null) { this.out.write(b); } checkLimits(); }
public void closeRecorder() throws IOException { recording = false; closeDiskStream(); // if any // This setting of size is important. Its passed to ReplayInputStream // on creation. It uses it to know EOS. if (this.size == 0) { this.size = this.position; } }
public void clearForReuse() throws IOException { recordingOutputStream.clearForReuse(); } }
/** * Return the digest value for any recorded, digested data. Call * only after all data has been recorded; otherwise, the running * digest state is ruined. * * @return the digest final value */ public byte[] getDigestValue() { return this.recordingOutputStream.getDigestValue(); }
/** * Expose the amount of in-memory buffering used by the internal * recording stream. * @return int buffer size */ public int getRecordedBufferLength() { return recordingOutputStream.getBufferLength(); }
/** * Close all streams. */ public void close() { logger.fine(Thread.currentThread().getName() + " closing"); try { this.ris.close(); } catch (IOException e) { // TODO: Can we not let the exception out of here and report it // higher up in the caller? DevUtils.logger.log(Level.SEVERE, "close() ris" + DevUtils.extraInfo(), e); } try { this.ros.close(); } catch (IOException e) { DevUtils.logger.log(Level.SEVERE, "close() ros" + DevUtils.extraInfo(), e); } }
int mark = findMessageBodyBeginMark(b, off, len); if (mark > 0) { if(recording) { record(b, off, mark - off); markMessageBodyBegin(); len = len - (mark - off); off = mark; record(b, off, len); checkLimits();
public void write(int b) throws IOException { if(position<maxPosition) { // revisiting previous content; do nothing but advance position position++; return; } if(recording) { record(b); } if (this.out != null) { this.out.write(b); } // see comment on int[] lastTwoBytes if (messageBodyBeginMark < 0l) { // looking for "\n\n" or "\n\r\n" if (b == '\n' && (lastTwoBytes[1] == '\n' || (lastTwoBytes[0] == '\n' && lastTwoBytes[1] == '\r'))) { markMessageBodyBegin(); } else { lastTwoBytes[0] = lastTwoBytes[1]; lastTwoBytes[1] = b; } } checkLimits(); }
long totalBytes = recordingOutputStream.position - recordingOutputStream.getMessageBodyBegin(); long bytesRead = -1L; long maxToRead = -1; : Math.min(drainBuffer.length, contentLength - totalBytes); maxToRead = Math.min(maxToRead, recordingOutputStream.getRemainingLength()); recordingOutputStream.checkLimits(); } catch (SocketException se) { throw se;
public long getSize() { return this.recordingOutputStream.getSize(); }