protected URI writeRequest(final WARCWriter w, final String timestamp, final String mimetype, final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { WARCRecordInfo recordInfo = new WARCRecordInfo(); recordInfo.setType(WARCRecordType.request); recordInfo.setUrl(curi.toString()); recordInfo.setCreate14DigitDate(timestamp); recordInfo.setMimetype(mimetype); recordInfo.setExtraHeaders(namedFields); recordInfo.setContentLength(curi.getRecorder().getRecordedOutput().getSize()); recordInfo.setEnforceLength(true); final URI uid = qualifyRecordID(baseid, TYPE, WARCRecordType.request.toString()); recordInfo.setRecordId(uid); ReplayInputStream ris = curi.getRecorder().getRecordedOutput().getReplayInputStream(); recordInfo.setContentStream(ris); try { w.writeRecord(recordInfo); } finally { IOUtils.closeQuietly(ris); } return recordInfo.getRecordId(); }
public long getSize() { return this.recordingOutputStream.getSize(); }
public long getSize() { return this.recordingOutputStream.getSize(); }
public long getSize() { return this.recordingOutputStream.getSize(); }
public long readFully() throws IOException { while(read(drainBuffer) != -1) { // Empty out stream. continue; } return this.recordingOutputStream.getSize(); }
public long readFully() throws IOException { while(read(drainBuffer) != -1) { // Empty out stream. continue; } return this.recordingOutputStream.getSize(); }
public long readFully() throws IOException { while(read(drainBuffer) != -1) { // Empty out stream. continue; } return this.recordingOutputStream.getSize(); }
protected URI writeRequest(final WARCWriter w, final String timestamp, final String mimetype, final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { WARCRecordInfo recordInfo = new WARCRecordInfo(); recordInfo.setType(WARCRecordType.request); recordInfo.setUrl(curi.toString()); recordInfo.setCreate14DigitDate(timestamp); recordInfo.setMimetype(mimetype); recordInfo.setExtraHeaders(namedFields); recordInfo.setContentLength(curi.getRecorder().getRecordedOutput().getSize()); recordInfo.setEnforceLength(true); final URI uid = qualifyRecordID(baseid, TYPE, WARCRecordType.request.toString()); recordInfo.setRecordId(uid); ReplayInputStream ris = curi.getRecorder().getRecordedOutput().getReplayInputStream(); recordInfo.setContentStream(ris); try { w.writeRecord(recordInfo); } finally { IOUtils.closeQuietly(ris); } return recordInfo.getRecordId(); }