/** Open a writer appending to an existing file. * <strong>Since 1.9.0 this method does not close in.</strong> * @param in reading the existing file. * @param out positioned at the end of the existing file. */ public DataFileWriter<D> appendTo(SeekableInput in, OutputStream out) throws IOException { assertNotOpen(); DataFileReader<D> reader = new DataFileReader<>(in, new GenericDatumReader<>()); this.schema = reader.getSchema(); this.sync = reader.getHeader().sync; this.meta.putAll(reader.getHeader().meta); byte[] codecBytes = this.meta.get(DataFileConstants.CODEC); if (codecBytes != null) { String strCodec = new String(codecBytes, "UTF-8"); this.codec = CodecFactory.fromString(strCodec).createInstance(); } else { this.codec = CodecFactory.nullCodec().createInstance(); } init(out); return this; }
/** Open a writer appending to an existing file. * @param in reading the existing file. * @param out positioned at the end of the existing file. */ public DataFileWriter<D> appendTo(SeekableInput in, OutputStream out) throws IOException { assertNotOpen(); DataFileReader<D> reader = new DataFileReader<D>(in, new GenericDatumReader<D>()); this.schema = reader.getSchema(); this.sync = reader.getHeader().sync; this.meta.putAll(reader.getHeader().meta); byte[] codecBytes = this.meta.get(DataFileConstants.CODEC); if (codecBytes != null) { String strCodec = new String(codecBytes, "UTF-8"); this.codec = CodecFactory.fromString(strCodec).createInstance(); } else { this.codec = CodecFactory.nullCodec().createInstance(); } reader.close(); init(out); return this; }
int j = 0; for (; j < SYNC_SIZE; j++) { if (getHeader().sync[j] != syncBuffer[(i+j)%SYNC_SIZE]) break;
int j = 0; for (; j < SYNC_SIZE; j++) { if (getHeader().sync[j] != syncBuffer[(i+j)%SYNC_SIZE]) break;
fakeHeader = new DataFileReader<>(new SeekableByteArrayInput(this.header), datumReader).getHeader(); dataFileReader = DataFileReader.openReader(sbai, datumReader, fakeHeader, true); Schema schema = dataFileReader.getSchema();
public void testReadWithHeader() throws IOException { File file = makeFile(); DataFileReader<Object> reader = new DataFileReader<>(file, new GenericDatumReader<>()); // get a header for this file DataFileStream.Header header = reader.getHeader(); // re-open to an arbitrary position near the middle, with sync == true SeekableFileInput sin = new SeekableFileInput(file); sin.seek(sin.length() / 2); reader = DataFileReader.openReader(sin, new GenericDatumReader<>(), header, true); assertNotNull("Should be able to reopen from arbitrary point", reader.next()); long validPos = reader.previousSync(); // post sync, we know of a valid sync point: re-open with seek (sync == false) sin.seek(validPos); reader = DataFileReader.openReader(sin, new GenericDatumReader<>(), header, false); assertEquals("Should not move from sync point on reopen", validPos, sin.tell()); assertNotNull("Should be able to reopen at sync point", reader.next()); }
/** Open a writer appending to an existing file. * @param in reading the existing file. * @param out positioned at the end of the existing file. */ public DataFileWriter<D> appendTo(SeekableInput in, OutputStream out) throws IOException { assertNotOpen(); DataFileReader<D> reader = new DataFileReader<D>(in, new GenericDatumReader<D>()); this.schema = reader.getSchema(); this.sync = reader.getHeader().sync; this.meta.putAll(reader.getHeader().meta); byte[] codecBytes = this.meta.get(DataFileConstants.CODEC); if (codecBytes != null) { String strCodec = new String(codecBytes, "UTF-8"); this.codec = CodecFactory.fromString(strCodec).createInstance(); } else { this.codec = CodecFactory.nullCodec().createInstance(); } reader.close(); init(out); return this; }
/** Open a writer appending to an existing file. * @param in reading the existing file. * @param out positioned at the end of the existing file. */ public DataFileWriter<D> appendTo(SeekableInput in, OutputStream out) throws IOException { assertNotOpen(); DataFileReader<D> reader = new DataFileReader<D>(in, new GenericDatumReader<D>()); this.schema = reader.getSchema(); this.sync = reader.getHeader().sync; this.meta.putAll(reader.getHeader().meta); byte[] codecBytes = this.meta.get(DataFileConstants.CODEC); if (codecBytes != null) { String strCodec = new String(codecBytes, "UTF-8"); this.codec = CodecFactory.fromString(strCodec).createInstance(); } else { this.codec = CodecFactory.nullCodec().createInstance(); } reader.close(); init(out); return this; }
int j = 0; for (; j < SYNC_SIZE; j++) { if (getHeader().sync[j] != syncBuffer[(i+j)%SYNC_SIZE]) break;
int j = 0; for (; j < SYNC_SIZE; j++) { if (getHeader().sync[j] != syncBuffer[(i+j)%SYNC_SIZE]) break;
fakeHeader = new DataFileReader<>(new SeekableByteArrayInput(this.header), datumReader).getHeader(); dataFileReader = DataFileReader.openReader(sbai, datumReader, fakeHeader, true); Schema schema = dataFileReader.getSchema();