/** * Writes an automaton to a file. */ public void save(final Path path) throws IOException { try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))) { save(new OutputStreamDataOutput(os)); } }
@Override public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException { if (weight < -1 || weight > Integer.MAX_VALUE) { throw new IllegalArgumentException("weight must be >= -1 && <= Integer.MAX_VALUE"); } for (int i = 0; i < surfaceForm.length; i++) { if (surfaceForm.bytes[i] == UNIT_SEPARATOR) { throw new IllegalArgumentException( "surface form cannot contain unit separator character U+001F; this character is reserved"); } } ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream); output.writeVLong(weight + 1); output.writeVInt(surfaceForm.length); output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length); output.writeVInt(payload.length); output.writeBytes(payload.bytes, 0, payload.length); output.close(); return new BytesRef(byteArrayOutputStream.toByteArray()); }
private BytesRef buildSuggestPayload() { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); try (OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream)) { output.writeVInt(surfaceForm.length); output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length); output.writeVInt(weight + 1); output.writeByte(type()); } catch (IOException e) { throw new RuntimeException(e); // not possible, it's a ByteArrayOutputStream! } return new BytesRef(byteArrayOutputStream.toByteArray()); }
protected URI createFst(SortedSet<String> values) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { FST fst = DatawaveFieldIndexListIteratorJexl.getFST(values); // now serialize to our file system CompressionCodec codec = null; String extension = ""; if (config.getHdfsFileCompressionCodec() != null) { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { classLoader = this.getClass().getClassLoader(); } Class<? extends CompressionCodec> clazz = Class.forName(config.getHdfsFileCompressionCodec(), true, classLoader).asSubclass(CompressionCodec.class); codec = clazz.newInstance(); extension = codec.getDefaultExtension(); } int fstCount = config.getFstCount().incrementAndGet(); Path fstFile = new Path(fstHdfsUri, "PushdownLargeFileFst." + fstCount + ".fst" + extension); OutputStream fstFileOut = new BufferedOutputStream(fs.create(fstFile, false)); if (codec != null) { fstFileOut = codec.createOutputStream(fstFileOut); } OutputStreamDataOutput outStream = new OutputStreamDataOutput(fstFileOut); fst.save(outStream); outStream.close(); return fstFile.toUri(); }
/** * Serialize a MonitorQuery into a BytesRef * @param mq the MonitorQuery * @return the serialized bytes */ public static BytesRef serialize(MonitorQuery mq) { ByteArrayOutputStream os = new ByteArrayOutputStream(); try (OutputStreamDataOutput data = new OutputStreamDataOutput(os)) { data.writeString(mq.getId()); data.writeString(mq.getQuery()); data.writeInt(mq.getMetadata().size()); for (Map.Entry<String, String> entry : mq.getMetadata().entrySet()) { data.writeString(entry.getKey()); data.writeString(entry.getValue()); } return new BytesRef(os.toByteArray()); } catch (IOException e) { throw new RuntimeException(e); // shouldn't happen, we're writing to a bytearray! } }
public static TranslogWriter create(Type type, ShardId shardId, String translogUUID, long fileGeneration, Path file, Callback<ChannelReference> onClose, int bufferSize, ChannelFactory channelFactory) throws IOException { final BytesRef ref = new BytesRef(translogUUID); final int headerLength = getHeaderLength(ref.length); final FileChannel channel = channelFactory.open(file); try { // This OutputStreamDataOutput is intentionally not closed because // closing it will close the FileChannel final OutputStreamDataOutput out = new OutputStreamDataOutput(java.nio.channels.Channels.newOutputStream(channel)); CodecUtil.writeHeader(out, TRANSLOG_CODEC, VERSION); out.writeInt(ref.length); out.writeBytes(ref.bytes, ref.offset, ref.length); channel.force(true); writeCheckpoint(headerLength, 0, file.getParent(), fileGeneration, StandardOpenOption.WRITE); final TranslogWriter writer = type.create(shardId, fileGeneration, new ChannelReference(file, fileGeneration, channel, onClose), bufferSize); return writer; } catch (Throwable throwable){ // if we fail to bake the file-generation into the checkpoint we stick with the file and once we recover and that // file exists we remove it. We only apply this logic to the checkpoint.generation+1 any other file with a higher generation is an error condition IOUtils.closeWhileHandlingException(channel); throw throwable; } } /** If this {@code TranslogWriter} was closed as a side-effect of a tragic exception,
@Override public void close() throws IOException { closed = true; out.close(); if (count != countWritten) { throw new IllegalStateException("wrote " + countWritten + " values, but expected " + count); } }
static void writeHeader(OutputStreamDataOutput out, BytesRef ref) throws IOException { CodecUtil.writeHeader(out, TRANSLOG_CODEC, VERSION); out.writeInt(ref.length); out.writeBytes(ref.bytes, ref.offset, ref.length); }
@Override public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException { if (weight < -1 || weight > Integer.MAX_VALUE) { throw new IllegalArgumentException("weight must be >= -1 && <= Integer.MAX_VALUE"); } for (int i = 0; i < surfaceForm.length; i++) { if (surfaceForm.bytes[i] == UNIT_SEPARATOR) { throw new IllegalArgumentException( "surface form cannot contain unit separator character U+001F; this character is reserved"); } } ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream); output.writeVLong(weight + 1); output.writeVInt(surfaceForm.length); output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length); output.writeVInt(payload.length); output.writeBytes(payload.bytes, 0, payload.length); output.close(); return new BytesRef(byteArrayOutputStream.toByteArray()); }
private static void serialize(String file, BytesRef [] all) throws IOException { final Object nothing = NoOutputs.getSingleton().getNoOutput(); final Builder<Object> builder = new Builder<Object>(INPUT_TYPE.BYTE4, NoOutputs.getSingleton()); final IntsRefBuilder intsRef = new IntsRefBuilder(); for (BytesRef br : all) { intsRef.clear(); intsRef.copyUTF8Bytes(br); builder.add(intsRef.get(), nothing); } final FST<Object> fst = builder.finish(); final OutputStreamDataOutput out = new OutputStreamDataOutput(new FileOutputStream(file)); fst.save(out); out.close(); } }
@Override public void close() throws IOException { closed = true; out.close(); if (count != countWritten) { throw new IllegalStateException("wrote " + countWritten + " values, but expected " + count); } }
@Override public boolean store(OutputStream output) throws IOException { DataOutput dataOut = new OutputStreamDataOutput(output); try { if (fst == null) { return false; } fst.save(dataOut); dataOut.writeVInt(maxAnalyzedPathsForOneInput); dataOut.writeByte((byte) (hasPayloads ? 1 : 0)); } finally { IOUtils.close(output); } return true; }
@Override public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException { if (weight < -1 || weight > Integer.MAX_VALUE) { throw new IllegalArgumentException("weight must be >= -1 && <= Integer.MAX_VALUE"); } for (int i = 0; i < surfaceForm.length; i++) { if (surfaceForm.bytes[i] == UNIT_SEPARATOR) { throw new IllegalArgumentException( "surface form cannot contain unit separator character U+001F; this character is reserved"); } } ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream); output.writeVLong(weight + 1); output.writeVInt(surfaceForm.length); output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length); output.writeVInt(payload.length); output.writeBytes(payload.bytes, 0, payload.length); output.close(); return new BytesRef(byteArrayOutputStream.toByteArray()); }
@Override public void close() throws IOException { closed = true; out.close(); if (count != countWritten) { throw new IllegalStateException("wrote " + countWritten + " values, but expected " + count); } }
/** * Writes this header with the latest format into the file channel */ void write(final FileChannel channel) throws IOException { // This output is intentionally not closed because closing it will close the FileChannel. @SuppressWarnings({"IOResourceOpenedButNotSafelyClosed", "resource"}) final BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput( new OutputStreamStreamOutput(java.nio.channels.Channels.newOutputStream(channel))); CodecUtil.writeHeader(new OutputStreamDataOutput(out), TRANSLOG_CODEC, CURRENT_VERSION); // Write uuid final BytesRef uuid = new BytesRef(translogUUID); out.writeInt(uuid.length); out.writeBytes(uuid.bytes, uuid.offset, uuid.length); // Write primary term out.writeLong(primaryTerm); // Checksum header out.writeInt((int) out.getChecksum()); out.flush(); channel.force(true); assert channel.position() == headerSizeInBytes : "Header is not fully written; header size [" + headerSizeInBytes + "], channel position [" + channel.position() + "]"; } }
/** * Writes an automaton to a file. */ public void save(final Path path) throws IOException { try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))) { save(new OutputStreamDataOutput(os)); } }
public OfflineWriter(long count) throws IOException { tempFile = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "size" + count + ".", ""); out = new OutputStreamDataOutput(new BufferedOutputStream(Files.newOutputStream(tempFile))); this.count = count; }