Refine search
/** * Create a {@link CompressedXContent} out of a {@link ToXContent} instance. */ public CompressedXContent(ToXContent xcontent, XContentType type, ToXContent.Params params) throws IOException { BytesStreamOutput bStream = new BytesStreamOutput(); OutputStream compressedStream = CompressorFactory.COMPRESSOR.streamOutput(bStream); CRC32 crc32 = new CRC32(); OutputStream checkedStream = new CheckedOutputStream(compressedStream, crc32); try (XContentBuilder builder = XContentFactory.contentBuilder(type, checkedStream)) { builder.startObject(); xcontent.toXContent(builder, params); builder.endObject(); } this.bytes = BytesReference.toBytes(bStream.bytes()); this.crc32 = (int) crc32.getValue(); assertConsistent(); }
private BytesReference writeHeader(int numFieldsWritten, boolean getTermStatistics, boolean getFieldStatistics, boolean scores) throws IOException { // now, write the information about offset of the terms in the // termVectors field BytesStreamOutput header = new BytesStreamOutput(); header.writeString(HEADER); header.writeInt(CURRENT_VERSION); header.writeBoolean(getTermStatistics); header.writeBoolean(getFieldStatistics); header.writeBoolean(scores); header.writeVInt(numFieldsWritten); for (int i = 0; i < fields.size(); i++) { header.writeString(fields.get(i)); header.writeVLong(fieldOffset.get(i).longValue()); } header.close(); return header.bytes(); }
/** * Writes the Tcp message header into a bytes reference. * * @param requestId the request ID * @param status the request status * @param protocolVersion the protocol version used to serialize the data in the message * @param length the payload length in bytes * @see TcpHeader */ private BytesReference buildHeader(long requestId, byte status, Version protocolVersion, int length) throws IOException { try (BytesStreamOutput headerOutput = new BytesStreamOutput(TcpHeader.HEADER_SIZE)) { headerOutput.setVersion(protocolVersion); TcpHeader.writeHeader(headerOutput, requestId, status, protocolVersion, length); final BytesReference bytes = headerOutput.bytes(); assert bytes.length() == TcpHeader.HEADER_SIZE : "header size mismatch expected: " + TcpHeader.HEADER_SIZE + " but was: " + bytes.length(); return bytes; } }
TransportKeepAlive(ThreadPool threadPool, AsyncBiFunction<TcpChannel, BytesReference, Void> pingSender) { this.threadPool = threadPool; this.pingSender = pingSender; try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeByte((byte) 'E'); out.writeByte((byte) 'S'); out.writeInt(PING_DATA_SIZE); pingMessage = out.bytes(); } catch (IOException e) { throw new AssertionError(e.getMessage(), e); // won't happen } this.lifecycle.moveToStarted(); }
public static byte[] toBytes(ClusterState state) throws IOException { BytesStreamOutput os = new BytesStreamOutput(); state.writeTo(os); return BytesReference.toBytes(os.bytes()); }
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { boolean includeDefaults = params.paramAsBoolean("include_defaults", false); if (!includeDefaults) { // simulate the generation to make sure we don't add unnecessary content if all is default // if all are defaults, no need to write it at all - generating is twice is ok though BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(0); XContentBuilder b = new XContentBuilder(builder.contentType().xContent(), bytesStreamOutput); b.startObject().flush(); long pos = bytesStreamOutput.position(); innerToXContent(b, false); b.flush(); if (pos == bytesStreamOutput.position()) { return builder; } } builder.startObject(CONTENT_TYPE); innerToXContent(builder, includeDefaults); builder.endObject(); return builder; }
protected XContentBuilder notEndedDocument(String messageId, long timestamp, String topicName, String status, String hostname) throws IOException { return jsonBuilder(new BytesStreamOutput(DOCUMENT_EXPECTED_SIZE)) .startObject() .field(MESSAGE_ID, messageId) .field(TIMESTAMP, timestamp) .field(TIMESTAMP_SECONDS, toSeconds(timestamp)) .field(TOPIC_NAME, topicName) .field(STATUS, status) .field(CLUSTER, clusterName) .field(SOURCE_HOSTNAME, this.hostname) .field(REMOTE_HOSTNAME, hostname); }
final int initialCapacity = nestedHit ? 1024 : Math.min(1024, source.internalSourceRef().length()); BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity); XContentBuilder builder = new XContentBuilder(source.sourceContentType().xContent(), streamOutput); if (value != null) { builder.value(value); } else { builder.startObject(); builder.endObject(); hitContext.hit().sourceRef(BytesReference.bytes(builder)); } catch (IOException e) { throw new ElasticsearchException("Error filtering source", e);
public void testWriteToOutputStream() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * 4); BytesReference pbr = newBytesReference(length); BytesStreamOutput out = new BytesStreamOutput(); pbr.writeTo(out); assertEquals(pbr.length(), out.size()); assertArrayEquals(BytesReference.toBytes(pbr), BytesReference.toBytes(out.bytes())); out.close(); }
@Override protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException { BytesReference originalSource = context.sourceToParse().source(); BytesReference source = originalSource; if (enabled && fieldType().stored() && source != null) { // Percolate and tv APIs may not set the source and that is ok, because these APIs will not index any data if (filter != null) { // we don't update the context source if we filter, we want to keep it as is... Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(source, true, context.sourceToParse().getXContentType()); Map<String, Object> filteredSource = filter.apply(mapTuple.v2()); BytesStreamOutput bStream = new BytesStreamOutput(); XContentType contentType = mapTuple.v1(); XContentBuilder builder = XContentFactory.contentBuilder(contentType, bStream).map(filteredSource); builder.close(); source = bStream.bytes(); } BytesRef ref = source.toBytesRef(); fields.add(new StoredField(fieldType().name(), ref.bytes, ref.offset, ref.length)); } else { source = null; } if (originalSource != null && source != originalSource && context.indexSettings().isSoftDeleteEnabled()) { // if we omitted source or modified it we add the _recovery_source to ensure we have it for ops based recovery BytesRef ref = originalSource.toBytesRef(); fields.add(new StoredField(RECOVERY_SOURCE_NAME, ref.bytes, ref.offset, ref.length)); fields.add(new NumericDocValuesField(RECOVERY_SOURCE_NAME, 1)); } }
protected void sendRequest(LocalTransport targetTransport, final DiscoveryNode node, final long requestId, final String action, final TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { final Version version = Version.min(node.getVersion(), getVersion()); try (BytesStreamOutput stream = new BytesStreamOutput()) { stream.setVersion(version); stream.writeLong(requestId); byte status = 0; status = TransportStatus.setRequest(status); stream.writeByte(status); // 0 for request, 1 for response. threadPool.getThreadContext().writeTo(stream); stream.writeString(action); request.writeTo(stream); stream.close(); final byte[] data = BytesReference.toBytes(stream.bytes()); transportServiceAdapter.addBytesSent(data.length); transportServiceAdapter.onRequestSent(node, requestId, action, request, options); targetTransport.receiveMessage(version, data, action, requestId, this); } }
@Override public void hitExecute(SearchContext context, HitContext hitContext) { FetchSourceContext fetchSourceContext = context.fetchSourceContext(); assert fetchSourceContext.fetchSource(); if (fetchSourceContext.includes().length == 0 && fetchSourceContext.excludes().length == 0) { hitContext.hit().sourceRef(context.lookup().source().internalSourceRef()); return; } SourceLookup source = context.lookup().source(); Object value = source.filter(fetchSourceContext.includes(), fetchSourceContext.excludes()); try { final int initialCapacity = Math.min(1024, source.internalSourceRef().length()); BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity); XContentBuilder builder = new XContentBuilder(context.lookup().source().sourceContentType().xContent(), streamOutput); builder.value(value); hitContext.hit().sourceRef(builder.bytes()); } catch (IOException e) { throw new ElasticsearchException("Error filtering source", e); } } }
Object value = sourceLookup.filter(request.fetchSource()); try { final int initialCapacity = Math.min(1024, sourceAsBytes.length()); BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity); try (XContentBuilder builder = new XContentBuilder(sourceContentType.xContent(), streamOutput)) { builder.value(value); sourceFilteredAsBytes = BytesReference.bytes(builder);
public static XContentBuilder restContentBuilder(RestRequest request, @Nullable BytesReference autoDetectSource) throws IOException { XContentType contentType = XContentType.fromRestContentType(request.param("format", request.header("Content-Type"))); if (contentType == null) { // try and guess it from the auto detect source if (autoDetectSource != null) { contentType = XContentFactory.xContentType(autoDetectSource); } } if (contentType == null) { // default to JSON contentType = XContentType.JSON; } XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), new BytesStreamOutput()); if (request.paramAsBoolean("pretty", false)) { builder.prettyPrint().lfAtEnd(); } builder.humanReadable(request.paramAsBoolean("human", builder.humanReadable())); String casing = request.param("case"); if (casing != null && "camelCase".equals(casing)) { builder.fieldCaseConversion(XContentBuilder.FieldCaseConversion.CAMELCASE); } else { // we expect all REST interfaces to write results in underscore casing, so // no need for double casing builder.fieldCaseConversion(XContentBuilder.FieldCaseConversion.NONE); } return builder; }
/** * Serialize the given query builder and asserts that both are equal */ protected static QueryBuilder assertSerialization(QueryBuilder testQuery, Version version) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { output.setVersion(version); output.writeNamedWriteable(testQuery); try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), serviceHolder.namedWriteableRegistry)) { in.setVersion(version); QueryBuilder deserializedQuery = in.readNamedWriteable(QueryBuilder.class); assertEquals(testQuery, deserializedQuery); assertEquals(testQuery.hashCode(), deserializedQuery.hashCode()); assertNotSame(testQuery, deserializedQuery); return deserializedQuery; } } }
private static BytesReference uncompress(BytesReference bytes, Compressor compressor) throws IOException { StreamInput compressed = compressor.streamInput(bytes.streamInput()); BytesStreamOutput bStream = new BytesStreamOutput(); Streams.copy(compressed, bStream); compressed.close(); return bStream.bytes(); } }
@Override public void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException { try (BytesStreamOutput stream = new BytesStreamOutput()) { stream.setVersion(version); stream.writeLong(requestId); byte status = 0; status = TransportStatus.setResponse(status); stream.writeByte(status); // 0 for request, 1 for response. response.writeTo(stream); sendResponseData(stream.bytes().toBytes()); sourceTransportServiceAdapter.onResponseSent(requestId, action, response, options); } }
@Override public BytesReference cacheKey() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); this.innerWriteTo(out, true); // copy it over, most requests are small, we might as well copy to make sure we are not sliced... // we could potentially keep it without copying, but then pay the price of extra unused bytes up to a page return new BytesArray(out.bytes().toBytesRef(), true);// do a deep copy }
/** * Writes the incompatible snapshot ids list to the `incompatible-snapshots` blob in the repository. * * Package private for testing. */ void writeIncompatibleSnapshots(RepositoryData repositoryData) throws IOException { assert isReadOnly() == false; // can not write to a read only repository final BytesReference bytes; try (BytesStreamOutput bStream = new BytesStreamOutput()) { try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream); repositoryData.incompatibleSnapshotsToXContent(builder, ToXContent.EMPTY_PARAMS); builder.close(); } bytes = bStream.bytes(); } // write the incompatible snapshots blob writeAtomic(INCOMPATIBLE_SNAPSHOTS_BLOB, bytes, false); }
/** * Marks this store as corrupted. This method writes a {@code corrupted_${uuid}} file containing the given exception * message. If a store contains a {@code corrupted_${uuid}} file {@link #isMarkedCorrupted()} will return <code>true</code>. */ public void markStoreCorrupted(IOException exception) throws IOException { ensureOpen(); if (!isMarkedCorrupted()) { String uuid = CORRUPTED + UUIDs.randomBase64UUID(); try (IndexOutput output = this.directory().createOutput(uuid, IOContext.DEFAULT)) { CodecUtil.writeHeader(output, CODEC, VERSION); BytesStreamOutput out = new BytesStreamOutput(); out.writeException(exception); BytesReference bytes = out.bytes(); output.writeVInt(bytes.length()); BytesRef ref = bytes.toBytesRef(); output.writeBytes(ref.bytes, ref.offset, ref.length); CodecUtil.writeFooter(output); } catch (IOException ex) { logger.warn("Can't mark store as corrupted", ex); } directory().sync(Collections.singleton(uuid)); } }