/** * Filter a snapshot, provided via the InputStream, to each of the OutputStreams. * * The number of provided OutputStreams should be equal to the number of provided HollowFilterConfigs at instantiation. * * @param in the input stream to the snapshot * @param out the output streams to write the filtered snapshot * @throws IOException if the snapshot cannot be filtered */ public void filterSnapshot(InputStream in, OutputStream... out) throws IOException { filter(false, in, out); }
public static DataOutputStream[] streamsOnly(FilteredHollowBlobWriterStreamAndFilter[] streamAndFilters) { DataOutputStream streams[] = new DataOutputStream[streamAndFilters.length]; for(int i=0;i<streams.length;i++) streams[i] = streamAndFilters[i].getStream(); return streams; }
private List<HollowSchema> getFilteredSchemaList(List<HollowSchema> schemaList, HollowFilterConfig filterConfig) { List<HollowSchema> filteredList = new ArrayList<HollowSchema>(); for(HollowSchema schema : schemaList) { HollowSchema filteredSchema = getFilteredSchema(schema, filterConfig); if(filteredSchema != null) filteredList.add(filteredSchema); } return filteredList; }
FilteredHollowBlobWriterStreamAndFilter allStreamAndFilters[] = FilteredHollowBlobWriterStreamAndFilter.combine(out, configs); List<HollowSchema> filteredSchemaList = getFilteredSchemaList(unfilteredSchemaList, streamAndFilter.getConfig()); header.setSchemas(filteredSchemaList); headerWriter.writeHeader(header, streamAndFilter.getStream()); VarInt.writeVInt(streamAndFilter.getStream(), filteredSchemaList.size()); int numShards = readNumShards(in); FilteredHollowBlobWriterStreamAndFilter[] streamsWithType = FilteredHollowBlobWriterStreamAndFilter.withType(schema.getName(), allStreamAndFilters); HollowObjectTypeReadState.discardType(dis, (HollowObjectSchema)schema, numShards, delta); else copyFilteredObjectState(delta, dis, streamsWithType, (HollowObjectSchema)schema, numShards); } else { for(int j=0;j<streamsWithType.length;j++) { schema.writeTo(streamsWithType[j].getStream()); VarInt.writeVInt(streamsWithType[j].getStream(), 1 + VarInt.sizeOfVInt(numShards)); VarInt.writeVInt(streamsWithType[j].getStream(), 0); /// forwards compatibility VarInt.writeVInt(streamsWithType[j].getStream(), numShards); HollowListTypeReadState.discardType(dis, numShards, delta); else copyListState(delta, dis, streamsOnly(streamsWithType), numShards); } else if(schema instanceof HollowSetSchema) { if(streamsWithType.length == 0) HollowSetTypeReadState.discardType(dis, numShards, delta); else copySetState(delta, dis, streamsOnly(streamsWithType), numShards);
@SuppressWarnings("unchecked") private void copyFilteredObjectState(boolean delta, DataInputStream is, FilteredHollowBlobWriterStreamAndFilter[] streamAndFilters, HollowObjectSchema schema, int numShards) throws IOException { DataOutputStream[] os = streamsOnly(streamAndFilters); HollowObjectSchema[] filteredObjectSchemas = new HollowObjectSchema[os.length]; HollowObjectSchema filteredObjectSchema = getFilteredObjectSchema(schema, streamAndFilters[i].getConfig()); filteredObjectSchemas[i] = filteredObjectSchema; filteredObjectSchema.writeTo(streamAndFilters[i].getStream()); VarInt.writeVInt(streamAndFilters[i].getStream(), 1 + VarInt.sizeOfVInt(numShards)); VarInt.writeVInt(streamAndFilters[i].getStream(), 0); /// forwards compatibility VarInt.writeVInt(streamAndFilters[i].getStream(), numShards); long bitsPerRecord = writeBitsPerField(schema, bitsPerField, filteredObjectSchemas[i], streamAndFilters[i].getStream()); FixedLengthArrayWriter filteredArrayWriter = new FixedLengthArrayWriter(fixedLengthArraysPerStream[i]); fixedLengthArraysPerField[fieldCursor].get(i).writeField(fieldValue, bitsPerField[fieldCursor]); List<DataOutputStream> streamsWithFieldList = new ArrayList<DataOutputStream>(); for(int j=0;j<streamAndFilters.length;j++) { ObjectFilterConfig objectTypeConfig = streamAndFilters[j].getConfig().getObjectTypeConfig(schema.getName()); if(objectTypeConfig.includesField(schema.getFieldName(i))) streamsWithFieldList.add(streamAndFilters[j].getStream()); copySnapshotPopulatedOrdinals(is, os);
filterConfig.addType("NonexistentType"); FilteredHollowBlobWriter blobWriter = new FilteredHollowBlobWriter(filterConfig); ByteArrayOutputStream filteredBlobStream = new ByteArrayOutputStream(); blobWriter.filterSnapshot(new ByteArrayInputStream(snapshotData), filteredBlobStream); blobWriter.filterDelta(new ByteArrayInputStream(deltaData), filteredBlobStream); blobWriter.filterDelta(new ByteArrayInputStream(removeOnlyDeltaData), filteredBlobStream);
HollowObjectMapper mapper = new HollowObjectMapper(writeEngine); mapper.add(new TypeA(1, "one")); mapper.add(new TypeA(2, "two")); mapper.add(new TypeA(3, "three")); mapper.add(new TypeB(1, 1.1f)); mapper.add(new TypeB(2, 2.2f)); mapper.add(new TypeB(3, 3.3f)); mapper.add(new TypeA(1, "one")); mapper.add(new TypeA(2, "two")); mapper.add(new TypeA(3, "four")); mapper.add(new TypeB(1, 1.1f)); mapper.add(new TypeB(2, 2.2f)); mapper.add(new TypeB(3, 4.4f)); mapper.add(new TypeA(2, "two")); mapper.add(new TypeA(3, "four")); mapper.add(new TypeB(2, 2.2f)); mapper.add(new TypeB(3, 4.4f));
private void copyListState(boolean delta, DataInputStream is, DataOutputStream[] os, int numShards) throws IOException { if(numShards > 1) copyVInt(is, os); for(int shard=0;shard<numShards;shard++) { copyVInt(is, os); /// maxOrdinal if(delta) { GapEncodedVariableLengthIntegerReader.copyEncodedDeltaOrdinals(is, os); GapEncodedVariableLengthIntegerReader.copyEncodedDeltaOrdinals(is, os); } copyVInt(is, os); /// bitsPerListPointer copyVInt(is, os); /// bitsPerElement copyVLong(is, os); /// totalNumberOfElements copySegmentedLongArray(is, os); copySegmentedLongArray(is, os); } if(!delta) copySnapshotPopulatedOrdinals(is, os); }
public static FilteredHollowBlobWriterStreamAndFilter[] combine(OutputStream streams[], HollowFilterConfig configs[]) { if(streams.length != configs.length) throw new IllegalArgumentException("Must provide exactly the same number of streams as configs"); FilteredHollowBlobWriterStreamAndFilter streamAndFilters[] = new FilteredHollowBlobWriterStreamAndFilter[streams.length]; for(int i=0;i<streams.length;i++) streamAndFilters[i] = new FilteredHollowBlobWriterStreamAndFilter(new DataOutputStream(streams[i]), configs[i]); return streamAndFilters; }
public static FilteredHollowBlobWriterStreamAndFilter[] withType(String typeName, FilteredHollowBlobWriterStreamAndFilter[] allStreamAndFilters) { int countConfigsWithType = 0; for(int i=0;i<allStreamAndFilters.length;i++) { if(allStreamAndFilters[i].getConfig().doesIncludeType(typeName)) countConfigsWithType++; } FilteredHollowBlobWriterStreamAndFilter[] streamAndFiltersWithType = new FilteredHollowBlobWriterStreamAndFilter[countConfigsWithType]; int withTypeCounter = 0; for(int i=0;i<allStreamAndFilters.length;i++) { if(allStreamAndFilters[i].getConfig().doesIncludeType(typeName)) streamAndFiltersWithType[withTypeCounter++] = allStreamAndFilters[i]; } return streamAndFiltersWithType; }
private int readNumShards(InputStream is) throws IOException { int backwardsCompatibilityBytes = VarInt.readVInt(is); if(backwardsCompatibilityBytes == 0) return 1; /// produced by a version of hollow prior to 2.1.0, always only 1 shard. skipForwardsCompatibilityBytes(is); return VarInt.readVInt(is); }
private HollowSchema getFilteredSchema(HollowSchema schema, HollowFilterConfig filterConfig) { if(filterConfig.doesIncludeType(schema.getName())) { if(schema.getSchemaType() == SchemaType.OBJECT) return getFilteredObjectSchema((HollowObjectSchema) schema, filterConfig); return schema; } return null; }
private void copySetState(boolean delta, DataInputStream is, DataOutputStream[] os, int numShards) throws IOException { if(numShards > 1) copyVInt(is, os); for(int shard=0;shard<numShards;shard++) { copyVInt(is, os); /// max ordinal if(delta) { GapEncodedVariableLengthIntegerReader.copyEncodedDeltaOrdinals(is, os); GapEncodedVariableLengthIntegerReader.copyEncodedDeltaOrdinals(is, os); } copyVInt(is, os); /// bitsPerSetPointer copyVInt(is, os); /// bitsPerSetSizeValue copyVInt(is, os); /// bitsPerElement copyVLong(is, os); /// totalNumberOfBuckets copySegmentedLongArray(is, os); copySegmentedLongArray(is, os); } if(!delta) copySnapshotPopulatedOrdinals(is, os); }
/** * Filter a delta (or reversedelta), provided via the InputStream, to each of the OutputStreams. * * The number of provided OutputStreams should be equal to the number of provided HollowFilterConfigs at instantiation. * * @param in the input stream to the delta * @param out the output streams to write the filtered delta * @throws IOException if the delta cannot be filtered */ public void filterDelta(InputStream in, OutputStream... out) throws IOException { filter(true, in, out); }
private void copyMapState(boolean delta, DataInputStream is, DataOutputStream[] os, int numShards) throws IOException { if(numShards > 1) copyVInt(is, os); for(int shard=0;shard<numShards;shard++) { copyVInt(is, os); /// max ordinal if(delta) { GapEncodedVariableLengthIntegerReader.copyEncodedDeltaOrdinals(is, os); GapEncodedVariableLengthIntegerReader.copyEncodedDeltaOrdinals(is, os); } copyVInt(is, os); /// bitsPerMapPointer copyVInt(is, os); /// bitsPerMapSizeValue copyVInt(is, os); /// bitsPerKeyElement copyVInt(is, os); /// bitsPerValueElement copyVLong(is, os); /// totalNumberOfBuckets copySegmentedLongArray(is, os); copySegmentedLongArray(is, os); } if(!delta) copySnapshotPopulatedOrdinals(is, os); }