@Override public T fromJson(String v) throws IOException { return ProtostuffUtil.fromJSON(v, schema, false); } }
@Override public String toJson(T v) { return ProtostuffUtil.toJSON(v, schema, false); }
public DatasetStateMutator(String username, VirtualDatasetState virtualDatasetState, boolean preview) { super(); this.virtualDatasetState = ProtostuffUtil.copy(virtualDatasetState); this.preview = preview; }
@Override public byte[] serialize(final T instance) throws IOException { try(final ByteArrayOutputStream out = new ByteArrayOutputStream()) { ProtostuffUtil.toJSON(out, instance, writeSchema, false); return out.toByteArray(); } }
/** * Convert a JSON string into a Java object * * @param data the JSON data * @param the object to update * @param schema the Protostuff schema for the object * @param numeric if true, use field id as key */ public static <T> T fromJSON(String data, Schema<T> schema, boolean numeric) throws IOException { T message = schema.newMessage(); fromJSON(data.getBytes(UTF_8), message, schema, numeric); return message; }
/** * Return clone of the sourceConfig * @return */ public SourceConfig getConfig() { return ProtostuffUtil.copy(sourceConfig); }
/** * Convert a message into a JSON string * * @param the message to write * @param schema the protostuff schema for the message * @param numeric if true, use field id as keys * @return a JSON string */ public static <T> String toJSON(T message, Schema<T> schema, boolean numeric) { try(java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream()) { toJSON(baos, message, schema, numeric); return new String(baos.toByteArray(), UTF_8); } catch (IOException e) { throw new AssertionError("IOException not expected with ByteArrayOutputStream", e); } } }
private static Map<String, BackupFileInfo> scanInfoFiles(FileSystem fs, Path backupDir) throws IOException { final Map<String, BackupFileInfo> tableToInfo = Maps.newHashMap(); final FileStatus[] backupFiles = fs.listStatus(backupDir, BACKUP_INFO_FILES_GLOB); for (FileStatus backupFile : backupFiles) { final String tableName = getTableName(backupFile.getPath().getName(), BACKUP_INFO_FILE_SUFFIX); // read backup info file final byte[] headerBytes = new byte[(int) backupFile.getLen()]; IOUtils.readFully(fs.open(backupFile.getPath()), headerBytes, 0, headerBytes.length); final BackupFileInfo backupFileInfo = new BackupFileInfo(); ProtostuffUtil.fromJSON(headerBytes, backupFileInfo, BackupFileInfo.getSchema(), false); tableToInfo.put(tableName, backupFileInfo); } return tableToInfo; }
public VirtualDatasetState getNewState() { return copy(newState); } public Set<String> getAddedColumns() {
private QueryProfile recordProfile(OutputStream out, JobId id, int attempt) throws IOException, JobNotFoundException { QueryProfile profile = jobsService.get().getProfile(id, attempt); ProtostuffUtil.toJSON(out, profile, SchemaUserBitShared.QueryProfile.WRITE, false); return profile; }
@Override public T deserialize(final byte[] raw) throws IOException { final B builder = readSchema.newMessage(); ProtostuffUtil.fromJSON(raw, builder, readSchema, false); return (T)builder.build(); }
public TransformActor( VirtualDatasetState initialState, boolean preview, String username, QueryExecutor executor) { super(); this.m = new DatasetStateMutator(username, copy(initialState), preview); this.preview = preview; this.username = username; this.executor = executor; }
private void createMetaDataFile(StoreBuilderConfig builderConfig) throws IOException { final KVStoreInfo kvStoreInfo = DataStoreUtils.toInfo(builderConfig); final Path metadataFile = new Path(metaDataFilesDir.getAbsolutePath(), format("%s%s", builderConfig.getName(), METADATA_FILE_SUFFIX)); final FileSystem fs = FileSystem.getLocal(new Configuration()); try (FSDataOutputStream metaDataOut = fs.create(metadataFile, true)) { ProtostuffUtil.toJSON(metaDataOut, kvStoreInfo, KVStoreInfo.getSchema(), false); } }
/** * Scan dbDirectory to read kvstore definitions and load all stores in memory. */ public void scan() throws Exception { final FileSystem fs = FileSystem.getLocal(new Configuration()); final FileStatus[] metaDataFiles = fs.listStatus(new Path(metaDataFilesDir.getPath()), METADATA_FILES_GLOB); for (FileStatus fileStatus : metaDataFiles) { final byte[] headerBytes = new byte[(int) fileStatus.getLen()]; IOUtils.readFully(fs.open(fileStatus.getPath()), headerBytes, 0, headerBytes.length); final KVStoreInfo metadata = new KVStoreInfo(); ProtostuffUtil.fromJSON(headerBytes, metadata, KVStoreInfo.getSchema(), false); final StoreBuilderConfig storeBuilderConfig = DataStoreUtils.toBuilderConfig(metadata); getOrCreateStore(storeBuilderConfig); } }
/** * * @param to immutable * @param from immutable * @param <T> * @return result of merge from into to */ public static <T extends Message<T>> T merge(T to, T from) throws IOException { Schema<T> schema = from.cachedSchema(); ByteArrayOutputStream out = new ByteArrayOutputStream(); GraphIOUtil.writeDelimitedTo(new DataOutputStream(out), from, schema); ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); T cloneTo = copy(to); GraphIOUtil.mergeDelimitedFrom(in, cloneTo, schema); return cloneTo; }
final Path backupInfoFile = new Path(backupRootDir, format("%s%s", backupFileInfo.getKvstoreInfo().getTablename(), BACKUP_INFO_FILE_SUFFIX)); try (FSDataOutputStream backupInfoOut = fs.create(backupInfoFile, true)) { ProtostuffUtil.toJSON(backupInfoOut, backupFileInfo, BackupFileInfo.getSchema(), false);
@Test public void testNanDeserialization() throws IOException { Foo foo = ProtostuffUtil.fromJSON("{ \"bar\": NaN }", SCHEMA, false); assertEquals(Double.NaN, foo.bar, 0); }
@Override public SubScan getSpecificScan(List<SplitWork> work) throws ExecutionSetupException { final BatchSchema schema = cachedRelDataType == null ? getDataset().getSchema(): BatchSchema.fromCalciteRowType(cachedRelDataType); // Create an abridged version of the splits to save network bytes. List<DatasetSplit> splits = work.stream().map( workSplit -> ProtostuffUtil.copy(workSplit.getSplit()) .setExtendedProperty(convertToScanXAttr(workSplit.getSplit().getExtendedProperty())) ).collect(Collectors.toList()); return new ParquetSubScan(dataset.getFormatSettings(), splits, getUserName(), schema, getDataset().getName().getPathComponents(), filter == null ? null : filter.getConditions(), dataset.getStoragePluginId(), columns, dataset.getReadDefinition().getPartitionColumnsList(), globalDictionaryEncodedColumns, dataset.getReadDefinition().getExtendedProperty()); }
@Test public void testNanSerialization() throws IOException { Foo foo = new Foo(); foo.bar = Double.NaN; assertThat(ProtostuffUtil.toJSON(foo, SCHEMA, false), CoreMatchers.containsString("NaN")); assertThat(ProtostuffUtil.toJSON(foo, SCHEMA, false), CoreMatchers.not(CoreMatchers.containsString("\"NaN\""))); }