@Override public void setBytes(final String key, final byte[] bytes) { this.backingConfig.setBytes(this.prefix + key, bytes); }
public static void writeObjectToConfig(Object o, Configuration config, String key) throws IOException { byte[] bytes = serializeObject(o); config.setBytes(key, bytes); }
public void setOperatorID(OperatorID operatorID) { this.config.setBytes(OPERATOR_ID, operatorID.getBytes()); }
public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) { int num = conf.getInteger(CACHE_FILE_NUM, 0) + 1; conf.setInteger(CACHE_FILE_NUM, num); conf.setString(CACHE_FILE_NAME + num, name); conf.setString(CACHE_FILE_PATH + num, e.filePath); conf.setBoolean(CACHE_FILE_EXE + num, e.isExecutable || new File(e.filePath).canExecute()); conf.setBoolean(CACHE_FILE_DIR + num, e.isZipped || new File(e.filePath).isDirectory()); if (e.blobKey != null) { conf.setBytes(CACHE_FILE_BLOB_KEY + num, e.blobKey); } }
orig.setDouble("E", Math.E); orig.setBoolean("shouldbetrue", true); orig.setBytes("bytes sequence", new byte[] { 1, 2, 3, 4, 5 }); orig.setClass("myclass", this.getClass());
@Override public void setBytes(final String key, final byte[] bytes) { this.backingConfig.setBytes(this.prefix + key, bytes); }
@Override public void setBytes(final String key, final byte[] bytes) { this.backingConfig.setBytes(this.prefix + key, bytes); }
public static void writeObjectToConfig(Object o, Configuration config, String key) throws IOException { byte[] bytes = serializeObject(o); config.setBytes(key, bytes); }
public static void writeObjectToConfig(Object o, Configuration config, String key) throws IOException { byte[] bytes = serializeObject(o); config.setBytes(key, bytes); }
public void setOperatorID(OperatorID operatorID) { this.config.setBytes(OPERATOR_ID, operatorID.getBytes()); }
public void setOperatorID(OperatorID operatorID) { this.config.setBytes(OPERATOR_ID, operatorID.getBytes()); }
public void setOutputDataDistribution(DataDistribution distribution, int outputNum) { this.config.setString(OUTPUT_DATA_DISTRIBUTION_CLASS, distribution.getClass().getName()); try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) { distribution.write(out); config.setBytes(OUTPUT_DATA_DISTRIBUTION_PREFIX + outputNum, baos.toByteArray()); } catch (IOException e) { throw new RuntimeException("Error serializing the DataDistribution: " + e.getMessage(), e); } }
public void setOutputDataDistribution(DataDistribution distribution, int outputNum) { this.config.setString(OUTPUT_DATA_DISTRIBUTION_CLASS, distribution.getClass().getName()); try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) { distribution.write(out); config.setBytes(OUTPUT_DATA_DISTRIBUTION_PREFIX + outputNum, baos.toByteArray()); } catch (IOException e) { throw new RuntimeException("Error serializing the DataDistribution: " + e.getMessage(), e); } }
public void setOutputDataDistribution(DataDistribution distribution, int outputNum) { this.config.setString(OUTPUT_DATA_DISTRIBUTION_CLASS, distribution.getClass().getName()); try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) { distribution.write(out); config.setBytes(OUTPUT_DATA_DISTRIBUTION_PREFIX + outputNum, baos.toByteArray()); } catch (IOException e) { throw new RuntimeException("Error serializing the DataDistribution: " + e.getMessage(), e); } }
public void setOutputDataDistribution(DataDistribution distribution, int outputNum) { this.config.setString(OUTPUT_DATA_DISTRIBUTION_CLASS, distribution.getClass().getName()); try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(baos)) { distribution.write(out); config.setBytes(OUTPUT_DATA_DISTRIBUTION_PREFIX + outputNum, baos.toByteArray()); } catch (IOException e) { throw new RuntimeException("Error serializing the DataDistribution: " + e.getMessage(), e); } }
public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) { int num = conf.getInteger(CACHE_FILE_NUM, 0) + 1; conf.setInteger(CACHE_FILE_NUM, num); conf.setString(CACHE_FILE_NAME + num, name); conf.setString(CACHE_FILE_PATH + num, e.filePath); conf.setBoolean(CACHE_FILE_EXE + num, e.isExecutable || new File(e.filePath).canExecute()); conf.setBoolean(CACHE_FILE_DIR + num, e.isZipped || new File(e.filePath).isDirectory()); if (e.blobKey != null) { conf.setBytes(CACHE_FILE_BLOB_KEY + num, e.blobKey); } }
public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) { int num = conf.getInteger(CACHE_FILE_NUM,0) + 1; conf.setInteger(CACHE_FILE_NUM, num); conf.setString(CACHE_FILE_NAME + num, name); conf.setString(CACHE_FILE_PATH + num, e.filePath); conf.setBoolean(CACHE_FILE_EXE + num, e.isExecutable || new File(e.filePath).canExecute()); conf.setBoolean(CACHE_FILE_DIR + num, e.isZipped || new File(e.filePath).isDirectory()); if (e.blobKey != null) { conf.setBytes(CACHE_FILE_BLOB_KEY + num, e.blobKey); } }
public static <T extends DataInstance> DataSet<DataOnMemory<T>> toBatches(DataFlink<T> data, int batchSize){ try{ Configuration config = new Configuration(); config.setInteger(BATCH_SIZE, batchSize); config.setBytes(ATTRIBUTES, Serialization.serializeObject(data.getAttributes())); return data.getDataSet().mapPartition(new DataBatch<T>()).withParameters(config); }catch(Exception ex){ throw new UndeclaredThrowableException(ex); } }
public static <T extends DataInstance> DataSet<DataOnMemory<T>> toBatchesBySeqID(DataFlink<T> data, int batchSize){ try{ Configuration config = new Configuration(); config.setInteger(BATCH_SIZE, batchSize); config.setBytes(ATTRIBUTES, Serialization.serializeObject(data.getAttributes())); return data.getDataSet().mapPartition(new DataBatchBySeqID<T>()).withParameters(config); }catch(Exception ex){ throw new UndeclaredThrowableException(ex); } }
public DataSet<DataPosterior> computePosterior(DataFlink<DataInstance> dataFlink){ Attribute seq_id = dataFlink.getAttributes().getSeq_id(); if (seq_id==null) throw new IllegalArgumentException("Functionality only available for data sets with a seq_id attribute"); try{ Configuration config = new Configuration(); config.setString(ParameterLearningAlgorithm.BN_NAME, this.getName()); config.setBytes(SVB, Serialization.serializeObject(svb)); return dataFlink .getBatchedDataSet(this.batchSize,batchConverter) .flatMap(new ParallelVBMapInference()) .withParameters(config); }catch(Exception ex){ throw new UndeclaredThrowableException(ex); } }