public static void serialize(Dictionary<?> dict, OutputStream outputStream) { try { DataOutputStream out = new DataOutputStream(outputStream); out.writeUTF(dict.getClass().getName()); dict.write(out); out.flush(); } catch (IOException e) { throw new RuntimeException(e); } }
public static ByteArray serialize(Dictionary<?> dict) { try { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream out = new DataOutputStream(baos); out.writeUTF(dict.getClass().getName()); dict.write(out); return new ByteArray(baos.toByteArray()); } catch (IOException e) { throw new RuntimeException(e); } }
private void outputDict(TblColRef col, Dictionary<String> dict) throws IOException, InterruptedException { // output written to baseDir/colName/colName.rldict-r-00000 (etc) String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos);) { outputStream.writeUTF(dict.getClass().getName()); dict.write(outputStream); mos.write(BatchConstants.CFG_OUTPUT_DICT, NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName); } }
@Override public void serialize(DictionaryInfo obj, DataOutputStream out) throws IOException { String json = JsonUtil.writeValueAsIndentString(obj); out.writeUTF(json); if (infoOnly == false) obj.getDictionaryObject().write(out); }
void writeData(DataOutput out) throws IOException { out.writeInt(rowIndices.size()); if (rowIndices.size() > 0) { int n = rowIndices.get(0).length; out.writeInt(n); if (this.useDictionary == true) { dict.write(out); for (int i = 0; i < rowIndices.size(); i++) { int[] row = rowIndices.get(i); for (int j = 0; j < n; j++) { out.writeInt(row[j]); } } } else { for (int i = 0; i < rowIndices.size(); i++) { int[] row = rowIndices.get(i); for (int j = 0; j < n; j++) { // NULL_STR is tricky, but we don't want to break the current snapshots out.writeUTF(dict.getValueFromId(row[j]) == null ? NULL_STR : dict.getValueFromId(row[j])); } } } } }
private void outputDict(TblColRef col, Dictionary<String> dict) throws IOException, InterruptedException { // output written to baseDir/colName/colName.rldict-r-00000 (etc) String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos);) { outputStream.writeUTF(dict.getClass().getName()); dict.write(outputStream); mos.write(BatchConstants.CFG_OUTPUT_DICT, NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName); } mos.close(); } }
private void outputDict(TblColRef col, Dictionary<String> dict, List<Tuple2<String, Tuple3<Writable, Writable, String>>> result) throws IOException { // output written to baseDir/colName/colName.rldict-r-00000 (etc) String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos)) { outputStream.writeUTF(dict.getClass().getName()); dict.write(outputStream); result.add(new Tuple2<String, Tuple3<Writable, Writable, String>>(BatchConstants.CFG_OUTPUT_DICT, new Tuple3<Writable, Writable, String>(NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName))); } }
shrunkenDict.write(dos);
private void readWriteTest(Dictionary<String> dict) throws Exception { final String path = "src/test/resources/dict/tmp_dict"; File f = new File(path); f.deleteOnExit(); f.createNewFile(); String dictClassName = dict.getClass().getName(); DataOutputStream out = new DataOutputStream(new FileOutputStream(f)); out.writeUTF(dictClassName); dict.write(out); out.close(); //read dict DataInputStream in = null; Dictionary<String> dict2 = null; try { File f2 = new File(path); in = new DataInputStream(new FileInputStream(f2)); String dictClassName2 = in.readUTF(); dict2 = (Dictionary<String>) ClassUtil.newInstance(dictClassName2); dict2.readFields(in); } finally { if (in != null) { in.close(); } } assertTrue(dict.equals(dict2)); } }
DataOutputStream dos = new DataOutputStream(bos); shrunkenDict.write(dos);
public static void serialize(Dictionary<?> dict, OutputStream outputStream) { try { DataOutputStream out = new DataOutputStream(outputStream); out.writeUTF(dict.getClass().getName()); dict.write(out); out.flush(); } catch (IOException e) { throw new RuntimeException(e); } }
public static ByteArray serialize(Dictionary<?> dict) { try { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream out = new DataOutputStream(baos); out.writeUTF(dict.getClass().getName()); dict.write(out); return new ByteArray(baos.toByteArray()); } catch (IOException e) { throw new RuntimeException(e); } }
@Override public void serialize(DictionaryInfo obj, DataOutputStream out) throws IOException { String json = JsonUtil.writeValueAsIndentString(obj); out.writeUTF(json); if (infoOnly == false) obj.getDictionaryObject().write(out); }
@Override public void serialize(DictionaryInfo obj, DataOutputStream out) throws IOException { String json = JsonUtil.writeValueAsIndentString(obj); out.writeUTF(json); if (infoOnly == false) obj.getDictionaryObject().write(out); }
private void outputDict(TblColRef col, Dictionary<String> dict) throws IOException, InterruptedException { // output written to baseDir/colName/colName.rldict-r-00000 (etc) String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos);) { outputStream.writeUTF(dict.getClass().getName()); dict.write(outputStream); mos.write(BatchConstants.CFG_OUTPUT_DICT, NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName); } }
void writeData(DataOutput out) throws IOException { out.writeInt(rowIndices.size()); if (rowIndices.size() > 0) { int n = rowIndices.get(0).length; out.writeInt(n); if (this.useDictionary == true) { dict.write(out); for (int i = 0; i < rowIndices.size(); i++) { int[] row = rowIndices.get(i); for (int j = 0; j < n; j++) { out.writeInt(row[j]); } } } else { for (int i = 0; i < rowIndices.size(); i++) { int[] row = rowIndices.get(i); for (int j = 0; j < n; j++) { out.writeUTF(dict.getValueFromId(row[j])); } } } } }
void writeData(DataOutput out) throws IOException { out.writeInt(rowIndices.size()); if (rowIndices.size() > 0) { int n = rowIndices.get(0).length; out.writeInt(n); if (this.useDictionary == true) { dict.write(out); for (int i = 0; i < rowIndices.size(); i++) { int[] row = rowIndices.get(i); for (int j = 0; j < n; j++) { out.writeInt(row[j]); } } } else { for (int i = 0; i < rowIndices.size(); i++) { int[] row = rowIndices.get(i); for (int j = 0; j < n; j++) { // NULL_STR is tricky, but we don't want to break the current snapshots out.writeUTF(dict.getValueFromId(row[j]) == null ? NULL_STR : dict.getValueFromId(row[j])); } } } } }
private void outputDict(TblColRef col, Dictionary<String> dict) throws IOException, InterruptedException { // output written to baseDir/colName/colName.rldict-r-00000 (etc) String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos);) { outputStream.writeUTF(dict.getClass().getName()); dict.write(outputStream); mos.write(BatchConstants.CFG_OUTPUT_DICT, NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName); } mos.close(); } }
private void outputDict(TblColRef col, Dictionary<String> dict, List<Tuple2<String, Tuple3<Writable, Writable, String>>> result) throws IOException { // output written to baseDir/colName/colName.rldict-r-00000 (etc) String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream outputStream = new DataOutputStream(baos)) { outputStream.writeUTF(dict.getClass().getName()); dict.write(outputStream); result.add(new Tuple2<String, Tuple3<Writable, Writable, String>>(BatchConstants.CFG_OUTPUT_DICT, new Tuple3<Writable, Writable, String>(NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName))); } }
shrunkenDict.write(dos);