@Override public void close(boolean abort) throws IOException { writer.close(); if (!abort) { length = target.getFileSystem(conf).getFileStatus(target).getLen(); } } };
@Override public void close() throws IOException { // close writer if (outWriter == null) { return; } outWriter.close(); outWriter = null; if (!exception) { FileStatus fss = fs.getFileStatus(outPath); LOG.info("renamed path " + outPath + " to " + finalPath + " . File size is " + fss.getLen()); if (!fs.rename(outPath, finalPath)) { throw new IOException("Unable to rename output to " + finalPath); } } else { if (!autoDelete) { fs.delete(outPath, true); } } }
@Override public void close() throws IOException { // close writer if (outWriter == null) { return; } outWriter.close(); outWriter = null; if (!exception) { FileStatus fss = fs.getFileStatus(outPath); LOG.info("renamed path " + outPath + " to " + finalPath + " . File size is " + fss.getLen()); if (!fs.rename(outPath, finalPath)) { throw new IOException("Unable to rename output to " + finalPath); } } else { if (!autoDelete) { fs.delete(outPath, true); } } }
@Override public void close(TaskAttemptContext task) throws IOException, InterruptedException { out.close(); } };
@Override public void close(boolean abort) throws IOException { outWriter.close(); } };
writer.close();
writer.close();
bytes.clear(); writer.close();
writer.close();
writer.append(bytes); writer.close(); long fileLen = fs.getFileStatus(testFile).getLen();
writer.append(bytes); writer.close();
private void writeTest(FileSystem fs, int count, Path file, byte[][] fieldsData, Configuration conf) throws IOException, SerDeException { cleanup(); RCFileOutputFormat.setColumnNumber(conf, fieldsData.length); RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, new DefaultCodec()); BytesRefArrayWritable bytes = new BytesRefArrayWritable(fieldsData.length); for (int i = 0; i < fieldsData.length; i++) { BytesRefWritable cu = null; cu = new BytesRefWritable(fieldsData[i], 0, fieldsData[i].length); bytes.set(i, cu); } for (int i = 0; i < count; i++) { writer.append(bytes); } writer.close(); long fileLen = fs.getFileStatus(file).getLen(); System.out.println("The file size of RCFile with " + bytes.size() + " number columns and " + count + " number rows is " + fileLen); }
private void writeRCFileTest(FileSystem fs, int rowCount, Path file, int columnNum, CompressionCodec codec) throws IOException { fs.delete(file, true); resetRandomGenerators(); RCFileOutputFormat.setColumnNumber(conf, columnNum); RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, codec); byte[][] columnRandom; BytesRefArrayWritable bytes = new BytesRefArrayWritable(columnNum); columnRandom = new byte[columnNum][]; for (int i = 0; i < columnNum; i++) { BytesRefWritable cu = new BytesRefWritable(); bytes.set(i, cu); } for (int i = 0; i < rowCount; i++) { nextRandomRow(columnRandom, bytes); writer.append(bytes); } writer.close(); }
@Override public void closeOp(boolean abort) throws HiveException { // close writer if (outWriter == null) { return; } try { outWriter.close(); } catch (IOException e) { throw new HiveException("Unable to close RCFileMergeOperator", e); } outWriter = null; super.closeOp(abort); }
@Override public void close(Reporter reporter) throws IOException { out.close(); }
@Override public void close(boolean abort) throws IOException { outWriter.close(); } };
@Override public void closeOp(boolean abort) throws HiveException { // close writer if (outWriter == null) { return; } try { outWriter.close(); } catch (IOException e) { throw new HiveException("Unable to close RCFileMergeOperator", e); } outWriter = null; super.closeOp(abort); }
@Override public void close(Reporter reporter) throws IOException { out.close(); }
@Override public void close(boolean abort) throws IOException { outWriter.close(); } };
@Override public void close(TaskAttemptContext task) throws IOException, InterruptedException { out.close(); } };