/** * Some filesystem requires path to be resolved in order to be able to write * * @param parent parent directory * @param fileName file name * @return */ private Path filePath(Path parent, String fileName) throws IOException { return FileSystemWrapper.canonicalizePath(fs, new Path(parent, fileName)); }
/** * Helper method to create a new {@link ParquetFileWriter} as impersonated user. * @throws IOException */ private void initRecordReader() throws IOException { this.path = fs.canonicalizePath(partition.qualified(location, prefix + "_" + index + "." + extension)); try { parquetFileWriter = proxyUserUGI.doAs(new PrivilegedExceptionAction<ParquetFileWriter>() { @Override public ParquetFileWriter run() throws Exception { final ParquetFileWriter parquetFileWriter = new ParquetFileWriter(checkNotNull(conf), checkNotNull(schema), path, ParquetFileWriter.Mode.CREATE, DEFAULT_BLOCK_SIZE, MAX_PADDING_SIZE_DEFAULT, true); parquetFileWriter.start(); return parquetFileWriter; } }); } catch (InterruptedException e) { // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the // interruption and respond to it if it wants to. Thread.currentThread().interrupt(); throw new IOException("Interrupted while creating the parquet record writer", e); } }
@Override public Path canonicalizePath(Path p) throws IOException { try { Path cp = canonicalizePath(underlyingFs, p); return new PathWrapperWithFileSystem(cp, this); } catch(FSError e) { throw propagateFSError(e); } }
@Override public void setup(VectorAccessible incoming, OutputEntryListener outputEntryListener, WriteStatsListener writeStatsListener) throws IOException { Preconditions.checkArgument(incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE, "SelectionVector remover is not supported."); this.incoming = incoming; this.outputEntryListener = outputEntryListener; this.writeStatsListener = writeStatsListener; this.fs = FileSystemWrapper.get(location, writerConfig.getFsConf(), stats); this.currentFile = fs.canonicalizePath(new Path(location, String.format("%s_%d.%s", prefix, nextFileIndex, extension))); this.relativePath = currentFile.getName(); this.currentFileOutputStream = fs.create(currentFile); listOfFilesCreated.add(currentFile); // write magic word bytes currentFileOutputStream.write(MAGIC_STRING.getBytes()); for(final VectorWrapper<? extends ValueVector> vw : incoming) { Preconditions.checkArgument(!vw.isHyper(), "Writing hyper vectors to arrow format is not supported."); footerBuilder.addField(TypeHelper.getMetadata(vw.getValueVector())); } nextFileIndex++; recordCount = 0; }
@Override public void startPartition(WritePartition partition) throws Exception { if(this.partition != null){ close(); } this.partition = partition; // open a new file for writing data with new schema try { this.path = fs.canonicalizePath(partition.qualified(location, prefix + "_" + index + "." + extension)); fos = fs.create(path); stream = new PrintStream(fos); stream.write(ByteOrderMark.UTF_8.getBytes(), 0, ByteOrderMark.UTF_8.length()); logger.debug("Created file: {}", path); } catch (IOException e) { throw UserException.dataWriteError(e) .message("Failure while attempting to write file %s.", path) .build(logger); } index++; String columns = Joiner.on(fieldDelimiter).join(columnNames); stream.print(columns); stream.print(lineDelimiter); }
@Override public void startPartition(WritePartition partition) throws Exception { // close previous partition if open. if(this.partition != null){ close(); } this.partition = partition; try { this.fileName = fs.canonicalizePath(partition.qualified(location, prefix + "_0." + extension)); stream = fs.create(fileName); JsonGenerator generator = factory.createGenerator(stream.getWrappedStream()).useDefaultPrettyPrinter(); if (uglify) { generator = generator.setPrettyPrinter(new MinimalPrettyPrinter(LINE_FEED)); } if(useExtendedOutput){ gen = new ExtendedJsonOutput(generator); }else{ gen = new BasicJsonOutput(generator); } logger.debug("Created file: {}", fileName); } catch (IOException ex) { throw UserException.dataWriteError(ex) .message("Failure writing JSON file %s.", fileName) .build(logger); } }