String getLogDir(Configuration conf) { String logDir = conf.get("hive.log.dir"); if (logDir == null) { logDir = System.getProperty("hive.log.dir"); } if (logDir != null) { return logDir; } LoggerContext context = (LoggerContext)LogManager.getContext(false); for (Logger logger: context.getLoggers()) { for (Appender appender: logger.getAppenders().values()) { if (appender instanceof AbstractOutputStreamAppender) { OutputStreamManager manager = ((AbstractOutputStreamAppender<?>)appender).getManager(); if (manager instanceof FileManager) { String fileName = ((FileManager)manager).getFileName(); if (fileName != null) { return fileName.substring(0, fileName.lastIndexOf('/')); } } } } } return null; }
@Test public void testLazyCreate() throws Exception { final Layout<String> layout = createPatternLayout(); // @formatter:off final FileAppender appender = FileAppender.newBuilder() .withFileName(FILE_NAME) .withName("test") .withImmediateFlush(false) .withIgnoreExceptions(false) .withBufferedIo(false) .withBufferSize(1) .withLayout(layout) .withCreateOnDemand(createOnDemand) .build(); // @formatter:on Assert.assertEquals(createOnDemand, appender.getManager().isCreateOnDemand()); try { Assert.assertNotEquals(createOnDemand, Files.exists(PATH)); appender.start(); Assert.assertNotEquals(createOnDemand, Files.exists(PATH)); } finally { appender.stop(); } Assert.assertNotEquals(createOnDemand, Files.exists(PATH)); }
@Override protected OutputStream createOutputStream() throws IOException { final String filename = getFileName(); LOGGER.debug("Now writing to {} at {}", filename, new Date()); final FileOutputStream fos = new FileOutputStream(filename, isAppend); defineAttributeView(Paths.get(filename)); return fos; }
private CompressedFileAppender(final String name, final Layout<? extends Serializable> layout, final Filter filter, final FileManager manager, final String filename, final boolean ignoreExceptions, final boolean immediateFlush, final Advertiser advertiser) { super(name, layout, filter, ignoreExceptions, immediateFlush, manager); if (advertiser != null) { final Map<String, String> configuration = new HashMap<>(layout.getContentFormat()); configuration.putAll(manager.getContentFormat()); configuration.put("contentType", layout.getContentType()); configuration.put("name", name); advertisement = advertiser.advertise(configuration); } this.fileName = filename; this.advertiser = advertiser; }
@Override protected synchronized void write(final byte[] bytes, final int offset, final int length, final boolean immediateFlush) { if (isLocking) { try { @SuppressWarnings("resource") final FileChannel channel = ((FileOutputStream) getOutputStream()).getChannel(); /* * Lock the whole file. This could be optimized to only lock from the current file position. Note that * locking may be advisory on some systems and mandatory on others, so locking just from the current * position would allow reading on systems where locking is mandatory. Also, Java 6 will throw an * exception if the region of the file is already locked by another FileChannel in the same JVM. * Hopefully, that will be avoided since every file should have a single file manager - unless two * different files strings are configured that somehow map to the same file. */ try (final FileLock lock = channel.lock(0, Long.MAX_VALUE, false)) { super.write(bytes, offset, length, immediateFlush); } } catch (final IOException ex) { throw new AppenderLoggingException("Unable to obtain lock on " + getName(), ex); } } else { super.write(bytes, offset, length, immediateFlush); } }
/** * Returns the FileManager. * @param fileName The name of the file to manage. * @param append true if the file should be appended to, false if it should be overwritten. * @param locking true if the file should be locked while writing, false otherwise. * @param bufferedIo true if the contents should be buffered as they are written. * @param createOnDemand true if you want to lazy-create the file (a.k.a. on-demand.) * @param advertiseUri the URI to use when advertising the file * @param layout The layout * @param bufferSize buffer size for buffered IO * @param filePermissions File permissions * @param fileOwner File owner * @param fileOwner File group * @param configuration The configuration. * @return A FileManager for the File. */ public static FileManager getFileManager(final String fileName, final boolean append, boolean locking, final boolean bufferedIo, final boolean createOnDemand, final String advertiseUri, final Layout<? extends Serializable> layout, final int bufferSize, final String filePermissions, final String fileOwner, final String fileGroup, final Configuration configuration) { if (locking && bufferedIo) { locking = false; } return narrow(FileManager.class, getManager(fileName, new FactoryData(append, locking, bufferedIo, bufferSize, createOnDemand, advertiseUri, layout, filePermissions, fileOwner, fileGroup, configuration), FACTORY)); }
/** * Creates a FileManager. * @param name The name of the File. * @param data The FactoryData * @return The FileManager for the File. */ @Override public FileManager createManager(final String name, final FactoryData data) { final File file = new File(name); try { FileUtils.makeParentDirs(file); final boolean writeHeader = !data.append || !file.exists(); final int actualSize = data.bufferedIo ? data.bufferSize : Constants.ENCODER_BYTE_BUFFER_SIZE; final ByteBuffer byteBuffer = ByteBuffer.wrap(new byte[actualSize]); final FileOutputStream fos = data.createOnDemand ? null : new FileOutputStream(file, data.append); final FileManager fm = new FileManager(data.getLoggerContext(), name, fos, data.append, data.locking, data.createOnDemand, data.advertiseURI, data.layout, data.filePermissions, data.fileOwner, data.fileGroup, writeHeader, byteBuffer); if (fos != null && fm.attributeViewEnabled) { fm.defineAttributeView(file.toPath()); } return fm; } catch (final IOException ex) { LOGGER.error("FileManager (" + name + ") " + ex, ex); } return null; } }
/** * Returns the name of the File being managed. * @return The name of the File being managed. */ public String getFileName() { return getName(); } /**
@Override public FileAppender build() { boolean bufferedIo = isBufferedIo(); final int bufferSize = getBufferSize(); if (locking && bufferedIo) { LOGGER.warn("Locking and buffering are mutually exclusive. No buffering will occur for {}", fileName); bufferedIo = false; } if (!bufferedIo && bufferSize > 0) { LOGGER.warn("The bufferSize is set to {} but bufferedIo is false: {}", bufferSize, bufferedIo); } final Layout<? extends Serializable> layout = getOrCreateLayout(); final FileManager manager = FileManager.getFileManager(fileName, append, locking, bufferedIo, createOnDemand, advertiseUri, layout, bufferSize, filePermissions, fileOwner, fileGroup, getConfiguration()); if (manager == null) { return null; } return new FileAppender(getName(), layout, getFilter(), manager, fileName, isIgnoreExceptions(), !bufferedIo || isImmediateFlush(), advertise ? getConfiguration().getAdvertiser() : null); }
private FileAppender(final String name, final Layout<? extends Serializable> layout, final Filter filter, final FileManager manager, final String filename, final boolean ignoreExceptions, final boolean immediateFlush, final Advertiser advertiser) { super(name, layout, filter, ignoreExceptions, immediateFlush, manager); if (advertiser != null) { final Map<String, String> configuration = new HashMap<>(layout.getContentFormat()); configuration.putAll(manager.getContentFormat()); configuration.put("contentType", layout.getContentType()); configuration.put("name", name); advertisement = advertiser.advertise(configuration); } else { advertisement = null; } this.fileName = filename; this.advertiser = advertiser; }
try { @SuppressWarnings("resource") final FileChannel channel = ((FileOutputStream) getOutputStream()).getChannel(); throw new AppenderLoggingException("Unable to obtain lock on " + getName(), ex);
String getLogDir(Configuration conf) { String logDir = conf.get("hive.log.dir"); if (logDir == null) { logDir = System.getProperty("hive.log.dir"); } if (logDir != null) { return logDir; } LoggerContext context = (LoggerContext)LogManager.getContext(false); for (Logger logger: context.getLoggers()) { for (Appender appender: logger.getAppenders().values()) { if (appender instanceof AbstractOutputStreamAppender) { OutputStreamManager manager = ((AbstractOutputStreamAppender<?>)appender).getManager(); if (manager instanceof FileManager) { String fileName = ((FileManager)manager).getFileName(); if (fileName != null) { return fileName.substring(0, fileName.lastIndexOf('/')); } } } } } return null; }
.build(); Assert.assertEquals(createOnDemand, appender.getManager().isCreateOnDemand()); try { appender.start();