private void log(Object m) { String indent = ""; for (int i = 0; i<currentLevel; ++i) { indent += " "; } logger.debug(indent + m); }
protected boolean isPigLoaded() { try { Class.forName("org.apache.pig.impl.logicalLayer.schema.Schema"); return true; } catch (ClassNotFoundException e) { LOG.info("Pig is not loaded, pig metadata will not be written"); return false; } }
/** * * @param c the current class * @return the corresponding logger */ public static Log getLog(Class<?> c) { return new Log(c); }
Log log = new Log(); Needed to set ID before adding it to mAdapter log.setmDate(date); log.setmAmount(amount); log.setmMiles(miles); mDataSource.insert(date, amount, miles, OilTable.TAG); log.setmId(mDataSource.getLastLogID(OilTable.TAG)); mAdapter.add(log);
private void checkBlockSizeReached() throws IOException { if (recordCount >= recordCountForNextMemCheck) { // checking the memory size is relatively expensive, so let's not do it for every record. long memSize = columnStore.getBufferedSize(); if (memSize > rowGroupSizeThreshold) { LOG.info(format("mem size %,d > %,d: flushing %,d records to disk.", memSize, rowGroupSizeThreshold, recordCount)); flushRowGroupToStore(); initStore(); recordCountForNextMemCheck = min(max(MINIMUM_RECORD_COUNT_FOR_CHECK, recordCount / 2), MAXIMUM_RECORD_COUNT_FOR_CHECK); } else { float recordSize = (float) memSize / recordCount; recordCountForNextMemCheck = min( max(MINIMUM_RECORD_COUNT_FOR_CHECK, (recordCount + (long)(rowGroupSizeThreshold / recordSize)) / 2), // will check halfway recordCount + MAXIMUM_RECORD_COUNT_FOR_CHECK // will not look more than max records ahead ); if (DEBUG) LOG.debug(format("Checked mem at %,d will check again at: %,d ", recordCount, recordCountForNextMemCheck)); } } }
private CompressionCodecName getHadoopCompressionCodec() { CompressionCodecName codec; try { // find the right codec Class<?> codecClass = getHadoopOutputCompressorClass(CompressionCodecName.UNCOMPRESSED.getHadoopCompressionCodecClass()); if (INFO) LOG.info("Compression set through hadoop codec: " + codecClass.getName()); codec = CompressionCodecName.fromCompressionCodec(codecClass); } catch (CompressionCodecNotSupportedException e) { if (WARN) LOG.warn("codec defined in hadoop config is not supported by parquet [" + e.getCodecClass().getName() + "] and will use UNCOMPRESSED", e); codec = CompressionCodecName.UNCOMPRESSED; } catch (IllegalArgumentException e) { if (WARN) LOG.warn("codec class not found: " + e.getMessage(), e); codec = CompressionCodecName.UNCOMPRESSED; } return codec; }
@Override public ResourceStatistics getStatistics(String location, Job job) throws IOException { if (DEBUG) LOG.debug("LoadMetadata.getStatistics(" + location + ", " + job + ")"); /* We need to call setInput since setLocation is not guaranteed to be called before this */ setInput(location, job); long length = 0; try { for (InputSplit split : getParquetInputFormat().getSplits(job)) { length += split.getLength(); } } catch (InterruptedException e) { LOG.warn("Interrupted: ", e); return null; } ResourceStatistics stats = new ResourceStatistics(); // TODO use pig-0.12 setBytes api when its available stats.setmBytes(length / 1024 / 1024); return stats; }
/** * Closes a (potentially null) closeable, swallowing any IOExceptions thrown by * c.close(). The exception will be logged. * @param c can be null */ public static void closeAndSwallowIOExceptions(Closeable c) { if (c == null) { return; } try { c.close(); } catch (IOException e) { LOG.warn("Encountered exception closing closeable", e); } } }
private void checkBlockSizeReached() throws IOException { if (recordCount >= recordCountForNextMemCheck) { // checking the memory size is relatively expensive, so let's not do it for every record. long memSize = store.memSize(); if (memSize > blockSize) { LOG.info(format("mem size %,d > %,d: flushing %,d records to disk.", memSize, blockSize, recordCount)); flushStore(); initStore(); recordCountForNextMemCheck = min(max(MINIMUM_RECORD_COUNT_FOR_CHECK, recordCount / 2), MAXIMUM_RECORD_COUNT_FOR_CHECK); } else { float recordSize = (float) memSize / recordCount; recordCountForNextMemCheck = min( max(MINIMUM_RECORD_COUNT_FOR_CHECK, (recordCount + (long)(blockSize / recordSize)) / 2), // will check halfway recordCount + MAXIMUM_RECORD_COUNT_FOR_CHECK // will not look more than max records ahead ); if (DEBUG) LOG.debug(format("Checked mem at %,d will check again at: %,d ", recordCount, recordCountForNextMemCheck)); } } }
private void flushStore() throws IOException { LOG.info(format("Flushing mem store to file. allocated memory: %,d", store.allocatedSize())); if (store.allocatedSize() > 3 * blockSize) { LOG.warn("Too much memory used: " + store.memUsageString()); } w.startBlock(recordCount); store.flush(); pageStore.flushToFileWriter(w); recordCount = 0; w.endBlock(); store = null; pageStore = null; } }
/** * Closes a (potentially null) closeable, swallowing any IOExceptions thrown by * c.close(). The exception will be logged. * @param c can be null */ public static void closeAndSwallowIOExceptions(Closeable c) { if (c == null) { return; } try { c.close(); } catch (IOException e) { LOG.warn("Encountered exception closing closeable", e); } } }
@Override public void initFromPage(int valueCount, byte[] in, int offset) throws IOException { if (DEBUG) LOG.debug("init from page at offset "+ offset + " for length " + (in.length - offset)); this.in = in; this.offset = offset; } }
/** * end a column (once all rep, def and data have been written) * @throws IOException */ public void endColumn() throws IOException { state = state.endColumn(); if (DEBUG) LOG.debug(out.getPos() + ": end column"); currentBlock.addColumn(ColumnChunkMetaData.get( currentChunkPath, currentChunkType, currentChunkCodec, currentEncodings, currentStatistics, currentChunkFirstDataPage, currentChunkDictionaryPageOffset, currentChunkValueCount, compressedLength, uncompressedLength)); if (DEBUG) LOG.info("ended Column chumk: " + currentColumn); currentColumn = null; this.currentBlock.setTotalByteSize(currentBlock.getTotalByteSize() + uncompressedLength); this.uncompressedLength = 0; this.compressedLength = 0; }
private void log(Object o) { LOG.info(o); }
private void flushRowGroupToStore() throws IOException { LOG.info(format("Flushing mem columnStore to file. allocated memory: %,d", columnStore.getAllocatedSize())); if (columnStore.getAllocatedSize() > 3 * (long)rowGroupSizeThreshold) { LOG.warn("Too much memory used: " + columnStore.memUsageString()); } if (recordCount > 0) { parquetFileWriter.startBlock(recordCount); columnStore.flush(); pageStore.flushToFileWriter(parquetFileWriter); recordCount = 0; parquetFileWriter.endBlock(); } columnStore = null; pageStore = null; }
aspect LogAllMethods { Log log = new Log(); // pseudocode -- replace with whatever logging mechanism you like before(): call(public * com.mycompany.*.*(..)) { log.log("Entered method: " + thisJoinPoint); } after(): call(public * com.mycomapny.*.*(..)) { log.log("Leaving method: " + thisJoinPoint); } }
public static void writeMetaDataFile(Configuration configuration, Path outputPath) { if (configuration.getBoolean(ParquetOutputFormat.ENABLE_JOB_SUMMARY, true)) { try { final FileSystem fileSystem = outputPath.getFileSystem(configuration); FileStatus outputStatus = fileSystem.getFileStatus(outputPath); List<Footer> footers = ParquetFileReader.readAllFootersInParallel(configuration, outputStatus); try { ParquetFileWriter.writeMetadataFile(configuration, outputPath, footers); } catch (Exception e) { LOG.warn("could not write summary file for " + outputPath, e); final Path metadataPath = new Path(outputPath, ParquetFileWriter.PARQUET_METADATA_FILE); if (fileSystem.exists(metadataPath)) { fileSystem.delete(metadataPath, true); } } } catch (Exception e) { LOG.warn("could not write summary file for " + outputPath, e); } } }
@SuppressWarnings("unchecked") @Override public void prepareToRead(@SuppressWarnings("rawtypes") RecordReader reader, PigSplit split) throws IOException { if (DEBUG) LOG.debug("LoadFunc.prepareToRead(" + reader + ", " + split + ")"); this.reader = reader; }
private static <T> List<T> runAllInParallel(int parallelism, List<Callable<T>> toRun) throws ExecutionException { LOG.info("Initiating action with parallelism: " + parallelism); ExecutorService threadPool = Executors.newFixedThreadPool(parallelism); try { List<Future<T>> futures = new ArrayList<Future<T>>(); for (Callable<T> callable : toRun) { futures.add(threadPool.submit(callable)); } List<T> result = new ArrayList<T>(toRun.size()); for (Future<T> future : futures) { try { result.add(future.get()); } catch (InterruptedException e) { throw new RuntimeException("The thread was interrupted", e); } } return result; } finally { threadPool.shutdownNow(); } }