/** * Create a {@link TimestampedHiveDatasetVersion} from a {@link Partition} based on the Modified time of underlying * hdfs data location * @throws IllegalArgumentException when argument is null * @throws IllegalArgumentException when data location of partition is null * @throws IllegalArgumentException when data location of partition doesn't exist * {@inheritDoc} */ @Override protected TimestampedHiveDatasetVersion getDatasetVersion(Partition partition) { try { Preconditions.checkArgument(partition != null, "Argument to method "); Path dataLocation = partition.getDataLocation(); Preconditions .checkArgument(dataLocation != null, "Data location is null for partition " + partition.getCompleteName()); boolean exists = this.fs.exists(dataLocation); Preconditions.checkArgument(exists, "Data location doesn't exist for partition " + partition.getCompleteName()); long modificationTS = this.fs.getFileStatus(dataLocation).getModificationTime(); return new TimestampedHiveDatasetVersion(new DateTime(modificationTS), partition); } catch (IOException e) { throw new RuntimeException(e); } } }
public static Predicate equalTo(final String target) { Preconditions.checkNotNull(target,"target"); return new Predicate() { @Override public boolean apply(ColumnReader input) { return target.equals(input.getBinary().toStringUsingUTF8()); } }; }
public static void validateBuffer(byte[] buffer, int off, int len) { Preconditions.checkNotNull(buffer, "buffer"); Preconditions.checkArgument(off >= 0 && len >= 0 && off <= buffer.length - len, "Invalid buffer offset or length: buffer.length=%s off=%s len=%s", buffer.length, off, len); } }
@Override protected PrimitiveType build(String name) { if (PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY == primitiveType) { Preconditions.checkArgument(length > 0, "Invalid FIXED_LEN_BYTE_ARRAY length: " + length); case JSON: case BSON: Preconditions.checkState( primitiveType == PrimitiveTypeName.BINARY, originalType.toString() + " can only annotate binary fields"); break; case DECIMAL: Preconditions.checkState( (primitiveType == PrimitiveTypeName.INT32) || (primitiveType == PrimitiveTypeName.INT64) || ); if (primitiveType == PrimitiveTypeName.INT32) { Preconditions.checkState( meta.getPrecision() <= MAX_PRECISION_INT32, "INT32 cannot store " + meta.getPrecision() + " digits " + "(max " + MAX_PRECISION_INT32 + ")"); } else if (primitiveType == PrimitiveTypeName.INT64) { Preconditions.checkState( meta.getPrecision() <= MAX_PRECISION_INT64, "INT64 cannot store " + meta.getPrecision() + " digits " + "(max " + MAX_PRECISION_INT64 + ")"); } else if (primitiveType == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) { Preconditions.checkState(
@Override protected GroupType build(String name) { Preconditions.checkState(!fields.isEmpty(), "Cannot build an empty group"); return new GroupType(repetition, name, originalType, fields, id); } }
/** * Precondition-style validation that throws {@link IllegalArgumentException}. * * @param isValid * {@code true} if valid, {@code false} if an exception should be * thrown * @param message * A String message for the exception. * @param args * Objects used to fill in {@code %s} placeholders in the message * @throws IllegalArgumentException if {@code isValid} is false */ public static void checkArgument(boolean isValid, String message, Object... args) throws IllegalArgumentException { if (!isValid) { throw new IllegalArgumentException( String.format(String.valueOf(message), strings(args))); } }
/** * @param readSupport Object which helps reads files of the given type, e.g. Thrift, Avro. * @param filter for filtering individual records */ public InternalParquetRecordReader(ReadSupport<T> readSupport, Filter filter) { this.readSupport = readSupport; this.filter = checkNotNull(filter, "filter"); }
protected final T repetition(Type.Repetition repetition) { Preconditions.checkArgument(!repetitionAlreadySet, "Repetition has already been set"); Preconditions.checkNotNull(repetition, "Repetition cannot be null"); this.repetition = repetition; this.repetitionAlreadySet = true; return self(); }
: HDFSMetadataManager.DEFAULT_NUM_METADATA_FILES_TO_RETAIN; Preconditions.checkState(numFilesToRetain > 0, "Number of files to retain cannot be <= 0");
/** * Precondition-style validation that throws {@link IllegalArgumentException}. * * @param isValid * {@code true} if valid, {@code false} if an exception should be * thrown * @param message * A String message for the exception. * @param args * Objects used to fill in {@code %s} placeholders in the message * @throws IllegalArgumentException if {@code isValid} is false */ public static void checkArgument(boolean isValid, String message, Object... args) throws IllegalArgumentException { if (!isValid) { throw new IllegalArgumentException( String.format(String.valueOf(message), strings(args))); } }
/** * If this is called, the thrift class is used. * If not, will use the thrift class based on the file * metadata if a thrift class information is present. */ public Builder<T> withThriftClass(Class<T> thriftClass) { this.thriftClass = checkNotNull(thriftClass, "thriftClass"); return this; }
public DeltaBinaryPackingConfig(int blockSizeInValues, int miniBlockNumInABlock) { this.blockSizeInValues = blockSizeInValues; this.miniBlockNumInABlock = miniBlockNumInABlock; double miniSize = (double) blockSizeInValues / miniBlockNumInABlock; Preconditions.checkArgument(miniSize % 8 == 0, "miniBlockSize must be multiple of 8, but it's " + miniSize); this.miniBlockSizeInValues = (int) miniSize; }
@Override public JavaRDD<AvroPayload> getData(@NonNull final ParquetWorkUnitCalculatorResult workUnitCalcResult) { Preconditions.checkState(workUnitCalcResult.hasWorkUnits(), "No work to process for: " + hiveConf.getDataPath()); /** * Current implementation of HiveSource assumes that only a single work unit exists which * corresponds to the single partition that is processed per job. */ final List<String> workUnits = workUnitCalcResult.getWorkUnits(); final String hdfsPath = new Path(this.hiveConf.getDataPath(), workUnits.get(0)).toString(); log.info("Reading data from path: {}", hdfsPath); final Dataset<Row> data = this.sqlContext.read().parquet(hdfsPath); final int numPartitions = calculateHiveNumPartitions(data); log.info("Using {} partitions", numPartitions); final JavaRDD<AvroPayload> hiveRawData = data .coalesce(numPartitions) .javaRDD() .flatMap(row -> { final List<AvroPayload> payloads = new ArrayList<>(); this.converter.convert(row).forEach(d -> payloads.add(d.getSuccessData().get().getData())); return payloads.iterator(); }); return hiveRawData; }
/** * Precondition-style validation that throws {@link IllegalStateException}. * * @param isValid * {@code true} if valid, {@code false} if an exception should be * thrown * @param message * A String message for the exception. * @param args * Objects used to fill in {@code %s} placeholders in the message * @throws IllegalStateException if {@code isValid} is false */ public static void checkState(boolean isValid, String message, Object... args) throws IllegalStateException { if (!isValid) { throw new IllegalStateException( String.format(String.valueOf(message), strings(args))); } }
/** * Construct a type builder that returns a "parent" object when the builder * is finished. The {@code parent} will be returned by * {@link #named(String)} so that builders can be chained. * * @param parent a non-null object to return from {@link #named(String)} */ protected Builder(P parent) { Preconditions.checkNotNull(parent, "Parent cannot be null"); this.parent = parent; this.returnClass = null; }
/** * @return the index of the last value written to this stream, which * can be passed to {@link #setByte(long, byte)} in order to change it */ public long getCurrentIndex() { checkArgument(bytesUsed > 0, "This is an empty stream"); return bytesUsed - 1; }
/** * Precondition-style validation that throws {@link IllegalStateException}. * * @param isValid * {@code true} if valid, {@code false} if an exception should be * thrown * @param message * A String message for the exception. * @param args * Objects used to fill in {@code %s} placeholders in the message * @throws IllegalStateException if {@code isValid} is false */ public static void checkState(boolean isValid, String message, Object... args) throws IllegalStateException { if (!isValid) { throw new IllegalStateException( String.format(String.valueOf(message), strings(args))); } }
public FilteringGroupConverter( GroupConverter delegate, List<Integer> indexFieldPath, Map<ColumnPath, List<ValueInspector>> valueInspectorsByColumn, Map<List<Integer>, PrimitiveColumnIO> columnIOsByIndexFieldPath) { this.delegate = checkNotNull(delegate, "delegate"); this.indexFieldPath = checkNotNull(indexFieldPath, "indexFieldPath"); this.columnIOsByIndexFieldPath = checkNotNull(columnIOsByIndexFieldPath, "columnIOsByIndexFieldPath"); this.valueInspectorsByColumn = checkNotNull(valueInspectorsByColumn, "valueInspectorsByColumn"); }
/** * @return the index of the last value written to this stream, which * can be passed to {@link #setByte(long, byte)} in order to change it */ public long getCurrentIndex() { checkArgument(bytesUsed > 0, "This is an empty stream"); return bytesUsed - 1; }
protected Column(ColumnPath columnPath, Class<T> columnType) { checkNotNull(columnPath, "columnPath"); checkNotNull(columnType, "columnType"); this.columnPath = columnPath; this.columnType = columnType; }