Refine search
/** * Constructor that let's the user specify additional options. * @param path pathname for file * @param options options for reading * @throws IOException */ public ReaderImpl(Path path, OrcFile.ReaderOptions options) throws IOException { super(path, options); this.inspector = OrcStruct.createObjectInspector(0, types); }
@Override public Object setStructFieldData(Object struct, StructField field, Object fieldValue) { OrcStruct orcStruct = (OrcStruct) struct; int offset = ((Field) field).offset; // if the offset is bigger than our current number of fields, grow it if (orcStruct.getNumFields() <= offset) { orcStruct.setNumFields(offset+1); } orcStruct.setFieldValue(offset, fieldValue); return struct; }
static OrcStruct nextStruct(ColumnVector vector, int row, TypeDescription schema, Object previous) { if (vector.isRepeating) { row = 0; } if (vector.noNulls || !vector.isNull[row]) { OrcStruct result; List<TypeDescription> childrenTypes = schema.getChildren(); int numChildren = childrenTypes.size(); if (previous == null || previous.getClass() != OrcStruct.class) { result = new OrcStruct(numChildren); } else { result = (OrcStruct) previous; result.setNumFields(numChildren); } StructColumnVector struct = (StructColumnVector) vector; for(int f=0; f < numChildren; ++f) { result.setFieldValue(f, nextValue(struct.fields[f], row, childrenTypes.get(f), result.getFieldValue(f))); } return result; } else { return null; } }
@Override protected OrcStruct createStructObject(Object previous, List<TypeInfo> childrenTypes) { int numChildren = childrenTypes.size(); if (previous == null || !(previous instanceof OrcStruct)) { return new OrcStruct(numChildren); } OrcStruct result = (OrcStruct) previous; result.setNumFields(numChildren); return result; }
private void addSimpleEvent(int operation, long currentTransaction, long rowId, Object row) throws IOException { this.operation.set(operation); this.currentTransaction.set(currentTransaction); // If this is an insert, originalTransaction should be set to this transaction. If not, // it will be reset by the following if anyway. long originalTransaction = currentTransaction; if (operation == DELETE_OPERATION || operation == UPDATE_OPERATION) { Object rowIdValue = rowInspector.getStructFieldData(row, recIdField); originalTransaction = origTxnInspector.get( recIdInspector.getStructFieldData(rowIdValue, originalTxnField)); rowId = rowIdInspector.get(recIdInspector.getStructFieldData(rowIdValue, rowIdField)); } else if(operation == INSERT_OPERATION) { rowId += rowIdOffset; } this.rowId.set(rowId); this.originalTransaction.set(originalTransaction); item.setFieldValue(OrcRecordUpdater.OPERATION, new IntWritable(operation)); item.setFieldValue(OrcRecordUpdater.ROW, (operation == DELETE_OPERATION ? null : row)); indexBuilder.addKey(operation, originalTransaction, bucket.get(), rowId); if (writer == null) { writer = OrcFile.createWriter(path, writerOptions); } writer.addRow(item); }
OrcFile.writerOptions(conf) .fileSystem(fs) .inspector(OrcStruct.createObjectInspector(typeInfo)) .compress(CompressionKind.NONE) .callback(indexBuilder) .stripeSize(128)); OrcStruct row = new OrcStruct(6); row.setFieldValue(0, new IntWritable(0)); row.setFieldValue(1, new LongWritable(1)); row.setFieldValue(2, new IntWritable(0)); LongWritable rowId = new LongWritable(); row.setFieldValue(3, rowId); row.setFieldValue(4, new LongWritable(1)); OrcStruct rowField = new OrcStruct(3); row.setFieldValue(5, rowField); IntWritable a = new IntWritable(); rowField.setFieldValue(0, a); OrcStruct b = new OrcStruct(1); rowField.setFieldValue(1, b); IntWritable c = new IntWritable(); b.setFieldValue(0, c); Text d = new Text(); rowField.setFieldValue(2, d);
nextRecord = new OrcStruct(OrcRecordUpdater.FIELDS); IntWritable operation = new IntWritable(OrcRecordUpdater.INSERT_OPERATION); nextRecord.setFieldValue(OrcRecordUpdater.OPERATION, operation); nextRecord.setFieldValue(OrcRecordUpdater.CURRENT_TRANSACTION, new LongWritable(0)); nextRecord.setFieldValue(OrcRecordUpdater.ORIGINAL_TRANSACTION, new LongWritable(0)); nextRecord.setFieldValue(OrcRecordUpdater.BUCKET, new IntWritable(bucket)); nextRecord.setFieldValue(OrcRecordUpdater.ROW_ID, new LongWritable(nextRowId)); nextRecord.setFieldValue(OrcRecordUpdater.ROW, recordReader.next(null)); } else { nextRecord = next; ((IntWritable) next.getFieldValue(OrcRecordUpdater.OPERATION)) .set(OrcRecordUpdater.INSERT_OPERATION); ((LongWritable) next.getFieldValue(OrcRecordUpdater.ORIGINAL_TRANSACTION)) .set(0); ((IntWritable) next.getFieldValue(OrcRecordUpdater.BUCKET)) .set(bucket); ((LongWritable) next.getFieldValue(OrcRecordUpdater.CURRENT_TRANSACTION)) .set(0); ((LongWritable) next.getFieldValue(OrcRecordUpdater.ROW_ID)) .set(nextRowId); nextRecord.setFieldValue(OrcRecordUpdater.ROW, recordReader.next(OrcRecordUpdater.getRow(next)));
actualValue = ((IntWritable) actualValue).get(); actualValue = ((LongWritable) actualValue).get(); List<Object> fields = new ArrayList<>(); OrcStruct structObject = (OrcStruct) actualValue; for (int fieldId = 0; fieldId < structObject.getNumFields(); fieldId++) { fields.add(OrcUtil.getFieldValue(structObject, fieldId));
@Override public void write(Writable w) throws IOException { OrcStruct orc = (OrcStruct) w; watcher.addKey( ((IntWritable) orc.getFieldValue(OrcRecordUpdater.OPERATION)).get(), ((LongWritable) orc.getFieldValue(OrcRecordUpdater.ORIGINAL_WRITEID)).get(), ((IntWritable) orc.getFieldValue(OrcRecordUpdater.BUCKET)).get(), ((LongWritable) orc.getFieldValue(OrcRecordUpdater.ROW_ID)).get()); writer.addRow(w); }
private static void setRow(OrcStruct event, int operation, long originalTransaction, int bucket, long rowId, long currentTransaction, String value) { event.setFieldValue(OrcRecordUpdater.OPERATION, new IntWritable(operation)); event.setFieldValue(OrcRecordUpdater.ORIGINAL_WRITEID, new LongWritable(originalTransaction)); event.setFieldValue(OrcRecordUpdater.BUCKET, new IntWritable(bucket)); event.setFieldValue(OrcRecordUpdater.ROW_ID, new LongWritable(rowId)); event.setFieldValue(OrcRecordUpdater.CURRENT_WRITEID, new LongWritable(currentTransaction)); OrcStruct row = new OrcStruct(1); row.setFieldValue(0, new Text(value)); event.setFieldValue(OrcRecordUpdater.ROW, row); }
static int getOperation(OrcStruct struct) { return ((IntWritable) struct.getFieldValue(OPERATION)).get(); }
static long getOriginalTransaction(OrcStruct struct) { return ((LongWritable) struct.getFieldValue(ORIGINAL_WRITEID)).get(); }
/** * Create an object of OrcStruct given a TypeInfo and a list of objects * * @param typeInfo The TypeInfo object representing the ORC record schema * @param objs ORC objects/Writables * @return an OrcStruct containing the specified objects for the specified schema */ public static OrcStruct createOrcStruct(TypeInfo typeInfo, Object... objs) { SettableStructObjectInspector oi = (SettableStructObjectInspector) OrcStruct .createObjectInspector(typeInfo); List<StructField> fields = (List<StructField>) oi.getAllStructFieldRefs(); OrcStruct result = (OrcStruct) oi.create(); result.setNumFields(fields.size()); for (int i = 0; i < fields.size(); i++) { oi.setStructFieldData(result, fields.get(i), objs[i]); } return result; }
inspector = OrcStruct.createObjectInspector(0, types); .bufferSize(10000) .blockPadding(false)); OrcStruct row = new OrcStruct(2); for (int year = minYear; year < maxYear; ++year) { for (int ms = 1000; ms < 2000; ++ms) { row.setFieldValue(0, new TimestampWritableV2(Timestamp.valueOf(year + "-05-05 12:34:56." + ms))); row.setFieldValue(1, new DateWritableV2(Date.of(year - 1900, 11, 25))); writer.addRow(row); assertEquals(new TimestampWritableV2 (Timestamp.valueOf(year + "-05-05 12:34:56." + ms)), row.getFieldValue(0)); assertEquals(new DateWritableV2(Date.of(year - 1900, 11, 25)), row.getFieldValue(1));
",c13:array<timestamp>>"); StructObjectInspector inspector = (StructObjectInspector) OrcStruct.createObjectInspector(typeInfo); assertEquals("struct<c1:boolean,c2:tinyint,c3:smallint,c4:int,c5:" + "bigint,c6:float,c7:double,c8:binary,c9:string,c10:struct<" + inspector.getAllStructFieldRefs().get(0).getFieldComment()); assertEquals(null, inspector.getStructFieldRef("UNKNOWN")); OrcStruct s1 = new OrcStruct(13); for(int i=0; i < 13; ++i) { s1.setFieldValue(i, i);
@Override public OrcStruct createValue() { return new OrcStruct(OrcRecordUpdater.FIELDS); }
@Test public void testStruct() throws Exception { OrcStruct st1 = new OrcStruct(4); OrcStruct st2 = new OrcStruct(4); OrcStruct st3 = new OrcStruct(3); st1.setFieldValue(0, "hop"); st1.setFieldValue(1, "on"); st1.setFieldValue(2, "pop"); st1.setFieldValue(3, 42); assertEquals(false, st1.equals(null)); st2.setFieldValue(0, "hop"); st2.setFieldValue(1, "on"); st2.setFieldValue(2, "pop"); st2.setFieldValue(3, 42); assertEquals(st1, st2); st3.setFieldValue(0, "hop"); st3.setFieldValue(1, "on"); st3.setFieldValue(2, "pop"); assertEquals(false, st1.equals(st3)); assertEquals(11241, st1.hashCode()); assertEquals(st1.hashCode(), st2.hashCode()); assertEquals(11204, st3.hashCode()); assertEquals("{hop, on, pop, 42}", st1.toString()); st1.setFieldValue(3, null); assertEquals(false, st1.equals(st2)); assertEquals(false, st2.equals(st1)); st2.setFieldValue(3, null); assertEquals(st1, st2); }
@Override protected int getStructLength(OrcStruct structObj) { return structObj.getNumFields(); }
private static OrcStruct createOriginalRow(String value) { OrcStruct result = new OrcStruct(1); result.setFieldValue(0, new Text(value)); return result; }
public static Object getFieldValue(OrcStruct orcStruct, int column) { if (column >= orcStruct.getNumFields()) { return null; } return orcStruct.getFieldValue(column); } }