public static ObjectInspector getReflectionObjectInspector(Type t, ObjectInspectorOptions options) { return getReflectionObjectInspector(t, options, true); }
private static synchronized ObjectInspector getInspector(Type type) { // ObjectInspectorFactory.getReflectionObjectInspector is not thread-safe although it // gives people a first impression that it is. This may have been fixed in HIVE-11586. // Presto only uses getReflectionObjectInspector here, in a test method. Therefore, we // choose to work around this issue by synchronizing this method. Before synchronizing // this method, test in this class fails approximately 1 out of 10 runs on Travis. return getReflectionObjectInspector(type, ObjectInspectorOptions.JAVA); }
@Override public void initialize(Configuration job, Properties tbl) throws SerDeException { cachedObjectInspector = ObjectInspectorFactory .getReflectionObjectInspector(S3LogStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); LOG.debug(getClass().getName() + ": initialized"); }
private ObjectInspector createIO() { synchronized (TestOrcFile.class) { return ObjectInspectorFactory .getReflectionObjectInspector(AllTypesRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } }
public ReflectiveMutatorFactory(Configuration configuration, Class<?> recordClass, int recordIdColumn, int[] bucketColumnIndexes) { this.configuration = configuration; this.recordIdColumn = recordIdColumn; this.bucketColumnIndexes = bucketColumnIndexes; objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(recordClass, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); }
@Override public ObjectInspector getObjectInspector() throws SerDeException { return ObjectInspectorFactory.getReflectionObjectInspector(objectType, getObjectInspectorOptions()); }
private StructObjectInspector createSoi() { synchronized (TestOrcFile.class) { return (StructObjectInspector)ObjectInspectorFactory.getReflectionObjectInspector( SimpleRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } }
protected List<? extends ObjectInspector> extractFieldObjectInspectors(Class<?> clazz, ObjectInspectorFactory.ObjectInspectorOptions options) { Field[] fields = ObjectInspectorUtils.getDeclaredNonStaticFields(clazz); ArrayList<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>( fields.length); for (int i = 0; i < fields.length; i++) { if (!shouldIgnoreField(fields[i].getName())) { structFieldObjectInspectors.add(ObjectInspectorFactory.getReflectionObjectInspector(fields[i] .getGenericType(), options, false)); } } return structFieldObjectInspectors; }
/** * Get the type name of the Java class. */ public static String getTypeNameFromJavaClass(Type t) { try { ObjectInspector oi = ObjectInspectorFactory.getReflectionObjectInspector(t, ObjectInspectorOptions.JAVA); return oi.getTypeName(); } catch (Throwable e) { LOG.info(StringUtils.stringifyException(e)); return "unknown"; } }
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { try { udf = (UDF)getUdfClassInternal().newInstance(); } catch (Exception e) { throw new UDFArgumentException( "Unable to instantiate UDF implementation class " + udfClassName + ": " + e); } // Resolve for the method based on argument types ArrayList<TypeInfo> argumentTypeInfos = new ArrayList<TypeInfo>( arguments.length); for (ObjectInspector argument : arguments) { argumentTypeInfos.add(TypeInfoUtils .getTypeInfoFromObjectInspector(argument)); } udfMethod = udf.getResolver().getEvalMethod(argumentTypeInfos); udfMethod.setAccessible(true); // Create parameter converters conversionHelper = new ConversionHelper(udfMethod, arguments); // Create the non-deferred realArgument realArguments = new Object[arguments.length]; // Get the return ObjectInspector. ObjectInspector returnOI = ObjectInspectorFactory .getReflectionObjectInspector(udfMethod.getGenericReturnType(), ObjectInspectorOptions.JAVA); return returnOI; }
@Test(expected = IllegalArgumentException.class) public void testNoBucketColumns() { new BucketIdResolverImpl(ObjectInspectorFactory.getReflectionObjectInspector(MutableRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA), RECORD_ID_COLUMN, TOTAL_BUCKETS, new int[0]); }
fs.delete(root, true); synchronized (TestOrcFile.class) { inspector = ObjectInspectorFactory.getReflectionObjectInspector (DummyRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); originalInspector = ObjectInspectorFactory.getReflectionObjectInspector(DummyOriginalRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); bigRowInspector = ObjectInspectorFactory.getReflectionObjectInspector(BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); bigOriginalRowInspector = ObjectInspectorFactory.getReflectionObjectInspector(BigOriginalRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
private void createOrcFile(int stripSize, int numberOfRows) throws IOException { ObjectInspector inspector; synchronized (TestOrcFileStripeMergeRecordReader.class) { inspector = ObjectInspectorFactory.getReflectionObjectInspector (StringIntIntIntRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } Writer writer = OrcFile.createWriter(tmpPath, OrcFile.writerOptions(conf) .inspector(inspector) .stripeSize(stripSize) .compress(CompressionKind.ZLIB) .bufferSize(5000) .rowIndexStride(1000)); Random rand = new Random(157); for (int i = 0; i < numberOfRows; i++) { writer.addRow(new StringIntIntIntRow( Integer.toBinaryString(i), rand.nextInt(), rand.nextInt(), rand.nextInt() )); } writer.close(); }
@Test public void emptyFile() throws Exception { ObjectInspector inspector; synchronized (TestOrcFile.class) { inspector = ObjectInspectorFactory.getReflectionObjectInspector (BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf) .inspector(inspector) .stripeSize(1000) .compress(CompressionKind.NONE) .bufferSize(100)); writer.close(); Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs)); assertEquals(false, reader.rows().hasNext()); assertEquals(CompressionKind.NONE, reader.getCompression()); assertEquals(0, reader.getNumberOfRows()); assertEquals(0, reader.getCompressionSize()); assertEquals(false, reader.getMetadataKeys().iterator().hasNext()); assertEquals(3, reader.getContentLength()); assertEquals(false, reader.getStripes().iterator().hasNext()); }
ObjectInspector inspector; synchronized (TestOrcSerDeStats.class) { inspector = ObjectInspectorFactory.getReflectionObjectInspector (SimpleStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
ObjectInspector inspector; synchronized (TestOrcFile.class) { inspector = ObjectInspectorFactory.getReflectionObjectInspector(Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
@Test public void testOrcSerDeStatsList() throws Exception { ObjectInspector inspector; synchronized (TestOrcSerDeStats.class) { inspector = ObjectInspectorFactory.getReflectionObjectInspector (ListStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf) .inspector(inspector) .stripeSize(10000) .bufferSize(10000)); for (int row = 0; row < 5000; row++) { List<String> test = new ArrayList<String>(); for (int i = 0; i < 1000; i++) { test.add("hi"); } writer.addRow(new ListStruct(test)); } writer.close(); assertEquals(5000, writer.getNumberOfRows()); assertEquals(430000000, writer.getRawDataSize()); Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs)); // stats from reader assertEquals(5000, reader.getNumberOfRows()); assertEquals(430000000, reader.getRawDataSize()); assertEquals(430000000, reader.getRawDataSizeOfColumns(Lists.newArrayList("list1"))); }
@Test public void testOrcSerDeStatsMap() throws Exception { ObjectInspector inspector; synchronized (TestOrcSerDeStats.class) { inspector = ObjectInspectorFactory.getReflectionObjectInspector (MapStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf) .inspector(inspector) .stripeSize(10000) .bufferSize(10000)); for (int row = 0; row < 1000; row++) { Map<String, Double> test = new HashMap<String, Double>(); for (int i = 0; i < 10; i++) { test.put("hi" + i, 2.0); } writer.addRow(new MapStruct(test)); } writer.close(); // stats from writer assertEquals(1000, writer.getNumberOfRows()); assertEquals(950000, writer.getRawDataSize()); Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs)); // stats from reader assertEquals(1000, reader.getNumberOfRows()); assertEquals(950000, reader.getRawDataSize()); assertEquals(950000, reader.getRawDataSizeOfColumns(Lists.newArrayList("map1"))); }
@Test public void testBitPack64Large() throws Exception { ObjectInspector inspector; synchronized (TestOrcFile.class) { inspector = ObjectInspectorFactory.getReflectionObjectInspector(Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } int size = 1080832; long[] inp = new long[size]; Random rand = new Random(1234); for (int i = 0; i < size; i++) { inp[i] = rand.nextLong(); } List<Long> input = Lists.newArrayList(Longs.asList(inp)); Writer writer = OrcFile.createWriter(testFilePath, OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.ZLIB)); for (Long l : input) { writer.addRow(l); } writer.close(); Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs)); RecordReader rows = reader.rows(); int idx = 0; while (rows.hasNext()) { Object row = rows.next(null); Assert.assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get()); } }
ObjectInspector inspector; synchronized (TestOrcFile.class) { inspector = ObjectInspectorFactory.getReflectionObjectInspector (DecimalStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);