/** * Gets the input file format. * * @return the input file format */ public String getInputFileFormat() { return sd.getInputFormat(); }
/** * Gets the input format. * * @return the input format */ public String getInputFormat() { return this.sd.getInputFormat(); }
private static boolean isAcidInputOutputFormat(String fullTableName, StorageDescriptor sd) { try { Class inputFormatClass = sd.getInputFormat() == null ? null : Class.forName(sd.getInputFormat()); Class outputFormatClass = sd.getOutputFormat() == null ? null : Class.forName(sd.getOutputFormat()); if (inputFormatClass != null && outputFormatClass != null && Class.forName("org.apache.hadoop.hive.ql.io.AcidInputFormat") .isAssignableFrom(inputFormatClass) && Class.forName("org.apache.hadoop.hive.ql.io.AcidOutputFormat") .isAssignableFrom(outputFormatClass)) { return true; } } catch (ClassNotFoundException e) { //if a table is using some custom I/O format and it's not in the classpath, we won't mark //the table for Acid, but today (Hive 3.1 and earlier) OrcInput/OutputFormat is the only //Acid format LOG.error("Could not determine if " + fullTableName + " can be made Acid due to: " + e.getMessage(), e); return false; } return false; } private static void makeConvertTableScript(List<String> alterTableAcid, List<String> alterTableMm,
/** * Check that InputFormatClass/OutputFormatClass should implement * AcidInputFormat/AcidOutputFormat */ public static boolean conformToAcid(Table table) throws MetaException { StorageDescriptor sd = table.getSd(); try { Class inputFormatClass = sd.getInputFormat() == null ? null : Class.forName(sd.getInputFormat()); Class outputFormatClass = sd.getOutputFormat() == null ? null : Class.forName(sd.getOutputFormat()); if (inputFormatClass == null || outputFormatClass == null || !Class.forName("org.apache.hadoop.hive.ql.io.AcidInputFormat").isAssignableFrom(inputFormatClass) || !Class.forName("org.apache.hadoop.hive.ql.io.AcidOutputFormat").isAssignableFrom(outputFormatClass)) { return false; } } catch (ClassNotFoundException e) { LOG.warn("Could not verify InputFormat=" + sd.getInputFormat() + " or OutputFormat=" + sd.getOutputFormat() + " for " + Warehouse.getQualifiedName(table)); return false; } return true; }
/** * Will try to find reader class based on Hive table input format. * If reader class was not find, will use default reader class. * * @param config Hive table config * @return reader class */ private static Class<? extends HiveAbstractReader> getReaderClass(HiveSubScan config) { final String formatName = config.getTable().getSd().getInputFormat(); Class<? extends HiveAbstractReader> readerClass = HiveDefaultReader.class; if (READER_MAP.containsKey(formatName)) { readerClass = READER_MAP.get(formatName); } return readerClass; }
static StorerInfo extractStorerInfo(StorageDescriptor sd, Map<String, String> properties) throws IOException { Properties hcatProperties = new Properties(); for (String key : properties.keySet()) { hcatProperties.put(key, properties.get(key)); } // also populate with StorageDescriptor->SerDe.Parameters for (Map.Entry<String, String> param : sd.getSerdeInfo().getParameters().entrySet()) { hcatProperties.put(param.getKey(), param.getValue()); } return new StorerInfo( sd.getInputFormat(), sd.getOutputFormat(), sd.getSerdeInfo().getSerializationLib(), properties.get(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE), hcatProperties); }
/** * @return an instance of the {@link InputFormat} in this {@link StorageDescriptor}. */ public static InputFormat<?, ?> getInputFormat(StorageDescriptor sd) throws IOException { try { InputFormat<?, ?> inputFormat = ConstructorUtils.invokeConstructor((Class<? extends InputFormat>) Class.forName(sd.getInputFormat())); if (inputFormat instanceof JobConfigurable) { ((JobConfigurable) inputFormat).configure(new JobConf(getHadoopConfiguration())); } return inputFormat; } catch (ReflectiveOperationException re) { throw new IOException("Failed to instantiate input format.", re); } }
StorageDescriptorKey(String baseLocation, StorageDescriptor sd) { this.sd = sd; this.baseLocation = baseLocation; if (sd == null) { hashCode = Objects.hashCode(baseLocation); } else { // use the baseLocation provided instead of sd.getLocation() hashCode = Objects.hash(sd.getSerdeInfo() == null ? null : sd.getSerdeInfo().getSerializationLib(), sd.getInputFormat(), sd.getOutputFormat(), baseLocation, sd.getCols()); } }
final public Class<? extends InputFormat> getInputFormatClass() { if (inputFormatClass == null) { try { String className = tTable.getSd().getInputFormat(); if (className == null) { if (getStorageHandler() == null) { return null; } inputFormatClass = getStorageHandler().getInputFormatClass(); } else { inputFormatClass = (Class<? extends InputFormat>) Class.forName(className, true, Utilities.getSessionSpecifiedClassLoader()); } } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } return inputFormatClass; }
final public Class<? extends InputFormat> getInputFormatClass() { if (inputFormatClass == null) { try { String className = tTable.getSd().getInputFormat(); if (className == null) { if (getStorageHandler() == null) { return null; } inputFormatClass = getStorageHandler().getInputFormatClass(); } else { inputFormatClass = (Class<? extends InputFormat>) Class.forName(className, true, Utilities.getSessionSpecifiedClassLoader()); } } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } return inputFormatClass; }
final public Class<? extends InputFormat> getInputFormatClass() throws HiveException { if (inputFormatClass == null) { // sd can be null for views String clsName = tPartition.getSd() == null ? null : tPartition.getSd().getInputFormat(); if (clsName == null) { return inputFormatClass = table.getInputFormatClass(); } try { inputFormatClass = ((Class<? extends InputFormat>) Class.forName(clsName, true, Utilities.getSessionSpecifiedClassLoader())); } catch (ClassNotFoundException e) { throw new HiveException("Class not found: " + clsName, e); } } return inputFormatClass; }
/** * Tell whether a hive table is actually an Avro table * @param targetTable * @return * @throws IOException */ public static boolean isHiveTableAvroType(Table targetTable) throws IOException { String serializationLib = targetTable.getTTable().getSd().getSerdeInfo().getSerializationLib(); String inputFormat = targetTable.getTTable().getSd().getInputFormat(); String outputFormat = targetTable.getTTable().getSd().getOutputFormat(); return inputFormat.endsWith("AvroContainerInputFormat") || outputFormat.endsWith("AvroContainerOutputFormat") || serializationLib.endsWith("AvroSerDe"); } }
final public Class<? extends InputFormat> getInputFormatClass() throws HiveException { if (inputFormatClass == null) { // sd can be null for views String clsName = tPartition.getSd() == null ? null : tPartition.getSd().getInputFormat(); if (clsName == null) { return inputFormatClass = table.getInputFormatClass(); } try { inputFormatClass = ((Class<? extends InputFormat>) Class.forName(clsName, true, Utilities.getSessionSpecifiedClassLoader())); } catch (ClassNotFoundException e) { throw new HiveException("Class not found: " + clsName, e); } } return inputFormatClass; }
public static void fromMetastoreApiStorageDescriptor(StorageDescriptor storageDescriptor, Storage.Builder builder, String tablePartitionName) { SerDeInfo serdeInfo = storageDescriptor.getSerdeInfo(); if (serdeInfo == null) { throw new PrestoException(HIVE_INVALID_METADATA, "Table storage descriptor is missing SerDe info"); } builder.setStorageFormat(StorageFormat.createNullable(serdeInfo.getSerializationLib(), storageDescriptor.getInputFormat(), storageDescriptor.getOutputFormat())) .setLocation(nullToEmpty(storageDescriptor.getLocation())) .setBucketProperty(HiveBucketProperty.fromStorageDescriptor(storageDescriptor, tablePartitionName)) .setSkewed(storageDescriptor.isSetSkewedInfo() && storageDescriptor.getSkewedInfo().isSetSkewedColNames() && !storageDescriptor.getSkewedInfo().getSkewedColNames().isEmpty()) .setSerdeParameters(serdeInfo.getParameters() == null ? ImmutableMap.of() : serdeInfo.getParameters()); }
@Test public void testAlterTblFFpart() throws Exception { driver.run("drop table junit_sem_analysis"); driver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as TEXTFILE"); driver.run("alter table junit_sem_analysis add partition (b='2010-10-10')"); hcatDriver.run("alter table junit_sem_analysis partition (b='2010-10-10') set fileformat RCFILE"); Table tbl = client.getTable(Warehouse.DEFAULT_DATABASE_NAME, TBL_NAME); assertEquals(TextInputFormat.class.getName(), tbl.getSd().getInputFormat()); assertEquals(HiveIgnoreKeyTextOutputFormat.class.getName(), tbl.getSd().getOutputFormat()); List<String> partVals = new ArrayList<String>(1); partVals.add("2010-10-10"); Partition part = client.getPartition(Warehouse.DEFAULT_DATABASE_NAME, TBL_NAME, partVals); assertEquals(RCFileInputFormat.class.getName(), part.getSd().getInputFormat()); assertEquals(RCFileOutputFormat.class.getName(), part.getSd().getOutputFormat()); hcatDriver.run("drop table junit_sem_analysis"); }
@Test public void testAlterTableSetFF() throws Exception { hcatDriver.run("drop table junit_sem_analysis"); hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE"); Table tbl = client.getTable(Warehouse.DEFAULT_DATABASE_NAME, TBL_NAME); assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat()); assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat()); hcatDriver.run("alter table junit_sem_analysis set fileformat INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' OUTPUTFORMAT " + "'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'mydriver' outputdriver 'yourdriver'"); hcatDriver.run("desc extended junit_sem_analysis"); tbl = client.getTable(Warehouse.DEFAULT_DATABASE_NAME, TBL_NAME); assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat()); assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat()); hcatDriver.run("drop table junit_sem_analysis"); }
@Test public void testAddDriverInfo() throws Exception { hcatDriver.run("drop table junit_sem_analysis"); query = "create table junit_sem_analysis (a int) partitioned by (b string) stored as " + "INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' OUTPUTFORMAT " + "'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'mydriver' outputdriver 'yourdriver' "; assertEquals(0, hcatDriver.run(query).getResponseCode()); Table tbl = client.getTable(Warehouse.DEFAULT_DATABASE_NAME, TBL_NAME); assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat()); assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat()); hcatDriver.run("drop table junit_sem_analysis"); }
@Test public void testCreateTableIfNotExists() throws Exception { hcatDriver.run("drop table " + TBL_NAME); hcatDriver.run("create table " + TBL_NAME + " (a int) stored as RCFILE"); Table tbl = client.getTable(Warehouse.DEFAULT_DATABASE_NAME, TBL_NAME); List<FieldSchema> cols = tbl.getSd().getCols(); assertEquals(1, cols.size()); assertTrue(cols.get(0).equals(new FieldSchema("a", "int", null))); assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat()); assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat()); CommandProcessorResponse resp = hcatDriver.run("create table if not exists junit_sem_analysis (a int) stored as RCFILE"); assertEquals(0, resp.getResponseCode()); assertNull(resp.getErrorMessage()); tbl = client.getTable(Warehouse.DEFAULT_DATABASE_NAME, TBL_NAME); cols = tbl.getSd().getCols(); assertEquals(1, cols.size()); assertTrue(cols.get(0).equals(new FieldSchema("a", "int", null))); assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat()); assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat()); hcatDriver.run("drop table junit_sem_analysis"); }
private void alterPartitionSpecInMemory(Table tbl, Map<String, String> partSpec, org.apache.hadoop.hive.metastore.api.Partition tpart, boolean inheritTableSpecs, String partPath) throws HiveException, InvalidOperationException { LOG.debug("altering partition for table " + tbl.getTableName() + " with partition spec : " + partSpec); if (inheritTableSpecs) { tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat()); tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat()); tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib()); tpart.getSd().getSerdeInfo().setParameters( tbl.getTTable().getSd().getSerdeInfo().getParameters()); tpart.getSd().setBucketCols(tbl.getBucketCols()); tpart.getSd().setNumBuckets(tbl.getNumBuckets()); tpart.getSd().setSortCols(tbl.getSortCols()); } if (partPath == null || partPath.trim().equals("")) { throw new HiveException("new partition path should not be null or empty."); } tpart.getSd().setLocation(partPath); }
private void verifyPartitionSharedSD(Table table, String name, List<String> values, int index) throws Exception { Partition part = client.getPartition(table.getDbName(), table.getTableName(), name); Assert.assertNotNull(part); Assert.assertEquals(table.getTableName(), part.getTableName()); List<String> partValues = part.getValues(); Assert.assertEquals(values.size(), partValues.size()); Assert.assertTrue(partValues.containsAll(values)); Assert.assertEquals(table.getDbName(), part.getDbName()); Assert.assertEquals(DEFAULT_CREATE_TIME, part.getLastAccessTime()); Assert.assertEquals(DEFAULT_PARAM_VALUE + index, part.getParameters().get(DEFAULT_PARAM_KEY + index)); Assert.assertFalse(part.getParameters().keySet().contains(table.getParameters().keySet())); StorageDescriptor sd = part.getSd(); Assert.assertNotNull(sd); Assert.assertEquals("TestInputFormat", sd.getInputFormat()); Assert.assertEquals("TestOutputFormat", sd.getOutputFormat()); Assert.assertEquals("sharedSDPartSerde", sd.getSerdeInfo().getName()); Assert.assertEquals("testSDParamValue", sd.getParameters().get("testSDParamKey")); Assert.assertEquals( metaStore.getWarehouseRoot() + "/" + TABLE_NAME + "/sharedSDTest/partwithoutsd" + index, sd.getLocation()); Assert.assertTrue(metaStore.isPathExists(new Path(sd.getLocation()))); }