@Override public void addSerDeProperties(HiveRegistrationUnit source, HiveRegistrationUnit target) throws IOException { if (source.getSerDeType().isPresent()) { target.setSerDeType(source.getSerDeType().get()); } if (source.getInputFormat().isPresent()) { target.setInputFormat(source.getInputFormat().get()); } if (source.getOutputFormat().isPresent()) { target.setOutputFormat(source.getOutputFormat().get()); } if (source.getSerDeProps().contains(SCHEMA_LITERAL)) { target.setSerDeProp(SCHEMA_LITERAL, source.getSerDeProps().getProp(SCHEMA_LITERAL)); } if (source.getSerDeProps().contains(SCHEMA_URL)) { target.setSerDeProp(SCHEMA_URL, source.getSerDeProps().getProp(SCHEMA_URL)); } }
private static StorageDescriptor getStorageDescriptor(HiveRegistrationUnit unit) { State props = unit.getStorageProps(); StorageDescriptor sd = new StorageDescriptor(); sd.setParameters(getParameters(props)); sd.setCols(getFieldSchemas(unit)); if (unit.getLocation().isPresent()) { sd.setLocation(unit.getLocation().get()); } if (unit.getInputFormat().isPresent()) { sd.setInputFormat(unit.getInputFormat().get()); } if (unit.getOutputFormat().isPresent()) { sd.setOutputFormat(unit.getOutputFormat().get()); } if (unit.getIsCompressed().isPresent()) { sd.setCompressed(unit.getIsCompressed().get()); } if (unit.getNumBuckets().isPresent()) { sd.setNumBuckets(unit.getNumBuckets().get()); } if (unit.getBucketColumns().isPresent()) { sd.setBucketCols(unit.getBucketColumns().get()); } if (unit.getIsStoredAsSubDirs().isPresent()) { sd.setStoredAsSubDirectories(unit.getIsStoredAsSubDirs().get()); } sd.setSerdeInfo(getSerDeInfo(unit)); return sd; }
/** * First tries getting the {@code FieldSchema}s from the {@code HiveRegistrationUnit}'s columns, if set. * Else, gets the {@code FieldSchema}s from the deserializer. */ private static List<FieldSchema> getFieldSchemas(HiveRegistrationUnit unit) { List<Column> columns = unit.getColumns(); List<FieldSchema> fieldSchemas = new ArrayList<>(); if (columns != null && columns.size() > 0) { fieldSchemas = getFieldSchemas(columns); } else { Deserializer deserializer = getDeserializer(unit); if (deserializer != null) { try { fieldSchemas = MetaStoreUtils.getFieldsFromDeserializer(unit.getTableName(), deserializer); } catch (SerDeException | MetaException e) { LOG.warn("Encountered exception while getting fields from deserializer.", e); } } } return fieldSchemas; }
@SuppressWarnings("unchecked") public T compareParameters() { if (!this.result) { checkExistingIsSuperstate(this.existingUnit.getProps(), this.newUnit.getProps()); checkExistingIsSuperstate(this.existingUnit.getStorageProps(), this.newUnit.getStorageProps()); checkExistingIsSuperstate(this.existingUnit.getSerDeProps(), this.newUnit.getSerDeProps()); } return (T) this; }
@Override public void updateSchema(HiveRegistrationUnit existingUnit, HiveRegistrationUnit newUnit) throws IOException { Preconditions.checkArgument( newUnit.getSerDeProps().contains(SCHEMA_LITERAL) || newUnit.getSerDeProps().contains(SCHEMA_URL)); if (newUnit.getSerDeProps().contains(SCHEMA_LITERAL)) { existingUnit.setSerDeProp(SCHEMA_LITERAL, newUnit.getSerDeProps().getProp(SCHEMA_LITERAL)); } else { existingUnit.setSerDeProp(SCHEMA_URL, newUnit.getSerDeProps().getProp(SCHEMA_URL)); } }
Optional<String> serdeClass = unit.getSerDeType(); if (!serdeClass.isPresent()) { return null; props.putAll(unit.getProps().getProperties()); props.putAll(unit.getStorageProps().getProperties()); props.putAll(unit.getSerDeProps().getProperties()); LOG.warn("Failed to initialize serde " + serde + " with properties " + props + " for table " + unit.getDbName() + "." + unit.getTableName()); return null;
private static SerDeInfo getSerDeInfo(HiveRegistrationUnit unit) { State props = unit.getSerDeProps(); SerDeInfo si = new SerDeInfo(); si.setParameters(getParameters(props)); si.setName(unit.getTableName()); if (unit.getSerDeType().isPresent()) { si.setSerializationLib(unit.getSerDeType().get()); } return si; }
@Override public boolean haveSameSchema(HiveRegistrationUnit unit1, HiveRegistrationUnit unit2) { if (unit1.getSerDeProps().contains(HiveAvroSerDeManager.SCHEMA_LITERAL) && unit2.getSerDeProps().contains(HiveAvroSerDeManager.SCHEMA_LITERAL)) { return unit1.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_LITERAL) .equals(unit2.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_LITERAL)); } else if (unit1.getSerDeProps().contains(HiveAvroSerDeManager.SCHEMA_URL) && unit2.getSerDeProps().contains(HiveAvroSerDeManager.SCHEMA_URL)) { return unit1.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_URL) .equals(unit2.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_URL)); } return false; }
@SuppressWarnings("unchecked") public T compareIsCompressed() { if (!this.result) { compare(this.existingUnit.getIsCompressed(), this.newUnit.getIsCompressed()); } return (T) this; }
@SuppressWarnings("unchecked") public T compareBucketCols() { if (!this.result) { compare(this.existingUnit.getBucketColumns(), this.newUnit.getBucketColumns()); } return (T) this; }
@SuppressWarnings("unchecked") public T compareNumBuckets() { if (!this.result) { compare(this.existingUnit.getNumBuckets(), this.newUnit.getNumBuckets()); } return (T) this; }
@SuppressWarnings("unchecked") public T compareInputFormat() { if (!this.result) { compare(this.existingUnit.getInputFormat(), this.newUnit.getInputFormat()); } return (T) this; }
@SuppressWarnings("unchecked") public T compareIsStoredAsSubDirs() { if (!this.result) { compare(this.existingUnit.getIsStoredAsSubDirs(), this.newUnit.getIsStoredAsSubDirs()); } return (T) this; }
@SuppressWarnings("unchecked") public T compareOutputFormat() { if (!this.result) { compare(this.existingUnit.getOutputFormat(), this.newUnit.getOutputFormat()); } return (T) this; }
/** * Compare the raw locations (without schema and authority). * * <p> * This is useful since existing tables/partitions in the Hive metastore have absolute paths in the location * property, but the new table/partition may have a raw path. * </p> */ @SuppressWarnings("unchecked") public T compareRawLocation() { if (!this.result) { this.result |= (!new Path(this.existingUnit.getLocation().get()).toUri().getRawPath() .equals(new Path(this.newUnit.getLocation().get()).toUri().getRawPath())); } return (T) this; }
Optional<String> serdeClass = unit.getSerDeType(); if (!serdeClass.isPresent()) { return null; props.putAll(unit.getProps().getProperties()); props.putAll(unit.getStorageProps().getProperties()); props.putAll(unit.getSerDeProps().getProperties()); LOG.warn("Failed to initialize serde " + serde + " with properties " + props + " for table " + unit.getDbName() + "." + unit.getTableName()); return null;
@SuppressWarnings("unchecked") public T compareParameters() { if (!this.result) { checkExistingIsSuperstate(this.existingUnit.getProps(), this.newUnit.getProps()); checkExistingIsSuperstate(this.existingUnit.getStorageProps(), this.newUnit.getStorageProps()); checkExistingIsSuperstate(this.existingUnit.getSerDeProps(), this.newUnit.getSerDeProps()); } return (T) this; }
private static SerDeInfo getSerDeInfo(HiveRegistrationUnit unit) { State props = unit.getSerDeProps(); SerDeInfo si = new SerDeInfo(); si.setParameters(getParameters(props)); si.setName(unit.getTableName()); if (unit.getSerDeType().isPresent()) { si.setSerializationLib(unit.getSerDeType().get()); } return si; }
@Override public void updateSchema(HiveRegistrationUnit existingUnit, HiveRegistrationUnit newUnit) throws IOException { Preconditions.checkArgument( newUnit.getSerDeProps().contains(SCHEMA_LITERAL) || newUnit.getSerDeProps().contains(SCHEMA_URL)); if (newUnit.getSerDeProps().contains(SCHEMA_LITERAL)) { existingUnit.setSerDeProp(SCHEMA_LITERAL, newUnit.getSerDeProps().getProp(SCHEMA_LITERAL)); } else { existingUnit.setSerDeProp(SCHEMA_URL, newUnit.getSerDeProps().getProp(SCHEMA_URL)); } }
private void validateSchemaUrl(State state, String targetSchemaFileName, boolean createConflictingFile) throws IOException { HiveAvroSerDeManager manager = new HiveAvroSerDeManager(state); HiveRegistrationUnit registrationUnit = (new HiveTable.Builder()).withDbName(TEST_DB).withTableName(TEST_TABLE).build(); // Clean up existing file String targetPathStr = new Path(this.testBasePath, targetSchemaFileName).toString(); File targetFile = new File(targetPathStr); targetFile.delete(); // create a conflicting file if (createConflictingFile) { targetFile.createNewFile(); } manager.addSerDeProperties(this.testBasePath, registrationUnit); Assert.assertNull(registrationUnit.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_LITERAL)); String schemaUrl = registrationUnit.getSerDeProps().getProp(HiveAvroSerDeManager.SCHEMA_URL); Assert.assertEquals(schemaUrl, targetPathStr); Assert.assertTrue(IOUtils.contentEquals(this.getClass().getResourceAsStream("/test-hive-table/hive-test.avsc"), new FileInputStream(schemaUrl))); }