static String getSqoopDBStoreName(SqoopJobDataPublisher.Data data) { StringBuilder name = new StringBuilder(String.format("%s --url %s", data.getStoreType(), data.getUrl())); if (StringUtils.isNotEmpty(data.getHiveTable())) { name.append(" --table ").append(data.getStoreTable()); } else { name.append(" --database ").append(data.getHiveDB()); } if (StringUtils.isNotEmpty(data.getStoreQuery())) { name.append(" --query ").append(data.getStoreQuery()); } return name.toString(); }
static String getSqoopProcessName(Data data, String clusterName) { StringBuilder name = new StringBuilder(String.format("sqoop %s --connect %s", data.getOperation(), data.getUrl())); if (StringUtils.isNotEmpty(data.getHiveTable())) { name.append(" --table ").append(data.getStoreTable()); } else { name.append(" --database ").append(data.getHiveDB()); } if (StringUtils.isNotEmpty(data.getStoreQuery())) { name.append(" --query ").append(data.getStoreQuery()); } if (data.getHiveTable() != null) { name.append(String.format(" --hive-%s --hive-database %s --hive-table %s --hive-cluster %s", data.getOperation(), data.getHiveDB().toLowerCase(), data.getHiveTable().toLowerCase(), clusterName)); } else { name.append(String.format("--hive-%s --hive-database %s --hive-cluster %s", data.getOperation(), data.getHiveDB(), clusterName)); } return name.toString(); }
@Override public void publish(SqoopJobDataPublisher.Data data) throws AtlasHookException { try { Configuration atlasProperties = ApplicationProperties.get(); String clusterName = atlasProperties.getString(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME); AtlasEntity entDbStore = toSqoopDBStoreEntity(data); AtlasEntity entHiveDb = toHiveDatabaseEntity(clusterName, data.getHiveDB()); AtlasEntity entHiveTable = data.getHiveTable() != null ? toHiveTableEntity(entHiveDb, data.getHiveTable()) : null; AtlasEntity entProcess = toSqoopProcessEntity(entDbStore, entHiveDb, entHiveTable, data, clusterName); AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo(entProcess); entities.addReferredEntity(entDbStore); entities.addReferredEntity(entHiveDb); if (entHiveTable != null) { entities.addReferredEntity(entHiveTable); } HookNotification message = new EntityCreateRequestV2(AtlasHook.getUser(), entities); atlasHook.sendNotification(message); } catch(Exception e) { LOG.error("SqoopHook.publish() failed", e); throw new AtlasHookException("SqoopHook.publish() failed.", e); } }
@Override public void publish(Data data) { hiveTable = data.getHiveTable(); storeTable = data.getStoreTable(); storeType = data.getStoreType(); operation = data.getOperation(); } }