static String getSqoopProcessName(Data data, String clusterName) { StringBuilder name = new StringBuilder(String.format("sqoop %s --connect %s", data.getOperation(), data.getUrl())); if (StringUtils.isNotEmpty(data.getHiveTable())) { name.append(" --table ").append(data.getStoreTable()); } else { name.append(" --database ").append(data.getHiveDB()); } if (StringUtils.isNotEmpty(data.getStoreQuery())) { name.append(" --query ").append(data.getStoreQuery()); } if (data.getHiveTable() != null) { name.append(String.format(" --hive-%s --hive-database %s --hive-table %s --hive-cluster %s", data.getOperation(), data.getHiveDB().toLowerCase(), data.getHiveTable().toLowerCase(), clusterName)); } else { name.append(String.format("--hive-%s --hive-database %s --hive-cluster %s", data.getOperation(), data.getHiveDB(), clusterName)); } return name.toString(); }
static String getSqoopDBStoreName(SqoopJobDataPublisher.Data data) { StringBuilder name = new StringBuilder(String.format("%s --url %s", data.getStoreType(), data.getUrl())); if (StringUtils.isNotEmpty(data.getHiveTable())) { name.append(" --table ").append(data.getStoreTable()); } else { name.append(" --database ").append(data.getHiveDB()); } if (StringUtils.isNotEmpty(data.getStoreQuery())) { name.append(" --query ").append(data.getStoreQuery()); } return name.toString(); }
procRef.set(SqoopHook.START_TIME, new Date(data.getStartTime())); procRef.set(SqoopHook.END_TIME, new Date(data.getEndTime())); Properties options = data.getOptions(); for (Object k : options.keySet()) { sqoopOptionsMap.put((String)k, (String) options.get(k));
private AtlasEntity toSqoopProcessEntity(AtlasEntity entDbStore, AtlasEntity entHiveDb, AtlasEntity entHiveTable, SqoopJobDataPublisher.Data data, String clusterName) { AtlasEntity entProcess = new AtlasEntity(SqoopDataTypes.SQOOP_PROCESS.getName()); String sqoopProcessName = getSqoopProcessName(data, clusterName); Map<String, String> sqoopOptionsMap = new HashMap<>(); Properties options = data.getOptions(); for (Object k : options.keySet()) { sqoopOptionsMap.put((String)k, (String) options.get(k)); } entProcess.setAttribute(AtlasClient.NAME, sqoopProcessName); entProcess.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, sqoopProcessName); entProcess.setAttribute(SqoopHook.OPERATION, data.getOperation()); List<AtlasObjectId> sqoopObjects = Collections.singletonList(AtlasTypeUtil.getAtlasObjectId(entDbStore)); List<AtlasObjectId> hiveObjects = Collections.singletonList(AtlasTypeUtil.getAtlasObjectId(entHiveTable != null ? entHiveTable : entHiveDb)); if (isImportOperation(data)) { entProcess.setAttribute(SqoopHook.INPUTS, sqoopObjects); entProcess.setAttribute(SqoopHook.OUTPUTS, hiveObjects); } else { entProcess.setAttribute(SqoopHook.INPUTS, hiveObjects); entProcess.setAttribute(SqoopHook.OUTPUTS, sqoopObjects); } entProcess.setAttribute(SqoopHook.USER, data.getUser()); entProcess.setAttribute(SqoopHook.START_TIME, new Date(data.getStartTime())); entProcess.setAttribute(SqoopHook.END_TIME, new Date(data.getEndTime())); entProcess.setAttribute(SqoopHook.CMD_LINE_OPTS, sqoopOptionsMap); return entProcess; }
@Override public void publish(SqoopJobDataPublisher.Data data) throws AtlasHookException { try { Configuration atlasProperties = ApplicationProperties.get(); String clusterName = atlasProperties.getString(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME); AtlasEntity entDbStore = toSqoopDBStoreEntity(data); AtlasEntity entHiveDb = toHiveDatabaseEntity(clusterName, data.getHiveDB()); AtlasEntity entHiveTable = data.getHiveTable() != null ? toHiveTableEntity(entHiveDb, data.getHiveTable()) : null; AtlasEntity entProcess = toSqoopProcessEntity(entDbStore, entHiveDb, entHiveTable, data, clusterName); AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo(entProcess); entities.addReferredEntity(entDbStore); entities.addReferredEntity(entHiveDb); if (entHiveTable != null) { entities.addReferredEntity(entHiveTable); } HookNotification message = new EntityCreateRequestV2(AtlasHook.getUser(), entities); atlasHook.sendNotification(message); } catch(Exception e) { LOG.error("SqoopHook.publish() failed", e); throw new AtlasHookException("SqoopHook.publish() failed.", e); } }
private AtlasEntity toSqoopDBStoreEntity(SqoopJobDataPublisher.Data data) throws ImportException { String table = data.getStoreTable(); String query = data.getStoreQuery(); if (StringUtils.isBlank(table) && StringUtils.isBlank(query)) { throw new ImportException("Both table and query cannot be empty for DBStoreInstance"); } String usage = table != null ? "TABLE" : "QUERY"; String source = table != null ? table : query; String name = getSqoopDBStoreName(data); AtlasEntity entDbStore = new AtlasEntity(SqoopDataTypes.SQOOP_DBDATASTORE.getName()); entDbStore.setAttribute(AtlasClient.NAME, name); entDbStore.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name); entDbStore.setAttribute(SqoopHook.DB_STORE_TYPE, data.getStoreType()); entDbStore.setAttribute(SqoopHook.DB_STORE_USAGE, usage); entDbStore.setAttribute(SqoopHook.STORE_URI, data.getUrl()); entDbStore.setAttribute(SqoopHook.SOURCE, source); entDbStore.setAttribute(SqoopHook.DESCRIPTION, ""); entDbStore.setAttribute(AtlasClient.OWNER, data.getUser()); return entDbStore; }
private boolean isImportOperation(SqoopJobDataPublisher.Data data) { return data.getOperation().toLowerCase().equals("import"); }
static String getSqoopProcessName(Data data, String clusterName) { StringBuilder name = new StringBuilder(String.format("sqoop %s --connect %s", data.getOperation(), data.getUrl())); if (StringUtils.isNotEmpty(data.getStoreTable())) { name.append(" --table ").append(data.getStoreTable()); } if (StringUtils.isNotEmpty(data.getStoreQuery())) { name.append(" --query ").append(data.getStoreQuery()); } name.append(String.format(" --hive-%s --hive-database %s --hive-table %s --hive-cluster %s", data.getOperation(), data.getHiveDB().toLowerCase(), data.getHiveTable().toLowerCase(), clusterName)); return name.toString(); }
static boolean isImportOperation(SqoopJobDataPublisher.Data data) { return data.getOperation().toLowerCase().equals("import"); }
@Override public void publish(Data data) { hiveTable = data.getHiveTable(); storeTable = data.getStoreTable(); storeType = data.getStoreType(); operation = data.getOperation(); } }