@Override public void publish(SqoopJobDataPublisher.Data data) throws AtlasHookException { try { Configuration atlasProperties = ApplicationProperties.get(); String clusterName = atlasProperties.getString(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME); Referenceable dbStoreRef = createDBStoreInstance(data); Referenceable dbRef = createHiveDatabaseInstance(clusterName, data.getHiveDB()); Referenceable hiveTableRef = createHiveTableInstance(clusterName, dbRef, data.getHiveTable(), data.getHiveDB()); Referenceable procRef = createSqoopProcessInstance(dbStoreRef, hiveTableRef, data, clusterName); int maxRetries = atlasProperties.getInt(HOOK_NUM_RETRIES, 3); HookNotification.HookNotificationMessage message = new HookNotification.EntityCreateRequest(AtlasHook.getUser(), dbStoreRef, dbRef, hiveTableRef, procRef); AtlasHook.notifyEntities(Arrays.asList(message), maxRetries); } catch(Exception e) { throw new AtlasHookException("SqoopHook.publish() failed.", e); } } }
private AtlasEntity toSqoopProcessEntity(AtlasEntity entDbStore, AtlasEntity entHiveDb, AtlasEntity entHiveTable, SqoopJobDataPublisher.Data data, String clusterName) { AtlasEntity entProcess = new AtlasEntity(SqoopDataTypes.SQOOP_PROCESS.getName()); String sqoopProcessName = getSqoopProcessName(data, clusterName); Map<String, String> sqoopOptionsMap = new HashMap<>(); Properties options = data.getOptions(); for (Object k : options.keySet()) { sqoopOptionsMap.put((String)k, (String) options.get(k)); } entProcess.setAttribute(AtlasClient.NAME, sqoopProcessName); entProcess.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, sqoopProcessName); entProcess.setAttribute(SqoopHook.OPERATION, data.getOperation()); List<AtlasObjectId> sqoopObjects = Collections.singletonList(AtlasTypeUtil.getAtlasObjectId(entDbStore)); List<AtlasObjectId> hiveObjects = Collections.singletonList(AtlasTypeUtil.getAtlasObjectId(entHiveTable != null ? entHiveTable : entHiveDb)); if (isImportOperation(data)) { entProcess.setAttribute(SqoopHook.INPUTS, sqoopObjects); entProcess.setAttribute(SqoopHook.OUTPUTS, hiveObjects); } else { entProcess.setAttribute(SqoopHook.INPUTS, hiveObjects); entProcess.setAttribute(SqoopHook.OUTPUTS, sqoopObjects); } entProcess.setAttribute(SqoopHook.USER, data.getUser()); entProcess.setAttribute(SqoopHook.START_TIME, new Date(data.getStartTime())); entProcess.setAttribute(SqoopHook.END_TIME, new Date(data.getEndTime())); entProcess.setAttribute(SqoopHook.CMD_LINE_OPTS, sqoopOptionsMap); return entProcess; }
private AtlasEntity toSqoopDBStoreEntity(SqoopJobDataPublisher.Data data) throws ImportException { String table = data.getStoreTable(); String query = data.getStoreQuery(); if (StringUtils.isBlank(table) && StringUtils.isBlank(query)) { throw new ImportException("Both table and query cannot be empty for DBStoreInstance"); } String usage = table != null ? "TABLE" : "QUERY"; String source = table != null ? table : query; String name = getSqoopDBStoreName(data); AtlasEntity entDbStore = new AtlasEntity(SqoopDataTypes.SQOOP_DBDATASTORE.getName()); entDbStore.setAttribute(AtlasClient.NAME, name); entDbStore.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name); entDbStore.setAttribute(SqoopHook.DB_STORE_TYPE, data.getStoreType()); entDbStore.setAttribute(SqoopHook.DB_STORE_USAGE, usage); entDbStore.setAttribute(SqoopHook.STORE_URI, data.getUrl()); entDbStore.setAttribute(SqoopHook.SOURCE, source); entDbStore.setAttribute(SqoopHook.DESCRIPTION, ""); entDbStore.setAttribute(AtlasClient.OWNER, data.getUser()); return entDbStore; }