protected void parseAuthenticationResponse(JsonObject jsonRet) throws RestApiConnectionException { if (!hasId(jsonRet)) { throw new RestApiConnectionException("Failed on authentication with the following HTTP response received:" + jsonRet.toString()); } this.instanceUrl = jsonRet.get("instance_url").getAsString(); this.accessToken = jsonRet.get("access_token").getAsString(); this.createdAt = System.currentTimeMillis(); } }
@Override public Extractor<JsonArray, JsonElement> getExtractor(WorkUnitState state) throws IOException { Extractor<JsonArray, JsonElement> extractor = null; try { extractor = new MysqlExtractor(state).build(); } catch (ExtractPrepareException e) { LOG.error("Failed to prepare extractor: error - " + e.getMessage()); throw new IOException(e); } return extractor; }
@Override public long getSourceCount(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws RecordCountException { this.log.info("Get source record count using JDBC"); long count = 0; try { List<Command> cmds = this.getCountMetadata(schema, entity, workUnit, predicateList); CommandOutput<?, ?> response = this.executeSql(cmds); count = this.getCount(response); this.log.info("Source record count:" + count); return count; } catch (Exception e) { throw new RecordCountException("Failed to get source record count using JDBC; error - " + e.getMessage(), e); } }
throw new ExtractPrepareException("Failed to get schema for this object; error - " + e.getMessage(), e); } catch (HighWatermarkException e) { throw new ExtractPrepareException("Failed to get high watermark; error - " + e.getMessage(), e); } catch (RecordCountException e) { throw new ExtractPrepareException("Failed to get record count; error - " + e.getMessage(), e); } catch (Exception e) { throw new ExtractPrepareException("Failed to prepare the extract build; error - " + e.getMessage(), e);
@Override public long getMaxWatermark(String schema, String entity, String watermarkColumn, List<Predicate> predicateList, String watermarkSourceFormat) throws HighWatermarkException { this.log.info("Get high watermark using JDBC"); long calculatedHighWatermark = ConfigurationKeys.DEFAULT_WATERMARK_VALUE; try { List<Command> cmds = this.getHighWatermarkMetadata(schema, entity, watermarkColumn, predicateList); CommandOutput<?, ?> response = this.executeSql(cmds); calculatedHighWatermark = this.getHighWatermark(response, watermarkColumn, watermarkSourceFormat); return calculatedHighWatermark; } catch (Exception e) { throw new HighWatermarkException("Failed to get high watermark using JDBC; error - " + e.getMessage(), e); } }
resultset = itr.next(); } else { throw new SchemaException("Failed to get schema from database - Resultset has no records"); throw new SchemaException("Failed to get schema from database; error - " + e.getMessage(), e);
private static String buildUrl(String path, List<NameValuePair> qparams) throws RestApiClientException { URIBuilder builder = new URIBuilder(); builder.setPath(path); ListIterator<NameValuePair> i = qparams.listIterator(); while (i.hasNext()) { NameValuePair keyValue = i.next(); builder.setParameter(keyValue.getName(), keyValue.getValue()); } URI uri; try { uri = builder.build(); } catch (Exception e) { throw new RestApiClientException("Failed to build url; error - " + e.getMessage(), e); } return new HttpGet(uri).getURI().toString(); }
@Override public long getSourceCount(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws RecordCountException { log.info("Get source record count using Rest Api"); long count = 0; try { boolean success = this.connector.connect(); if (!success) { throw new RecordCountException("Failed to connect."); } log.debug("Connected successfully."); List<Command> cmds = this.getCountMetadata(schema, entity, workUnit, predicateList); CommandOutput<?, ?> response = this.connector.getResponse(cmds); count = getCount(response); log.info("Source record count:" + count); return count; } catch (Exception e) { throw new RecordCountException("Failed to get record count using rest api; error - " + e.getMessage(), e); } }
@Override public long getMaxWatermark(String schema, String entity, String watermarkColumn, List<Predicate> predicateList, String watermarkSourceFormat) throws HighWatermarkException { log.info("Get high watermark using Rest Api"); long CalculatedHighWatermark = -1; try { boolean success = this.connector.connect(); if (!success) { throw new HighWatermarkException("Failed to connect."); } log.debug("Connected successfully."); List<Command> cmds = this.getHighWatermarkMetadata(schema, entity, watermarkColumn, predicateList); CommandOutput<?, ?> response = this.connector.getResponse(cmds); CalculatedHighWatermark = this.getHighWatermark(response, watermarkColumn, watermarkSourceFormat); log.info("High watermark:" + CalculatedHighWatermark); return CalculatedHighWatermark; } catch (Exception e) { throw new HighWatermarkException("Failed to get high watermark using rest api; error - " + e.getMessage(), e); } }
resultset = itr.next(); } else { throw new SchemaException("Failed to get schema from Teradata - empty schema resultset"); throw new SchemaException("Failed to get schema from Teradaa; error - " + e.getMessage(), e);
public Extractor<JsonArray, JsonElement> getExtractor(WorkUnitState state) throws IOException { Extractor<JsonArray, JsonElement> extractor = null; try { extractor = new TeradataExtractor(state).build(); } catch (ExtractPrepareException e) { log.error("Failed to prepare extractor: error - {}", e.getMessage()); throw new IOException(e); } return extractor; } }
throw new RestApiConnectionException("Failed to authenticate salesforce host:" + host + "; error-" + e.getMessage(), e);
@Override public long getCount(CommandOutput<?, ?> response) throws RecordCountException { log.info("Get source record count from salesforce"); String output; Iterator<String> itr = (Iterator<String>) response.getResults().values().iterator(); if (itr.hasNext()) { output = itr.next(); } else { throw new RecordCountException("Failed to get count from salesforce; REST response has no output"); } JsonElement element = GSON.fromJson(output, JsonObject.class); long count; try { JsonObject jsonObject = element.getAsJsonObject(); count = jsonObject.get("totalSize").getAsLong(); } catch (Exception e) { throw new RecordCountException("Failed to get record count from salesforce; error - " + e.getMessage(), e); } return count; }
resultset = itr.next(); } else { throw new HighWatermarkException("Failed to get high watermark from database - Resultset has no records"); throw new HighWatermarkException("Failed to get high watermark from database; error - " + e.getMessage(), e);
@Override public Extractor<JsonArray, JsonElement> getExtractor(WorkUnitState state) throws IOException { Extractor<JsonArray, JsonElement> extractor = null; try { extractor = new SqlServerExtractor(state).build(); } catch (ExtractPrepareException e) { LOG.error("Failed to prepare extractor: error - " + e.getMessage()); throw new IOException(e); } return extractor; } }
@Override public long getCount(CommandOutput<?, ?> response) throws RecordCountException { this.log.debug("Extract source record count from resultset"); ResultSet resultset = null; long count = 0; Iterator<ResultSet> itr = (Iterator<ResultSet>) response.getResults().values().iterator(); if (itr.hasNext()) { resultset = itr.next(); try { if (resultset.next()) { count = resultset.getLong(1); } } catch (Exception e) { throw new RecordCountException("Failed to get source record count from database; error - " + e.getMessage(), e); } } else { throw new RuntimeException("Failed to get source record count from database - Resultset has no records"); } return count; }
return constructGetCommand(this.sfConnector.getFullUri(getSoqlUrl(query))); } catch (Exception e) { throw new HighWatermarkException("Failed to get salesforce url for high watermark; error - " + e.getMessage(), e);
@Override public Extractor<JsonArray, JsonElement> getExtractor(WorkUnitState state) throws IOException { Extractor<JsonArray, JsonElement> extractor = null; try { extractor = new OracleExtractor(state).build(); } catch (ExtractPrepareException e) { LOG.error("Failed to prepare extractor: error - " + e.getMessage()); throw new IOException(e); } return extractor; } }
return constructGetCommand(this.sfConnector.getFullUri(getSoqlUrl(query))); } catch (Exception e) { throw new RecordCountException("Failed to get salesforce url for record count; error - " + e.getMessage(), e);
@Override public Extractor<JsonArray, JsonElement> getExtractor(WorkUnitState state) throws IOException { Extractor<JsonArray, JsonElement> extractor; try { extractor = new PostgresqlExtractor(state).build(); } catch (ExtractPrepareException e) { LOG.error("Failed to prepare extractor: error - " + e.getMessage()); throw new IOException(e); } return extractor; } }