private Extract getExtractForFile(PartitionAwareFileRetriever.FileInfo file, String topicName, String namespace, Map<Long, Extract> extractMap) { Extract extract = extractMap.get(file.getWatermarkMsSinceEpoch()); if (extract == null) { // Create an extract object for the dayPath extract = new Extract(this.tableType, namespace, topicName); LOG.info("Created extract: " + extract.getExtractId() + " for path " + topicName); extractMap.put(file.getWatermarkMsSinceEpoch(), extract); } return extract; }
@Override public int hashCode() { return (this.getNamespace() + this.getTable() + this.getExtractId()).hashCode(); }
@Override public boolean equals(Object object) { if (!(object instanceof Extract)) { return false; } Extract other = (Extract) object; return super.equals(other) && this.getNamespace().equals(other.getNamespace()) && this.getTable().equals(other.getTable()) && this.getExtractId().equals(other.getExtractId()); }
/** * Get the writer output file path corresponding to this {@link Extract}. * * @return writer output file path corresponding to this {@link Extract} * @deprecated As {@code this.getIsFull} is deprecated. */ @Deprecated public String getOutputFilePath() { return this.getNamespace().replaceAll("\\.", "/") + "/" + this.getTable() + "/" + this.getExtractId() + "_" + (this.getIsFull() ? "full" : "append"); }
/** * Returns a unique {@link Extract} instance. * Any two calls of this method from the same {@link ExtractFactory} instance guarantees to * return {@link Extract}s with different IDs. * * @param type {@link TableType} * @param namespace dot separated namespace path * @param table table name * @return a unique {@link Extract} instance */ public synchronized Extract getUniqueExtract(TableType type, String namespace, String table) { Extract newExtract = new Extract(type, namespace, table); while (this.createdInstances.contains(newExtract)) { if (Strings.isNullOrEmpty(newExtract.getExtractId())) { newExtract.setExtractId(this.dtf.print(new DateTime())); } else { DateTime extractDateTime = this.dtf.parseDateTime(newExtract.getExtractId()); newExtract.setExtractId(this.dtf.print(extractDateTime.plusSeconds(1))); } } this.createdInstances.add(newExtract); return newExtract; } }
/** * Create a new properly populated {@link Extract} instance. * * <p> * This method should always return a new unique {@link Extract} instance. * </p> * * @param type {@link org.apache.gobblin.source.workunit.Extract.TableType} * @param namespace namespace of the table this extract belongs to * @param table name of the table this extract belongs to * @return a new unique {@link Extract} instance * * @Deprecated Use {@link org.apache.gobblin.source.extractor.extract.AbstractSource#createExtract( *org.apache.gobblin.source.workunit.Extract.TableType, String, String)} */ @Deprecated public synchronized Extract createExtract(Extract.TableType type, String namespace, String table) { Extract extract = new Extract(this, type, namespace, table); while (EXTRACT_SET.contains(extract)) { if (Strings.isNullOrEmpty(extract.getExtractId())) { extract.setExtractId(DTF.print(new DateTime())); } else { DateTime extractDateTime = DTF.parseDateTime(extract.getExtractId()); extract.setExtractId(DTF.print(extractDateTime.plusSeconds(1))); } } EXTRACT_SET.add(extract); return extract; }
/** * Verify that each {@link Extract} created by an {@ExtractFactory} has a unique ID. */ @Test public void testGetUniqueExtract() { ExtractFactory extractFactory = new ExtractFactory("yyyyMMddHHmmss"); Set<String> extractIDs = Sets.newHashSet(); int numOfExtracts = 100; for (int i = 0; i < numOfExtracts; i++) { extractIDs .add(extractFactory.getUniqueExtract(Extract.TableType.APPEND_ONLY, "namespace", "table").getExtractId()); } Assert.assertEquals(extractIDs.size(), numOfExtracts); } }
@Override public boolean equals(Object object) { if (!(object instanceof Extract)) { return false; } Extract other = (Extract) object; return super.equals(other) && this.getNamespace().equals(other.getNamespace()) && this.getTable().equals(other.getTable()) && this.getExtractId().equals(other.getExtractId()); }
@Override public int hashCode() { return (this.getNamespace() + this.getTable() + this.getExtractId()).hashCode(); }
private Extract getExtractForFile(PartitionAwareFileRetriever.FileInfo file, String topicName, String namespace, Map<Long, Extract> extractMap) { Extract extract = extractMap.get(file.getWatermarkMsSinceEpoch()); if (extract == null) { // Create an extract object for the dayPath extract = new Extract(this.tableType, namespace, topicName); LOG.info("Created extract: " + extract.getExtractId() + " for path " + topicName); extractMap.put(file.getWatermarkMsSinceEpoch(), extract); } return extract; }
/** * Get the writer output file path corresponding to this {@link Extract}. * * @return writer output file path corresponding to this {@link Extract} * @deprecated As {@code this.getIsFull} is deprecated. */ @Deprecated public String getOutputFilePath() { return this.getNamespace().replaceAll("\\.", "/") + "/" + this.getTable() + "/" + this.getExtractId() + "_" + (this.getIsFull() ? "full" : "append"); }
/** * Returns a unique {@link Extract} instance. * Any two calls of this method from the same {@link ExtractFactory} instance guarantees to * return {@link Extract}s with different IDs. * * @param type {@link TableType} * @param namespace dot separated namespace path * @param table table name * @return a unique {@link Extract} instance */ public synchronized Extract getUniqueExtract(TableType type, String namespace, String table) { Extract newExtract = new Extract(type, namespace, table); while (this.createdInstances.contains(newExtract)) { if (Strings.isNullOrEmpty(newExtract.getExtractId())) { newExtract.setExtractId(this.dtf.print(new DateTime())); } else { DateTime extractDateTime = this.dtf.parseDateTime(newExtract.getExtractId()); newExtract.setExtractId(this.dtf.print(extractDateTime.plusSeconds(1))); } } this.createdInstances.add(newExtract); return newExtract; } }
/** * Create a new properly populated {@link Extract} instance. * * <p> * This method should always return a new unique {@link Extract} instance. * </p> * * @param type {@link org.apache.gobblin.source.workunit.Extract.TableType} * @param namespace namespace of the table this extract belongs to * @param table name of the table this extract belongs to * @return a new unique {@link Extract} instance * * @Deprecated Use {@link org.apache.gobblin.source.extractor.extract.AbstractSource#createExtract( *org.apache.gobblin.source.workunit.Extract.TableType, String, String)} */ @Deprecated public synchronized Extract createExtract(Extract.TableType type, String namespace, String table) { Extract extract = new Extract(this, type, namespace, table); while (EXTRACT_SET.contains(extract)) { if (Strings.isNullOrEmpty(extract.getExtractId())) { extract.setExtractId(DTF.print(new DateTime())); } else { DateTime extractDateTime = DTF.parseDateTime(extract.getExtractId()); extract.setExtractId(DTF.print(extractDateTime.plusSeconds(1))); } } EXTRACT_SET.add(extract); return extract; }