private void initInnerTapIfNotSet(Object target, String hadoopTypeName) { if (actualTap != null) { return; } Class<?> clz = null; try { clz = Class.forName(hadoopTypeName, false, getClass().getClassLoader()); if (clz.isInstance(target)) { runningInHadoop = true; } } catch (ClassNotFoundException e) { runningInHadoop = false; } actualTap = (runningInHadoop ? new EsHadoopTap(host, port, resource, query, fields, props) : new EsLocalTap(host, port, resource, query, fields, props)); setScheme(actualTap.getScheme()); if (log.isDebugEnabled()) { log.debug(String.format("Detected %s environment; initializing [%s]", (runningInHadoop ? "Hadoop" : "local"), actualTap.getClass().getSimpleName())); } // use SLF4J just like Cascading if (!logVersion) { logVersion = true; Logger esTapLogger = LoggerFactory.getLogger(EsTap.class); esTapLogger.info(String.format("Elasticsearch Hadoop %s initialized", Version.version())); esTapLogger.warn("ES-Hadoop Cascading Integration is Deprecated as of 6.6.0 and will be removed in a later release."); } } }
@Override public Scheme<Config, Input, Output, ?, ?> getScheme() { return original.getScheme(); }
/** * Method isSink returns true if this Tap instance can be used as a sink. * * @return boolean */ public boolean isSink() { return getScheme().isSink(); }
public HadoopTupleEntrySchemeIterator( FlowProcess<? extends Configuration> flowProcess, Tap parentTap, RecordReader recordReader ) throws IOException { this( flowProcess, parentTap, parentTap.getScheme(), makeIterator( flowProcess, parentTap, recordReader ) ); }
/** * A hook for allowing a Scheme to lazily retrieve its source fields. * * @param flowProcess of type FlowProcess * @return the found Fields */ public Fields retrieveSourceFields( FlowProcess<? extends Config> flowProcess ) { return getScheme().retrieveSourceFields( flowProcess, this ); }
/** * Method isSource returns true if this Tap instance can be used as a source. * * @return boolean */ public boolean isSource() { return getScheme().isSource(); }
/** * Method getSourceFields returns the sourceFields of this Tap object. * * @return the sourceFields (type Fields) of this Tap object. */ public Fields getSourceFields() { return getScheme().getSourceFields(); }
@Override public Scheme getScheme() { Scheme scheme = super.getScheme(); if( scheme != null ) return scheme; return taps[ 0 ].getScheme(); // they should all be equivalent per verifyTaps }
public HadoopTupleEntrySchemeIterator( FlowProcess<? extends Configuration> flowProcess, Tap parentTap, RecordReader recordReader ) throws IOException { this( flowProcess, parentTap, parentTap.getScheme(), makeIterator( flowProcess, parentTap, recordReader ) ); }
@Override public String toString() { if( getIdentifier() != null ) return getClass().getSimpleName() + "[\"" + original.getScheme() + "\"]" + "[\"" + Util.sanitizeUrl( getIdentifier() ) + "\"]"; // sanitize else return getClass().getSimpleName() + "[\"" + original.getScheme() + "\"]" + "[not initialized]"; } }
private static int hash( FlowElement flowElement ) { int lhs = flowElement.getClass().getName().hashCode(); int rhs = 0; if( flowElement instanceof Operator && ( (Operator) flowElement ).getOperation() != null ) rhs = ( (Operator) flowElement ).getOperation().getClass().getName().hashCode(); else if( flowElement instanceof Tap && ( (Tap) flowElement ).getScheme() != null ) rhs = ( (Tap) flowElement ).getScheme().getClass().getName().hashCode(); else if( flowElement instanceof Splice ) rhs = ( (Splice) flowElement ).getJoiner().getClass().getName().hashCode() + 31 * ( (Splice) flowElement ).getNumSelfJoins(); return pair( lhs, rhs ); }
private void verifyTaps() { Tap tap = taps[ 0 ]; for( int i = 1; i < taps.length; i++ ) { if( tap.getClass() != taps[ i ].getClass() ) throw new TapException( "all taps must be of the same type" ); if( !tap.getScheme().equals( taps[ i ].getScheme() ) ) throw new TapException( "all tap schemes must be equivalent" ); } }
protected BasePartitionTap( Tap parent, Partition partition, SinkMode sinkMode, boolean keepParentOnDelete, int openWritesThreshold ) { super( new PartitionScheme( parent.getScheme(), partition.getPartitionFields() ), sinkMode ); this.parent = parent; this.partition = partition; this.keepParentOnDelete = keepParentOnDelete; this.openWritesThreshold = openWritesThreshold; }
public HadoopTupleEntrySchemeCollector( FlowProcess<? extends Configuration> flowProcess, Tap<Configuration, RecordReader, OutputCollector> tap ) throws IOException { super( flowProcess, tap, tap.getScheme(), makeCollector( flowProcess, tap, null ), tap.getIdentifier() ); }
protected BasePartitionTap( Tap parent, Partition partition, SinkMode sinkMode ) { super( new PartitionScheme( parent.getScheme(), partition.getPartitionFields() ), sinkMode ); this.parent = parent; this.partition = partition; }
private static String createMessage( Tap tap, Fields incomingFields, Fields selectorFields ) { String message = "unable to resolve scheme sink selector: " + selectorFields.printVerbose() + ", with incoming: " + incomingFields.printVerbose(); return TraceUtil.formatTrace( tap.getScheme(), message ); } }
@Test public void testCoGroupAroundCoGroupWith() throws Exception { // hack to get classname runCoGroupAroundCoGroup( getPlatform().getDelimitedFile( new Fields( "num" ), "\t", inputFileNums10 ).getScheme().getClass(), "cogroupacogroupopt2" ); }