Codota Logo
org.apache.storm.hdfs.spout
Code IndexAdd Codota to your IDE (free)

How to use org.apache.storm.hdfs.spout

Best Java code snippets using org.apache.storm.hdfs.spout (Showing top 20 results out of 315)

  • Add the Codota plugin to your IDE and get smart completions
private void myMethod () {
SimpleDateFormat s =
  • Codota IconString pattern;new SimpleDateFormat(pattern)
  • Codota IconString template;Locale locale;new SimpleDateFormat(template, locale)
  • Codota Iconnew SimpleDateFormat()
  • Smart code suggestions by Codota
}
origin: apache/storm

private FileLock(FileSystem fs, Path lockFile, FSDataOutputStream lockFileStream, String spoutId)
  throws IOException {
  this.fs = fs;
  this.lockFile = lockFile;
  this.lockFileStream = lockFileStream;
  this.componentID = spoutId;
  logProgress("0", false);
}
origin: apache/storm

  @Override
  public void validateField(String name, Object o) {
    HdfsSpout.checkValidReader((String) o);
  }
}
origin: apache/storm

  @Override
  public Offset clone() {
    return new Offset(charOffset, lineNumber);
  }
} //class Offset
origin: apache/storm

private static void releaseLockAndLog(FileLock fLock, String spoutId) {
  try {
    if (fLock != null) {
      fLock.release();
      LOG.debug("Spout {} released FileLock. SpoutId = {}", fLock.getLockFile(), spoutId);
    }
  } catch (IOException e) {
    LOG.error("Unable to delete lock file : " + fLock.getLockFile() + " SpoutId =" + spoutId, e);
  }
}
origin: apache/storm

  @Override
  public List<Object> next() throws IOException, ParseException {
    readAttempts++;
    if (readAttempts == 3 || readAttempts == 4) {
      throw new IOException("mock test exception");
    } else if (readAttempts > 5) {
      throw new ParseException("mock test exception", null);
    }
    return super.next();
  }
}
origin: apache/storm

private void markFileAsDone(Path filePath) {
  try {
    Path newFile = renameCompletedFile(reader.getFilePath());
    LOG.info("Completed processing {}. Spout Id = {}", newFile, spoutId);
  } catch (IOException e) {
    LOG.error("Unable to archive completed file" + filePath + " Spout ID " + spoutId, e);
  }
  closeReaderAndResetTrackers();
}
origin: apache/storm

private void openSpout(HdfsSpout spout, int spoutId, Map<String, Object> topoConf) {
  MockCollector collector = new MockCollector();
  spout.open(topoConf, new MockTopologyContext(spoutId, topoConf), collector);
}
origin: apache/storm

public synchronized void recordAckedOffset(FileOffset newOffset) {
  if (newOffset == null) {
    return;
  }
  offsets.add(newOffset);
  FileOffset currHead = offsets.first();
  if (currHead.isNextOffset(newOffset)) { // check is a minor optimization
    trimHead();
  }
}
origin: apache/storm

private static String getFileProgress(FileReader reader) {
  return reader.getFilePath() + " " + reader.getFileOffset();
}
origin: apache/storm

private DirLockingThread[] startThreads(int thdCount, Path dir)
  throws IOException {
  DirLockingThread[] result = new DirLockingThread[thdCount];
  for (int i = 0; i < thdCount; i++) {
    result[i] = new DirLockingThread(i, fs, dir);
  }
  for (DirLockingThread thd : result) {
    thd.start();
  }
  return result;
}
origin: apache/storm

private FileDeletionThread[] startThreads(int thdCount, Path file)
  throws IOException {
  FileDeletionThread[] result = new FileDeletionThread[thdCount];
  for (int i = 0; i < thdCount; i++) {
    result[i] = new FileDeletionThread(i, fs, file);
  }
  for (FileDeletionThread thd : result) {
    thd.start();
  }
  return result;
}
origin: apache/storm

@Override
public Offset clone() {
  return new Offset(lastSyncPoint, recordsSinceLastSync, currentRecord, currRecordEndOffset, prevRecordEndOffset);
}
origin: apache/storm

public Offset getFileOffset() {
  return offset.clone();
}
origin: apache/storm

  @Override
  public void close() throws Exception {
    spout.close();
  }
}
origin: apache/storm

@Override
public List<Integer> emit(String streamId, List<Object> tuple, Object messageId) {
  return emit(tuple, messageId);
}
origin: apache/storm

  @SuppressWarnings("deprecation")
  @Test
  public void testBad() {
    verifyBad(Configs.READER_TYPE, "SomeString");
    verifyBad(Configs.HDFS_URI, 100);
    verifyBad(Configs.COMMIT_FREQ_COUNT, -10);
  }
}
origin: apache/storm

private synchronized void trimHead() {
  if (offsets.size() <= 1) {
    return;
  }
  FileOffset head = offsets.first();
  FileOffset head2 = offsets.higher(head);
  if (head.isNextOffset(head2)) {
    offsets.pollFirst();
    trimHead();
  }
  return;
}
origin: apache/storm

public TextFileReader(FileSystem fs, Path file, Map<String, Object> conf) throws IOException {
  this(fs, file, conf, new TextFileReader.Offset(0, 0));
}
origin: apache/storm

public void heartbeat(String fileOffset) throws IOException {
  logProgress(fileOffset, true);
}
origin: apache/storm

public TextFileReader(FileSystem fs, Path file, Map<String, Object> conf, String startOffset) throws IOException {
  this(fs, file, conf, new TextFileReader.Offset(startOffset));
}
org.apache.storm.hdfs.spout

Most used classes

  • HdfsSpout
  • TextFileReader
  • DirLock
    Facility to synchronize access to HDFS directory. The lock itself is represented as a file in the sa
  • FileLock
    Facility to synchronize access to HDFS files. Thread gains exclusive access to a file by acquiring a
  • ParseException
  • TextFileReader$Offset,
  • ConfigsTest,
  • FileLock$LogEntry,
  • FileOffset,
  • FileReader,
  • HdfsSpout$MessageId,
  • HdfsSpout$RenameException,
  • HdfsSpoutTopology$ConstBolt,
  • HdfsSpoutTopology,
  • SequenceFileReader$Offset,
  • SequenceFileReader,
  • TestDirLock$DirLockingThread,
  • TestFileLock$FileLockingThread,
  • TestFileLock
Codota Logo
  • Products

    Search for Java codeSearch for JavaScript codeEnterprise
  • IDE Plugins

    IntelliJ IDEAWebStormAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimAtomGoLandRubyMineEmacsJupyter
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogCodota Academy Plugin user guide Terms of usePrivacy policyJava Code IndexJavascript Code Index
Get Codota for your IDE now