@Override public Iterator<String> iterator() { try { return new FileLineIterator(is, encoding, skipFirstLine, this.origFilename); } catch (IOException ioe) { throw new IllegalStateException(ioe); } }
@Override public void close() throws IOException { endOfData(); Closeables.close(reader, true); }
/** Creates a {@link FileLineIterable} over a given file, using the given encoding. */ public FileLineIterable(File file, Charset encoding, boolean skipFirstLine) throws IOException { this(FileLineIterator.getFileInputStream(file), encoding, skipFirstLine); }
protected void processFileWithoutID(FileLineIterator dataOrUpdateFileIterator, FastByIDMap<FastIDSet> data, FastByIDMap<FastByIDMap<Long>> timestamps) { log.info("Reading file info..."); int count = 0; while (dataOrUpdateFileIterator.hasNext()) { String line = dataOrUpdateFileIterator.next(); if (!line.isEmpty()) { processLineWithoutID(line, data, timestamps); if (++count % 100000 == 0) { log.info("Processed {} lines", count); } } } log.info("Read lines: {}", count); }
this.lastUpdateFileModified = readLastUpdateFileModified(); FileLineIterator iterator = new FileLineIterator(dataFile, false); String firstLine = iterator.peek(); while (firstLine.isEmpty() || firstLine.charAt(0) == COMMENT_CHAR) { iterator.next(); firstLine = iterator.peek();
/** * Read in a dictionary file. Format is: First line is the number of entries * * <pre> * term DocFreq Index * </pre> */ private static String[] loadTermDictionary(InputStream is) throws IOException { FileLineIterator it = new FileLineIterator(is); int numEntries = Integer.parseInt(it.next()); String[] result = new String[numEntries]; while (it.hasNext()) { String line = it.next(); if (line.startsWith("#")) { continue; } String[] tokens = VectorHelper.TAB_PATTERN.split(line); if (tokens.length < 3) { continue; } int index = Integer.parseInt(tokens[2]); // tokens[1] is the doc freq result[index] = tokens[0]; } return result; } }
@Override public void skip(int n) { try { for (int i = 0; i < n; i++) { if (reader.readLine() == null) { break; } } } catch (IOException ioe) { try { close(); } catch (IOException e) { throw new IllegalStateException(e); } } }
protected void processFileWithoutID(FileLineIterator dataOrUpdateFileIterator, FastByIDMap<FastIDSet> data, FastByIDMap<FastByIDMap<Long>> timestamps) { log.info("Reading file info..."); int count = 0; while (dataOrUpdateFileIterator.hasNext()) { String line = dataOrUpdateFileIterator.next(); if (!line.isEmpty()) { processLineWithoutID(line, data, timestamps); if (++count % 100000 == 0) { log.info("Processed {} lines", count); } } } log.info("Read lines: {}", count); }
this.lastUpdateFileModified = readLastUpdateFileModified(); FileLineIterator iterator = new FileLineIterator(dataFile, false); String firstLine = iterator.peek(); while (firstLine.isEmpty() || firstLine.charAt(0) == COMMENT_CHAR) { iterator.next(); firstLine = iterator.peek();
@Override public void skip(int n) { try { for (int i = 0; i < n; i++) { if (reader.readLine() == null) { break; } } } catch (IOException ioe) { try { close(); } catch (IOException e) { throw new IllegalStateException(e); } } }
protected void processFileWithoutID(FileLineIterator dataOrUpdateFileIterator, FastByIDMap<FastIDSet> data, FastByIDMap<FastByIDMap<Long>> timestamps) { log.info("Reading file info..."); int count = 0; while (dataOrUpdateFileIterator.hasNext()) { String line = dataOrUpdateFileIterator.next(); if (!line.isEmpty()) { processLineWithoutID(line, data, timestamps); if (++count % 100000 == 0) { log.info("Processed {} lines", count); } } } log.info("Read lines: {}", count); }
this.lastUpdateFileModified = readLastUpdateFileModified(); FileLineIterator iterator = new FileLineIterator(dataFile, false); String firstLine = iterator.peek(); while (firstLine.isEmpty() || firstLine.charAt(0) == COMMENT_CHAR) { iterator.next(); firstLine = iterator.peek();
@Override public Iterator<String> iterator() { try { return new FileLineIterator(is, encoding, skipFirstLine, this.origFilename); } catch (IOException ioe) { throw new IllegalStateException(ioe); } }
/** Creates a {@link FileLineIterable} over a given file, using the given encoding. */ public FileLineIterable(File file, Charset encoding, boolean skipFirstLine) throws IOException { this(FileLineIterator.getFileInputStream(file), encoding, skipFirstLine); }
@Override public void close() throws IOException { endOfData(); Closeables.close(reader, true); }
@Override public void skip(int n) { try { for (int i = 0; i < n; i++) { if (reader.readLine() == null) { break; } } } catch (IOException ioe) { try { close(); } catch (IOException e) { throw new IllegalStateException(e); } } }
protected void processFile(FileLineIterator dataOrUpdateFileIterator, FastByIDMap<?> data, FastByIDMap<FastByIDMap<Long>> timestamps, boolean fromPriorData) { log.info("Reading file info..."); int count = 0; while (dataOrUpdateFileIterator.hasNext()) { String line = dataOrUpdateFileIterator.next(); if (!line.isEmpty()) { processLine(line, data, timestamps, fromPriorData); if (++count % 1000000 == 0) { log.info("Processed {} lines", count); } } } log.info("Read lines: {}", count); }