@Override public boolean nextKeyValue() { return carbonIterator.hasNext(); }
private boolean internalHasNext() { if (firstTime) { firstTime = false; currentIterator.initialize(); } boolean hasNext = currentIterator.hasNext(); // If iterator is finished then check for next iterator. if (!hasNext) { currentIterator.close(); // Check next iterator is available in the list. if (counter < inputIterators.size()) { // Get the next iterator from the list. currentIterator = inputIterators.get(counter++); // Initialize the new iterator currentIterator.initialize(); hasNext = internalHasNext(); } } return hasNext; }
@Override public void close() { if (!closed) { super.close(); for (CarbonIterator inputIterator : inputIterators) { inputIterator.close(); } } }
columns[i] = csvHeader.get(i).getColName(); optionsFinal.put("fileheader", Strings.mkString(columns, ",")); Maps.getOrDefault(options, "sort_scope", CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT)); CarbonLoadModel model = new CarbonLoadModel(); model.setCarbonTransactionalTable(table.isTransactionalTable()); model.setUseOnePass(Boolean.parseBoolean(Maps.getOrDefault(options, "onepass", "false"))); model.setDictionaryServerHost(Maps.getOrDefault(options, "dicthost", null)); try { model.setDictionaryServerPort(Integer.parseInt(Maps.getOrDefault(options, "dictport", "-1"))); } catch (NumberFormatException e) { throw new InvalidLoadOptionException(e.getMessage());
LoggerAction.REDIRECT.name().equalsIgnoreCase(bad_records_action)) { if (!StringUtils.isEmpty(bad_record_path)) { bad_record_path = CarbonUtil.checkAndAppendHDFSUrl(bad_record_path); } else { throw new InvalidLoadOptionException( "Cannot redirect bad records as bad record location is not provided."); if (!headerOption.equalsIgnoreCase("true") && !headerOption.equalsIgnoreCase("false")) { throw new InvalidLoadOptionException( "'header' option should be either 'true' or 'false'."); throw new InvalidLoadOptionException( "When 'header' option is true, 'fileheader' option is not required."); fileHeader = Strings.mkString(columnNames.toArray(new String[columnNames.size()]), ","); complex_delimeter_level1.equalsIgnoreCase(complex_delimeter_level2) || delimeter.equalsIgnoreCase(complex_delimeter_level2)) { throw new InvalidLoadOptionException("Field Delimiter and Complex types delimiter are same"); } else { carbonLoadModel.setComplexDelimiterLevel1(complex_delimeter_level1);
Map<String, String> options) throws InvalidLoadOptionException { Map<String, String> optionsFinal = new HashMap<>(); optionsFinal.put("delimiter", Maps.getOrDefault(options, "delimiter", ",")); optionsFinal.put("quotechar", Maps.getOrDefault(options, "quotechar", "\"")); optionsFinal.put("fileheader", Maps.getOrDefault(options, "fileheader", "")); optionsFinal.put("commentchar", Maps.getOrDefault(options, "commentchar", "#")); optionsFinal.put("columndict", Maps.getOrDefault(options, "columndict", null)); CarbonLoaderUtil.getEscapeChar(Maps.getOrDefault(options,"escapechar", "\\"))); Maps.getOrDefault(options, "serialization_null_format", "\\N")); Maps.getOrDefault( options, "bad_records_logger_enable", Maps.getOrDefault( options, "bad_records_action", Maps.getOrDefault( options, "is_empty_data_bad_record", Maps.getOrDefault( options, "skip_empty_line", Maps.getOrDefault(options, "all_dictionary_path", ""));
/** * Returns {@code true} if the iteration has more elements. (In other words, * returns {@code true} if {@link #next} would return an element rather than * throwing an exception.) * * @return {@code true} if the iteration has more elements */ @Override public boolean hasNext() { if (currentChunk != null && currentChunk.hasNext()) { return true; } else if (iterator != null && iterator.hasNext()) { currentChunk = iterator.next(); return hasNext(); } return false; }
private boolean internalHasNext() { if (firstTime) { firstTime = false; currentIterator.initialize(); } boolean hasNext = currentIterator.hasNext(); // If iterator is finished then check for next iterator. if (!hasNext) { currentIterator.close(); // Check next iterator is available in the list. if (counter < inputIterators.size()) { // Get the next iterator from the list. currentIterator = inputIterators.get(counter++); // Initialize the new iterator currentIterator.initialize(); hasNext = internalHasNext(); } } return hasNext; }
@Override public void close() { if (!closed) { super.close(); for (CarbonIterator inputIterator : inputIterators) { inputIterator.close(); } } }
@Override public T getCurrentValue() throws IOException, InterruptedException { rowCount += 1; if (null != inputMetricsStats) { inputMetricsStats.incrementRecordRead(1L); } return readSupport.readRow(carbonIterator.next()); }
@Override public void close() { if (!closed) { super.close(); if (null != executorService) { executorService.shutdownNow(); } for (CarbonIterator inputIterator : inputIterators) { inputIterator.close(); } } }
private CarbonRowBatch getBatch() { // Create batch and fill it. CarbonRowBatch carbonRowBatch = new CarbonRowBatch(batchSize); int count = 0; while (internalHasNext() && count < batchSize) { carbonRowBatch.addRow( new CarbonRow(convertToNoDictionaryToBytes(currentIterator.next(), dataFields))); count++; } rowCounter.getAndAdd(carbonRowBatch.getSize()); return carbonRowBatch; }
queryIterator.close(); } catch (Throwable e) { exceptionOccurred = e;
private CarbonRowBatch getBatch() { // Create batch and fill it. CarbonRowBatch carbonRowBatch = new CarbonRowBatch(batchSize); int count = 0; if (isRawDataRequired) { while (internalHasNext() && count < batchSize) { Object[] rawRow = currentIterator.next(); carbonRowBatch.addRow(new CarbonRow(rowParser.parseRow(rawRow), rawRow)); count++; } } else { while (internalHasNext() && count < batchSize) { carbonRowBatch.addRow(new CarbonRow(rowParser.parseRow(currentIterator.next()))); count++; } } rowCounter.getAndAdd(carbonRowBatch.getSize()); return carbonRowBatch; } }
@Override public void close() throws IOException { logStatistics(rowCount, queryModel.getStatisticsRecorder()); // clear dictionary cache Map<String, Dictionary> columnToDictionaryMapping = queryModel.getColumnToDictionaryMapping(); if (null != columnToDictionaryMapping) { for (Map.Entry<String, Dictionary> entry : columnToDictionaryMapping.entrySet()) { CarbonUtil.clearDictionaryCache(entry.getValue()); } } if (!skipClearDataMapAtClose) { // Clear the datamap cache DataMapStoreManager.getInstance().clearDataMaps( queryModel.getTable().getAbsoluteTableIdentifier()); } // close read support readSupport.close(); carbonIterator.close(); try { queryExecutor.finish(); } catch (QueryExecutionException e) { throw new IOException(e); } }