/** * Return length of the fragment. * In the UNKNOWN_LENGTH case get FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH from the configuration. * * @param conf Tajo system property * @param fragment Fragment * @return */ public static long guessFragmentVolume(TajoConf conf, Fragment fragment) { if (fragment.getLength() == TajoConstants.UNKNOWN_LENGTH) { return conf.getLongVar(TajoConf.ConfVars.FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH); } else { return fragment.getLength(); } }
/** * Return length of the fragment. * In the UNKNOWN_LENGTH case get FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH from the configuration. * * @param conf Tajo system property * @param fragment Fragment * @return */ public static long guessFragmentVolume(TajoConf conf, Fragment fragment) { if (fragment.getLength() == TajoConstants.UNKNOWN_LENGTH) { return conf.getLongVar(TajoConf.ConfVars.FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH); } else { return fragment.getLength(); } }
@Override public long getTableVolume(TableDesc table, Optional<EvalNode> filter) { long totalVolume; try { totalVolume = getSplits("", table, false, filter.orElse(null)).stream() .map(f -> f.getLength()) .filter(size -> size > 0) // eliminate unknown sizes (-1) .reduce(0L, Long::sum); } catch (TajoException e) { throw new TajoRuntimeException(e); } catch (Throwable ioe) { throw new TajoInternalError(ioe); } return totalVolume; }
public DelimitedTextFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final Fragment fragment) throws IOException { super(conf, schema, meta, fragment); reader = new DelimitedLineReader(conf, this.fragment, conf.getInt(READ_BUFFER_SIZE, 128 * StorageUnit.KB)); if (!reader.isCompressed()) { splittable = true; } startOffset = this.fragment.getStartKey(); endOffset = startOffset + fragment.getLength(); errorTorrenceMaxNum = Integer.parseInt(meta.getOption(TEXT_ERROR_TOLERANCE_MAXNUM, DEFAULT_TEXT_ERROR_TOLERANCE_MAXNUM)); }
/** * Test for getSplit. * * @throws Exception */ @Test public void testGetSplit() throws Exception { TableMeta meta = CatalogUtil.newTableMeta("KAFKA", new TajoConf()); Map<String, String> option = new java.util.HashMap<String, String>(); option.put(KafkaStorageConstants.KAFKA_TOPIC, TOPIC_NAME); option.put(KafkaStorageConstants.KAFKA_FRAGMENT_SIZE, "10"); meta.setPropertySet(new KeyValueSet(option)); TableDesc td = new TableDesc("test_table", null, meta, null); KafkaTablespace kafkaTablespace = TablespaceManager.getByName("cluster1"); List<Fragment> fragmentList = kafkaTablespace.getSplits("", td, false, null); long totalCount = 0; for (int i = 0; i < fragmentList.size(); i++) { totalCount += fragmentList.get(i).getLength(); } assertEquals(100, totalCount); } }
assertTrue(fragment.getLength() <= fileStatus.getBlockSize());