/** * Return length of the fragment. * In the UNKNOWN_LENGTH case get FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH from the configuration. * * @param conf Tajo system property * @param fragment Fragment * @return */ public static long guessFragmentVolume(TajoConf conf, Fragment fragment) { if (fragment.getLength() == TajoConstants.UNKNOWN_LENGTH) { return conf.getLongVar(TajoConf.ConfVars.FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH); } else { return fragment.getLength(); } }
public void updateAssignedFragments(String tableId, Fragment[] fragments) { fragmentMap.remove(tableId); for(Fragment t : fragments) { if (fragmentMap.containsKey(t.getTableName())) { fragmentMap.get(t.getTableName()).add(t.getProto()); } else { List<FragmentProto> frags = new ArrayList<FragmentProto>(); frags.add(t.getProto()); fragmentMap.put(t.getTableName(), frags); } } }
public Object clone() throws CloneNotSupportedException { HBaseFragment frag = (HBaseFragment) super.clone(); frag.hbaseTableName = hbaseTableName; frag.last = last; return frag; }
public static List<FragmentProto> toFragmentProtoList(Fragment... fragments) { List<FragmentProto> list = Lists.newArrayList(); if (fragments == null) { return list; } for (Fragment fragment : fragments) { list.add(fragment.getProto()); } return list; }
public static FragmentProto toFragmentProto(Configuration conf, Fragment fragment) { FragmentProto.Builder fragmentBuilder = FragmentProto.newBuilder(); fragmentBuilder.setId(fragment.getInputSourceId()); fragmentBuilder.setKind(fragment.getKind()); fragmentBuilder.setContents(getFragmentSerde(conf, fragment.getKind()).serialize(fragment).toByteString()); return fragmentBuilder.build(); }
private void addDataLocation(Fragment fragment) { ImmutableList<String> hosts = fragment.getHostNames(); Integer[] diskIds = null; if (fragment instanceof FileFragment) { diskIds = ((FileFragment)fragment).getDiskIds(); } for (int i = 0; i < hosts.size(); i++) { dataLocations.add(new DataLocation(hosts.get(i), diskIds == null ? DataLocation.UNKNOWN_VOLUME_ID : diskIds[i])); } }
@Override public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, @Nullable Schema target) throws IOException { if (!(fragment instanceof JdbcFragment)) { throw new TajoInternalError("fragment must be JdbcFragment"); } if (target == null) { target = schema; } Scanner scanner; if (fragment.isEmpty()) { scanner = new NullScanner(conf, schema, meta, fragment); } else { scanner = new PgSQLJdbcScanner(getDatabaseMetaData(), connProperties, schema, meta, (JdbcFragment) fragment); } scanner.setTarget(target.toArray()); return scanner; }
public void addFragment(Fragment fragment, boolean useDataLocation) { Set<FragmentProto> fragmentProtos; if (fragMap.containsKey(fragment.getInputSourceId())) { fragmentProtos = fragMap.get(fragment.getInputSourceId()); } else { fragmentProtos = new HashSet<>(); fragMap.put(fragment.getInputSourceId(), fragmentProtos); } fragmentProtos.add(FragmentConvertor.toFragmentProto(systemConf, fragment)); if (useDataLocation) { addDataLocation(fragment); } totalFragmentNum++; }
public static List<FragmentProto> toFragmentProtoList(Fragment... fragments) { List<FragmentProto> list = Lists.newArrayList(); if (fragments == null) { return list; } for (Fragment fragment : fragments) { list.add(fragment.getProto()); } return list; }
partitions.subList(0, partitions.size() / 2).toArray(new Path[partitions.size() / 2]))); assertEquals(testCount / 2, splits.size()); assertEquals(1, splits.get(0).getHostNames().size()); assertEquals(DataLocation.UNKNOWN_VOLUME_ID, ((FileFragment)splits.get(0)).getDiskIds()[0].intValue());
/** * Returns Scanner instance. * * @param meta The table meta * @param schema The input schema * @param fragment The fragment for scanning * @param target The output schema * @return Scanner instance * @throws java.io.IOException */ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, @Nullable Schema target) throws IOException { if (target == null) { target = schema; } if (fragment.isEmpty()) { Scanner scanner = new NullScanner(conf, schema, meta, fragment); scanner.setTarget(target.toArray()); return scanner; } Scanner scanner; Class<? extends Scanner> scannerClass = getScannerClass(meta.getDataFormat()); scanner = OldStorageManager.newScannerInstance(scannerClass, conf, schema, meta, fragment); scanner.setTarget(target.toArray()); return scanner; }
public void updateAssignedFragments(String tableId, Fragment[] fragments) { fragmentMap.remove(tableId); for(Fragment t : fragments) { if (fragmentMap.containsKey(t.getInputSourceId())) { fragmentMap.get(t.getInputSourceId()).add(FragmentConvertor.toFragmentProto(getConf(), t)); } else { List<FragmentProto> frags = new ArrayList<>(); frags.add(FragmentConvertor.toFragmentProto(getConf(), t)); fragmentMap.put(t.getInputSourceId(), frags); } } }
/** * Return length of the fragment. * In the UNKNOWN_LENGTH case get FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH from the configuration. * * @param conf Tajo system property * @param fragment Fragment * @return */ public static long guessFragmentVolume(TajoConf conf, Fragment fragment) { if (fragment.getLength() == TajoConstants.UNKNOWN_LENGTH) { return conf.getLongVar(TajoConf.ConfVars.FRAGMENT_ALTERNATIVE_UNKNOWN_LENGTH); } else { return fragment.getLength(); } }
public void addFragment(Fragment fragment, boolean useDataLocation) { Set<FragmentProto> fragmentProtos; if (fragMap.containsKey(fragment.getTableName())) { fragmentProtos = fragMap.get(fragment.getTableName()); } else { fragmentProtos = new HashSet<FragmentProto>(); fragMap.put(fragment.getTableName(), fragmentProtos); } fragmentProtos.add(fragment.getProto()); if (useDataLocation) { addDataLocation(fragment); } totalFragmentNum++; }
assertEquals(2, splits.get(0).getHostNames().size()); assertEquals(2, ((FileFragment)splits.get(0)).getDiskIds().length); assertNotEquals(DataLocation.UNKNOWN_VOLUME_ID, ((FileFragment)splits.get(0)).getDiskIds()[0].intValue());
/** * Returns Scanner instance. * * @param meta The table meta * @param schema The input schema * @param fragment The fragment for scanning * @param target The output schema * @return Scanner instance * @throws java.io.IOException */ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, @Nullable Schema target) throws IOException { if (target == null) { target = schema; } if (fragment.isEmpty()) { Scanner scanner = new NullScanner(conf, schema, meta, fragment); scanner.setTarget(target.toArray()); return scanner; } Scanner scanner; Class<? extends Scanner> scannerClass = getScannerClass(meta.getDataFormat()); scanner = OldStorageManager.newScannerInstance(scannerClass, conf, schema, meta, fragment); scanner.setTarget(target.toArray()); return scanner; }
@Override public Object clone() throws CloneNotSupportedException { KafkaFragment frag = (KafkaFragment) super.clone(); frag.topicName = topicName; frag.last = last; return frag; }
@Override public long getTableVolume(TableDesc table, Optional<EvalNode> filter) { long totalVolume; try { totalVolume = getSplits("", table, false, filter.orElse(null)).stream() .map(f -> f.getLength()) .filter(size -> size > 0) // eliminate unknown sizes (-1) .reduce(0L, Long::sum); } catch (TajoException e) { throw new TajoRuntimeException(e); } catch (Throwable ioe) { throw new TajoInternalError(ioe); } return totalVolume; }
public DelimitedTextFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final Fragment fragment) throws IOException { super(conf, schema, meta, fragment); reader = new DelimitedLineReader(conf, this.fragment, conf.getInt(READ_BUFFER_SIZE, 128 * StorageUnit.KB)); if (!reader.isCompressed()) { splittable = true; } startOffset = this.fragment.getStartKey(); endOffset = startOffset + fragment.getLength(); errorTorrenceMaxNum = Integer.parseInt(meta.getOption(TEXT_ERROR_TOLERANCE_MAXNUM, DEFAULT_TEXT_ERROR_TOLERANCE_MAXNUM)); }
/** * Test for getSplit. * * @throws Exception */ @Test public void testGetSplit() throws Exception { TableMeta meta = CatalogUtil.newTableMeta("KAFKA", new TajoConf()); Map<String, String> option = new java.util.HashMap<String, String>(); option.put(KafkaStorageConstants.KAFKA_TOPIC, TOPIC_NAME); option.put(KafkaStorageConstants.KAFKA_FRAGMENT_SIZE, "10"); meta.setPropertySet(new KeyValueSet(option)); TableDesc td = new TableDesc("test_table", null, meta, null); KafkaTablespace kafkaTablespace = TablespaceManager.getByName("cluster1"); List<Fragment> fragmentList = kafkaTablespace.getSplits("", td, false, null); long totalCount = 0; for (int i = 0; i < fragmentList.size(); i++) { totalCount += fragmentList.get(i).getLength(); } assertEquals(100, totalCount); } }