public TableMeta getTableMeta(Path tablePath) throws IOException { TableMeta meta; FileSystem fs = tablePath.getFileSystem(conf); Path tableMetaPath = new Path(tablePath, ".meta"); if (!fs.exists(tableMetaPath)) { throw new FileNotFoundException(".meta file not found in " + tablePath.toString()); } FSDataInputStream tableMetaIn = fs.open(tableMetaPath); CatalogProtos.TableProto tableProto = (CatalogProtos.TableProto) FileUtil.loadProto(tableMetaIn, CatalogProtos.TableProto.getDefaultInstance()); meta = new TableMeta(tableProto); return meta; }
public int hashCode() { return Objects.hashCode(getDataFormat(), getPropertySet()); }
public CSVLineDeserializer(Schema schema, TableMeta meta, Column [] projected) { super(schema, meta); targetColumnIndexes = PlannerUtil.getTargetIds(schema, projected); // The quote char must be a single ASCII character. hasQuoteChar = meta.containsProperty(StorageConstants.QUOTE_CHAR); quoteChar = meta.getProperty(StorageConstants.QUOTE_CHAR, "\0").getBytes()[0]; }
public void mergeProtoToLocal() { getDataFormat(); toMap(); } }
public int hashCode() { return Objects.hashCode(getDataFormat(), getOptions()); }
table.setDbName(databaseName); table.setTableName(tableName); table.setParameters(new HashMap<>(tableDesc.getMeta().getPropertySet().getAllKeyValus())); if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.RCFILE)) { StorageFormatDescriptor descriptor = storageFormatFactory.get(IOConstants.RCFILE); sd.setInputFormat(descriptor.getInputFormat()); sd.setOutputFormat(descriptor.getOutputFormat()); String serde = tableDesc.getMeta().getProperty(StorageConstants.RCFILE_SERDE); if (StorageConstants.DEFAULT_TEXT_SERDE.equals(serde)) { sd.getSerdeInfo().setSerializationLib(ColumnarSerDe.class.getName()); if (tableDesc.getMeta().getPropertySet().containsKey(StorageConstants.RCFILE_NULL)) { table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getProperty(StorageConstants.RCFILE_NULL))); } else if (tableDesc.getMeta().getDataFormat().equals(BuiltinStorages.TEXT)) { String fieldDelimiter = tableDesc.getMeta().getProperty(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); if (tableDesc.getMeta().containsProperty(StorageConstants.TEXT_NULL)) { table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getProperty(StorageConstants.TEXT_NULL))); table.getParameters().remove(StorageConstants.TEXT_NULL); } else if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { StorageFormatDescriptor descriptor = storageFormatFactory.get(IOConstants.SEQUENCEFILE);
@Override public URI getTableUri(TableMeta meta, String databaseName, String tableName) { String tablespaceUriString = uri.toASCIIString(); String tablePath = meta.getProperty(PATH); if (!tablespaceUriString.endsWith("/") && !tablePath.startsWith("/")) { tablePath = "/" + tablePath; } return URI.create(tablespaceUriString + tablePath); }
tableMeta = (TableMeta) meta.clone(); if (!tableMeta.containsProperty(StorageConstants.TEXT_DELIMITER)) { tableMeta.putProperty(StorageConstants.TEXT_DELIMITER, tableMeta.getProperty(StorageConstants .SEQUENCEFILE_DELIMITER)); if (!tableMeta.containsProperty(StorageConstants.TEXT_NULL) && tableMeta.containsProperty(StorageConstants .SEQUENCEFILE_NULL)) { tableMeta.putProperty(StorageConstants.TEXT_NULL, tableMeta.getProperty(StorageConstants.SEQUENCEFILE_NULL)); String delim = tableMeta.getProperty(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); this.delimiter = StringEscapeUtils.unescapeJava(delim).charAt(0); String nullCharacters = StringEscapeUtils.unescapeJava(tableMeta.getProperty(StorageConstants.TEXT_NULL, NullDatum.DEFAULT_TEXT)); if (StringUtils.isEmpty(nullCharacters)) { String serdeClass = tableMeta.getProperty(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName()); serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance();
public static Schema getAvroSchema(TableMeta meta, Configuration conf) throws IOException { boolean isSchemaLiteral = meta.containsOption(StorageConstants.AVRO_SCHEMA_LITERAL); boolean isSchemaUrl = meta.containsOption(StorageConstants.AVRO_SCHEMA_URL); if (!isSchemaLiteral && !isSchemaUrl) { throw new RuntimeException("No Avro schema for table."); } if (isSchemaLiteral) { String schema = meta.getOption(StorageConstants.AVRO_SCHEMA_LITERAL); return new Schema.Parser().parse(schema); } String schemaURL = meta.getOption(StorageConstants.AVRO_SCHEMA_URL); Path schemaPath = new Path(schemaURL); FileSystem fs = schemaPath.getFileSystem(conf); FSDataInputStream inputStream = fs.open(schemaPath); return new Schema.Parser().parse(inputStream); } }
/** * Set nullChar to TableMeta according to data format * * @param meta TableMeta * @param nullChar A character for NULL representation */ private static void setNullCharForTextSerializer(TableMeta meta, String nullChar) { String dataFormat = meta.getDataFormat(); if (dataFormat.equalsIgnoreCase(BuiltinStorages.TEXT)) { meta.putProperty(StorageConstants.TEXT_NULL, nullChar); } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) { meta.putProperty(StorageConstants.RCFILE_NULL, nullChar); } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { meta.putProperty(StorageConstants.TEXT_NULL, nullChar); } }
/** * Check if TableMeta contains NULL char property according to data format * * @param meta Table Meta * @return True if TableMeta contains NULL char property according to data format */ public static boolean containsNullChar(TableMeta meta) { String dataFormat = meta.getDataFormat(); if (dataFormat.equalsIgnoreCase(BuiltinStorages.TEXT)) { return meta.containsProperty(StorageConstants.TEXT_NULL); } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) { return meta.containsProperty(StorageConstants.RCFILE_NULL); } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { if (!meta.containsProperty(StorageConstants.TEXT_NULL) && meta.containsProperty(StorageConstants.SEQUENCEFILE_NULL)) { return meta.containsProperty(StorageConstants.SEQUENCEFILE_NULL); } else { return meta.containsProperty(StorageConstants.TEXT_NULL); } } else { return false; } }
/** * Check if TableMeta contains NULL char property according to data format * * @param meta Table Meta * @return True if TableMeta contains NULL char property according to data format */ public static boolean containsNullChar(TableMeta meta) { String dataFormat = meta.getDataFormat(); if (dataFormat.equalsIgnoreCase(BuiltinStorages.TEXT)) { return meta.containsOption(StorageConstants.TEXT_NULL); } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.RCFILE)) { return meta.containsOption(StorageConstants.RCFILE_NULL); } else if (dataFormat.equalsIgnoreCase(BuiltinStorages.SEQUENCE_FILE)) { return meta.containsOption(StorageConstants.SEQUENCEFILE_NULL); } else { return false; } }
public static ByteBuf getNullChars(TableMeta meta) { byte[] nullCharByteArray; if (meta.getDataFormat().equals(BuiltinStorages.SEQUENCE_FILE)) { nullCharByteArray = getNullCharsAsBytes(meta, StorageConstants.SEQUENCEFILE_NULL, "\\"); } else { nullCharByteArray = getNullCharsAsBytes(meta); } ByteBuf nullChars = BufferPool.directBuffer(nullCharByteArray.length, nullCharByteArray.length); nullChars.writeBytes(nullCharByteArray); return nullChars; }
private List<HBaseFragment> getRawSplits(String inputSourceId, TableDesc table, @Nullable EvalNode filterCondition) throws IOException, TajoException { final ColumnMapping columnMapping = new ColumnMapping(table.getSchema(), table.getMeta().getPropertySet()); try (final HTable htable = new HTable(hbaseConf, table.getMeta().getProperty(HBaseStorageConstants.META_TABLE_KEY))) { final RegionSizeCalculator sizeCalculator = new RegionSizeCalculator(htable); final org.apache.hadoop.hbase.util.Pair<byte[][], byte[][]> tableRange = htable.getStartEndKeys(); if (isEmptyRegion(tableRange)) { return createEmptyFragment(table, inputSourceId, htable, sizeCalculator); } final Pair<List<byte []>, List<byte []>> selectedRange = getSelectedKeyRange( columnMapping, getIndexPredications(columnMapping, table, filterCondition)); // region startkey -> HBaseFragment List<HBaseFragment> fragments = new ArrayList<>(convertRangeToFragment(table, inputSourceId, htable, sizeCalculator, tableRange, selectedRange)); Collections.sort(fragments); if (!fragments.isEmpty()) { fragments.get(fragments.size() - 1).setLast(true); } return ImmutableList.copyOf(fragments); } }
public Map<String,String> toMap() { return getPropertySet().getAllKeyValus(); }
@Test public void testIncompleteQuote() throws IOException, CloneNotSupportedException { TableMeta meta = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, conf); meta.putProperty(StorageUtil.TEXT_DELIMITER, ","); meta.putProperty(StorageUtil.QUOTE_CHAR, "\""); FileFragment fragment = getFileFragment("testIncompleteQuote.txt"); Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment, null); scanner.init(); Tuple tuple; int i = 0; while ((tuple = scanner.next()) != null) { assertEquals("(f,hyunsik\",NULL,NULL,NULL,NULL,0.0,\"hyunsik,hyunsik)", tuple.toString()); i++; } assertEquals(1, i); scanner.close(); }
/** * If there is no table property for the propertyKey, set default property from system conf to the table. * * @param conf TajoConf * @param confVarKey system variable key * @param meta TableMeta * @param propertyKey table property key */ private static void setProperty(TajoConf conf, TajoConf.ConfVars confVarKey, TableMeta meta, String propertyKey) { if (!meta.containsProperty(propertyKey)) { meta.putProperty(propertyKey, conf.getVar(confVarKey)); } } }
@Override public void init() throws IOException { // Buffer size, Delimiter this.bufSize = DEFAULT_BUFFER_SIZE; String delim = meta.getOption(DELIMITER, DELIMITER_DEFAULT); this.delimiter = delim.charAt(0); super.init(); }
public Map<String,String> toMap() { return getOptions().getAllKeyValus(); }
private List<HBaseFragment> getRawSplits(String inputSourceId, TableDesc table, @Nullable EvalNode filterCondition) throws IOException, TajoException { final ColumnMapping columnMapping = new ColumnMapping(table.getSchema(), table.getMeta().getOptions()); try (final HTable htable = new HTable(hbaseConf, table.getMeta().getOption(HBaseStorageConstants.META_TABLE_KEY))) { final RegionSizeCalculator sizeCalculator = new RegionSizeCalculator(htable); final org.apache.hadoop.hbase.util.Pair<byte[][], byte[][]> tableRange = htable.getStartEndKeys(); if (isEmptyRegion(tableRange)) { return createEmptyFragment(table, inputSourceId, htable, sizeCalculator); } final Pair<List<byte []>, List<byte []>> selectedRange = getSelectedKeyRange( columnMapping, getIndexPredications(columnMapping, table, filterCondition)); // region startkey -> HBaseFragment List<HBaseFragment> fragments = new ArrayList<>(convertRangeToFragment(table, inputSourceId, htable, sizeCalculator, tableRange, selectedRange)); Collections.sort(fragments); if (!fragments.isEmpty()) { fragments.get(fragments.size() - 1).setLast(true); } return ImmutableList.copyOf(fragments); } }